Add shrink paths (#14208)

(cherry picked from commit a5db6399ad)

account-decoder/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-account-decoder"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -18,11 +18,11 @@ lazy_static = "1.4.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.16" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.16" }
-spl-token-v2-0 = { package = "spl-token", version = "=3.0.0", features = ["no-entrypoint"] }
+solana-config-program = { path = "../programs/config", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-stake-program = { path = "../programs/stake", version = "1.4.19" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.19" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.0.1", features = ["no-entrypoint"] }
 thiserror = "1.0"
 zstd = "0.5.1"
 

accounts-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,11 +11,11 @@ publish = false
 [dependencies]
 log = "0.4.6"
 rayon = "1.4.0"
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-runtime = { path = "../runtime", version = "1.4.16" }
-solana-measure = { path = "../measure", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
+solana-measure = { path = "../measure", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

accounts-bench/src/main.rs

@@ -98,7 +98,10 @@ fn main() {
         } else {
             let mut pubkeys: Vec<Pubkey> = vec![];
             let mut time = Measure::start("hash");
-            let hash = accounts.accounts_db.update_accounts_hash(0, &ancestors).0;
+            let hash = accounts
+                .accounts_db
+                .update_accounts_hash(0, &ancestors, true)
+                .0;
             time.stop();
             println!("hash: {} {}", hash, time);
             create_test_accounts(&accounts, &mut pubkeys, 1, 0);

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,16 +14,16 @@ crossbeam-channel = "0.4"
 log = "0.4.6"
 rand = "0.7.0"
 rayon = "1.4.0"
-solana-core = { path = "../core", version = "1.4.16" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-streamer = { path = "../streamer", version = "1.4.16" }
-solana-perf = { path = "../perf", version = "1.4.16" }
-solana-ledger = { path = "../ledger", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-runtime = { path = "../runtime", version = "1.4.16" }
-solana-measure = { path = "../measure", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
+solana-core = { path = "../core", version = "1.4.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-streamer = { path = "../streamer", version = "1.4.19" }
+solana-perf = { path = "../perf", version = "1.4.19" }
+solana-ledger = { path = "../ledger", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
+solana-measure = { path = "../measure", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banks-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-client"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,15 +12,16 @@ edition = "2018"
 async-trait = "0.1.36"
 bincode = "1.3.1"
 futures = "0.3"
-solana-banks-interface = { path = "../banks-interface", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
+mio = "0.7.6"
+solana-banks-interface = { path = "../banks-interface", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
 tarpc = { version = "0.23.0", features = ["full"] }
 tokio = { version = "0.3", features = ["full"] }
 tokio-serde = { version = "0.6", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "1.4.16" }
-solana-banks-server = { path = "../banks-server", version = "1.4.16" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
+solana-banks-server = { path = "../banks-server", version = "1.4.19" }
 
 [lib]
 crate-type = ["lib"]

banks-interface/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-interface"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,8 +9,9 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
+mio = "0.7.6"
 serde = { version = "1.0.112", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
 tarpc = { version = "0.23.0", features = ["full"] }
 
 [lib]

banks-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-banks-server"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,10 +12,11 @@ edition = "2018"
 bincode = "1.3.1"
 futures = "0.3"
 log = "0.4.8"
-solana-banks-interface = { path = "../banks-interface", version = "1.4.16" }
-solana-runtime = { path = "../runtime", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-metrics = { path = "../metrics", version = "1.4.16" }
+mio = "0.7.6"
+solana-banks-interface = { path = "../banks-interface", version = "1.4.19" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-metrics = { path = "../metrics", version = "1.4.19" }
 tarpc = { version = "0.23.0", features = ["full"] }
 tokio = { version = "0.3", features = ["full"] }
 tokio-serde = { version = "0.6", features = ["bincode"] }

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -18,21 +18,21 @@ rand = "0.7.0"
 rayon = "1.4.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-core = { path = "../core", version = "1.4.16" }
-solana-genesis = { path = "../genesis", version = "1.4.16" }
-solana-client = { path = "../client", version = "1.4.16" }
-solana-faucet = { path = "../faucet", version = "1.4.16" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-metrics = { path = "../metrics", version = "1.4.16" }
-solana-net-utils = { path = "../net-utils", version = "1.4.16" }
-solana-runtime = { path = "../runtime", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-core = { path = "../core", version = "1.4.19" }
+solana-genesis = { path = "../genesis", version = "1.4.19" }
+solana-client = { path = "../client", version = "1.4.19" }
+solana-faucet = { path = "../faucet", version = "1.4.19" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-metrics = { path = "../metrics", version = "1.4.19" }
+solana-net-utils = { path = "../net-utils", version = "1.4.19" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.4.16" }
+solana-local-cluster = { path = "../local-cluster", version = "1.4.19" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-streamer = { path = "../streamer", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-net-utils = { path = "../net-utils", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-streamer = { path = "../streamer", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-net-utils = { path = "../net-utils", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,23 +15,23 @@ log = "0.4.8"
 rayon = "1.4.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-core = { path = "../core", version = "1.4.16" }
-solana-genesis = { path = "../genesis", version = "1.4.16" }
-solana-client = { path = "../client", version = "1.4.16" }
-solana-faucet = { path = "../faucet", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-metrics = { path = "../metrics", version = "1.4.16" }
-solana-measure = { path = "../measure", version = "1.4.16" }
-solana-net-utils = { path = "../net-utils", version = "1.4.16" }
-solana-runtime = { path = "../runtime", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-core = { path = "../core", version = "1.4.19" }
+solana-genesis = { path = "../genesis", version = "1.4.19" }
+solana-client = { path = "../client", version = "1.4.19" }
+solana-faucet = { path = "../faucet", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-metrics = { path = "../metrics", version = "1.4.19" }
+solana-measure = { path = "../measure", version = "1.4.19" }
+solana-net-utils = { path = "../net-utils", version = "1.4.19" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.4.16" }
+solana-local-cluster = { path = "../local-cluster", version = "1.4.19" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
 thiserror = "1.0.20"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli-output/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -17,13 +17,13 @@ indicatif = "0.15.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.4.16" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-client = { path = "../client", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.16" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.16" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.16" }
+solana-account-decoder = { path = "../account-decoder", version = "1.4.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-client = { path = "../client", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-stake-program = { path = "../programs/stake", version = "1.4.19" }
+solana-transaction-status = { path = "../transaction-status", version = "1.4.19" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.19" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

cli-output/src/cli_output.rs

@@ -592,7 +592,7 @@ pub struct CliEpochReward {
     pub amount: u64,       // lamports
     pub post_balance: u64, // lamports
     pub percent_change: f64,
-    pub apr: f64,
+    pub apr: Option<f64>,
 }
 
 fn show_epoch_rewards(
@@ -613,13 +613,16 @@ fn show_epoch_rewards(
         for reward in epoch_rewards {
             writeln!(
                 f,
-                "  {:<8}  {:<11}  ◎{:<14.9}  ◎{:<14.9}  {:>13.9}%  {:>13.9}%",
+                "  {:<8}  {:<11}  ◎{:<14.9}  ◎{:<14.9}  {:>13.9}%  {}",
                 reward.epoch,
                 reward.effective_slot,
                 lamports_to_sol(reward.amount),
                 lamports_to_sol(reward.post_balance),
                 reward.percent_change,
-                reward.apr,
+                reward
+                    .apr
+                    .map(|apr| format!("{:>13.9}%", apr))
+                    .unwrap_or_default(),
             )?;
         }
     }

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -27,30 +27,30 @@ reqwest = { version = "0.10.8", default-features = false, features = ["blocking"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.4.16" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.4.16" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-cli-config = { path = "../cli-config", version = "1.4.16" }
-solana-cli-output = { path = "../cli-output", version = "1.4.16" }
-solana-client = { path = "../client", version = "1.4.16" }
-solana-config-program = { path = "../programs/config", version = "1.4.16" }
-solana-faucet = { path = "../faucet", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-net-utils = { path = "../net-utils", version = "1.4.16" }
+solana-account-decoder = { path = "../account-decoder", version = "1.4.19" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.4.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-cli-config = { path = "../cli-config", version = "1.4.19" }
+solana-cli-output = { path = "../cli-output", version = "1.4.19" }
+solana-client = { path = "../client", version = "1.4.19" }
+solana-config-program = { path = "../programs/config", version = "1.4.19" }
+solana-faucet = { path = "../faucet", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-net-utils = { path = "../net-utils", version = "1.4.19" }
 solana_rbpf = "=0.1.34"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.16" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.16" }
-solana-vote-signer = { path = "../vote-signer", version = "1.4.16" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-stake-program = { path = "../programs/stake", version = "1.4.19" }
+solana-transaction-status = { path = "../transaction-status", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.19" }
+solana-vote-signer = { path = "../vote-signer", version = "1.4.19" }
 thiserror = "1.0.20"
 tiny-bip39 = "0.7.0"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.4.16" }
+solana-core = { path = "../core", version = "1.4.19" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cli.rs

@@ -154,6 +154,9 @@ pub enum CliCommand {
         limit: usize,
         show_transactions: bool,
     },
+    WaitForMaxStake {
+        max_stake_percent: f32,
+    },
     // Nonce commands
     AuthorizeNonceAccount {
         nonce_account: Pubkey,
@@ -503,6 +506,7 @@ impl CliConfig<'_> {
         config.commitment = CommitmentConfig::recent();
         config.send_transaction_config = RpcSendTransactionConfig {
             skip_preflight: true,
+            preflight_commitment: Some(CommitmentConfig::recent().commitment),
             ..RpcSendTransactionConfig::default()
         };
         config
@@ -630,6 +634,13 @@ pub fn parse_command(
                 signers,
             })
         }
+        ("wait-for-max-stake", Some(matches)) => {
+            let max_stake_percent = value_t_or_exit!(matches, "max_percent", f32);
+            Ok(CliCommandInfo {
+                command: CliCommand::WaitForMaxStake { max_stake_percent },
+                signers: vec![],
+            })
+        }
         // Stake Commands
         ("create-stake-account", Some(matches)) => {
             parse_stake_create_account(matches, default_signer, wallet_manager)
@@ -1405,6 +1416,7 @@ fn do_process_deploy(
             config.commitment,
             RpcSendTransactionConfig {
                 skip_preflight: true,
+                preflight_commitment: Some(config.commitment.commitment),
                 ..RpcSendTransactionConfig::default()
             },
         )
@@ -1583,6 +1595,9 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             *use_lamports_unit,
             vote_account_pubkeys.as_deref(),
         ),
+        CliCommand::WaitForMaxStake { max_stake_percent } => {
+            process_wait_for_max_stake(&rpc_client, config, *max_stake_percent)
+        }
         CliCommand::ShowValidators { use_lamports_unit } => {
             process_show_validators(&rpc_client, config, *use_lamports_unit)
         }

cli/src/cluster_query.rs

@@ -351,6 +351,17 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .help("Display the full transactions"),
                 )
         )
+        .subcommand(
+            SubCommand::with_name("wait-for-max-stake")
+                .about("Wait for the max stake of any one node to drop below a percentage of total.")
+                .arg(
+                    Arg::with_name("max_percent")
+                        .long("max-percent")
+                        .value_name("PERCENT")
+                        .takes_value(true)
+                        .index(1),
+                ),
+        )
     }
 }
 
@@ -789,8 +800,9 @@ pub fn process_get_block(
                     format!(
                         "◎{:<19.9}  {:>13.9}%",
                         lamports_to_sol(reward.post_balance),
-                        reward.lamports.abs() as f64
-                            / (reward.post_balance as f64 - reward.lamports as f64)
+                        (reward.lamports.abs() as f64
+                            / (reward.post_balance as f64 - reward.lamports as f64))
+                            * 100.0
                     )
                 }
             );
@@ -1533,6 +1545,16 @@ pub fn process_show_stakes(
         .formatted_string(&CliStakeVec::new(stake_accounts)))
 }
 
+pub fn process_wait_for_max_stake(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    max_stake_percent: f32,
+) -> ProcessResult {
+    let now = std::time::Instant::now();
+    rpc_client.wait_for_max_stake(config.commitment, max_stake_percent)?;
+    Ok(format!("Done waiting, took: {}s", now.elapsed().as_secs()))
+}
+
 pub fn process_show_validators(
     rpc_client: &RpcClient,
     config: &CliConfig,

cli/src/stake.rs

@@ -1647,29 +1647,37 @@ pub(crate) fn fetch_epoch_rewards(
         let previous_epoch_rewards = first_confirmed_block.rewards;
 
         if let Some((effective_slot, epoch_end_time, epoch_rewards)) = epoch_info {
-            let wallclock_epoch_duration =
-                { Local.timestamp(epoch_end_time, 0) - Local.timestamp(epoch_start_time, 0) }
-                    .to_std()?
-                    .as_secs_f64();
-
-            let wallclock_epochs_per_year =
-                (SECONDS_PER_DAY * 356) as f64 / wallclock_epoch_duration;
+            let wallclock_epoch_duration = if epoch_end_time > epoch_start_time {
+                Some(
+                    { Local.timestamp(epoch_end_time, 0) - Local.timestamp(epoch_start_time, 0) }
+                        .to_std()?
+                        .as_secs_f64(),
+                )
+            } else {
+                None
+            };
 
             if let Some(reward) = epoch_rewards
                 .into_iter()
                 .find(|reward| reward.pubkey == address.to_string())
             {
                 if reward.post_balance > reward.lamports.try_into().unwrap_or(0) {
-                    let balance_increase_percent = reward.lamports.abs() as f64
+                    let rate_change = reward.lamports.abs() as f64
                         / (reward.post_balance as f64 - reward.lamports as f64);
 
+                    let apr = wallclock_epoch_duration.map(|wallclock_epoch_duration| {
+                        let wallclock_epochs_per_year =
+                            (SECONDS_PER_DAY * 356) as f64 / wallclock_epoch_duration;
+                        rate_change * wallclock_epochs_per_year
+                    });
+
                     all_epoch_rewards.push(CliEpochReward {
                         epoch,
                         effective_slot,
                         amount: reward.lamports.abs() as u64,
                         post_balance: reward.post_balance,
-                        percent_change: balance_increase_percent,
-                        apr: balance_increase_percent * wallclock_epochs_per_year,
+                        percent_change: rate_change * 100.0,
+                        apr: apr.map(|r| r * 100.0),
                     });
                 }
             }

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -16,19 +16,20 @@ clap = "2.33.0"
 indicatif = "0.15.0"
 jsonrpc-core = "15.0.0"
 log = "0.4.8"
+net2 = "0.2.37"
 rayon = "1.4.0"
 reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
 semver = "0.11.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.4.16" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-net-utils = { path = "../net-utils", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.16" }
+solana-account-decoder = { path = "../account-decoder", version = "1.4.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-net-utils = { path = "../net-utils", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-transaction-status = { path = "../transaction-status", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.19" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -37,7 +38,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "15.0.0"
 jsonrpc-http-server = "15.0.0"
-solana-logger = { path = "../logger", version = "1.4.16" }
+solana-logger = { path = "../logger", version = "1.4.19" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/rpc_client.rs

@@ -384,6 +384,38 @@ impl RpcClient {
         self.send(RpcRequest::GetVoteAccounts, json!([commitment_config]))
     }
 
+    pub fn wait_for_max_stake(
+        &self,
+        commitment: CommitmentConfig,
+        max_stake_percent: f32,
+    ) -> ClientResult<()> {
+        let mut current_percent;
+        loop {
+            let vote_accounts = self.get_vote_accounts_with_commitment(commitment)?;
+
+            let mut max = 0;
+            let total_active_stake = vote_accounts
+                .current
+                .iter()
+                .chain(vote_accounts.delinquent.iter())
+                .map(|vote_account| {
+                    max = std::cmp::max(max, vote_account.activated_stake);
+                    vote_account.activated_stake
+                })
+                .sum::<u64>();
+            current_percent = 100f32 * max as f32 / total_active_stake as f32;
+            if current_percent < max_stake_percent {
+                break;
+            }
+            info!(
+                "Waiting for stake to drop below {} current: {:.1}",
+                max_stake_percent, current_percent
+            );
+            sleep(Duration::from_secs(10));
+        }
+        Ok(())
+    }
+
     pub fn get_cluster_nodes(&self) -> ClientResult<Vec<RpcContactInfo>> {
         self.send(RpcRequest::GetClusterNodes, Value::Null)
     }

client/src/rpc_custom_error.rs

@@ -10,6 +10,7 @@ pub const JSON_RPC_SERVER_ERROR_TRANSACTION_SIGNATURE_VERIFICATION_FAILURE: i64
 pub const JSON_RPC_SERVER_ERROR_BLOCK_NOT_AVAILABLE: i64 = -32004;
 pub const JSON_RPC_SERVER_ERROR_NODE_UNHEALTHLY: i64 = -32005;
 pub const JSON_RPC_SERVER_ERROR_TRANSACTION_PRECOMPILE_VERIFICATION_FAILURE: i64 = -32006;
+pub const JSON_RPC_SERVER_ERROR_SLOT_SKIPPED: i64 = -32007;
 
 pub enum RpcCustomError {
     BlockCleanedUp {
@@ -26,6 +27,9 @@ pub enum RpcCustomError {
     },
     RpcNodeUnhealthy,
     TransactionPrecompileVerificationFailure(solana_sdk::transaction::TransactionError),
+    SlotSkipped {
+        slot: Slot,
+    },
 }
 
 impl From<RpcCustomError> for Error {
@@ -73,6 +77,14 @@ impl From<RpcCustomError> for Error {
                 message: format!("Transaction precompile verification failure {:?}", e),
                 data: None,
             },
+            RpcCustomError::SlotSkipped { slot } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_SLOT_SKIPPED),
+                message: format!(
+                    "Slot {} was skipped, or missing due to ledger jump to recent snapshot",
+                    slot
+                ),
+                data: None,
+            },
         }
     }
 }

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.16"
+version = "1.4.19"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -36,6 +36,8 @@ jsonrpc-pubsub = "15.0.0"
 jsonrpc-ws-server = "15.0.0"
 log = "0.4.8"
 lru = "0.6.0"
+miow = "0.2.2"
+net2 = "0.2.37"
 num_cpus = "1.13.0"
 num-traits = "0.2"
 rand = "0.7.0"
@@ -46,38 +48,40 @@ regex = "1.3.9"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-account-decoder = { path = "../account-decoder", version = "1.4.16" }
-solana-banks-server = { path = "../banks-server", version = "1.4.16" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-client = { path = "../client", version = "1.4.16" }
-solana-faucet = { path = "../faucet", version = "1.4.16" }
-solana-frozen-abi = { path = "../frozen-abi", version = "1.4.16" }
-solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "1.4.16" }
-solana-ledger = { path = "../ledger", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.4.16" }
-solana-metrics = { path = "../metrics", version = "1.4.16" }
-solana-measure = { path = "../measure", version = "1.4.16" }
-solana-net-utils = { path = "../net-utils", version = "1.4.16" }
-solana-perf = { path = "../perf", version = "1.4.16" }
-solana-runtime = { path = "../runtime", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.16" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "1.4.16" }
-solana-streamer = { path = "../streamer", version = "1.4.16" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.4.16" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.16" }
-solana-vote-signer = { path = "../vote-signer", version = "1.4.16" }
-spl-token-v2-0 = { package = "spl-token", version = "=3.0.0", features = ["no-entrypoint"] }
+solana-account-decoder = { path = "../account-decoder", version = "1.4.19" }
+solana-banks-server = { path = "../banks-server", version = "1.4.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-client = { path = "../client", version = "1.4.19" }
+solana-faucet = { path = "../faucet", version = "1.4.19" }
+solana-frozen-abi = { path = "../frozen-abi", version = "1.4.19" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "1.4.19" }
+solana-ledger = { path = "../ledger", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.4.19" }
+solana-metrics = { path = "../metrics", version = "1.4.19" }
+solana-measure = { path = "../measure", version = "1.4.19" }
+solana-net-utils = { path = "../net-utils", version = "1.4.19" }
+solana-perf = { path = "../perf", version = "1.4.19" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-stake-program = { path = "../programs/stake", version = "1.4.19" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.4.19" }
+solana-streamer = { path = "../streamer", version = "1.4.19" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.4.19" }
+solana-transaction-status = { path = "../transaction-status", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.19" }
+solana-vote-signer = { path = "../vote-signer", version = "1.4.19" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.0.1", features = ["no-entrypoint"] }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = { version = "0.2", features = ["full"] }
 tokio_01 = { version = "0.1", package = "tokio" }
+tokio_01_bytes = { version = "0.4.7", package = "bytes" }
 tokio_fs_01 = { version = "0.1", package = "tokio-fs" }
 tokio_io_01 = { version = "0.1", package = "tokio-io" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.4.16" }
+tokio_codec_01 = { version = "0.1", package = "tokio-codec" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.4.19" }
 trees = "0.2.1"
 
 [dev-dependencies]

core/src/accounts_hash_verifier.rs

@@ -254,6 +254,9 @@ mod tests {
                 0,
                 100,
             );
+            // sleep for 1ms to create a newer timestmap for gossip entry
+            // otherwise the timestamp won't be newer.
+            std::thread::sleep(Duration::from_millis(1));
         }
         cluster_info.flush_push_queue();
         let cluster_hashes = cluster_info

core/src/banking_stage.rs

@@ -53,7 +53,7 @@ type PacketsAndOffsets = (Packets, Vec<usize>);
 pub type UnprocessedPackets = Vec<PacketsAndOffsets>;
 
 /// Transaction forwarding
-pub const FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET: u64 = 1;
+pub const FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET: u64 = 2;
 
 // Fixed thread size seems to be fastest on GCP setup
 pub const NUM_THREADS: u32 = 4;

core/src/lib.rs

@@ -42,6 +42,7 @@ pub mod local_vote_signer_service;
 pub mod non_circulating_supply;
 pub mod optimistic_confirmation_verifier;
 pub mod optimistically_confirmed_bank_tracker;
+pub mod packet_hasher;
 pub mod ping_pong;
 pub mod poh_recorder;
 pub mod poh_service;

core/src/packet_hasher.rs

@@ -0,0 +1,34 @@
+// Get a unique hash value for a packet
+// Used in retransmit and shred fetch to prevent dos with same packet data.
+
+use ahash::AHasher;
+use rand::{thread_rng, Rng};
+use solana_perf::packet::Packet;
+use std::hash::Hasher;
+
+#[derive(Clone)]
+pub struct PacketHasher {
+    seed1: u128,
+    seed2: u128,
+}
+
+impl Default for PacketHasher {
+    fn default() -> Self {
+        Self {
+            seed1: thread_rng().gen::<u128>(),
+            seed2: thread_rng().gen::<u128>(),
+        }
+    }
+}
+
+impl PacketHasher {
+    pub fn hash_packet(&self, packet: &Packet) -> u64 {
+        let mut hasher = AHasher::new_with_keys(self.seed1, self.seed2);
+        hasher.write(&packet.data[0..packet.meta.size]);
+        hasher.finish()
+    }
+
+    pub fn reset(&mut self) {
+        *self = Self::default();
+    }
+}

core/src/replay_stage.rs

@@ -2542,10 +2542,10 @@ pub(crate) mod tests {
 
     #[test]
     fn test_replay_commitment_cache() {
-        fn leader_vote(bank: &Arc<Bank>, pubkey: &Pubkey) {
+        fn leader_vote(vote_slot: Slot, bank: &Arc<Bank>, pubkey: &Pubkey) {
             let mut leader_vote_account = bank.get_account(&pubkey).unwrap();
             let mut vote_state = VoteState::from(&leader_vote_account).unwrap();
-            vote_state.process_slot_vote_unchecked(bank.slot());
+            vote_state.process_slot_vote_unchecked(vote_slot);
             let versioned = VoteStateVersions::Current(Box::new(vote_state));
             VoteState::to(&versioned, &mut leader_vote_account).unwrap();
             bank.store_account(&pubkey, &leader_vote_account);
@@ -2565,10 +2565,7 @@ pub(crate) mod tests {
         }
         bank0.freeze();
         let arc_bank0 = Arc::new(bank0);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new_from_banks(
-            &[arc_bank0.clone()],
-            0,
-        )));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new_from_banks(&[arc_bank0], 0)));
 
         let exit = Arc::new(AtomicBool::new(false));
         let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
@@ -2592,44 +2589,33 @@ pub(crate) mod tests {
             .get_block_commitment(1)
             .is_none());
 
-        let bank1 = Bank::new_from_parent(&arc_bank0, &Pubkey::default(), arc_bank0.slot() + 1);
-        let _res = bank1.transfer(
-            10,
-            &genesis_config_info.mint_keypair,
-            &solana_sdk::pubkey::new_rand(),
-        );
-        for _ in 0..genesis_config.ticks_per_slot {
-            bank1.register_tick(&Hash::default());
+        for i in 1..=3 {
+            let prev_bank = bank_forks.read().unwrap().get(i - 1).unwrap().clone();
+            let bank = Bank::new_from_parent(&prev_bank, &Pubkey::default(), prev_bank.slot() + 1);
+            let _res = bank.transfer(
+                10,
+                &genesis_config_info.mint_keypair,
+                &solana_sdk::pubkey::new_rand(),
+            );
+            for _ in 0..genesis_config.ticks_per_slot {
+                bank.register_tick(&Hash::default());
+            }
+            bank_forks.write().unwrap().insert(bank);
+            let arc_bank = bank_forks.read().unwrap().get(i).unwrap().clone();
+            leader_vote(i - 1, &arc_bank, &leader_voting_pubkey);
+            ReplayStage::update_commitment_cache(
+                arc_bank.clone(),
+                0,
+                leader_lamports,
+                &lockouts_sender,
+            );
+            arc_bank.freeze();
         }
-        bank1.freeze();
-        bank_forks.write().unwrap().insert(bank1);
-        let arc_bank1 = bank_forks.read().unwrap().get(1).unwrap().clone();
-        leader_vote(&arc_bank1, &leader_voting_pubkey);
-        ReplayStage::update_commitment_cache(
-            arc_bank1.clone(),
-            0,
-            leader_lamports,
-            &lockouts_sender,
-        );
 
-        let bank2 = Bank::new_from_parent(&arc_bank1, &Pubkey::default(), arc_bank1.slot() + 1);
-        let _res = bank2.transfer(
-            10,
-            &genesis_config_info.mint_keypair,
-            &solana_sdk::pubkey::new_rand(),
-        );
-        for _ in 0..genesis_config.ticks_per_slot {
-            bank2.register_tick(&Hash::default());
-        }
-        bank2.freeze();
-        bank_forks.write().unwrap().insert(bank2);
-        let arc_bank2 = bank_forks.read().unwrap().get(2).unwrap().clone();
-        leader_vote(&arc_bank2, &leader_voting_pubkey);
-        ReplayStage::update_commitment_cache(arc_bank2, 0, leader_lamports, &lockouts_sender);
         thread::sleep(Duration::from_millis(200));
 
         let mut expected0 = BlockCommitment::default();
-        expected0.increase_confirmation_stake(2, leader_lamports);
+        expected0.increase_confirmation_stake(3, leader_lamports);
         assert_eq!(
             block_commitment_cache
                 .read()

core/src/retransmit_stage.rs

@@ -1,7 +1,5 @@
 //! The `retransmit_stage` retransmits shreds between validators
 
-use crate::shred_fetch_stage::ShredFetchStage;
-use crate::shred_fetch_stage::ShredFetchStats;
 use crate::{
     cluster_info::{compute_retransmit_peers, ClusterInfo, DATA_PLANE_FANOUT},
     cluster_info_vote_listener::VerifiedVoteReceiver,
@@ -14,10 +12,9 @@ use crate::{
     result::{Error, Result},
     window_service::{should_retransmit_and_persist, WindowService},
 };
-use ahash::AHasher;
 use crossbeam_channel::Receiver;
 use lru::LruCache;
-use rand::{thread_rng, Rng};
+use solana_ledger::shred::{get_shred_slot_index_type, ShredFetchStats};
 use solana_ledger::{
     blockstore::{Blockstore, CompletedSlotsReceiver},
     leader_schedule_cache::LeaderScheduleCache,
@@ -25,14 +22,13 @@ use solana_ledger::{
 };
 use solana_measure::measure::Measure;
 use solana_metrics::inc_new_counter_error;
-use solana_perf::packet::Packets;
+use solana_perf::packet::{Packet, Packets};
 use solana_runtime::bank_forks::BankForks;
 use solana_sdk::clock::{Epoch, Slot};
 use solana_sdk::epoch_schedule::EpochSchedule;
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::timing::timestamp;
 use solana_streamer::streamer::PacketReceiver;
-use std::hash::Hasher;
 use std::{
     cmp,
     collections::hash_set::HashSet,
@@ -189,7 +185,42 @@ struct EpochStakesCache {
     stakes_and_index: Vec<(u64, usize)>,
 }
 
-pub type ShredFilterAndSeeds = (LruCache<(Slot, u32), Vec<u64>>, u128, u128);
+use crate::packet_hasher::PacketHasher;
+// Map of shred (slot, index, is_data) => list of hash values seen for that key.
+pub type ShredFilter = LruCache<(Slot, u32, bool), Vec<u64>>;
+
+pub type ShredFilterAndHasher = (ShredFilter, PacketHasher);
+
+// Return true if shred is already received and should skip retransmit
+fn check_if_already_received(
+    packet: &Packet,
+    shreds_received: &Arc<Mutex<ShredFilterAndHasher>>,
+) -> bool {
+    match get_shred_slot_index_type(packet, &mut ShredFetchStats::default()) {
+        Some(slot_index) => {
+            let mut received = shreds_received.lock().unwrap();
+            let hasher = received.1.clone();
+            if let Some(sent) = received.0.get_mut(&slot_index) {
+                if sent.len() < MAX_DUPLICATE_COUNT {
+                    let hash = hasher.hash_packet(packet);
+                    if sent.contains(&hash) {
+                        return true;
+                    }
+
+                    sent.push(hash);
+                } else {
+                    return true;
+                }
+            } else {
+                let hash = hasher.hash_packet(&packet);
+                received.0.put(slot_index, vec![hash]);
+            }
+
+            false
+        }
+        None => true,
+    }
+}
 
 #[allow(clippy::too_many_arguments)]
 fn retransmit(
@@ -202,7 +233,7 @@ fn retransmit(
     stats: &Arc<RetransmitStats>,
     epoch_stakes_cache: &Arc<RwLock<EpochStakesCache>>,
     last_peer_update: &Arc<AtomicU64>,
-    shreds_received: &Arc<Mutex<ShredFilterAndSeeds>>,
+    shreds_received: &Arc<Mutex<ShredFilterAndHasher>>,
 ) -> Result<()> {
     let timer = Duration::new(1, 0);
     let r_lock = r.lock().unwrap();
@@ -254,8 +285,7 @@ fn retransmit(
         {
             let mut sr = shreds_received.lock().unwrap();
             sr.0.clear();
-            sr.1 = thread_rng().gen::<u128>();
-            sr.2 = thread_rng().gen::<u128>();
+            sr.1.reset();
         }
     }
     let mut peers_len = 0;
@@ -282,33 +312,10 @@ fn retransmit(
                 continue;
             }
 
-            match ShredFetchStage::get_slot_index(packet, &mut ShredFetchStats::default()) {
-                Some(slot_index) => {
-                    let mut received = shreds_received.lock().unwrap();
-                    let seed1 = received.1;
-                    let seed2 = received.2;
-                    if let Some(sent) = received.0.get_mut(&slot_index) {
-                        if sent.len() < MAX_DUPLICATE_COUNT {
-                            let mut hasher = AHasher::new_with_keys(seed1, seed2);
-                            hasher.write(&packet.data[0..packet.meta.size]);
-                            let hash = hasher.finish();
-                            if sent.contains(&hash) {
-                                continue;
-                            }
-
-                            sent.push(hash);
-                        } else {
-                            continue;
-                        }
-                    } else {
-                        let mut hasher = AHasher::new_with_keys(seed1, seed2);
-                        hasher.write(&packet.data[0..packet.meta.size]);
-                        let hash = hasher.finish();
-                        received.0.put(slot_index, vec![hash]);
-                    }
-                }
-                None => continue,
+            if check_if_already_received(packet, shreds_received) {
+                continue;
             }
+
             let mut compute_turbine_peers = Measure::start("turbine_start");
             let (my_index, mut shuffled_stakes_and_index) = ClusterInfo::shuffle_peers_and_index(
                 &my_id,
@@ -397,7 +404,10 @@ pub fn retransmitter(
     r: Arc<Mutex<PacketReceiver>>,
 ) -> Vec<JoinHandle<()>> {
     let stats = Arc::new(RetransmitStats::default());
-    let shreds_received = Arc::new(Mutex::new((LruCache::new(DEFAULT_LRU_SIZE), 0, 0)));
+    let shreds_received = Arc::new(Mutex::new((
+        LruCache::new(DEFAULT_LRU_SIZE),
+        PacketHasher::default(),
+    )));
     (0..sockets.len())
         .map(|s| {
             let sockets = sockets.clone();
@@ -551,6 +561,7 @@ mod tests {
     use solana_ledger::blockstore_processor::{process_blockstore, ProcessOptions};
     use solana_ledger::create_new_tmp_ledger;
     use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use solana_ledger::shred::Shred;
     use solana_net_utils::find_available_port_in_range;
     use solana_perf::packet::{Packet, Packets};
     use std::net::{IpAddr, Ipv4Addr};
@@ -599,8 +610,7 @@ mod tests {
         );
         let _thread_hdls = vec![t_retransmit];
 
-        let mut shred =
-            solana_ledger::shred::Shred::new_from_data(0, 0, 0, None, true, true, 0, 0x20, 0);
+        let mut shred = Shred::new_from_data(0, 0, 0, None, true, true, 0, 0x20, 0);
         let mut packet = Packet::default();
         shred.copy_to_packet(&mut packet);
 
@@ -625,4 +635,52 @@ mod tests {
         assert_eq!(packets.packets.len(), 1);
         assert_eq!(packets.packets[0].meta.repair, false);
     }
+
+    #[test]
+    fn test_already_received() {
+        let mut packet = Packet::default();
+        let slot = 1;
+        let index = 5;
+        let version = 0x40;
+        let shred = Shred::new_from_data(slot, index, 0, None, true, true, 0, version, 0);
+        shred.copy_to_packet(&mut packet);
+        let shreds_received = Arc::new(Mutex::new((LruCache::new(100), PacketHasher::default())));
+        // unique shred for (1, 5) should pass
+        assert!(!check_if_already_received(&packet, &shreds_received));
+        // duplicate shred for (1, 5) blocked
+        assert!(check_if_already_received(&packet, &shreds_received));
+
+        let shred = Shred::new_from_data(slot, index, 2, None, true, true, 0, version, 0);
+        shred.copy_to_packet(&mut packet);
+        // first duplicate shred for (1, 5) passed
+        assert!(!check_if_already_received(&packet, &shreds_received));
+        // then blocked
+        assert!(check_if_already_received(&packet, &shreds_received));
+
+        let shred = Shred::new_from_data(slot, index, 8, None, true, true, 0, version, 0);
+        shred.copy_to_packet(&mut packet);
+        // 2nd duplicate shred for (1, 5) blocked
+        assert!(check_if_already_received(&packet, &shreds_received));
+        assert!(check_if_already_received(&packet, &shreds_received));
+
+        let shred = Shred::new_empty_coding(slot, index, 0, 1, 1, 0, version);
+        shred.copy_to_packet(&mut packet);
+        // Coding at (1, 5) passes
+        assert!(!check_if_already_received(&packet, &shreds_received));
+        // then blocked
+        assert!(check_if_already_received(&packet, &shreds_received));
+
+        let shred = Shred::new_empty_coding(slot, index, 2, 1, 1, 0, version);
+        shred.copy_to_packet(&mut packet);
+        // 2nd unique coding at (1, 5) passes
+        assert!(!check_if_already_received(&packet, &shreds_received));
+        // same again is blocked
+        assert!(check_if_already_received(&packet, &shreds_received));
+
+        let shred = Shred::new_empty_coding(slot, index, 3, 1, 1, 0, version);
+        shred.copy_to_packet(&mut packet);
+        // Another unique coding at (1, 5) always blocked
+        assert!(check_if_already_received(&packet, &shreds_received));
+        assert!(check_if_already_received(&packet, &shreds_received));
+    }
 }

core/src/rpc.rs

@@ -237,7 +237,7 @@ impl JsonRpcRequestProcessor {
         let cluster_info = Arc::new(ClusterInfo::default());
         let tpu_address = cluster_info.my_contact_info().tpu;
         let (sender, receiver) = channel();
-        SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
 
         Self {
             config: JsonRpcConfig::default(),
@@ -347,13 +347,12 @@ impl JsonRpcRequestProcessor {
         let bank = self.bank(None);
         let epoch = bank.epoch();
         let inflation = bank.inflation();
-        let year =
-            (bank.epoch_schedule().get_last_slot_in_epoch(epoch)) as f64 / bank.slots_per_year();
+        let slot_in_year = bank.slot_in_year_for_inflation();
 
         RpcInflationRate {
-            total: inflation.total(year),
-            validator: inflation.validator(year),
-            foundation: inflation.foundation(year),
+            total: inflation.total(slot_in_year),
+            validator: inflation.validator(slot_in_year),
+            foundation: inflation.foundation(slot_in_year),
             epoch,
         }
     }
@@ -602,7 +601,7 @@ impl JsonRpcRequestProcessor {
         }
     }
 
-    fn check_blockstore_max_root<T>(
+    fn check_blockstore_root<T>(
         &self,
         result: &std::result::Result<T, BlockstoreError>,
         slot: Slot,
@@ -613,7 +612,7 @@ impl JsonRpcRequestProcessor {
         if result.is_err() {
             let err = result.as_ref().unwrap_err();
             debug!(
-                "check_blockstore_max_root, slot: {:?}, max root: {:?}, err: {:?}",
+                "check_blockstore_root, slot: {:?}, max root: {:?}, err: {:?}",
                 slot,
                 self.blockstore.max_root(),
                 err
@@ -621,6 +620,9 @@ impl JsonRpcRequestProcessor {
             if slot >= self.blockstore.max_root() {
                 return Err(RpcCustomError::BlockNotAvailable { slot }.into());
             }
+            if self.blockstore.is_skipped(slot) {
+                return Err(RpcCustomError::SlotSkipped { slot }.into());
+            }
         }
         Ok(())
     }
@@ -663,7 +665,7 @@ impl JsonRpcRequestProcessor {
                     .highest_confirmed_root()
         {
             let result = self.blockstore.get_confirmed_block(slot);
-            self.check_blockstore_max_root(&result, slot)?;
+            self.check_blockstore_root(&result, slot)?;
             if result.is_err() {
                 if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
                     return Ok(self
@@ -769,8 +771,8 @@ impl JsonRpcRequestProcessor {
                 .highest_confirmed_root()
         {
             let result = self.blockstore.get_block_time(slot);
-            self.check_blockstore_max_root(&result, slot)?;
-            if result.is_err() {
+            self.check_blockstore_root(&result, slot)?;
+            if result.is_err() || matches!(result, Ok(None)) {
                 if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage {
                     return Ok(self
                         .runtime_handle
@@ -2320,8 +2322,13 @@ impl RpcSol for RpcSolImpl {
         let config = config.unwrap_or_default();
         let encoding = config.encoding.unwrap_or(UiTransactionEncoding::Base58);
         let (wire_transaction, transaction) = deserialize_transaction(data, encoding)?;
-        let bank = &*meta.bank(None);
-        let last_valid_slot = bank
+
+        let preflight_commitment = config
+            .preflight_commitment
+            .map(|commitment| CommitmentConfig { commitment });
+        let preflight_bank = &*meta.bank(preflight_commitment);
+
+        let last_valid_slot = preflight_bank
             .get_blockhash_last_valid_slot(&transaction.message.recent_blockhash)
             .unwrap_or(0);
 
@@ -2333,11 +2340,6 @@ impl RpcSol for RpcSolImpl {
             if meta.health.check() != RpcHealthStatus::Ok {
                 return Err(RpcCustomError::RpcNodeUnhealthy.into());
             }
-
-            let preflight_commitment = config
-                .preflight_commitment
-                .map(|commitment| CommitmentConfig { commitment });
-            let preflight_bank = &*meta.bank(preflight_commitment);
             if let (Err(err), logs) = preflight_bank.simulate_transaction(transaction.clone()) {
                 return Err(RpcCustomError::SendTransactionPreflightFailure {
                     message: format!("Transaction simulation failed: {}", err),
@@ -2899,7 +2901,7 @@ pub mod tests {
             None,
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
         );
-        SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
 
         cluster_info.insert_info(ContactInfo::new_with_pubkey_socketaddr(
             &leader_pubkey,
@@ -3307,12 +3309,11 @@ pub mod tests {
         };
         let inflation = bank.inflation();
         let epoch = bank.epoch();
-        let year =
-            (bank.epoch_schedule().get_last_slot_in_epoch(epoch)) as f64 / bank.slots_per_year();
+        let slot_in_year = bank.slot_in_year_for_inflation();
         let expected_inflation_rate = RpcInflationRate {
-            total: inflation.total(year),
-            validator: inflation.validator(year),
-            foundation: inflation.foundation(year),
+            total: inflation.total(slot_in_year),
+            validator: inflation.validator(slot_in_year),
+            foundation: inflation.foundation(slot_in_year),
             epoch,
         };
         assert_eq!(inflation_rate, expected_inflation_rate);
@@ -4300,7 +4301,7 @@ pub mod tests {
             None,
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
         );
-        SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
 
         let mut bad_transaction = system_transaction::transfer(
             &mint_keypair,
@@ -4496,7 +4497,7 @@ pub mod tests {
             None,
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
         );
-        SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
         assert_eq!(request_processor.validator_exit(), false);
         assert_eq!(exit.load(Ordering::Relaxed), false);
     }
@@ -4526,7 +4527,7 @@ pub mod tests {
             None,
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
         );
-        SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
         assert_eq!(request_processor.validator_exit(), true);
         assert_eq!(exit.load(Ordering::Relaxed), true);
     }
@@ -4615,7 +4616,7 @@ pub mod tests {
             None,
             OptimisticallyConfirmedBank::locked_from_bank_forks_root(&bank_forks),
         );
-        SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+        SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
         assert_eq!(
             request_processor.get_block_commitment(0),
             RpcBlockCommitment {

core/src/rpc_pubsub.rs

@@ -24,6 +24,8 @@ use std::{
     sync::{atomic, Arc},
 };
 
+const MAX_ACTIVE_SUBSCRIPTIONS: usize = 100_000;
+
 // Suppress needless_return due to
 //   https://github.com/paritytech/jsonrpc/blob/2d38e6424d8461cdf72e78425ce67d51af9c6586/derive/src/lib.rs#L204
 // Once https://github.com/paritytech/jsonrpc/issues/418 is resolved, try to remove this clippy allow
@@ -179,6 +181,22 @@ impl RpcSolPubSubImpl {
         let subscriptions = Arc::new(RpcSubscriptions::default_with_bank_forks(bank_forks));
         Self { uid, subscriptions }
     }
+
+    fn check_subscription_count(&self) -> Result<()> {
+        let num_subscriptions = self.subscriptions.total();
+        debug!("Total existing subscriptions: {}", num_subscriptions);
+        if num_subscriptions >= MAX_ACTIVE_SUBSCRIPTIONS {
+            info!("Node subscription limit reached");
+            Err(Error {
+                code: ErrorCode::InternalError,
+                message: "Internal Error: Subscription refused. Node subscription limit reached"
+                    .into(),
+                data: None,
+            })
+        } else {
+            Ok(())
+        }
+    }
 }
 
 fn param<T: FromStr>(param_str: &str, thing: &str) -> Result<T> {
@@ -199,6 +217,10 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
         pubkey_str: String,
         config: Option<RpcAccountInfoConfig>,
     ) {
+        if let Err(err) = self.check_subscription_count() {
+            subscriber.reject(err).unwrap_or_default();
+            return;
+        }
         match param::<Pubkey>(&pubkey_str, "pubkey") {
             Ok(pubkey) => {
                 let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
@@ -207,7 +229,7 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
                 self.subscriptions
                     .add_account_subscription(pubkey, config, sub_id, subscriber)
             }
-            Err(e) => subscriber.reject(e).unwrap(),
+            Err(e) => subscriber.reject(e).unwrap_or_default(),
         }
     }
 
@@ -235,6 +257,10 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
         pubkey_str: String,
         config: Option<RpcProgramAccountsConfig>,
     ) {
+        if let Err(err) = self.check_subscription_count() {
+            subscriber.reject(err).unwrap_or_default();
+            return;
+        }
         match param::<Pubkey>(&pubkey_str, "pubkey") {
             Ok(pubkey) => {
                 let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
@@ -243,7 +269,7 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
                 self.subscriptions
                     .add_program_subscription(pubkey, config, sub_id, subscriber)
             }
-            Err(e) => subscriber.reject(e).unwrap(),
+            Err(e) => subscriber.reject(e).unwrap_or_default(),
         }
     }
 
@@ -272,6 +298,10 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
         config: Option<RpcTransactionLogsConfig>,
     ) {
         info!("logs_subscribe");
+        if let Err(err) = self.check_subscription_count() {
+            subscriber.reject(err).unwrap_or_default();
+            return;
+        }
 
         let (address, include_votes) = match filter {
             RpcTransactionLogsFilter::All => (None, false),
@@ -281,7 +311,7 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
                     1 => match param::<Pubkey>(&addresses[0], "mentions") {
                         Ok(address) => (Some(address), false),
                         Err(e) => {
-                            subscriber.reject(e).unwrap();
+                            subscriber.reject(e).unwrap_or_default();
                             return;
                         }
                     },
@@ -294,7 +324,7 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
                                 message: "Invalid Request: Only 1 address supported".into(),
                                 data: None,
                             })
-                            .unwrap();
+                            .unwrap_or_default();
                         return;
                     }
                 }
@@ -333,6 +363,10 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
         signature_subscribe_config: Option<RpcSignatureSubscribeConfig>,
     ) {
         info!("signature_subscribe");
+        if let Err(err) = self.check_subscription_count() {
+            subscriber.reject(err).unwrap_or_default();
+            return;
+        }
         match param::<Signature>(&signature_str, "signature") {
             Ok(signature) => {
                 let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
@@ -348,7 +382,7 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
                     subscriber,
                 );
             }
-            Err(e) => subscriber.reject(e).unwrap(),
+            Err(e) => subscriber.reject(e).unwrap_or_default(),
         }
     }
 
@@ -371,6 +405,10 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
 
     fn slot_subscribe(&self, _meta: Self::Metadata, subscriber: Subscriber<SlotInfo>) {
         info!("slot_subscribe");
+        if let Err(err) = self.check_subscription_count() {
+            subscriber.reject(err).unwrap_or_default();
+            return;
+        }
         let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
         let sub_id = SubscriptionId::Number(id as u64);
         info!("slot_subscribe: id={:?}", sub_id);
@@ -392,6 +430,10 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
 
     fn vote_subscribe(&self, _meta: Self::Metadata, subscriber: Subscriber<RpcVote>) {
         info!("vote_subscribe");
+        if let Err(err) = self.check_subscription_count() {
+            subscriber.reject(err).unwrap_or_default();
+            return;
+        }
         let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
         let sub_id = SubscriptionId::Number(id as u64);
         info!("vote_subscribe: id={:?}", sub_id);
@@ -413,6 +455,10 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
 
     fn root_subscribe(&self, _meta: Self::Metadata, subscriber: Subscriber<Slot>) {
         info!("root_subscribe");
+        if let Err(err) = self.check_subscription_count() {
+            subscriber.reject(err).unwrap_or_default();
+            return;
+        }
         let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
         let sub_id = SubscriptionId::Number(id as u64);
         info!("root_subscribe: id={:?}", sub_id);

core/src/rpc_service.rs

@@ -17,6 +17,7 @@ use jsonrpc_http_server::{
 };
 use regex::Regex;
 use solana_ledger::blockstore::Blockstore;
+use solana_metrics::inc_new_counter_info;
 use solana_runtime::{
     bank_forks::{BankForks, SnapshotConfig},
     commitment::BlockCommitmentCache,
@@ -61,7 +62,7 @@ impl RpcRequestMiddleware {
         Self {
             ledger_path,
             snapshot_archive_path_regex: Regex::new(
-                r"/snapshot-\d+-[[:alnum:]]+\.tar\.(bz2|zst|gz)$",
+                r"/snapshot-\d+-[[:alnum:]]+\.(tar|tar\.bz2|tar\.zst|tar\.gz)$",
             )
             .unwrap(),
             snapshot_config,
@@ -85,6 +86,7 @@ impl RpcRequestMiddleware {
             .unwrap()
     }
 
+    #[allow(dead_code)]
     fn internal_server_error() -> hyper::Response<hyper::Body> {
         hyper::Response::builder()
             .status(hyper::StatusCode::INTERNAL_SERVER_ERROR)
@@ -112,27 +114,42 @@ impl RpcRequestMiddleware {
         let stem = path.split_at(1).1; // Drop leading '/' from path
         let filename = {
             match path {
-                "/genesis.tar.bz2" => self.ledger_path.join(stem),
-                _ => self
-                    .snapshot_config
-                    .as_ref()
-                    .unwrap()
-                    .snapshot_package_output_path
-                    .join(stem),
+                "/genesis.tar.bz2" => {
+                    inc_new_counter_info!("rpc-get_genesis", 1);
+                    self.ledger_path.join(stem)
+                }
+                _ => {
+                    inc_new_counter_info!("rpc-get_snapshot", 1);
+                    self.snapshot_config
+                        .as_ref()
+                        .unwrap()
+                        .snapshot_package_output_path
+                        .join(stem)
+                }
             }
         };
 
-        info!("get {} -> {:?}", path, filename);
+        let file_length = std::fs::metadata(&filename)
+            .map(|m| m.len())
+            .unwrap_or(0)
+            .to_string();
+        info!("get {} -> {:?} ({} bytes)", path, filename, file_length);
 
         RequestMiddlewareAction::Respond {
             should_validate_hosts: true,
             response: Box::new(
                 tokio_fs_01::file::File::open(filename)
                     .and_then(|file| {
-                        let buf: Vec<u8> = Vec::new();
-                        tokio_io_01::io::read_to_end(file, buf)
-                            .and_then(|item| Ok(hyper::Response::new(item.1.into())))
-                            .or_else(|_| Ok(RpcRequestMiddleware::internal_server_error()))
+                        use tokio_codec_01::{BytesCodec, FramedRead};
+
+                        let stream = FramedRead::new(file, BytesCodec::new())
+                            .map(tokio_01_bytes::BytesMut::freeze);
+                        let body = hyper::Body::wrap_stream(stream);
+
+                        Ok(hyper::Response::builder()
+                            .header(hyper::header::CONTENT_LENGTH, file_length)
+                            .body(body)
+                            .unwrap())
                     })
                     .or_else(|_| Ok(RpcRequestMiddleware::not_found())),
             ),
@@ -251,6 +268,8 @@ impl JsonRpcService {
         trusted_validators: Option<HashSet<Pubkey>>,
         override_health_check: Arc<AtomicBool>,
         optimistically_confirmed_bank: Arc<RwLock<OptimisticallyConfirmedBank>>,
+        send_transaction_retry_ms: u64,
+        send_transaction_leader_forward_count: u64,
     ) -> Self {
         info!("rpc bound to {:?}", rpc_addr);
         info!("rpc configuration: {:?}", config);
@@ -323,6 +342,8 @@ impl JsonRpcService {
             &bank_forks,
             leader_info,
             receiver,
+            send_transaction_retry_ms,
+            send_transaction_leader_forward_count,
         ));
 
         #[cfg(test)]
@@ -456,6 +477,8 @@ mod tests {
             None,
             Arc::new(AtomicBool::new(false)),
             optimistically_confirmed_bank,
+            1000,
+            1,
         );
         let thread = rpc_service.thread_hdl.thread();
         assert_eq!(thread.name().unwrap(), "solana-jsonrpc");
@@ -524,6 +547,13 @@ mod tests {
         assert!(rrm_with_snapshot_config.is_file_get_path(
             "/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"
         ));
+        assert!(rrm_with_snapshot_config.is_file_get_path(
+            "/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.zst"
+        ));
+        assert!(rrm_with_snapshot_config
+            .is_file_get_path("/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.gz"));
+        assert!(rrm_with_snapshot_config
+            .is_file_get_path("/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar"));
 
         assert!(!rrm.is_file_get_path(
             "/snapshot-notaslotnumber-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"

core/src/rpc_subscriptions.rs

@@ -371,6 +371,16 @@ fn filter_logs_results(
     }
 }
 
+fn total_nested_subscriptions<K, L, V>(
+    subscription_map: &RwLock<HashMap<K, HashMap<L, V>>>,
+) -> usize {
+    subscription_map
+        .read()
+        .unwrap()
+        .iter()
+        .fold(0, |acc, x| acc + x.1.len())
+}
+
 #[derive(Clone)]
 struct Subscriptions {
     account_subscriptions: Arc<RpcAccountSubscriptions>,
@@ -386,6 +396,24 @@ struct Subscriptions {
     root_subscriptions: Arc<RpcRootSubscriptions>,
 }
 
+impl Subscriptions {
+    fn total(&self) -> usize {
+        let mut total = 0;
+        total += total_nested_subscriptions(&self.account_subscriptions);
+        total += total_nested_subscriptions(&self.program_subscriptions);
+        total += total_nested_subscriptions(&self.logs_subscriptions);
+        total += total_nested_subscriptions(&self.signature_subscriptions);
+        total += total_nested_subscriptions(&self.gossip_account_subscriptions);
+        total += total_nested_subscriptions(&self.gossip_logs_subscriptions);
+        total += total_nested_subscriptions(&self.gossip_program_subscriptions);
+        total += total_nested_subscriptions(&self.gossip_signature_subscriptions);
+        total += self.slot_subscriptions.read().unwrap().len();
+        total += self.vote_subscriptions.read().unwrap().len();
+        total += self.root_subscriptions.read().unwrap().len();
+        total
+    }
+}
+
 pub struct RpcSubscriptions {
     subscriptions: Subscriptions,
     notification_sender: Arc<Mutex<Sender<NotificationEntry>>>,
@@ -594,6 +622,10 @@ impl RpcSubscriptions {
         notified_ids
     }
 
+    pub fn total(&self) -> usize {
+        self.subscriptions.total()
+    }
+
     pub fn add_account_subscription(
         &self,
         pubkey: Pubkey,
@@ -2073,4 +2105,150 @@ pub(crate) mod tests {
             .unwrap()
             .contains_key(&alice.pubkey()));
     }
+
+    #[test]
+    fn test_total_nested_subscriptions() {
+        let mock_subscriptions = RwLock::new(HashMap::new());
+        assert_eq!(total_nested_subscriptions(&mock_subscriptions), 0);
+
+        mock_subscriptions
+            .write()
+            .unwrap()
+            .insert(0, HashMap::new());
+        assert_eq!(total_nested_subscriptions(&mock_subscriptions), 0);
+
+        mock_subscriptions
+            .write()
+            .unwrap()
+            .entry(0)
+            .and_modify(|map| {
+                map.insert(0, "test");
+            });
+        assert_eq!(total_nested_subscriptions(&mock_subscriptions), 1);
+
+        mock_subscriptions
+            .write()
+            .unwrap()
+            .entry(0)
+            .and_modify(|map| {
+                map.insert(1, "test");
+            });
+        assert_eq!(total_nested_subscriptions(&mock_subscriptions), 2);
+
+        mock_subscriptions
+            .write()
+            .unwrap()
+            .insert(1, HashMap::new());
+        assert_eq!(total_nested_subscriptions(&mock_subscriptions), 2);
+
+        mock_subscriptions
+            .write()
+            .unwrap()
+            .entry(1)
+            .and_modify(|map| {
+                map.insert(0, "test");
+            });
+        assert_eq!(total_nested_subscriptions(&mock_subscriptions), 3);
+    }
+
+    #[test]
+    fn test_total_subscriptions() {
+        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(100);
+        let bank = Bank::new(&genesis_config);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
+        let subscriptions = RpcSubscriptions::default_with_bank_forks(bank_forks);
+
+        let (subscriber, _id_receiver, _transport_receiver) =
+            Subscriber::new_test("accountNotification");
+        let account_sub_id = SubscriptionId::Number(0u64);
+        subscriptions.add_account_subscription(
+            Pubkey::default(),
+            None,
+            account_sub_id.clone(),
+            subscriber,
+        );
+        assert_eq!(subscriptions.total(), 1);
+
+        let (subscriber, _id_receiver, _transport_receiver) =
+            Subscriber::new_test("programNotification");
+        let program_sub_id = SubscriptionId::Number(1u64);
+        subscriptions.add_program_subscription(
+            Pubkey::default(),
+            None,
+            program_sub_id.clone(),
+            subscriber,
+        );
+        assert_eq!(subscriptions.total(), 2);
+
+        let (subscriber, _id_receiver, _transport_receiver) =
+            Subscriber::new_test("logsNotification");
+        let logs_sub_id = SubscriptionId::Number(2u64);
+        subscriptions.add_logs_subscription(None, false, None, logs_sub_id.clone(), subscriber);
+        assert_eq!(subscriptions.total(), 3);
+
+        let (subscriber, _id_receiver, _transport_receiver) =
+            Subscriber::new_test("signatureNotification");
+        let sig_sub_id = SubscriptionId::Number(3u64);
+        subscriptions.add_signature_subscription(
+            Signature::default(),
+            None,
+            sig_sub_id.clone(),
+            subscriber,
+        );
+        assert_eq!(subscriptions.total(), 4);
+
+        let (subscriber, _id_receiver, _transport_receiver) =
+            Subscriber::new_test("slotNotification");
+        let slot_sub_id = SubscriptionId::Number(4u64);
+        subscriptions.add_slot_subscription(slot_sub_id.clone(), subscriber);
+        assert_eq!(subscriptions.total(), 5);
+
+        let (subscriber, _id_receiver, _transport_receiver) =
+            Subscriber::new_test("voteNotification");
+        let vote_sub_id = SubscriptionId::Number(5u64);
+        subscriptions.add_vote_subscription(vote_sub_id.clone(), subscriber);
+        assert_eq!(subscriptions.total(), 6);
+
+        let (subscriber, _id_receiver, _transport_receiver) =
+            Subscriber::new_test("rootNotification");
+        let root_sub_id = SubscriptionId::Number(6u64);
+        subscriptions.add_root_subscription(root_sub_id.clone(), subscriber);
+        assert_eq!(subscriptions.total(), 7);
+
+        // Add duplicate account subscription to ensure totals include all subscriptions on all keys
+        let (subscriber, _id_receiver, _transport_receiver) =
+            Subscriber::new_test("accountNotification2");
+        let account_dupe_sub_id = SubscriptionId::Number(7u64);
+        subscriptions.add_account_subscription(
+            Pubkey::default(),
+            None,
+            account_dupe_sub_id.clone(),
+            subscriber,
+        );
+        assert_eq!(subscriptions.total(), 8);
+
+        subscriptions.remove_account_subscription(&account_sub_id);
+        assert_eq!(subscriptions.total(), 7);
+
+        subscriptions.remove_account_subscription(&account_dupe_sub_id);
+        assert_eq!(subscriptions.total(), 6);
+
+        subscriptions.remove_program_subscription(&program_sub_id);
+        assert_eq!(subscriptions.total(), 5);
+
+        subscriptions.remove_logs_subscription(&logs_sub_id);
+        assert_eq!(subscriptions.total(), 4);
+
+        subscriptions.remove_signature_subscription(&sig_sub_id);
+        assert_eq!(subscriptions.total(), 3);
+
+        subscriptions.remove_slot_subscription(&slot_sub_id);
+        assert_eq!(subscriptions.total(), 2);
+
+        subscriptions.remove_vote_subscription(&vote_sub_id);
+        assert_eq!(subscriptions.total(), 1);
+
+        subscriptions.remove_root_subscription(&root_sub_id);
+        assert_eq!(subscriptions.total(), 0);
+    }
 }

core/src/send_transaction_service.rs

@@ -4,7 +4,11 @@ use crate::poh_recorder::PohRecorder;
 use log::*;
 use solana_metrics::{datapoint_warn, inc_new_counter_info};
 use solana_runtime::{bank::Bank, bank_forks::BankForks};
-use solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature};
+use solana_sdk::{
+    clock::{Slot, NUM_CONSECUTIVE_LEADER_SLOTS},
+    pubkey::Pubkey,
+    signature::Signature,
+};
 use std::sync::Mutex;
 use std::{
     collections::HashMap,
@@ -64,12 +68,21 @@ impl LeaderInfo {
             .collect();
     }
 
-    pub fn get_leader_tpu(&self) -> Option<&SocketAddr> {
-        self.poh_recorder
-            .lock()
-            .unwrap()
-            .leader_after_n_slots(0)
-            .and_then(|leader| self.recent_peers.get(&leader))
+    pub fn get_leader_tpus(&self, max_count: u64) -> Vec<&SocketAddr> {
+        let recorder = self.poh_recorder.lock().unwrap();
+        let leaders: Vec<_> = (0..max_count)
+            .filter_map(|i| recorder.leader_after_n_slots(i * NUM_CONSECUTIVE_LEADER_SLOTS))
+            .collect();
+        drop(recorder);
+        let mut unique_leaders = vec![];
+        for leader in leaders.iter() {
+            if let Some(addr) = self.recent_peers.get(leader) {
+                if !unique_leaders.contains(&addr) {
+                    unique_leaders.push(addr);
+                }
+            }
+        }
+        unique_leaders
     }
 }
 
@@ -88,8 +101,17 @@ impl SendTransactionService {
         bank_forks: &Arc<RwLock<BankForks>>,
         leader_info: Option<LeaderInfo>,
         receiver: Receiver<TransactionInfo>,
+        retry_rate_ms: u64,
+        leader_forward_count: u64,
     ) -> Self {
-        let thread = Self::retry_thread(tpu_address, receiver, bank_forks.clone(), leader_info);
+        let thread = Self::retry_thread(
+            tpu_address,
+            receiver,
+            bank_forks.clone(),
+            leader_info,
+            retry_rate_ms,
+            leader_forward_count,
+        );
         Self { thread }
     }
 
@@ -98,8 +120,11 @@ impl SendTransactionService {
         receiver: Receiver<TransactionInfo>,
         bank_forks: Arc<RwLock<BankForks>>,
         mut leader_info: Option<LeaderInfo>,
+        retry_rate_ms: u64,
+        leader_forward_count: u64,
     ) -> JoinHandle<()> {
         let mut last_status_check = Instant::now();
+        let mut last_leader_refresh = Instant::now();
         let mut transactions = HashMap::new();
         let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
 
@@ -110,19 +135,21 @@ impl SendTransactionService {
         Builder::new()
             .name("send-tx-sv2".to_string())
             .spawn(move || loop {
-                match receiver.recv_timeout(Duration::from_secs(1)) {
+                match receiver.recv_timeout(Duration::from_millis(1000.min(retry_rate_ms))) {
                     Err(RecvTimeoutError::Disconnected) => break,
                     Err(RecvTimeoutError::Timeout) => {}
                     Ok(transaction_info) => {
-                        let address = leader_info
+                        let addresses = leader_info
                             .as_ref()
-                            .and_then(|leader_info| leader_info.get_leader_tpu())
-                            .unwrap_or(&tpu_address);
-                        Self::send_transaction(
-                            &send_socket,
-                            address,
-                            &transaction_info.wire_transaction,
-                        );
+                            .map(|leader_info| leader_info.get_leader_tpus(leader_forward_count));
+                        let addresses = addresses.unwrap_or_else(|| vec![&tpu_address]);
+                        for address in addresses {
+                            Self::send_transaction(
+                                &send_socket,
+                                address,
+                                &transaction_info.wire_transaction,
+                            );
+                        }
                         if transactions.len() < MAX_TRANSACTION_QUEUE_SIZE {
                             transactions.insert(transaction_info.signature, transaction_info);
                         } else {
@@ -131,15 +158,19 @@ impl SendTransactionService {
                     }
                 }
 
-                if Instant::now().duration_since(last_status_check).as_secs() >= 5 {
+                if last_status_check.elapsed().as_millis() as u64 >= retry_rate_ms {
                     if !transactions.is_empty() {
                         datapoint_info!(
                             "send_transaction_service-queue-size",
                             ("len", transactions.len(), i64)
                         );
-                        let bank_forks = bank_forks.read().unwrap();
-                        let root_bank = bank_forks.root_bank();
-                        let working_bank = bank_forks.working_bank();
+                        let (root_bank, working_bank) = {
+                            let bank_forks = bank_forks.read().unwrap();
+                            (
+                                bank_forks.root_bank().clone(),
+                                bank_forks.working_bank().clone(),
+                            )
+                        };
 
                         let _result = Self::process_transactions(
                             &working_bank,
@@ -151,8 +182,11 @@ impl SendTransactionService {
                         );
                     }
                     last_status_check = Instant::now();
-                    if let Some(leader_info) = leader_info.as_mut() {
-                        leader_info.refresh_recent_peers();
+                    if last_leader_refresh.elapsed().as_millis() > 1000 {
+                        if let Some(leader_info) = leader_info.as_mut() {
+                            leader_info.refresh_recent_peers();
+                        }
+                        last_leader_refresh = Instant::now();
                     }
                 }
             })
@@ -188,12 +222,21 @@ impl SendTransactionService {
                         info!("Retrying transaction: {}", signature);
                         result.retried += 1;
                         inc_new_counter_info!("send_transaction_service-retry", 1);
+                        let leaders = leader_info
+                            .as_ref()
+                            .map(|leader_info| leader_info.get_leader_tpus(1));
+                        let leader = if let Some(leaders) = leaders {
+                            if leaders.is_empty() {
+                                &tpu_address
+                            } else {
+                                leaders[0]
+                            }
+                        } else {
+                            &tpu_address
+                        };
                         Self::send_transaction(
                             &send_socket,
-                            leader_info
-                                .as_ref()
-                                .and_then(|leader_info| leader_info.get_leader_tpu())
-                                .unwrap_or(&tpu_address),
+                            leader,
                             &transaction_info.wire_transaction,
                         );
                         true
@@ -234,9 +277,20 @@ impl SendTransactionService {
 #[cfg(test)]
 mod test {
     use super::*;
+    use crate::contact_info::ContactInfo;
+    use solana_ledger::{
+        blockstore::Blockstore, get_tmp_ledger_path, leader_schedule_cache::LeaderScheduleCache,
+    };
+    use solana_runtime::genesis_utils::{
+        create_genesis_config_with_vote_accounts, GenesisConfigInfo, ValidatorVoteKeypairs,
+    };
     use solana_sdk::{
-        genesis_config::create_genesis_config, pubkey::Pubkey, signature::Signer,
+        genesis_config::create_genesis_config,
+        poh_config::PohConfig,
+        pubkey::Pubkey,
+        signature::{Keypair, Signer},
         system_transaction,
+        timing::timestamp,
     };
     use std::sync::mpsc::channel;
 
@@ -248,7 +302,7 @@ mod test {
         let (sender, receiver) = channel();
 
         let send_tranaction_service =
-            SendTransactionService::new(tpu_address, &bank_forks, None, receiver);
+            SendTransactionService::new(tpu_address, &bank_forks, None, receiver, 1000, 1);
 
         drop(sender);
         send_tranaction_service.join().unwrap();
@@ -401,4 +455,113 @@ mod test {
             }
         );
     }
+
+    #[test]
+    fn test_get_leader_tpus() {
+        let ledger_path = get_tmp_ledger_path!();
+        {
+            let blockstore = Blockstore::open(&ledger_path).unwrap();
+
+            let validator_vote_keypairs0 = ValidatorVoteKeypairs::new_rand();
+            let validator_vote_keypairs1 = ValidatorVoteKeypairs::new_rand();
+            let validator_vote_keypairs2 = ValidatorVoteKeypairs::new_rand();
+            let validator_keypairs = vec![
+                &validator_vote_keypairs0,
+                &validator_vote_keypairs1,
+                &validator_vote_keypairs2,
+            ];
+            let GenesisConfigInfo {
+                genesis_config,
+                mint_keypair: _,
+                voting_keypair: _,
+            } = create_genesis_config_with_vote_accounts(
+                1_000_000_000,
+                &validator_keypairs,
+                vec![10_000; 3],
+            );
+            let bank = Arc::new(Bank::new(&genesis_config));
+
+            let (poh_recorder, _entry_receiver) = PohRecorder::new(
+                0,
+                bank.last_blockhash(),
+                0,
+                Some((2, 2)),
+                bank.ticks_per_slot(),
+                &Pubkey::default(),
+                &Arc::new(blockstore),
+                &Arc::new(LeaderScheduleCache::new_from_bank(&bank)),
+                &Arc::new(PohConfig::default()),
+            );
+
+            let node_keypair = Arc::new(Keypair::new());
+            let cluster_info = Arc::new(ClusterInfo::new(
+                ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()),
+                node_keypair,
+            ));
+
+            let validator0_socket = SocketAddr::from(([127, 0, 0, 1], 1111));
+            let validator1_socket = SocketAddr::from(([127, 0, 0, 1], 2222));
+            let validator2_socket = SocketAddr::from(([127, 0, 0, 1], 3333));
+            let recent_peers: HashMap<_, _> = vec![
+                (
+                    validator_vote_keypairs0.node_keypair.pubkey(),
+                    validator0_socket,
+                ),
+                (
+                    validator_vote_keypairs1.node_keypair.pubkey(),
+                    validator1_socket,
+                ),
+                (
+                    validator_vote_keypairs2.node_keypair.pubkey(),
+                    validator2_socket,
+                ),
+            ]
+            .iter()
+            .cloned()
+            .collect();
+            let leader_info = LeaderInfo {
+                cluster_info,
+                poh_recorder: Arc::new(Mutex::new(poh_recorder)),
+                recent_peers: recent_peers.clone(),
+            };
+
+            let slot = bank.slot();
+            let first_leader =
+                solana_ledger::leader_schedule_utils::slot_leader_at(slot, &bank).unwrap();
+            assert_eq!(
+                leader_info.get_leader_tpus(1),
+                vec![recent_peers.get(&first_leader).unwrap()]
+            );
+
+            let second_leader = solana_ledger::leader_schedule_utils::slot_leader_at(
+                slot + NUM_CONSECUTIVE_LEADER_SLOTS,
+                &bank,
+            )
+            .unwrap();
+            let mut expected_leader_sockets = vec![
+                recent_peers.get(&first_leader).unwrap(),
+                recent_peers.get(&second_leader).unwrap(),
+            ];
+            expected_leader_sockets.dedup();
+            assert_eq!(leader_info.get_leader_tpus(2), expected_leader_sockets);
+
+            let third_leader = solana_ledger::leader_schedule_utils::slot_leader_at(
+                slot + (2 * NUM_CONSECUTIVE_LEADER_SLOTS),
+                &bank,
+            )
+            .unwrap();
+            let mut expected_leader_sockets = vec![
+                recent_peers.get(&first_leader).unwrap(),
+                recent_peers.get(&second_leader).unwrap(),
+                recent_peers.get(&third_leader).unwrap(),
+            ];
+            expected_leader_sockets.dedup();
+            assert_eq!(leader_info.get_leader_tpus(3), expected_leader_sockets);
+
+            for x in 4..8 {
+                assert!(leader_info.get_leader_tpus(x).len() <= recent_peers.len());
+            }
+        }
+        Blockstore::destroy(&ledger_path).unwrap();
+    }
 }

core/src/shred_fetch_stage.rs

@@ -1,17 +1,10 @@
 //! The `shred_fetch_stage` pulls shreds from UDP sockets and sends it to a channel.
 
-use ahash::AHasher;
+use crate::packet_hasher::PacketHasher;
 use lru::LruCache;
-use rand::{thread_rng, Rng};
-use std::hash::Hasher;
-
-use solana_ledger::blockstore::MAX_DATA_SHREDS_PER_SLOT;
-use solana_ledger::shred::{
-    CODING_SHRED, DATA_SHRED, OFFSET_OF_SHRED_INDEX, OFFSET_OF_SHRED_SLOT, OFFSET_OF_SHRED_TYPE,
-    SIZE_OF_SHRED_INDEX, SIZE_OF_SHRED_SLOT,
-};
+use solana_ledger::shred::{get_shred_slot_index_type, ShredFetchStats};
 use solana_perf::cuda_runtime::PinnedVec;
-use solana_perf::packet::{limited_deserialize, Packet, PacketsRecycler};
+use solana_perf::packet::{Packet, PacketsRecycler};
 use solana_perf::recycler::Recycler;
 use solana_runtime::bank_forks::BankForks;
 use solana_sdk::clock::{Slot, DEFAULT_MS_PER_SLOT};
@@ -27,48 +20,11 @@ use std::time::Instant;
 const DEFAULT_LRU_SIZE: usize = 10_000;
 pub type ShredsReceived = LruCache<u64, ()>;
 
-#[derive(Default)]
-pub struct ShredFetchStats {
-    index_overrun: usize,
-    shred_count: usize,
-    index_bad_deserialize: usize,
-    index_out_of_bounds: usize,
-    slot_bad_deserialize: usize,
-    duplicate_shred: usize,
-    slot_out_of_range: usize,
-}
-
 pub struct ShredFetchStage {
     thread_hdls: Vec<JoinHandle<()>>,
 }
 
 impl ShredFetchStage {
-    pub fn get_slot_index(p: &Packet, stats: &mut ShredFetchStats) -> Option<(u64, u32)> {
-        let index_start = OFFSET_OF_SHRED_INDEX;
-        let index_end = index_start + SIZE_OF_SHRED_INDEX;
-        let slot_start = OFFSET_OF_SHRED_SLOT;
-        let slot_end = slot_start + SIZE_OF_SHRED_SLOT;
-
-        if index_end <= p.meta.size {
-            if let Ok(index) = limited_deserialize::<u32>(&p.data[index_start..index_end]) {
-                if index < MAX_DATA_SHREDS_PER_SLOT as u32 && slot_end <= p.meta.size {
-                    if let Ok(slot) = limited_deserialize::<Slot>(&p.data[slot_start..slot_end]) {
-                        return Some((slot, index));
-                    } else {
-                        stats.slot_bad_deserialize += 1;
-                    }
-                } else {
-                    stats.index_out_of_bounds += 1;
-                }
-            } else {
-                stats.index_bad_deserialize += 1;
-            }
-        } else {
-            stats.index_overrun += 1;
-        }
-        None
-    }
-
     fn process_packet<F>(
         p: &mut Packet,
         shreds_received: &mut ShredsReceived,
@@ -77,32 +33,24 @@ impl ShredFetchStage {
         last_slot: Slot,
         slots_per_epoch: u64,
         modify: &F,
-        seeds: (u128, u128),
+        packet_hasher: &PacketHasher,
     ) where
         F: Fn(&mut Packet),
     {
         p.meta.discard = true;
-        if let Some((slot, _index)) = Self::get_slot_index(p, stats) {
+        if let Some((slot, _index, _shred_type)) = get_shred_slot_index_type(p, stats) {
             // Seems reasonable to limit shreds to 2 epochs away
-            if slot > last_root
-                && slot < (last_slot + 2 * slots_per_epoch)
-                && p.meta.size > OFFSET_OF_SHRED_TYPE
-            {
-                let shred_type = p.data[OFFSET_OF_SHRED_TYPE];
-                if shred_type == DATA_SHRED || shred_type == CODING_SHRED {
-                    // Shred filter
+            if slot > last_root && slot < (last_slot + 2 * slots_per_epoch) {
+                // Shred filter
 
-                    let mut hasher = AHasher::new_with_keys(seeds.0, seeds.1);
-                    hasher.write(&p.data[0..p.meta.size]);
-                    let hash = hasher.finish();
+                let hash = packet_hasher.hash_packet(p);
 
-                    if shreds_received.get(&hash).is_none() {
-                        shreds_received.put(hash, ());
-                        p.meta.discard = false;
-                        modify(p);
-                    } else {
-                        stats.duplicate_shred += 1;
-                    }
+                if shreds_received.get(&hash).is_none() {
+                    shreds_received.put(hash, ());
+                    p.meta.discard = false;
+                    modify(p);
+                } else {
+                    stats.duplicate_shred += 1;
                 }
             } else {
                 stats.slot_out_of_range += 1;
@@ -130,12 +78,12 @@ impl ShredFetchStage {
 
         let mut last_stats = Instant::now();
         let mut stats = ShredFetchStats::default();
-        let mut seeds = (thread_rng().gen::<u128>(), thread_rng().gen::<u128>());
+        let mut packet_hasher = PacketHasher::default();
 
         while let Some(mut p) = recvr.iter().next() {
             if last_updated.elapsed().as_millis() as u64 > DEFAULT_MS_PER_SLOT {
                 last_updated = Instant::now();
-                seeds = (thread_rng().gen::<u128>(), thread_rng().gen::<u128>());
+                packet_hasher.reset();
                 shreds_received.clear();
                 if let Some(bank_forks) = bank_forks.as_ref() {
                     let bank_forks_r = bank_forks.read().unwrap();
@@ -156,7 +104,7 @@ impl ShredFetchStage {
                     last_slot,
                     slots_per_epoch,
                     &modify,
-                    seeds,
+                    &packet_hasher,
                 );
             });
             if last_stats.elapsed().as_millis() > 1000 {
@@ -274,6 +222,7 @@ impl ShredFetchStage {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use solana_ledger::blockstore::MAX_DATA_SHREDS_PER_SLOT;
     use solana_ledger::shred::Shred;
 
     #[test]
@@ -287,7 +236,7 @@ mod tests {
         let shred = Shred::new_from_data(slot, 3, 0, None, true, true, 0, 0, 0);
         shred.copy_to_packet(&mut packet);
 
-        let seeds = (thread_rng().gen::<u128>(), thread_rng().gen::<u128>());
+        let hasher = PacketHasher::default();
 
         let last_root = 0;
         let last_slot = 100;
@@ -300,7 +249,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(!packet.meta.discard);
 
@@ -315,7 +264,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(!packet.meta.discard);
     }
@@ -329,7 +278,9 @@ mod tests {
         let last_root = 0;
         let last_slot = 100;
         let slots_per_epoch = 10;
-        let seeds = (thread_rng().gen::<u128>(), thread_rng().gen::<u128>());
+
+        let hasher = PacketHasher::default();
+
         // packet size is 0, so cannot get index
         ShredFetchStage::process_packet(
             &mut packet,
@@ -339,7 +290,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert_eq!(stats.index_overrun, 1);
         assert!(packet.meta.discard);
@@ -355,7 +306,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(packet.meta.discard);
 
@@ -368,7 +319,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(!packet.meta.discard);
 
@@ -381,7 +332,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(packet.meta.discard);
 
@@ -397,7 +348,7 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(packet.meta.discard);
 
@@ -412,20 +363,8 @@ mod tests {
             last_slot,
             slots_per_epoch,
             &|_p| {},
-            seeds,
+            &hasher,
         );
         assert!(packet.meta.discard);
     }
-
-    #[test]
-    fn test_shred_offsets() {
-        let shred = Shred::new_from_data(1, 3, 0, None, true, true, 0, 0, 0);
-        let mut packet = Packet::default();
-        shred.copy_to_packet(&mut packet);
-        let mut stats = ShredFetchStats::default();
-        assert_eq!(
-            Some((1, 3)),
-            ShredFetchStage::get_slot_index(&packet, &mut stats)
-        );
-    }
 }

core/src/validator.rs

@@ -80,6 +80,7 @@ pub struct ValidatorConfig {
     pub expected_shred_version: Option<u16>,
     pub voting_disabled: bool,
     pub account_paths: Vec<PathBuf>,
+    pub account_shrink_paths: Option<Vec<PathBuf>>,
     pub rpc_config: JsonRpcConfig,
     pub rpc_addrs: Option<(SocketAddr, SocketAddr)>, // (JsonRpc, JsonRpcPubSub)
     pub pubsub_config: PubSubConfig,
@@ -104,6 +105,8 @@ pub struct ValidatorConfig {
     pub cuda: bool,
     pub require_tower: bool,
     pub debug_keys: Option<Arc<HashSet<Pubkey>>>,
+    pub send_transaction_retry_ms: u64,
+    pub send_transaction_leader_forward_count: u64,
 }
 
 impl Default for ValidatorConfig {
@@ -116,6 +119,7 @@ impl Default for ValidatorConfig {
             voting_disabled: false,
             max_ledger_shreds: None,
             account_paths: Vec::new(),
+            account_shrink_paths: None,
             rpc_config: JsonRpcConfig::default(),
             rpc_addrs: None,
             pubsub_config: PubSubConfig::default(),
@@ -139,6 +143,8 @@ impl Default for ValidatorConfig {
             cuda: false,
             require_tower: false,
             debug_keys: None,
+            send_transaction_retry_ms: 2000,
+            send_transaction_leader_forward_count: 2,
         }
     }
 }
@@ -254,6 +260,11 @@ impl Validator {
         for accounts_path in &config.account_paths {
             cleanup_accounts_path(accounts_path);
         }
+        if let Some(ref shrink_paths) = config.account_shrink_paths {
+            for accounts_path in shrink_paths {
+                cleanup_accounts_path(accounts_path);
+            }
+        }
         start.stop();
         info!("done. {}", start);
 
@@ -292,6 +303,9 @@ impl Validator {
 
         let leader_schedule_cache = Arc::new(leader_schedule_cache);
         let bank = bank_forks.working_bank();
+        if let Some(ref shrink_paths) = config.account_shrink_paths {
+            bank.set_shrink_paths(shrink_paths.clone());
+        }
         let bank_forks = Arc::new(RwLock::new(bank_forks));
 
         let sample_performance_service =
@@ -419,6 +433,8 @@ impl Validator {
                         config.trusted_validators.clone(),
                         rpc_override_health_check.clone(),
                         optimistically_confirmed_bank.clone(),
+                        config.send_transaction_retry_ms,
+                        config.send_transaction_leader_forward_count,
                     ),
                     pubsub_service: PubSubService::new(
                         config.pubsub_config.clone(),
@@ -862,6 +878,7 @@ fn new_banks_from_ledger(
         &genesis_config,
         &blockstore,
         config.account_paths.clone(),
+        config.account_shrink_paths.clone(),
         config.snapshot_config.as_ref(),
         process_options,
         transaction_history_services

crate-features/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-crate-features"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana Crate Features"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

docs/src/developing/deployed-programs/debugging.md

@@ -47,16 +47,16 @@ get more information:
   - `InstructionError::InvalidAccountData` will be returned as part of the
     transaction error.
 - The BPF loader may fail to setup the program's execution environment
-  - `InstrucitonError::Custom(0x0b9f_0001)` will be returned as part of the
+  - `InstructionError::Custom(0x0b9f_0001)` will be returned as part of the
     transaction error.  "0x0b9f_0001" is the hexadecimal representation of
     [`VirtualMachineCreationFailed`](https://github.com/solana-labs/solana/blob/bc7133d7526a041d1aaee807b80922baa89b6f90/programs/bpf_loader/src/lib.rs#L44).
 - The BPF loader may have detected a fatal error during program executions
   (things like panics, memory violations, system call errors, etc...)
-  - `InstrucitonError::Custom(0x0b9f_0002)` will be returned as part of the
+  - `InstructionError::Custom(0x0b9f_0002)` will be returned as part of the
       transaction error.  "0x0b9f_0002" is the hexadecimal representation of
       [`VirtualMachineFailedToRunProgram`](https://github.com/solana-labs/solana/blob/bc7133d7526a041d1aaee807b80922baa89b6f90/programs/bpf_loader/src/lib.rs#L46).
 - The program itself may return an error
-  - `InstrucitonError::Custom(<user defined value>)` will be returned.  The
+  - `InstructionError::Custom(<user defined value>)` will be returned.  The
     "user defined value" must not conflict with any of the [builtin runtime
     program
     errors](https://github.com/solana-labs/solana/blob/bc7133d7526a041d1aaee807b80922baa89b6f90/sdk/program/src/program_error.rs#L87).

docs/src/terminology.md

@@ -58,7 +58,7 @@ A gossip network connecting all [nodes](terminology.md#node) of a [cluster](term
 
 ## cooldown period
 
-Some number of epochs after stake has been deactivated while it progressively becomes available for withdrawal. During this period, the stake is considered to be "deactivating". More info about: [warmup and cooldown](implemented-proposals/staking-rewards.md#stake-warmup-cooldown-withdrawal)
+Some number of [epochs](terminology.md#epoch) after [stake](terminology.md#stake) has been deactivated while it progressively becomes available for withdrawal. During this period, the stake is considered to be "deactivating". More info about: [warmup and cooldown](implemented-proposals/staking-rewards.md#stake-warmup-cooldown-withdrawal)
 
 ## credit
 
@@ -96,7 +96,7 @@ The fee account in the transaction is the account pays for the cost of including
 
 ## finality
 
-When nodes representing 2/3rd of the stake have a common [root](terminology.md#root).
+When nodes representing 2/3rd of the [stake](terminology.md#stake) have a common [root](terminology.md#root).
 
 ## fork
 
@@ -120,7 +120,7 @@ An increase in token supply over time used to fund rewards for validation and to
 
 ## instruction
 
-The smallest unit of a [program](terminology.md#program) that a [client](terminology.md#client) can include in a [transaction](terminology.md#instruction).
+The smallest unit of a [program](terminology.md#program) that a [client](terminology.md#client) can include in a [transaction](terminology.md#transaction).
 
 ## keypair
 
@@ -144,7 +144,7 @@ A list of [entries](terminology.md#entry) containing [transactions](terminology.
 
 ## ledger vote
 
-A [hash](terminology.md#hash) of the [validator's state](terminology.md#bank-state) at a given [tick height](terminology.md#tick-height). It comprises a validator's affirmation that a [block](terminology.md#block) it has received has been verified, as well as a promise not to vote for a conflicting [block](terminology.md#block) \(i.e. [fork](terminology.md#fork)\) for a specific amount of time, the [lockout](terminology.md#lockout) period.
+A [hash](terminology.md#hash) of the [validator's state](terminology.md#bank-state) at a given [tick height](terminology.md#tick-height). It comprises a [validator's](terminology.md#validator) affirmation that a [block](terminology.md#block) it has received has been verified, as well as a promise not to vote for a conflicting [block](terminology.md#block) \(i.e. [fork](terminology.md#fork)\) for a specific amount of time, the [lockout](terminology.md#lockout) period.
 
 ## light client
 
@@ -176,7 +176,7 @@ See [Proof of History](terminology.md#proof-of-history).
 
 ## point
 
-A weighted [credit](terminology.md#credit) in a rewards regime. In the validator [rewards regime](cluster/stake-delegation-and-rewards.md), the number of points owed to a stake during redemption is the product of the [vote credits](terminology.md#vote-credit) earned and the number of lamports staked.
+A weighted [credit](terminology.md#credit) in a rewards regime. In the [validator](terminology.md#validator) [rewards regime](cluster/stake-delegation-and-rewards.md), the number of points owed to a [stake](terminology.md#stake) during redemption is the product of the [vote credits](terminology.md#vote-credit) earned and the number of lamports staked.
 
 ## private key
 
@@ -200,7 +200,7 @@ The public key of a [keypair](terminology.md#keypair).
 
 ## root
 
-A [block](terminology.md#block) or [slot](terminology.md#slot) that has reached maximum [lockout](terminology.md#lockout) on a validator. The root is the highest block that is an ancestor of all active forks on a validator. All ancestor blocks of a root are also transitively a root. Blocks that are not an ancestor and not a descendant of the root are excluded from consideration for consensus and can be discarded.
+A [block](terminology.md#block) or [slot](terminology.md#slot) that has reached maximum [lockout](terminology.md#lockout) on a [validator](terminology.md#validator). The root is the highest block that is an ancestor of all active forks on a validator. All ancestor blocks of a root are also transitively a root. Blocks that are not an ancestor and not a descendant of the root are excluded from consideration for consensus and can be discarded.
 
 ## runtime
 
@@ -288,7 +288,7 @@ See [ledger vote](terminology.md#ledger-vote).
 
 ## vote credit
 
-A reward tally for validators. A vote credit is awarded to a validator in its vote account when the validator reaches a [root](terminology.md#root).
+A reward tally for [validators](terminology.md#validator). A vote credit is awarded to a validator in its vote account when the validator reaches a [root](terminology.md#root).
 
 ## wallet
 
@@ -296,4 +296,4 @@ A collection of [keypairs](terminology.md#keypair).
 
 ## warmup period
 
-Some number of epochs after stake has been delegated while it progressively becomes effective. During this period, the stake is considered to be "activating". More info about: [warmup and cooldown](cluster/stake-delegation-and-rewards.md#stake-warmup-cooldown-withdrawal)
+Some number of [epochs](terminology.md#epoch) after [stake](terminology.md#stake) has been delegated while it progressively becomes effective. During this period, the stake is considered to be "activating". More info about: [warmup and cooldown](cluster/stake-delegation-and-rewards.md#stake-warmup-cooldown-withdrawal)

dos/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-dos"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,15 +14,15 @@ clap = "2.33.1"
 log = "0.4.8"
 rand = "0.7.0"
 rayon = "1.4.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-core = { path = "../core", version = "1.4.16" }
-solana-ledger = { path = "../ledger", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-net-utils = { path = "../net-utils", version = "1.4.16" }
-solana-runtime = { path = "../runtime", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
-solana-client = { path = "../client", version = "1.4.16" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-core = { path = "../core", version = "1.4.19" }
+solana-ledger = { path = "../ledger", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-net-utils = { path = "../net-utils", version = "1.4.19" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
+solana-client = { path = "../client", version = "1.4.19" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

download-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-download-utils"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana Download Utils"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,8 +14,8 @@ console = "0.11.3"
 indicatif = "0.15.0"
 log = "0.4.8"
 reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-runtime = { path = "../runtime", version = "1.4.16" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
 tar = "0.4.28"
 
 [lib]

download-utils/src/lib.rs

@@ -168,52 +168,40 @@ pub fn download_snapshot(
     desired_snapshot_hash: (Slot, Hash),
     use_progress_bar: bool,
 ) -> Result<(), String> {
-    // Remove all snapshot not matching the desired hash
-    let snapshot_packages = snapshot_utils::get_snapshot_archives(ledger_path);
-    let mut found_package = false;
-    for (snapshot_package, (snapshot_slot, snapshot_hash, _compression)) in snapshot_packages.iter()
-    {
-        if (*snapshot_slot, *snapshot_hash) != desired_snapshot_hash {
-            info!("Removing old snapshot: {:?}", snapshot_package);
-            fs::remove_file(snapshot_package)
-                .unwrap_or_else(|err| info!("Failed to remove old snapshot: {:}", err));
-        } else {
-            found_package = true;
+    snapshot_utils::purge_old_snapshot_archives(ledger_path);
+
+    for compression in &[
+        CompressionType::Zstd,
+        CompressionType::Gzip,
+        CompressionType::Bzip2,
+    ] {
+        let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path(
+            ledger_path,
+            &desired_snapshot_hash,
+            compression,
+        );
+
+        if desired_snapshot_package.is_file() {
+            return Ok(());
+        }
+
+        if download_file(
+            &format!(
+                "http://{}/{}",
+                rpc_addr,
+                desired_snapshot_package
+                    .file_name()
+                    .unwrap()
+                    .to_str()
+                    .unwrap()
+            ),
+            &desired_snapshot_package,
+            use_progress_bar,
+        )
+        .is_ok()
+        {
+            return Ok(());
         }
     }
-
-    if found_package {
-        Ok(())
-    } else {
-        for compression in &[
-            CompressionType::Zstd,
-            CompressionType::Gzip,
-            CompressionType::Bzip2,
-        ] {
-            let desired_snapshot_package = snapshot_utils::get_snapshot_archive_path(
-                ledger_path,
-                &desired_snapshot_hash,
-                compression,
-            );
-
-            if download_file(
-                &format!(
-                    "http://{}/{}",
-                    rpc_addr,
-                    desired_snapshot_package
-                        .file_name()
-                        .unwrap()
-                        .to_str()
-                        .unwrap()
-                ),
-                &desired_snapshot_package,
-                use_progress_bar,
-            )
-            .is_ok()
-            {
-                return Ok(());
-            }
-        }
-        Err("Snapshot couldn't be downloaded".to_string())
-    }
+    Err("Snapshot couldn't be downloaded".to_string())
 }

explorer/wasm/Cargo.lock

@@ -98,7 +98,7 @@ dependencies = [
 
 [[package]]
 name = "solana-sdk-wasm"
-version = "1.4.14"
+version = "1.4.19"
 dependencies = [
  "bincode",
  "bs58",

explorer/wasm/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-sdk-wasm"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana SDK Wasm"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

faucet/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-faucet"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana Faucet"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -16,12 +16,12 @@ clap = "2.33"
 log = "0.4.8"
 serde = "1.0.112"
 serde_derive = "1.0.103"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-cli-config = { path = "../cli-config", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-metrics = { path = "../metrics", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-cli-config = { path = "../cli-config", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-metrics = { path = "../metrics", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
 tokio = "0.1"
 tokio-codec = "0.1"
 

frozen-abi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-frozen-abi"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana Frozen ABI"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,13 +15,13 @@ log = "0.4.8"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 sha2 = "0.8.2"
-solana-frozen-abi-macro = { path = "macro", version = "1.4.16" }
+solana-frozen-abi-macro = { path = "macro", version = "1.4.19" }
 thiserror = "1.0"
 
 [target.'cfg(not(target_arch = "bpf"))'.dependencies]
-solana-logger = { path = "../logger", version = "1.4.16" }
+solana-logger = { path = "../logger", version = "1.4.19" }
 generic-array = { version = "0.14.3", default-features = false, features = ["serde", "more_lengths"]}
-memmap = "0.7.0"
+memmap2 = "0.1.0"
 
 [build-dependencies]
 rustc_version = "0.2"

frozen-abi/macro/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-frozen-abi-macro"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana Frozen ABI Macro"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

frozen-abi/src/abi_example.rs

@@ -415,9 +415,9 @@ impl<T: std::cmp::Ord + AbiExample> AbiExample for BTreeSet<T> {
 }
 
 #[cfg(not(target_arch = "bpf"))]
-impl AbiExample for memmap::MmapMut {
+impl AbiExample for memmap2::MmapMut {
     fn example() -> Self {
-        memmap::MmapMut::map_anon(1).expect("failed to map the data file")
+        memmap2::MmapMut::map_anon(1).expect("failed to map the data file")
     }
 }
 

genesis/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-genesis"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,18 +15,18 @@ chrono = "0.4"
 serde = "1.0.112"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-budget-program = { path = "../programs/budget", version = "1.4.16" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-cli-config = { path = "../cli-config", version = "1.4.16" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.4.16" }
-solana-ledger = { path = "../ledger", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-runtime = { path = "../runtime", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
-solana-vest-program = { path = "../programs/vest", version = "1.4.16" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.16" }
+solana-budget-program = { path = "../programs/budget", version = "1.4.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-cli-config = { path = "../cli-config", version = "1.4.19" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.4.19" }
+solana-ledger = { path = "../ledger", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-stake-program = { path = "../programs/stake", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
+solana-vest-program = { path = "../programs/vest", version = "1.4.19" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.19" }
 tempfile = "3.1.0"
 
 [[bin]]

gossip/Cargo.toml

@@ -3,20 +3,20 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-gossip"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-core = { path = "../core", version = "1.4.16" }
-solana-client = { path = "../client", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-net-utils = { path = "../net-utils", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-core = { path = "../core", version = "1.4.19" }
+solana-client = { path = "../client", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-net-utils = { path = "../net-utils", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

install/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-install"
 description = "The solana cluster software installer"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -24,12 +24,12 @@ reqwest = { version = "0.10.8", default-features = false, features = ["blocking"
 serde = "1.0.112"
 serde_derive = "1.0.103"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-client = { path = "../client", version = "1.4.16" }
-solana-config-program = { path = "../programs/config", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-client = { path = "../client", version = "1.4.19" }
+solana-config-program = { path = "../programs/config", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
 semver = "0.9.0"
 tar = "0.4.28"
 tempfile = "3.1.0"

keygen/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-keygen"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana key generation utility"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,11 +13,11 @@ bs58 = "0.3.1"
 clap = "2.33"
 dirs-next = "2.0.0"
 num_cpus = "1.13.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-cli-config = { path = "../cli-config", version = "1.4.16" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-cli-config = { path = "../cli-config", version = "1.4.19" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
 tiny-bip39 = "0.7.0"
 
 [[bin]]

ledger-tool/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-ledger-tool"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -22,18 +22,18 @@ regex = "1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-cli-output = { path = "../cli-output", version = "1.4.16" }
-solana-ledger = { path = "../ledger", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-measure = { path = "../measure", version = "1.4.16" }
-solana-runtime = { path = "../runtime", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.16" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "1.4.16" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.16" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-cli-output = { path = "../cli-output", version = "1.4.19" }
+solana-ledger = { path = "../ledger", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-measure = { path = "../measure", version = "1.4.19" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-stake-program = { path = "../programs/stake", version = "1.4.19" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.4.19" }
+solana-transaction-status = { path = "../transaction-status", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.19" }
 tempfile = "3.1.0"
 tokio = { version = "0.2.22", features = ["full"] }
 

ledger-tool/src/main.rs

@@ -705,6 +705,7 @@ fn load_bank_forks(
         &genesis_config,
         &blockstore,
         account_paths,
+        None,
         snapshot_config.as_ref(),
         process_options,
         None,
@@ -1217,6 +1218,13 @@ fn main() {
                     .help("Enable stake program v2 (several inflation-related staking \
                            bugs are feature-gated behind this)"),
             )
+            .arg(
+                Arg::with_name("enable_simple_capitalization")
+                    .required(false)
+                    .long("enable-simple-capitalization")
+                    .takes_value(false)
+                    .help("Enable simple capitalization to test hardcoded cap adjustments"),
+            )
             .arg(
                 Arg::with_name("recalculate_capitalization")
                     .required(false)
@@ -2115,11 +2123,47 @@ fn main() {
                             .lazy_rent_collection
                             .store(true, std::sync::atomic::Ordering::Relaxed);
 
+                        let feature_account_balance = std::cmp::max(
+                            genesis_config.rent.minimum_balance(Feature::size_of()),
+                            1,
+                        );
+                        if arg_matches.is_present("enable_simple_capitalization") {
+                            if base_bank
+                                .get_account(&feature_set::simple_capitalization::id())
+                                .is_none()
+                            {
+                                base_bank.store_account(
+                                    &feature_set::simple_capitalization::id(),
+                                    &feature::create_account(
+                                        &Feature { activated_at: None },
+                                        feature_account_balance,
+                                    ),
+                                );
+                                if base_bank
+                                    .get_account(&feature_set::cumulative_rent_related_fixes::id())
+                                    .is_some()
+                                {
+                                    // steal some lamports from the pretty old feature not to affect
+                                    // capitalizaion, which doesn't affect inflation behavior!
+                                    base_bank.store_account(
+                                        &feature_set::cumulative_rent_related_fixes::id(),
+                                        &Account::default(),
+                                    );
+                                } else {
+                                    let old_cap = base_bank.set_capitalization();
+                                    let new_cap = base_bank.capitalization();
+                                    warn!(
+                                        "Skewing capitalization a bit to enable simple capitalization as \
+                                        requested: increasing {} from {} to {}",
+                                        feature_account_balance, old_cap, new_cap,
+                                    );
+                                    assert_eq!(old_cap + feature_account_balance, new_cap);
+                                }
+                            } else {
+                                warn!("Already simple_capitalization is activated (or scheduled)");
+                            }
+                        }
                         if arg_matches.is_present("enable_stake_program_v2") {
-                            let feature_account_balance = std::cmp::max(
-                                genesis_config.rent.minimum_balance(Feature::size_of()),
-                                1,
-                            );
                             let mut force_enabled_count = 0;
                             if base_bank
                                 .get_account(&feature_set::stake_program_v2::id())

ledger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-ledger"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana ledger"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -32,20 +32,20 @@ reed-solomon-erasure = { version = "4.0.2", features = ["simd-accel"] }
 serde = "1.0.112"
 serde_bytes = "0.11.4"
 sha2 = "0.8.2"
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.4.16" }
-solana-transaction-status = { path = "../transaction-status", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-measure = { path = "../measure", version = "1.4.16" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.4.16" }
-solana-metrics = { path = "../metrics", version = "1.4.16" }
-solana-perf = { path = "../perf", version = "1.4.16" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.4.16" }
-solana-runtime = { path = "../runtime", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.16" }
-solana-storage-bigtable = { path = "../storage-bigtable", version = "1.4.16" }
-solana-storage-proto = { path = "../storage-proto", version = "1.4.16" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.16" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.4.19" }
+solana-transaction-status = { path = "../transaction-status", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-measure = { path = "../measure", version = "1.4.19" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.4.19" }
+solana-metrics = { path = "../metrics", version = "1.4.19" }
+solana-perf = { path = "../perf", version = "1.4.19" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.4.19" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-stake-program = { path = "../programs/stake", version = "1.4.19" }
+solana-storage-bigtable = { path = "../storage-bigtable", version = "1.4.19" }
+solana-storage-proto = { path = "../storage-proto", version = "1.4.19" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.19" }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = { version = "0.2.22", features = ["full"] }
@@ -61,7 +61,7 @@ features = ["lz4"]
 [dev-dependencies]
 assert_matches = "1.3.0"
 matches = "0.1.6"
-solana-budget-program = { path = "../programs/budget", version = "1.4.16" }
+solana-budget-program = { path = "../programs/budget", version = "1.4.19" }
 
 [lib]
 crate-type = ["lib"]

ledger/src/bank_forks_utils.rs

@@ -33,6 +33,7 @@ pub fn load(
     genesis_config: &GenesisConfig,
     blockstore: &Blockstore,
     account_paths: Vec<PathBuf>,
+    shrink_paths: Option<Vec<PathBuf>>,
     snapshot_config: Option<&SnapshotConfig>,
     process_options: ProcessOptions,
     transaction_status_sender: Option<TransactionStatusSender>,
@@ -69,6 +70,9 @@ pub fn load(
                     Some(&crate::builtins::get(genesis_config.cluster_type)),
                 )
                 .expect("Load from snapshot failed");
+                if let Some(shrink_paths) = shrink_paths {
+                    deserialized_bank.set_shrink_paths(shrink_paths);
+                }
 
                 let deserialized_snapshot_hash = (
                     deserialized_bank.slot(),

ledger/src/blockstore.rs

@@ -2657,6 +2657,21 @@ impl Blockstore {
         matches!(self.db.get::<cf::Root>(slot), Ok(Some(true)))
     }
 
+    /// Returns true if a slot is between the rooted slot bounds of the ledger, but has not itself
+    /// been rooted. This is either because the slot was skipped, or due to a gap in ledger data,
+    /// as when booting from a newer snapshot.
+    pub fn is_skipped(&self, slot: Slot) -> bool {
+        let lowest_root = self
+            .rooted_slot_iterator(0)
+            .ok()
+            .and_then(|mut iter| iter.next())
+            .unwrap_or_default();
+        match self.db.get::<cf::Root>(slot).ok().flatten() {
+            Some(_) => false,
+            None => slot < self.max_root() && slot > lowest_root,
+        }
+    }
+
     pub fn set_roots(&self, rooted_slots: &[u64]) -> Result<()> {
         let mut write_batch = self.db.batch()?;
         for slot in rooted_slots {
@@ -5537,6 +5552,25 @@ pub mod tests {
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
 
+    #[test]
+    fn test_is_skipped() {
+        let blockstore_path = get_tmp_ledger_path!();
+        let blockstore = Blockstore::open(&blockstore_path).unwrap();
+        let roots = vec![2, 4, 7, 12, 15];
+        blockstore.set_roots(&roots).unwrap();
+
+        for i in 0..20 {
+            if i < 2 || roots.contains(&i) || i > 15 {
+                assert!(!blockstore.is_skipped(i));
+            } else {
+                assert!(blockstore.is_skipped(i));
+            }
+        }
+
+        drop(blockstore);
+        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
+    }
+
     #[test]
     fn test_iter_bounds() {
         let blockstore_path = get_tmp_ledger_path!();

ledger/src/blockstore_processor.rs

@@ -900,7 +900,7 @@ fn load_frozen_forks(
             leader_schedule_cache.set_root(&new_root_bank);
             new_root_bank.squash();
 
-            if last_free.elapsed() > Duration::from_secs(30) {
+            if last_free.elapsed() > Duration::from_secs(10) {
                 // This could take few secs; so update last_free later
                 new_root_bank.exhaustively_free_unused_resource();
                 last_free = Instant::now();

ledger/src/shred.rs

@@ -1,5 +1,6 @@
 //! The `shred` module defines data structures and methods to pull MTU sized data frames from the network.
 use crate::{
+    blockstore::MAX_DATA_SHREDS_PER_SLOT,
     entry::{create_ticks, Entry},
     erasure::Session,
 };
@@ -11,8 +12,7 @@ use rayon::{
     ThreadPool,
 };
 use serde::{Deserialize, Serialize};
-use solana_metrics::datapoint_debug;
-use solana_perf::packet::Packet;
+use solana_perf::packet::{limited_deserialize, Packet};
 use solana_rayon_threadlimit::get_thread_count;
 use solana_sdk::{
     clock::Slot,
@@ -281,6 +281,27 @@ impl Shred {
         Ok(shred)
     }
 
+    pub fn new_empty_coding(
+        slot: Slot,
+        index: u32,
+        fec_set_index: u32,
+        num_data: usize,
+        num_code: usize,
+        position: usize,
+        version: u16,
+    ) -> Self {
+        let (header, coding_header) = Shredder::new_coding_shred_header(
+            slot,
+            index,
+            fec_set_index,
+            num_data,
+            num_code,
+            position,
+            version,
+        );
+        Shred::new_empty_from_header(header, DataShredHeader::default(), coding_header)
+    }
+
     pub fn new_empty_from_header(
         common_header: ShredCommonHeader,
         data_header: DataShredHeader,
@@ -671,7 +692,7 @@ impl Shredder {
             // Create empty coding shreds, with correctly populated headers
             let mut coding_shreds = Vec::with_capacity(num_coding);
             (0..num_coding).for_each(|i| {
-                let (header, coding_header) = Self::new_coding_shred_header(
+                let shred = Shred::new_empty_coding(
                     slot,
                     start_index + i as u32,
                     start_index,
@@ -680,8 +701,6 @@ impl Shredder {
                     i,
                     version,
                 );
-                let shred =
-                    Shred::new_empty_from_header(header, DataShredHeader::default(), coding_header);
                 coding_shreds.push(shred.payload);
             });
 
@@ -702,7 +721,7 @@ impl Shredder {
                 .into_iter()
                 .enumerate()
                 .map(|(i, payload)| {
-                    let (common_header, coding_header) = Self::new_coding_shred_header(
+                    let mut shred = Shred::new_empty_coding(
                         slot,
                         start_index + i as u32,
                         start_index,
@@ -711,12 +730,8 @@ impl Shredder {
                         i,
                         version,
                     );
-                    Shred {
-                        common_header,
-                        data_header: DataShredHeader::default(),
-                        coding_header,
-                        payload,
-                    }
+                    shred.payload = payload;
+                    shred
                 })
                 .collect()
         } else {
@@ -935,6 +950,71 @@ impl Shredder {
     }
 }
 
+#[derive(Default, Debug, Eq, PartialEq)]
+pub struct ShredFetchStats {
+    pub index_overrun: usize,
+    pub shred_count: usize,
+    pub index_bad_deserialize: usize,
+    pub index_out_of_bounds: usize,
+    pub slot_bad_deserialize: usize,
+    pub duplicate_shred: usize,
+    pub slot_out_of_range: usize,
+    pub bad_shred_type: usize,
+}
+
+// Get slot, index, and type from a packet with partial deserialize
+pub fn get_shred_slot_index_type(
+    p: &Packet,
+    stats: &mut ShredFetchStats,
+) -> Option<(Slot, u32, bool)> {
+    let index_start = OFFSET_OF_SHRED_INDEX;
+    let index_end = index_start + SIZE_OF_SHRED_INDEX;
+    let slot_start = OFFSET_OF_SHRED_SLOT;
+    let slot_end = slot_start + SIZE_OF_SHRED_SLOT;
+
+    debug_assert!(index_end > slot_end);
+    debug_assert!(index_end > OFFSET_OF_SHRED_TYPE);
+
+    if index_end > p.meta.size {
+        stats.index_overrun += 1;
+        return None;
+    }
+
+    let index;
+    match limited_deserialize::<u32>(&p.data[index_start..index_end]) {
+        Ok(x) => index = x,
+        Err(_e) => {
+            stats.index_bad_deserialize += 1;
+            return None;
+        }
+    }
+
+    if index >= MAX_DATA_SHREDS_PER_SLOT as u32 {
+        stats.index_out_of_bounds += 1;
+        return None;
+    }
+
+    let slot;
+    match limited_deserialize::<Slot>(&p.data[slot_start..slot_end]) {
+        Ok(x) => {
+            slot = x;
+        }
+        Err(_e) => {
+            stats.slot_bad_deserialize += 1;
+            return None;
+        }
+    }
+
+    let shred_type = p.data[OFFSET_OF_SHRED_TYPE];
+    if shred_type == DATA_SHRED || shred_type == CODING_SHRED {
+        return Some((slot, index, shred_type == DATA_SHRED));
+    } else {
+        stats.bad_shred_type += 1;
+    }
+
+    None
+}
+
 pub fn max_ticks_per_n_shreds(num_shreds: u64, shred_data_size: Option<usize>) -> u64 {
     let ticks = create_ticks(1, 0, Hash::default());
     max_entries_per_n_shred(&ticks[0], num_shreds, shred_data_size)
@@ -1676,4 +1756,60 @@ pub mod tests {
             })
         );
     }
+
+    #[test]
+    fn test_shred_offsets() {
+        solana_logger::setup();
+        let mut packet = Packet::default();
+        let shred = Shred::new_from_data(1, 3, 0, None, true, true, 0, 0, 0);
+        shred.copy_to_packet(&mut packet);
+        let mut stats = ShredFetchStats::default();
+        let ret = get_shred_slot_index_type(&packet, &mut stats);
+        assert_eq!(Some((1, 3, true)), ret);
+        assert_eq!(stats, ShredFetchStats::default());
+
+        packet.meta.size = OFFSET_OF_SHRED_TYPE;
+        assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
+        assert_eq!(stats.index_overrun, 1);
+
+        packet.meta.size = OFFSET_OF_SHRED_INDEX;
+        assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
+        assert_eq!(stats.index_overrun, 2);
+
+        packet.meta.size = OFFSET_OF_SHRED_INDEX + 1;
+        assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
+        assert_eq!(stats.index_overrun, 3);
+
+        packet.meta.size = OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX - 1;
+        assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
+        assert_eq!(stats.index_overrun, 4);
+
+        packet.meta.size = OFFSET_OF_SHRED_INDEX + SIZE_OF_SHRED_INDEX;
+        assert_eq!(
+            Some((1, 3, true)),
+            get_shred_slot_index_type(&packet, &mut stats)
+        );
+        assert_eq!(stats.index_overrun, 4);
+
+        let shred = Shred::new_empty_coding(8, 2, 10, 30, 4, 7, 200);
+        shred.copy_to_packet(&mut packet);
+        assert_eq!(
+            Some((8, 2, false)),
+            get_shred_slot_index_type(&packet, &mut stats)
+        );
+
+        let shred = Shred::new_from_data(1, std::u32::MAX - 10, 0, None, true, true, 0, 0, 0);
+        shred.copy_to_packet(&mut packet);
+        assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
+        assert_eq!(1, stats.index_out_of_bounds);
+
+        let (mut header, coding_header) =
+            Shredder::new_coding_shred_header(8, 2, 10, 30, 4, 7, 200);
+        header.shred_type = ShredType(u8::MAX);
+        let shred = Shred::new_empty_from_header(header, DataShredHeader::default(), coding_header);
+        shred.copy_to_packet(&mut packet);
+
+        assert_eq!(None, get_shred_slot_index_type(&packet, &mut stats));
+        assert_eq!(1, stats.bad_shred_type);
+    }
 }

local-cluster/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-local-cluster"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,21 +15,21 @@ gag = "0.1.10"
 fs_extra = "1.1.0"
 log = "0.4.8"
 rand = "0.7.0"
-solana-config-program = { path = "../programs/config", version = "1.4.16" }
-solana-core = { path = "../core", version = "1.4.16" }
-solana-client = { path = "../client", version = "1.4.16" }
-solana-download-utils = { path = "../download-utils", version = "1.4.16" }
-solana-faucet = { path = "../faucet", version = "1.4.16" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.4.16" }
-solana-ledger = { path = "../ledger", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-runtime = { path = "../runtime", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.16" }
-solana-vest-program = { path = "../programs/vest", version = "1.4.16" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.16" }
+solana-config-program = { path = "../programs/config", version = "1.4.19" }
+solana-core = { path = "../core", version = "1.4.19" }
+solana-client = { path = "../client", version = "1.4.19" }
+solana-download-utils = { path = "../download-utils", version = "1.4.19" }
+solana-faucet = { path = "../faucet", version = "1.4.19" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.4.19" }
+solana-ledger = { path = "../ledger", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-stake-program = { path = "../programs/stake", version = "1.4.19" }
+solana-vest-program = { path = "../programs/vest", version = "1.4.19" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.19" }
 tempfile = "3.1.0"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.4.16" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.4.19" }
 
 [dev-dependencies]
 assert_matches = "1.3.0"

local-cluster/tests/local_cluster.rs

@@ -1350,6 +1350,26 @@ fn test_faulty_node(faulty_node_type: BroadcastStageType) {
     cluster.check_for_new_roots(16, &"test_faulty_node");
 }
 
+#[test]
+fn test_wait_for_max_stake() {
+    solana_logger::setup();
+    let mut validator_config = ValidatorConfig::default();
+    validator_config.rpc_config.enable_validator_exit = true;
+    let mut config = ClusterConfig {
+        cluster_lamports: 10_000,
+        node_stakes: vec![100; 4],
+        validator_configs: vec![validator_config; 4],
+        ..ClusterConfig::default()
+    };
+    let cluster = LocalCluster::new(&mut config);
+    let client = RpcClient::new_socket(cluster.entry_point_info.rpc);
+
+    assert!(client
+        .wait_for_max_stake(CommitmentConfig::default(), 33.0f32)
+        .is_ok());
+    assert!(client.get_slot().unwrap() > 10);
+}
+
 #[test]
 // Test that when a leader is leader for banks B_i..B_{i+n}, and B_i is not
 // votable, then B_{i+1} still chains to B_i

log-analyzer/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-log-analyzer"
 description = "The solana cluster network analysis tool"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,9 +14,9 @@ byte-unit = "4.0.8"
 clap = "2.33.1"
 serde = "1.0.112"
 serde_json = "1.0.56"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
 
 [[bin]]
 name = "solana-log-analyzer"

logger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-logger"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana Logger"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

measure/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-measure"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.16"
+version = "1.4.19"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -12,8 +12,8 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-metrics = { path = "../metrics", version = "1.4.16" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-metrics = { path = "../metrics", version = "1.4.19" }
 
 [target."cfg(unix)".dependencies]
 jemallocator = "0.3.2"

merkle-tree/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-merkle-tree"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana Merkle Tree"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../sdk", version = "1.4.16" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
 fast-math = "0.1"
 
 [dev-dependencies]

metrics/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-metrics"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana Metrics"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,7 +14,7 @@ gethostname = "0.2.1"
 lazy_static = "1.4.0"
 log = "0.4.8"
 reqwest = { version = "0.10.8", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
 
 [dev-dependencies]
 rand = "0.7.0"

net-shaper/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-net-shaper"
 description = "The solana cluster network shaping tool"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -13,8 +13,8 @@ publish = false
 clap = "2.33.1"
 serde = "1.0.112"
 serde_json = "1.0.56"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
 rand = "0.7.0"
 
 [[bin]]

net-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-net-utils"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana Network Utilities"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -17,10 +17,10 @@ nix = "0.17.0"
 rand = "0.7.0"
 serde = "1.0.112"
 serde_derive = "1.0.103"
-socket2 = "0.3.15"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
+socket2 = "0.3.17"
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
 tokio = "0.1"
 url = "2.1.1"
 

notifier/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-notifier"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana Notifier"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"

perf/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-perf"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana Performance APIs"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -17,11 +17,11 @@ serde = "1.0.112"
 dlopen_derive = "0.1.4"
 lazy_static = "1.4.0"
 log = "0.4.8"
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.4.16" }
-solana-budget-program = { path = "../programs/budget", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-metrics = { path = "../metrics", version = "1.4.16" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.4.19" }
+solana-budget-program = { path = "../programs/budget", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-metrics = { path = "../metrics", version = "1.4.19" }
 curve25519-dalek = { version = "2" }
 
 [lib]

poh-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-poh-bench"
-version = "1.4.16"
+version = "1.4.19"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -12,13 +12,13 @@ clap = "2.33.1"
 log = "0.4.6"
 rand = "0.7.0"
 rayon = "1.4.0"
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-ledger = { path = "../ledger", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.16" }
-solana-measure = { path = "../measure", version = "1.4.16" }
-solana-version = { path = "../version", version = "1.4.16" }
-solana-perf = { path = "../perf", version = "1.4.16" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-ledger = { path = "../ledger", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.19" }
+solana-measure = { path = "../measure", version = "1.4.19" }
+solana-version = { path = "../version", version = "1.4.19" }
+solana-perf = { path = "../perf", version = "1.4.19" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

program-test/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2018"
 license = "Apache-2.0"
 name = "solana-program-test"
 repository = "https://github.com/solana-labs/solana"
-version = "1.4.16"
+version = "1.4.19"
 
 [dependencies]
 async-trait = "0.1.36"
@@ -13,11 +13,12 @@ base64 = "0.12.3"
 chrono = "0.4.19"
 chrono-humanize = "0.1.1"
 log = "0.4.8"
-solana-banks-client = { path = "../banks-client", version = "1.4.16" }
-solana-banks-server = { path = "../banks-server", version = "1.4.16" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.4.16" }
-solana-logger = { path = "../logger", version = "1.4.16" }
-solana-program = { path = "../sdk/program", version = "1.4.16" }
-solana-runtime = { path = "../runtime", version = "1.4.16" }
-solana-sdk = { path = "../sdk", version = "1.4.16" }
+mio = "0.7.6"
+solana-banks-client = { path = "../banks-client", version = "1.4.19" }
+solana-banks-server = { path = "../banks-server", version = "1.4.19" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.4.19" }
+solana-logger = { path = "../logger", version = "1.4.19" }
+solana-program = { path = "../sdk/program", version = "1.4.19" }
+solana-runtime = { path = "../runtime", version = "1.4.19" }
+solana-sdk = { path = "../sdk", version = "1.4.19" }
 tokio = { version = "0.3", features = ["full"] }

programs/bpf/Cargo.lock

@@ -91,7 +91,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0df2f85c8a2abbe3b7d7e748052fdd9b76a0458fdeb16ad4223f5eca78c7c130"
 dependencies = [
  "addr2line",
- "cfg-if",
+ "cfg-if 0.1.10",
  "libc",
  "object",
  "rustc-demangle",
@@ -129,7 +129,7 @@ dependencies = [
  "arrayref",
  "arrayvec",
  "cc",
- "cfg-if",
+ "cfg-if 0.1.10",
  "constant_time_eq",
  "crypto-mac 0.8.0",
  "digest 0.9.0",
@@ -266,6 +266,12 @@ version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
 
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
 [[package]]
 name = "chrono"
 version = "0.4.11"
@@ -329,7 +335,7 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
 ]
 
 [[package]]
@@ -360,7 +366,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
 dependencies = [
  "autocfg",
- "cfg-if",
+ "cfg-if 0.1.10",
  "crossbeam-utils",
  "lazy_static",
  "maybe-uninit",
@@ -374,7 +380,7 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
  "crossbeam-utils",
  "maybe-uninit",
 ]
@@ -386,7 +392,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
 dependencies = [
  "autocfg",
- "cfg-if",
+ "cfg-if 0.1.10",
  "lazy_static",
 ]
 
@@ -449,7 +455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0f260e2fc850179ef410018660006951c1b55b79e8087e87111a2c388994b9b5"
 dependencies = [
  "ahash",
- "cfg-if",
+ "cfg-if 0.1.10",
  "num_cpus",
 ]
 
@@ -531,7 +537,7 @@ version = "0.8.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
 ]
 
 [[package]]
@@ -565,7 +571,7 @@ version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "affc17579b132fc2461adf7c575cc6e8b134ebca52c51f5411388965227dc695"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
  "libc",
  "redox_syscall",
  "winapi 0.3.8",
@@ -577,7 +583,7 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
  "crc32fast",
  "libc",
  "miniz_oxide",
@@ -719,7 +725,7 @@ version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
  "libc",
  "wasi",
 ]
@@ -1064,7 +1070,7 @@ version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
 ]
 
 [[package]]
@@ -1086,13 +1092,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
 
 [[package]]
-name = "memmap"
-version = "0.7.0"
+name = "memmap2"
+version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
+checksum = "d9b70ca2a6103ac8b665dc150b142ef0e4e89df640c9e6cf295d189c3caebe5a"
 dependencies = [
  "libc",
- "winapi 0.3.8",
 ]
 
 [[package]]
@@ -1135,7 +1140,7 @@ version = "0.6.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
  "fuchsia-zircon",
  "fuchsia-zircon-sys",
  "iovec",
@@ -1161,9 +1166,9 @@ dependencies = [
 
 [[package]]
 name = "miow"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
+checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d"
 dependencies = [
  "kernel32-sys",
  "net2",
@@ -1173,11 +1178,11 @@ dependencies = [
 
 [[package]]
 name = "net2"
-version = "0.2.34"
+version = "0.2.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7"
+checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
  "libc",
  "winapi 0.3.8",
 ]
@@ -1296,7 +1301,7 @@ version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
  "cloudabi",
  "libc",
  "redox_syscall",
@@ -1825,19 +1830,18 @@ checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252"
 
 [[package]]
 name = "socket2"
-version = "0.3.15"
+version = "0.3.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44"
+checksum = "97e0e9fd577458a4f61fb91fcb559ea2afecc54c934119421f9f5d3d5b1a1057"
 dependencies = [
- "cfg-if",
+ "cfg-if 1.0.0",
  "libc",
- "redox_syscall",
  "winapi 0.3.8",
 ]
 
 [[package]]
 name = "solana-bpf-loader-program"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "bincode",
  "byteorder 1.3.4",
@@ -1852,11 +1856,14 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-programs"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "bincode",
  "byteorder 1.3.4",
  "elf",
+ "miow",
+ "net2",
+ "socket2",
  "solana-bpf-loader-program",
  "solana-logger",
  "solana-measure",
@@ -1868,7 +1875,7 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-128bit"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-bpf-rust-128bit-dep",
  "solana-program",
@@ -1876,42 +1883,42 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-128bit-dep"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-alloc"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-call-depth"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-caller-access"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-custom-heap"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-dep-crate"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "byteorder 1.3.4",
  "solana-program",
@@ -1919,21 +1926,21 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-deprecated_loader"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-dup-accounts"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-error-handling"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "num-derive 0.2.5",
  "num-traits",
@@ -1943,21 +1950,21 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-external-spend"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-instruction-introspection"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-invoke"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-bpf-rust-invoked",
  "solana-program",
@@ -1965,35 +1972,35 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-invoke-and-error"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-invoke-and-ok"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-invoked"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-iter"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-many-args"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-bpf-rust-many-args-dep",
  "solana-program",
@@ -2001,35 +2008,35 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-many-args-dep"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-mem"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-noop"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-panic"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-param-passing"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-bpf-rust-param-passing-dep",
  "solana-program",
@@ -2037,14 +2044,14 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-param-passing-dep"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-rand"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "getrandom",
  "rand",
@@ -2053,7 +2060,7 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-ristretto"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "curve25519-dalek 3.0.0",
  "getrandom",
@@ -2062,35 +2069,35 @@ dependencies = [
 
 [[package]]
 name = "solana-bpf-rust-ro-modify"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-sanity"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-sha256"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-bpf-rust-sysval"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "solana-program",
 ]
 
 [[package]]
 name = "solana-config-program"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "bincode",
  "chrono",
@@ -2102,7 +2109,7 @@ dependencies = [
 
 [[package]]
 name = "solana-crate-features"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "backtrace",
  "bytes 0.4.12",
@@ -2124,13 +2131,13 @@ dependencies = [
 
 [[package]]
 name = "solana-frozen-abi"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "bs58",
  "bv",
  "generic-array 0.14.3",
  "log",
- "memmap",
+ "memmap2",
  "rustc_version",
  "serde",
  "serde_derive",
@@ -2142,7 +2149,7 @@ dependencies = [
 
 [[package]]
 name = "solana-frozen-abi-macro"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "lazy_static",
  "proc-macro2 1.0.24",
@@ -2153,7 +2160,7 @@ dependencies = [
 
 [[package]]
 name = "solana-logger"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "env_logger",
  "lazy_static",
@@ -2162,7 +2169,7 @@ dependencies = [
 
 [[package]]
 name = "solana-measure"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "jemalloc-ctl",
  "jemallocator",
@@ -2173,7 +2180,7 @@ dependencies = [
 
 [[package]]
 name = "solana-metrics"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "env_logger",
  "gethostname",
@@ -2185,7 +2192,7 @@ dependencies = [
 
 [[package]]
 name = "solana-program"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "bincode",
  "bs58",
@@ -2213,7 +2220,7 @@ dependencies = [
 
 [[package]]
 name = "solana-rayon-threadlimit"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "lazy_static",
  "num_cpus",
@@ -2221,7 +2228,7 @@ dependencies = [
 
 [[package]]
 name = "solana-runtime"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "bincode",
  "blake3",
@@ -2239,7 +2246,7 @@ dependencies = [
  "libc",
  "libloading",
  "log",
- "memmap",
+ "memmap2",
  "num-derive 0.3.0",
  "num-traits",
  "num_cpus",
@@ -2270,7 +2277,7 @@ dependencies = [
 
 [[package]]
 name = "solana-sdk"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "assert_matches",
  "bincode",
@@ -2287,7 +2294,7 @@ dependencies = [
  "lazy_static",
  "libsecp256k1",
  "log",
- "memmap",
+ "memmap2",
  "num-derive 0.3.0",
  "num-traits",
  "pbkdf2",
@@ -2312,7 +2319,7 @@ dependencies = [
 
 [[package]]
 name = "solana-sdk-macro"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "bs58",
  "proc-macro2 1.0.24",
@@ -2323,7 +2330,7 @@ dependencies = [
 
 [[package]]
 name = "solana-secp256k1-program"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "bincode",
  "digest 0.9.0",
@@ -2336,7 +2343,7 @@ dependencies = [
 
 [[package]]
 name = "solana-stake-program"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "bincode",
  "log",
@@ -2356,7 +2363,7 @@ dependencies = [
 
 [[package]]
 name = "solana-vote-program"
-version = "1.4.15"
+version = "1.4.19"
 dependencies = [
  "bincode",
  "log",
@@ -2473,7 +2480,7 @@ version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
  "libc",
  "rand",
  "redox_syscall",
@@ -2767,7 +2774,7 @@ version = "0.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
  "log",
  "pin-project-lite",
  "tracing-core",
@@ -2889,7 +2896,7 @@ version = "0.2.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4c2dc4aa152834bc334f506c1a06b866416a8b6697d5c9f75b9a689c8486def0"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
  "serde",
  "serde_json",
  "wasm-bindgen-macro",
@@ -2916,7 +2923,7 @@ version = "0.4.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "64487204d863f109eb77e8462189d111f27cb5712cc9fdb3461297a76963a2f6"
 dependencies = [
- "cfg-if",
+ "cfg-if 0.1.10",
  "js-sys",
  "wasm-bindgen",
  "web-sys",

programs/bpf/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-bpf-programs"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.4.16"
+version = "1.4.19"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "README.md"
@@ -22,11 +22,14 @@ walkdir = "2"
 bincode = "1.1.4"
 byteorder = "1.3.2"
 elf = "0.0.10"
-solana-bpf-loader-program = { path = "../bpf_loader", version = "1.4.16" }
-solana-logger = { path = "../../logger", version = "1.4.16" }
-solana-measure = { path = "../../measure", version = "1.4.16" }
-solana-runtime = { path = "../../runtime", version = "1.4.16" }
-solana-sdk = { path = "../../sdk", version = "1.4.16" }
+miow = "0.2.2"
+net2 = "0.2.37"
+socket2 = "0.3.17"
+solana-bpf-loader-program = { path = "../bpf_loader", version = "1.4.19" }
+solana-logger = { path = "../../logger", version = "1.4.19" }
+solana-measure = { path = "../../measure", version = "1.4.19" }
+solana-runtime = { path = "../../runtime", version = "1.4.19" }
+solana-sdk = { path = "../../sdk", version = "1.4.19" }
 solana_rbpf = "=0.1.34"
 
 [[bench]]

programs/bpf/c/src/invoke/invoke.c

@@ -11,6 +11,7 @@ static const uint8_t TEST_PPROGRAM_NOT_EXECUTABLE = 4;
 static const uint8_t TEST_EMPTY_ACCOUNTS_SLICE = 5;
 static const uint8_t TEST_CAP_SEEDS = 6;
 static const uint8_t TEST_CAP_SIGNERS = 7;
+static const uint8_t TEST_ALLOC_ACCESS_VIOLATION = 8;
 
 static const int MINT_INDEX = 0;
 static const int ARGUMENT_INDEX = 1;
@@ -283,6 +284,7 @@ extern uint64_t entrypoint(const uint8_t *input) {
                                         data, SOL_ARRAY_SIZE(data)};
 
     sol_assert(SUCCESS == sol_invoke(&instruction, 0, 0));
+    break;
   }
   case TEST_CAP_SEEDS: {
     sol_log("Test cap seeds");
@@ -307,6 +309,7 @@ extern uint64_t entrypoint(const uint8_t *input) {
     sol_assert(SUCCESS == sol_invoke_signed(
                               &instruction, accounts, SOL_ARRAY_SIZE(accounts),
                               signers_seeds, SOL_ARRAY_SIZE(signers_seeds)));
+    break;
   }
   case TEST_CAP_SIGNERS: {
     sol_log("Test cap signers");
@@ -346,6 +349,46 @@ extern uint64_t entrypoint(const uint8_t *input) {
     sol_assert(SUCCESS == sol_invoke_signed(
                               &instruction, accounts, SOL_ARRAY_SIZE(accounts),
                               signers_seeds, SOL_ARRAY_SIZE(signers_seeds)));
+    break;
+  }
+  case TEST_ALLOC_ACCESS_VIOLATION: {
+    sol_log("Test resize violation");
+    SolAccountMeta arguments[] = {
+        {accounts[FROM_INDEX].key, true, true},
+        {accounts[DERIVED_KEY1_INDEX].key, true, true}};
+    uint8_t data[4 + 8 + 8 + 32];
+    *(uint64_t *)(data + 4) = 42;
+    *(uint64_t *)(data + 4 + 8) = MAX_PERMITTED_DATA_INCREASE;
+    sol_memcpy(data + 4 + 8 + 8, params.program_id, SIZE_PUBKEY);
+    const SolInstruction instruction = {accounts[SYSTEM_PROGRAM_INDEX].key,
+                                        arguments, SOL_ARRAY_SIZE(arguments),
+                                        data, SOL_ARRAY_SIZE(data)};
+    uint8_t seed1[] = {'Y', 'o', 'u', ' ', 'p', 'a', 's', 's',
+                       ' ', 'b', 'u', 't', 't', 'e', 'r'};
+    const SolSignerSeed seeds1[] = {{seed1, SOL_ARRAY_SIZE(seed1)},
+                                    {&bump_seed1, 1}};
+    const SolSignerSeeds signers_seeds[] = {{seeds1, SOL_ARRAY_SIZE(seeds1)}};
+
+    SolAccountInfo derived_account = {
+        .key = accounts[DERIVED_KEY1_INDEX].key,
+        .lamports = accounts[DERIVED_KEY1_INDEX].lamports,
+        .data_len = accounts[DERIVED_KEY1_INDEX].data_len,
+        // Point to top edge of heap, attempt to allocate into unprivileged
+        // memory
+        .data = (uint8_t *)0x300007ff8,
+        .owner = accounts[DERIVED_KEY1_INDEX].owner,
+        .rent_epoch = accounts[DERIVED_KEY1_INDEX].rent_epoch,
+        .is_signer = accounts[DERIVED_KEY1_INDEX].is_signer,
+        .is_writable = accounts[DERIVED_KEY1_INDEX].is_writable,
+        .executable = accounts[DERIVED_KEY1_INDEX].executable,
+    };
+    const SolAccountInfo invoke_accounts[] = {
+        accounts[FROM_INDEX], accounts[SYSTEM_PROGRAM_INDEX], derived_account};
+    sol_assert(SUCCESS ==
+               sol_invoke_signed(&instruction,
+                                 (const SolAccountInfo *)invoke_accounts, 3,
+                                 signers_seeds, SOL_ARRAY_SIZE(signers_seeds)));
+    break;
   }
   default:
     sol_panic();

programs/bpf/rust/128bit/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-128bit"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,8 +9,8 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
-solana-bpf-rust-128bit-dep = { path = "../128bit_dep", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
+solana-bpf-rust-128bit-dep = { path = "../128bit_dep", version = "1.4.19" }
 
 [lib]
 name = "solana_bpf_rust_128bit"

programs/bpf/rust/128bit_dep/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-128bit-dep"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

programs/bpf/rust/alloc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-alloc"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 name = "solana_bpf_rust_alloc"

programs/bpf/rust/call_depth/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-call-depth"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 name = "solana_bpf_rust_call_depth"

programs/bpf/rust/caller_access/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-caller-access"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 crate-type = ["cdylib"]

programs/bpf/rust/custom_heap/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-custom-heap"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [features]
 default = ["custom-heap"]

programs/bpf/rust/dep_crate/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-dep-crate"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,7 +10,7 @@ edition = "2018"
 
 [dependencies]
 byteorder = { version = "1", default-features = false }
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 name = "solana_bpf_rust_dep_crate"

programs/bpf/rust/deprecated_loader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-deprecated_loader"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 name = "solana_bpf_rust_deprecated_loader"

programs/bpf/rust/dup_accounts/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-dup-accounts"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 name = "solana_bpf_rust_dup_accounts"

programs/bpf/rust/error_handling/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-error-handling"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,7 +11,7 @@ edition = "2018"
 [dependencies]
 num-derive = "0.2"
 num-traits = "0.2"
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 thiserror = "1.0"
 
 [lib]

programs/bpf/rust/external_spend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-external-spend"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 name = "solana_bpf_rust_external_spend"

programs/bpf/rust/instruction_introspection/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-instruction-introspection"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 name = "solana_bpf_rust_instruction_introspection"

programs/bpf/rust/invoke/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-invoke"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,7 +10,7 @@ edition = "2018"
 
 [dependencies]
 solana-bpf-rust-invoked = { path = "../invoked", default-features = false }
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 name = "solana_bpf_rust_invoke"

programs/bpf/rust/invoke/src/lib.rs

@@ -23,6 +23,7 @@ const TEST_PPROGRAM_NOT_EXECUTABLE: u8 = 4;
 const TEST_EMPTY_ACCOUNTS_SLICE: u8 = 5;
 const TEST_CAP_SEEDS: u8 = 6;
 const TEST_CAP_SIGNERS: u8 = 7;
+const TEST_ALLOC_ACCESS_VIOLATION: u8 = 8;
 
 // const MINT_INDEX: usize = 0;
 const ARGUMENT_INDEX: usize = 1;
@@ -33,7 +34,7 @@ const INVOKED_PROGRAM_DUP_INDEX: usize = 4;
 const DERIVED_KEY1_INDEX: usize = 6;
 const DERIVED_KEY2_INDEX: usize = 7;
 const DERIVED_KEY3_INDEX: usize = 8;
-// const SYSTEM_PROGRAM_INDEX: usize = 9;
+const SYSTEM_PROGRAM_INDEX: usize = 9;
 const FROM_INDEX: usize = 10;
 
 entrypoint!(process_instruction);
@@ -403,6 +404,72 @@ fn process_instruction(
                 ],
             )?;
         }
+        TEST_ALLOC_ACCESS_VIOLATION => {
+            msg!("Test resize violation");
+            let pubkey = *accounts[FROM_INDEX].key;
+            let owner = *accounts[FROM_INDEX].owner;
+            let ptr = accounts[FROM_INDEX].data.borrow().as_ptr() as u64 as *mut _;
+            let len = accounts[FROM_INDEX].data_len();
+            let mut data = unsafe { std::slice::from_raw_parts_mut(ptr, len) };
+            let mut lamports = accounts[FROM_INDEX].lamports();
+            let from_info = AccountInfo::new(
+                &pubkey,
+                false,
+                true,
+                &mut lamports,
+                &mut data,
+                &owner,
+                false,
+                0,
+            );
+
+            let pubkey = *accounts[DERIVED_KEY1_INDEX].key;
+            let owner = *accounts[DERIVED_KEY1_INDEX].owner;
+            // Point to top edge of heap, attempt to allocate into unprivileged memory
+            let mut data = unsafe { std::slice::from_raw_parts_mut(0x300007ff8 as *mut _, 0) };
+            let mut lamports = accounts[DERIVED_KEY1_INDEX].lamports();
+            let derived_info = AccountInfo::new(
+                &pubkey,
+                false,
+                true,
+                &mut lamports,
+                &mut data,
+                &owner,
+                false,
+                0,
+            );
+
+            let pubkey = *accounts[SYSTEM_PROGRAM_INDEX].key;
+            let owner = *accounts[SYSTEM_PROGRAM_INDEX].owner;
+            let ptr = accounts[SYSTEM_PROGRAM_INDEX].data.borrow().as_ptr() as u64 as *mut _;
+            let len = accounts[SYSTEM_PROGRAM_INDEX].data_len();
+            let mut data = unsafe { std::slice::from_raw_parts_mut(ptr, len) };
+            let mut lamports = accounts[SYSTEM_PROGRAM_INDEX].lamports();
+            let system_info = AccountInfo::new(
+                &pubkey,
+                false,
+                false,
+                &mut lamports,
+                &mut data,
+                &owner,
+                true,
+                0,
+            );
+
+            let instruction = system_instruction::create_account(
+                accounts[FROM_INDEX].key,
+                accounts[DERIVED_KEY1_INDEX].key,
+                42,
+                MAX_PERMITTED_DATA_INCREASE as u64,
+                program_id,
+            );
+
+            invoke_signed(
+                &instruction,
+                &[system_info.clone(), from_info.clone(), derived_info.clone()],
+                &[&[b"You pass butter", &[bump_seed1]]],
+            )?;
+        }
         _ => panic!(),
     }
 

programs/bpf/rust/invoke_and_error/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-invoke-and-error"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 crate-type = ["cdylib"]

programs/bpf/rust/invoke_and_ok/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-invoke-and-ok"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 crate-type = ["cdylib"]

programs/bpf/rust/invoked/Cargo.toml

@@ -2,7 +2,7 @@
 
 [package]
 name = "solana-bpf-rust-invoked"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,7 +11,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [features]
 default = ["program"]

programs/bpf/rust/iter/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-iter"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 name = "solana_bpf_rust_iter"

programs/bpf/rust/many_args/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-many-args"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,8 +9,8 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
-solana-bpf-rust-many-args-dep = { path = "../many_args_dep", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
+solana-bpf-rust-many-args-dep = { path = "../many_args_dep", version = "1.4.19" }
 
 [lib]
 name = "solana_bpf_rust_many_args"

programs/bpf/rust/many_args_dep/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-many-args-dep"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

programs/bpf/rust/mem/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-mem"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 name = "solana_bpf_rust_mem"

programs/bpf/rust/noop/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-noop"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [lib]
 name = "solana_bpf_rust_noop"

programs/bpf/rust/panic/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-panic"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,7 +9,7 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 [features]
 default = ["custom-panic"]

programs/bpf/rust/param_passing/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-rust-param-passing"
-version = "1.4.16"
+version = "1.4.19"
 description = "Solana BPF test program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,8 +9,8 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-bpf-rust-param-passing-dep = { path = "../param_passing_dep", version = "1.4.16" }
-solana-program = { path = "../../../../sdk/program", version = "1.4.16" }
+solana-bpf-rust-param-passing-dep = { path = "../param_passing_dep", version = "1.4.19" }
+solana-program = { path = "../../../../sdk/program", version = "1.4.19" }
 
 
 [lib]