Rpc: Add getCirculatingSupply endpoint, redux (#9953) (#9954)

automerge

.appveyor.yml

@@ -0,0 +1,42 @@
+version: '{build}'
+
+branches:
+  only:
+  - master
+  - /^v[0-9.]+\.[0-9.]+/
+
+cache:
+  - '%USERPROFILE%\.cargo'
+  - '%APPVEYOR_BUILD_FOLDER%\target'
+
+clone_folder: d:\projects\solana
+
+build_script:
+  - bash ci/publish-tarball.sh
+
+notifications:
+  - provider: Slack
+    incoming_webhook:
+      secure: GJsBey+F5apAtUm86MHVJ68Uqa6WN1SImcuIc4TsTZrDhA8K1QWUNw9FFQPybUWDyOcS5dly3kubnUqlGt9ux6Ad2efsfRIQYWv0tOVXKeY=
+    channel: ci-status
+    on_build_success: false
+    on_build_failure: true
+    on_build_status_changed: true
+
+deploy:
+  - provider: S3
+    access_key_id:
+      secure: fTbJl6JpFebR40J7cOWZ2mXBa3kIvEiXgzxAj6L3N7A=
+    secret_access_key:
+      secure: vItsBXb2rEFLvkWtVn/Rcxu5a5+2EwC+b7GsA0waJy9hXh6XuBAD0lnHd9re3g/4
+    bucket: release.solana.com
+    region: us-west-1
+    set_public: true
+
+  - provider: GitHub
+    auth_token:
+      secure: 81fEmPZ0cV1wLtNuUrcmtgxKF6ROQF1+/ft5m+fHX21z6PoeCbaNo8cTyLioWBj7
+    draft: false
+    prerelease: false
+    on:
+      appveyor_repo_tag: true

.gitignore

@@ -1,6 +1,5 @@
 /docs/html/
 /docs/src/tests.ok
-/docs/src/cli/usage.md
 /docs/src/.gitbook/assets/*.svg
 /farf/
 /solana-release/

.mergify.yml

@@ -19,13 +19,20 @@ pull_request_rules:
       label:
         add:
           - automerge
+  - name: v0.23 backport
+    conditions:
+      - base=master
+      - label=v0.23
+    actions:
+      backport:
+        branches:
+          - v0.23
   - name: v1.0 backport
     conditions:
       - base=master
       - label=v1.0
     actions:
       backport:
-        ignore_conflicts: true
         branches:
           - v1.0
   - name: v1.1 backport
@@ -34,7 +41,6 @@ pull_request_rules:
       - label=v1.1
     actions:
       backport:
-        ignore_conflicts: true
         branches:
           - v1.1
   - name: v1.2 backport
@@ -43,6 +49,5 @@ pull_request_rules:
       - label=v1.2
     actions:
       backport:
-        ignore_conflicts: true
         branches:
           - v1.2

CONTRIBUTING.md

@@ -45,7 +45,7 @@ $ git pull --rebase upstream master
 
 If there are no functional changes, PRs can be very large and that's no
 problem. If, however, your changes are making meaningful changes or additions,
-then about 1,000 lines of changes is about the most you should ask a Solana
+then about 1000 lines of changes is about the most you should ask a Solana
 maintainer to review.
 
 ### Should I send small PRs as I develop large, new components?

Cargo.toml

@@ -3,7 +3,6 @@ members = [
     "bench-exchange",
     "bench-streamer",
     "bench-tps",
-    "accounts-bench",
     "banking-bench",
     "chacha",
     "chacha-cuda",
@@ -11,8 +10,6 @@ members = [
     "cli-config",
     "client",
     "core",
-    "dos",
-    "download-utils",
     "faucet",
     "perf",
     "validator",
@@ -27,7 +24,6 @@ members = [
     "logger",
     "log-analyzer",
     "merkle-tree",
-    "streamer",
     "measure",
     "metrics",
     "net-shaper",
@@ -52,7 +48,6 @@ members = [
     "sdk",
     "sdk-c",
     "scripts",
-    "stake-accounts",
     "stake-monitor",
     "sys-tuner",
     "transaction-status",

README.md

@@ -9,7 +9,46 @@ Blockchain Rebuilt for Scale
 Solana™ is a new blockchain architecture built from the ground up for scale. The architecture supports
 up to 710 thousand transactions per second on a gigabit network.
 
-Read all about it at [Solana: Blockchain Rebuilt for Scale](https://docs.solana.com/v/master).
+Documentation
+===
+
+Before you jump into the code, review the documentation [Solana: Blockchain Rebuilt for Scale](https://docs.solana.com).
+
+(The _latest_ development version of the docs is [available here](https://docs.solana.com/v/master).)
+
+Release Binaries
+===
+Official release binaries are available at [Github Releases](https://github.com/solana-labs/solana/releases).
+
+Additionally we provide pre-release binaries for the latest code on the edge and
+beta channels.  Note that these pre-release binaries may be less stable than an
+official release.
+
+### Edge channel
+#### Linux (x86_64-unknown-linux-gnu)
+* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-unknown-linux-gnu.tar.bz2)
+* [solana-install-init](http://release.solana.com/edge/solana-install-init-x86_64-unknown-linux-gnu) as a stand-alone executable
+#### mac OS (x86_64-apple-darwin)
+* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-apple-darwin.tar.bz2)
+* [solana-install-init](http://release.solana.com/edge/solana-install-init-x86_64-apple-darwin) as a stand-alone executable
+#### Windows (x86_64-pc-windows-msvc)
+* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-pc-windows-msvc.tar.bz2)
+* [solana-install-init.exe](http://release.solana.com/edge/solana-install-init-x86_64-pc-windows-msvc.exe) as a stand-alone executable
+#### All platforms
+* [solana-metrics.tar.bz2](http://release.solana.com.s3.amazonaws.com/edge/solana-metrics.tar.bz2)
+
+### Beta channel
+#### Linux (x86_64-unknown-linux-gnu)
+* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-unknown-linux-gnu.tar.bz2)
+* [solana-install-init](http://release.solana.com/beta/solana-install-init-x86_64-unknown-linux-gnu) as a stand-alone executable
+#### mac OS (x86_64-apple-darwin)
+* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-apple-darwin.tar.bz2)
+* [solana-install-init](http://release.solana.com/beta/solana-install-init-x86_64-apple-darwin) as a stand-alone executable
+#### Windows (x86_64-pc-windows-msvc)
+* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-pc-windows-msvc.tar.bz2)
+* [solana-install-init.exe](http://release.solana.com/beta/solana-install-init-x86_64-pc-windows-msvc.exe) as a stand-alone executable
+#### All platforms
+* [solana-metrics.tar.bz2](http://release.solana.com.s3.amazonaws.com/beta/solana-metrics.tar.bz2)
 
 Developing
 ===

accounts-bench/Cargo.toml

@@ -1,19 +0,0 @@
-[package]
-authors = ["Solana Maintainers <maintainers@solana.com>"]
-edition = "2018"
-name = "solana-accounts-bench"
-version = "1.1.1"
-repository = "https://github.com/solana-labs/solana"
-license = "Apache-2.0"
-homepage = "https://solana.com/"
-
-[dependencies]
-log = "0.4.6"
-rayon = "1.3.0"
-solana-logger = { path = "../logger", version = "1.1.1" }
-solana-runtime = { path = "../runtime", version = "1.1.1" }
-solana-measure = { path = "../measure", version = "1.1.1" }
-solana-sdk = { path = "../sdk", version = "1.1.1" }
-rand = "0.6.5"
-clap = "2.33.0"
-crossbeam-channel = "0.4"

accounts-bench/src/main.rs

@@ -1,103 +0,0 @@
-use clap::{value_t, App, Arg};
-use rayon::prelude::*;
-use solana_measure::measure::Measure;
-use solana_runtime::accounts::{create_test_accounts, update_accounts, Accounts};
-use solana_sdk::pubkey::Pubkey;
-use std::collections::HashMap;
-use std::fs;
-use std::path::PathBuf;
-
-fn main() {
-    solana_logger::setup();
-
-    let matches = App::new("crate")
-        .about("about")
-        .version("version")
-        .arg(
-            Arg::with_name("num_slots")
-                .long("num_slots")
-                .takes_value(true)
-                .value_name("SLOTS")
-                .help("Number of slots to store to."),
-        )
-        .arg(
-            Arg::with_name("num_accounts")
-                .long("num_accounts")
-                .takes_value(true)
-                .value_name("NUM_ACCOUNTS")
-                .help("Total number of accounts"),
-        )
-        .arg(
-            Arg::with_name("iterations")
-                .long("iterations")
-                .takes_value(true)
-                .value_name("ITERATIONS")
-                .help("Number of bench iterations"),
-        )
-        .arg(
-            Arg::with_name("clean")
-                .long("clean")
-                .takes_value(false)
-                .help("Run clean"),
-        )
-        .get_matches();
-
-    let num_slots = value_t!(matches, "num_slots", usize).unwrap_or(4);
-    let num_accounts = value_t!(matches, "num_accounts", usize).unwrap_or(10_000);
-    let iterations = value_t!(matches, "iterations", usize).unwrap_or(20);
-    let clean = matches.is_present("clean");
-    println!("clean: {:?}", clean);
-
-    let path = PathBuf::from("farf/accounts-bench");
-    if fs::remove_dir_all(path.clone()).is_err() {
-        println!("Warning: Couldn't remove {:?}", path);
-    }
-    let accounts = Accounts::new(vec![path]);
-    println!("Creating {} accounts", num_accounts);
-    let mut create_time = Measure::start("create accounts");
-    let pubkeys: Vec<_> = (0..num_slots)
-        .into_par_iter()
-        .map(|slot| {
-            let mut pubkeys: Vec<Pubkey> = vec![];
-            create_test_accounts(
-                &accounts,
-                &mut pubkeys,
-                num_accounts / num_slots,
-                slot as u64,
-            );
-            pubkeys
-        })
-        .collect();
-    let pubkeys: Vec<_> = pubkeys.into_iter().flatten().collect();
-    create_time.stop();
-    println!(
-        "created {} accounts in {} slots {}",
-        (num_accounts / num_slots) * num_slots,
-        num_slots,
-        create_time
-    );
-    let mut ancestors: HashMap<u64, usize> = vec![(0, 0)].into_iter().collect();
-    for i in 1..num_slots {
-        ancestors.insert(i as u64, i - 1);
-        accounts.add_root(i as u64);
-    }
-    for x in 0..iterations {
-        if clean {
-            let mut time = Measure::start("clean");
-            accounts.accounts_db.clean_accounts();
-            time.stop();
-            println!("{}", time);
-            for slot in 0..num_slots {
-                update_accounts(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64);
-                accounts.add_root((x * num_slots + slot) as u64);
-            }
-        } else {
-            let mut pubkeys: Vec<Pubkey> = vec![];
-            let mut time = Measure::start("hash");
-            let hash = accounts.accounts_db.update_accounts_hash(0, &ancestors);
-            time.stop();
-            println!("hash: {} {}", hash, time);
-            create_test_accounts(&accounts, &mut pubkeys, 1, 0);
-        }
-    }
-}

archiver-lib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-archiver-lib"
-version = "1.1.1"
+version = "1.0.22"
 description = "Solana Archiver Library"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,31 +10,33 @@ edition = "2018"
 
 [dependencies]
 bincode = "1.2.1"
-crossbeam-channel = "0.4"
-ed25519-dalek = "=1.0.0-pre.1"
+crossbeam-channel = "0.3"
+ed25519-dalek = "=1.0.0-pre.3"
 log = "0.4.8"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
-solana-client = { path = "../client", version = "1.1.1" }
-solana-storage-program = { path = "../programs/storage", version = "1.1.1" }
+rand = "0.7.0"
+rand_chacha = "0.2.2"
+solana-client = { path = "../client", version = "1.0.22" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.22" }
 thiserror = "1.0"
-serde = "1.0.105"
-serde_json = "1.0.48"
+serde = "1.0.104"
+serde_json = "1.0.46"
 serde_derive = "1.0.103"
-solana-net-utils = { path = "../net-utils", version = "1.1.1" }
-solana-chacha = { path = "../chacha", version = "1.1.1" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.1.1" }
-solana-ledger = { path = "../ledger", version = "1.1.1" }
-solana-logger = { path = "../logger", version = "1.1.1" }
-solana-perf = { path = "../perf", version = "1.1.1" }
-solana-sdk = { path = "../sdk", version = "1.1.1" }
-solana-core = { path = "../core", version = "1.1.1" }
-solana-streamer = { path = "../streamer", version = "1.1.1" }
-solana-archiver-utils = { path = "../archiver-utils", version = "1.1.1" }
-solana-metrics = { path = "../metrics", version = "1.1.1" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-chacha = { path = "../chacha", version = "1.0.22" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.0.22" }
+solana-ledger = { path = "../ledger", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-perf = { path = "../perf", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
+solana-core = { path = "../core", version = "1.0.22" }
+solana-archiver-utils = { path = "../archiver-utils", version = "1.0.22" }
+solana-metrics = { path = "../metrics", version = "1.0.22" }
 
 [dev-dependencies]
-hex = "0.4.2"
+hex = "0.4.0"
 
 [lib]
 name = "solana_archiver_lib"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

archiver-lib/src/archiver.rs

@@ -1,7 +1,7 @@
 use crate::result::ArchiverError;
 use crossbeam_channel::unbounded;
-use rand::{thread_rng, Rng, SeedableRng};
-use rand_chacha::ChaChaRng;
+use rand::{thread_rng, Rng};
+use rand_chacha::{rand_core::SeedableRng, ChaChaRng};
 use solana_archiver_utils::sample_file;
 use solana_chacha::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE};
 use solana_client::{
@@ -10,15 +10,16 @@ use solana_client::{
 };
 use solana_core::{
     cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE},
-    cluster_slots::ClusterSlots,
     contact_info::ContactInfo,
     gossip_service::GossipService,
+    packet::{limited_deserialize, PACKET_DATA_SIZE},
     repair_service,
-    repair_service::{RepairService, RepairSlotRange, RepairStats, RepairStrategy},
+    repair_service::{RepairService, RepairSlotRange, RepairStrategy},
     serve_repair::ServeRepair,
     shred_fetch_stage::ShredFetchStage,
     sigverify_stage::{DisabledSigVerifier, SigVerifyStage},
     storage_stage::NUM_STORAGE_SAMPLES,
+    streamer::{receiver, responder, PacketReceiver},
     window_service::WindowService,
 };
 use solana_ledger::{
@@ -26,7 +27,6 @@ use solana_ledger::{
 };
 use solana_net_utils::bind_in_range;
 use solana_perf::packet::Packets;
-use solana_perf::packet::{limited_deserialize, PACKET_DATA_SIZE};
 use solana_perf::recycler::Recycler;
 use solana_sdk::packet::Packet;
 use solana_sdk::{
@@ -45,7 +45,6 @@ use solana_storage_program::{
     storage_contract::StorageContract,
     storage_instruction::{self, StorageAccountType},
 };
-use solana_streamer::streamer::{receiver, responder, PacketReceiver};
 use std::{
     io::{self, ErrorKind},
     net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
@@ -188,7 +187,7 @@ impl Archiver {
         let mut cluster_info = ClusterInfo::new(node.info.clone(), keypair.clone());
         cluster_info.set_entrypoint(cluster_entrypoint.clone());
         let cluster_info = Arc::new(RwLock::new(cluster_info));
-        let cluster_slots = Arc::new(ClusterSlots::default());
+
         // Note for now, this ledger will not contain any of the existing entries
         // in the ledger located at ledger_path, and will only append on newly received
         // entries after being passed to window_service
@@ -200,7 +199,7 @@ impl Archiver {
 
         info!("Connecting to the cluster via {:?}", cluster_entrypoint);
         let (nodes, _) =
-            match solana_core::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 1) {
+            match solana_core::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 2) {
                 Ok(nodes_and_archivers) => nodes_and_archivers,
                 Err(e) => {
                     //shutdown services before exiting
@@ -263,7 +262,6 @@ impl Archiver {
                     repair_socket,
                     shred_fetch_receiver,
                     slot_sender,
-                    cluster_slots,
                 ) {
                     Ok(window_service) => window_service,
                     Err(e) => {
@@ -402,7 +400,6 @@ impl Archiver {
     }
 
     // Find a segment to replicate and download it.
-    #[allow(clippy::too_many_arguments)]
     fn setup(
         meta: &mut ArchiverMeta,
         cluster_info: Arc<RwLock<ClusterInfo>>,
@@ -413,7 +410,6 @@ impl Archiver {
         repair_socket: Arc<UdpSocket>,
         shred_fetch_receiver: PacketReceiver,
         slot_sender: Sender<u64>,
-        cluster_slots: Arc<ClusterSlots>,
     ) -> Result<WindowService> {
         let slots_per_segment =
             match Self::get_segment_config(&cluster_info, meta.client_commitment) {
@@ -471,7 +467,6 @@ impl Archiver {
             RepairStrategy::RepairRange(repair_slot_range),
             &Arc::new(LeaderScheduleCache::default()),
             |_, _, _, _| true,
-            cluster_slots,
         );
         info!("waiting for ledger download");
         Self::wait_for_segment_download(
@@ -844,14 +839,13 @@ impl Archiver {
                 repair_service::MAX_REPAIR_LENGTH,
                 &repair_slot_range,
             );
-            let mut repair_stats = RepairStats::default();
             //iter over the repairs and send them
             if let Ok(repairs) = repairs {
                 let reqs: Vec<_> = repairs
                     .into_iter()
                     .filter_map(|repair_request| {
                         serve_repair
-                            .map_repair_request(&repair_request, &mut repair_stats)
+                            .map_repair_request(&repair_request)
                             .map(|result| ((archiver_info.gossip, result), repair_request))
                             .ok()
                     })

archiver-lib/src/result.rs

@@ -1,4 +1,3 @@
-use serde_json;
 use solana_client::client_error;
 use solana_ledger::blockstore;
 use solana_sdk::transport;

archiver-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-archiver-utils"
-version = "1.1.1"
+version = "1.0.22"
 description = "Solana Archiver Utils"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,16 +10,19 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-rand = "0.6.5"
-solana-chacha = { path = "../chacha", version = "1.1.1" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.1.1" }
-solana-ledger = { path = "../ledger", version = "1.1.1" }
-solana-logger = { path = "../logger", version = "1.1.1" }
-solana-perf = { path = "../perf", version = "1.1.1" }
-solana-sdk = { path = "../sdk", version = "1.1.1" }
+rand = "0.7.0"
+solana-chacha = { path = "../chacha", version = "1.0.22" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.0.22" }
+solana-ledger = { path = "../ledger", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-perf = { path = "../perf", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
 
 [dev-dependencies]
-hex = "0.4.2"
+hex = "0.4.0"
 
 [lib]
 name = "solana_archiver_utils"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

archiver/Cargo.toml

@@ -2,19 +2,22 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-archiver"
-version = "1.1.1"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-console = "0.10.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.1.1" }
-solana-core = { path = "../core", version = "1.1.1" }
-solana-logger = { path = "../logger", version = "1.1.1" }
-solana-metrics = { path = "../metrics", version = "1.1.1" }
-solana-archiver-lib = { path = "../archiver-lib", version = "1.1.1" }
-solana-net-utils = { path = "../net-utils", version = "1.1.1" }
-solana-sdk = { path = "../sdk", version = "1.1.1" }
+console = "0.9.2"
+solana-clap-utils = { path = "../clap-utils", version = "1.0.22" }
+solana-core = { path = "../core", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-metrics = { path = "../metrics", version = "1.0.22" }
+solana-archiver-lib = { path = "../archiver-lib", version = "1.0.22" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
 
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

banking-bench/Cargo.toml

@@ -2,21 +2,22 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.1.1"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 log = "0.4.6"
-rayon = "1.3.0"
-solana-core = { path = "../core", version = "1.1.1" }
-solana-streamer = { path = "../streamer", version = "1.1.1" }
-solana-perf = { path = "../perf", version = "1.1.1" }
-solana-ledger = { path = "../ledger", version = "1.1.1" }
-solana-logger = { path = "../logger", version = "1.1.1" }
-solana-runtime = { path = "../runtime", version = "1.1.1" }
-solana-measure = { path = "../measure", version = "1.1.1" }
-solana-sdk = { path = "../sdk", version = "1.1.1" }
-rand = "0.6.5"
-crossbeam-channel = "0.4"
+rayon = "1.2.0"
+solana-core = { path = "../core", version = "1.0.22" }
+solana-ledger = { path = "../ledger", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-runtime = { path = "../runtime", version = "1.0.22" }
+solana-measure = { path = "../measure", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
+rand = "0.7.0"
+crossbeam-channel = "0.3"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -2,36 +2,29 @@ use crossbeam_channel::unbounded;
 use log::*;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
-use solana_core::{
-    banking_stage::{create_test_recorder, BankingStage},
-    cluster_info::ClusterInfo,
-    cluster_info::Node,
-    poh_recorder::PohRecorder,
-    poh_recorder::WorkingBankEntry,
-};
-use solana_ledger::{
-    bank_forks::BankForks,
-    blockstore::Blockstore,
-    genesis_utils::{create_genesis_config, GenesisConfigInfo},
-    get_tmp_ledger_path,
-};
+use solana_core::banking_stage::{create_test_recorder, BankingStage};
+use solana_core::cluster_info::ClusterInfo;
+use solana_core::cluster_info::Node;
+use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+use solana_core::packet::to_packets_chunked;
+use solana_core::poh_recorder::PohRecorder;
+use solana_core::poh_recorder::WorkingBankEntry;
+use solana_ledger::bank_forks::BankForks;
+use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
 use solana_measure::measure::Measure;
-use solana_perf::packet::to_packets_chunked;
 use solana_runtime::bank::Bank;
-use solana_sdk::{
-    hash::Hash,
-    pubkey::Pubkey,
-    signature::Keypair,
-    signature::Signature,
-    system_transaction,
-    timing::{duration_as_us, timestamp},
-    transaction::Transaction,
-};
-use std::{
-    sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex, RwLock},
-    thread::sleep,
-    time::{Duration, Instant},
-};
+use solana_sdk::hash::Hash;
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::Keypair;
+use solana_sdk::signature::Signature;
+use solana_sdk::system_transaction;
+use solana_sdk::timing::{duration_as_us, timestamp};
+use solana_sdk::transaction::Transaction;
+use std::sync::atomic::Ordering;
+use std::sync::mpsc::Receiver;
+use std::sync::{Arc, Mutex, RwLock};
+use std::thread::sleep;
+use std::time::{Duration, Instant};
 
 fn check_txs(
     receiver: &Arc<Receiver<WorkingBankEntry>>,
@@ -253,7 +246,7 @@ fn main() {
                 poh_recorder.lock().unwrap().set_bank(&bank);
                 assert!(poh_recorder.lock().unwrap().bank().is_some());
                 if bank.slot() > 32 {
-                    bank_forks.set_root(root, &None);
+                    bank_forks.set_root(root, &None, None);
                     root += 1;
                 }
                 debug!(

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.1.1"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,25 +10,28 @@ publish = false
 
 [dependencies]
 clap = "2.32.0"
-itertools = "0.9.0"
+itertools = "0.8.2"
 log = "0.4.8"
 num-derive = "0.3"
 num-traits = "0.2"
-rand = "0.6.5"
-rayon = "1.3.0"
-serde_json = "1.0.48"
+rand = "0.7.0"
+rayon = "1.2.0"
+serde_json = "1.0.46"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.1.1" }
-solana-core = { path = "../core", version = "1.1.1" }
-solana-genesis = { path = "../genesis", version = "1.1.1" }
-solana-client = { path = "../client", version = "1.1.1" }
-solana-faucet = { path = "../faucet", version = "1.1.1" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.1.1" }
-solana-logger = { path = "../logger", version = "1.1.1" }
-solana-metrics = { path = "../metrics", version = "1.1.1" }
-solana-net-utils = { path = "../net-utils", version = "1.1.1" }
-solana-runtime = { path = "../runtime", version = "1.1.1" }
-solana-sdk = { path = "../sdk", version = "1.1.1" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.22" }
+solana-core = { path = "../core", version = "1.0.22" }
+solana-genesis = { path = "../genesis", version = "1.0.22" }
+solana-client = { path = "../client", version = "1.0.22" }
+solana-faucet = { path = "../faucet", version = "1.0.22" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-metrics = { path = "../metrics", version = "1.0.22" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-runtime = { path = "../runtime", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.1.1" }
+solana-local-cluster = { path = "../local-cluster", version = "1.0.22" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/Cargo.toml

@@ -2,14 +2,17 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.1.1"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.1.1" }
-solana-streamer = { path = "../streamer", version = "1.1.1" }
-solana-logger = { path = "../logger", version = "1.1.1" }
-solana-net-utils = { path = "../net-utils", version = "1.1.1" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.22" }
+solana-core = { path = "../core", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/src/main.rs

@@ -1,6 +1,6 @@
 use clap::{crate_description, crate_name, App, Arg};
-use solana_streamer::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
-use solana_streamer::streamer::{receiver, PacketReceiver};
+use solana_core::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
+use solana_core::streamer::{receiver, PacketReceiver};
 use std::cmp::max;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.1.1"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,27 +11,30 @@ homepage = "https://solana.com/"
 bincode = "1.2.1"
 clap = "2.33.0"
 log = "0.4.8"
-rayon = "1.3.0"
-serde_json = "1.0.48"
+rayon = "1.2.0"
+serde_json = "1.0.46"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.1.1" }
-solana-core = { path = "../core", version = "1.1.1" }
-solana-genesis = { path = "../genesis", version = "1.1.1" }
-solana-client = { path = "../client", version = "1.1.1" }
-solana-faucet = { path = "../faucet", version = "1.1.1" }
-solana-librapay = { path = "../programs/librapay", version = "1.1.1", optional = true }
-solana-logger = { path = "../logger", version = "1.1.1" }
-solana-metrics = { path = "../metrics", version = "1.1.1" }
-solana-measure = { path = "../measure", version = "1.1.1" }
-solana-net-utils = { path = "../net-utils", version = "1.1.1" }
-solana-runtime = { path = "../runtime", version = "1.1.1" }
-solana-sdk = { path = "../sdk", version = "1.1.1" }
-solana-move-loader-program = { path = "../programs/move_loader", version = "1.1.1", optional = true }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.22" }
+solana-core = { path = "../core", version = "1.0.22" }
+solana-genesis = { path = "../genesis", version = "1.0.22" }
+solana-client = { path = "../client", version = "1.0.22" }
+solana-faucet = { path = "../faucet", version = "1.0.22" }
+#solana-librapay = { path = "../programs/librapay", version = "1.0.20", optional = true }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-metrics = { path = "../metrics", version = "1.0.22" }
+solana-measure = { path = "../measure", version = "1.0.22" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-runtime = { path = "../runtime", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
+#solana-move-loader-program = { path = "../programs/move_loader", version = "1.0.20", optional = true }
 
 [dev-dependencies]
-serial_test = "0.4.0"
+serial_test = "0.3.2"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.1.1" }
+solana-local-cluster = { path = "../local-cluster", version = "1.0.22" }
 
-[features]
-move = ["solana-librapay", "solana-move-loader-program"]
+#[features]
+#move = ["solana-librapay", "solana-move-loader-program"]
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

chacha-cuda/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha-cuda"
-version = "1.1.1"
+version = "1.0.22"
 description = "Solana Chacha Cuda APIs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,15 +10,18 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-solana-archiver-utils = { path = "../archiver-utils", version = "1.1.1" }
-solana-chacha = { path = "../chacha", version = "1.1.1" }
-solana-ledger = { path = "../ledger", version = "1.1.1" }
-solana-logger = { path = "../logger", version = "1.1.1" }
-solana-perf = { path = "../perf", version = "1.1.1" }
-solana-sdk = { path = "../sdk", version = "1.1.1" }
+solana-archiver-utils = { path = "../archiver-utils", version = "1.0.22" }
+solana-chacha = { path = "../chacha", version = "1.0.22" }
+solana-ledger = { path = "../ledger", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-perf = { path = "../perf", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
 
 [dev-dependencies]
 hex-literal = "0.2.1"
 
 [lib]
 name = "solana_chacha_cuda"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

chacha-sys/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha-sys"
-version = "1.1.1"
+version = "1.0.22"
 description = "Solana chacha-sys"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,3 +10,6 @@ edition = "2018"
 
 [build-dependencies]
 cc = "1.0.49"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

chacha/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha"
-version = "1.1.1"
+version = "1.0.22"
 description = "Solana Chacha APIs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,16 +10,19 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
-solana-chacha-sys = { path = "../chacha-sys", version = "1.1.1" }
-solana-ledger = { path = "../ledger", version = "1.1.1" }
-solana-logger = { path = "../logger", version = "1.1.1" }
-solana-perf = { path = "../perf", version = "1.1.1" }
-solana-sdk = { path = "../sdk", version = "1.1.1" }
+rand = "0.7.0"
+rand_chacha = "0.2.2"
+solana-chacha-sys = { path = "../chacha-sys", version = "1.0.22" }
+solana-ledger = { path = "../ledger", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-perf = { path = "../perf", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
 
 [dev-dependencies]
 hex-literal = "0.2.1"
 
 [lib]
 name = "solana_chacha"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

ci/buildkite-secondary.yml

@@ -2,6 +2,16 @@
 # Build steps that run after the primary pipeline on pushes and tags.
 # Pull requests to not run these steps.
 steps:
+  - command: "ci/publish-tarball.sh"
+    timeout_in_minutes: 60
+    name: "publish tarball"
+  - command: "ci/publish-docs.sh"
+    timeout_in_minutes: 15
+    name: "publish docs"
+  - command: "ci/publish-bpf-sdk.sh"
+    timeout_in_minutes: 5
+    name: "publish bpf sdk"
+  - wait
   - command: "sdk/docker-solana/build.sh"
     timeout_in_minutes: 60
     name: "publish docker"
@@ -9,12 +19,6 @@ steps:
     timeout_in_minutes: 240
     name: "publish crate"
     branches: "!master"
-  - command: "ci/publish-bpf-sdk.sh"
-    timeout_in_minutes: 5
-    name: "publish bpf sdk"
-  - command: "ci/publish-tarball.sh"
-    timeout_in_minutes: 60
-    name: "publish tarball"
-  - command: "ci/publish-docs.sh"
-    timeout_in_minutes: 15
-    name: "publish docs"
+    #  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
+    #    name: "move"
+    #    timeout_in_minutes: 20

ci/buildkite-tests.yml

@@ -2,12 +2,15 @@
 # other than those in docs/ are modified
 
 steps:
-  - command: "ci/shellcheck.sh"
-    name: "shellcheck"
-    timeout_in_minutes: 5
-
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh"
+    name: "coverage"
+    timeout_in_minutes: 30
+  - wait
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh"
+    name: "stable"
+    timeout_in_minutes: 60
+    artifact_paths: "log-*.txt"
   - wait
-
   - command: "ci/test-stable-perf.sh"
     name: "stable-perf"
     timeout_in_minutes: 40
@@ -17,21 +20,7 @@ steps:
   - command: "ci/test-bench.sh"
     name: "bench"
     timeout_in_minutes: 30
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh"
-    name: "stable"
-    timeout_in_minutes: 60
-    artifact_paths: "log-*.txt"
-    agents:
-      - "queue=rpc-test-capable"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
-    name: "move"
-    timeout_in_minutes: 20
   - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-local-cluster.sh"
     name: "local-cluster"
     timeout_in_minutes: 45
     artifact_paths: "log-*.txt"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh"
-    name: "coverage"
-    timeout_in_minutes: 30
-    agents:
-      - "queue=rpc-test-capable"

ci/buildkite.yml

@@ -8,6 +8,9 @@ steps:
   - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-checks.sh"
     name: "checks"
     timeout_in_minutes: 20
+  - command: "ci/shellcheck.sh"
+    name: "shellcheck"
+    timeout_in_minutes: 5
 
   - wait
 

ci/docker-run.sh

@@ -49,7 +49,7 @@ else
   # ~/.cargo
   ARGS+=(--volume "$PWD:/home")
 fi
-ARGS+=(--env "CARGO_HOME=/home/.cargo")
+ARGS+=(--env "HOME=/home" --env "CARGO_HOME=/home/.cargo")
 
 # kcov tries to set the personality of the binary which docker
 # doesn't allow by default.

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.42.0
+FROM solanalabs/rust:1.43.0
 ARG date
 
 RUN set -x \

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.42.0
+FROM rust:1.43.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0
@@ -32,7 +32,6 @@ RUN set -x \
  && cargo install cargo-audit \
  && cargo install svgbob_cli \
  && cargo install mdbook \
- && cargo install mdbook-linkcheck \
  && rustc --version \
  && cargo --version \
  && curl -OL https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP \

ci/publish-tarball.sh

@@ -71,7 +71,7 @@ echo --- Creating release tarball
   export CHANNEL
 
   source ci/rust-version.sh stable
-  scripts/cargo-install-all.sh +"$rust_stable" --use-move solana-release
+  scripts/cargo-install-all.sh +"$rust_stable" solana-release
 
   tar cvf solana-release-$TARGET.tar solana-release
   bzip2 solana-release-$TARGET.tar
@@ -95,9 +95,8 @@ fi
 source ci/upload-ci-artifact.sh
 
 for file in solana-release-$TARGET.tar.bz2 solana-release-$TARGET.yml solana-install-init-"$TARGET"* $MAYBE_TARBALLS; do
-  upload-ci-artifact "$file"
-
   if [[ -n $DO_NOT_PUBLISH_TAR ]]; then
+    upload-ci-artifact "$file"
     echo "Skipped $file due to DO_NOT_PUBLISH_TAR"
     continue
   fi

ci/rust-version.sh

@@ -1,28 +1,30 @@
 #
 # This file maintains the rust versions for use by CI.
 #
-# Build with stable rust, updating the stable toolchain if necessary:
-#   $ source ci/rust-version.sh stable
-#   $ cargo +"$rust_stable" build
-#
-# Build with nightly rust, updating the nightly toolchain if necessary:
-#   $ source ci/rust-version.sh nightly
-#   $ cargo +"$rust_nightly" build
-#
 # Obtain the environment variables without any automatic toolchain updating:
 #   $ source ci/rust-version.sh
 #
+# Obtain the environment variables updating both stable and nightly, only stable, or
+# only nightly:
+#   $ source ci/rust-version.sh all
+#   $ source ci/rust-version.sh stable
+#   $ source ci/rust-version.sh nightly
+
+# Then to build with either stable or nightly:
+#   $ cargo +"$rust_stable" build
+#   $ cargo +"$rust_nightly" build
+#
 
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.42.0
+  stable_version=1.43.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2020-03-12
+  nightly_version=2020-04-23
 fi
 
 
@@ -51,6 +53,10 @@ export rust_nightly_docker_image=solanalabs/rust-nightly:"$nightly_version"
   nightly)
      rustup_install "$rust_nightly"
     ;;
+  all)
+     rustup_install "$rust_stable"
+     rustup_install "$rust_nightly"
+    ;;
   *)
     echo "Note: ignoring unknown argument: $1"
     ;;

ci/test-bench.sh

@@ -25,7 +25,7 @@ source ci/_
 source ci/upload-ci-artifact.sh
 
 eval "$(ci/channel-info.sh)"
-source ci/rust-version.sh nightly
+source ci/rust-version.sh all
 
 set -o pipefail
 export RUST_BACKTRACE=1
@@ -67,9 +67,8 @@ _ cargo +$rust_nightly bench --manifest-path core/Cargo.toml ${V:+--verbose} \
 _ cargo +$rust_nightly bench --manifest-path programs/bpf/Cargo.toml ${V:+--verbose} --features=bpf_c \
   -- -Z unstable-options --format=json --nocapture | tee -a "$BENCH_FILE"
 
-# Run banking/accounts bench. Doesn't require nightly, but use since it is already built.
+# Run banking bench. Doesn't require nightly, but use since it is already built.
 _ cargo +$rust_nightly run --release --manifest-path banking-bench/Cargo.toml ${V:+--verbose} | tee -a "$BENCH_FILE"
-_ cargo +$rust_nightly run --release --manifest-path accounts-bench/Cargo.toml ${V:+--verbose} -- --num_accounts 10000 --num_slots 4 | tee -a "$BENCH_FILE"
 
 # `solana-upload-perf` disabled as it can take over 30 minutes to complete for some
 # reason

ci/test-stable.sh

@@ -38,11 +38,16 @@ test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
 # Clear the BPF sysroot files, they are not automatically rebuilt
 rm -rf target/xargo # Issue #3105
 
+# Limit compiler jobs to reduce memory usage
+# on machines with 1gb/thread of memory
+NPROC=$(nproc)
+NPROC=$((NPROC>16 ? 16 : NPROC))
+
 echo "Executing $testName"
 case $testName in
 test-stable)
-  _ cargo +"$rust_stable" test --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
-  _ cargo +"$rust_stable" test --manifest-path bench-tps/Cargo.toml --features=move ${V:+--verbose} test_bench_tps_local_cluster_move -- --nocapture
+  _ cargo +"$rust_stable" test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
+  #_ cargo +"$rust_stable" test --manifest-path bench-tps/Cargo.toml --features=move ${V:+--verbose} test_bench_tps_local_cluster_move -- --nocapture
   ;;
 test-stable-perf)
   ci/affects-files.sh \

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.1.1"
+version = "1.0.22"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.1.1" }
-solana-sdk = { path = "../sdk", version = "1.1.1" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
 thiserror = "1.0.11"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"
@@ -20,3 +20,6 @@ chrono = "0.4"
 
 [lib]
 name = "solana_clap_utils"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

clap-utils/src/commitment.rs

@@ -0,0 +1,18 @@
+use crate::ArgConstant;
+use clap::Arg;
+
+pub const COMMITMENT_ARG: ArgConstant<'static> = ArgConstant {
+    name: "commitment",
+    long: "commitment",
+    help: "Return information at the selected commitment level",
+};
+
+pub fn commitment_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(COMMITMENT_ARG.name)
+        .long(COMMITMENT_ARG.long)
+        .takes_value(true)
+        .possible_values(&["recent", "root", "max"])
+        .default_value("recent")
+        .value_name("COMMITMENT_LEVEL")
+        .help(COMMITMENT_ARG.help)
+}

clap-utils/src/input_parsers.rs

@@ -7,6 +7,7 @@ use clap::ArgMatches;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     clock::UnixTimestamp,
+    commitment_config::CommitmentConfig,
     native_token::sol_to_lamports,
     pubkey::Pubkey,
     signature::{read_keypair_file, Keypair, Signature, Signer},
@@ -62,21 +63,6 @@ pub fn keypair_of(matches: &ArgMatches<'_>, name: &str) -> Option<Keypair> {
     }
 }
 
-pub fn keypairs_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<Keypair>> {
-    matches.values_of(name).map(|values| {
-        values
-            .filter_map(|value| {
-                if value == ASK_KEYWORD {
-                    let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
-                    keypair_from_seed_phrase(name, skip_validation, true).ok()
-                } else {
-                    read_keypair_file(value).ok()
-                }
-            })
-            .collect()
-    })
-}
-
 // Return a pubkey for an argument that can itself be parsed into a pubkey,
 // or is a filename that can be read as a keypair
 pub fn pubkey_of(matches: &ArgMatches<'_>, name: &str) -> Option<Pubkey> {
@@ -177,6 +163,15 @@ pub fn lamports_of_sol(matches: &ArgMatches<'_>, name: &str) -> Option<u64> {
     value_of(matches, name).map(sol_to_lamports)
 }
 
+pub fn commitment_of(matches: &ArgMatches<'_>, name: &str) -> Option<CommitmentConfig> {
+    matches.value_of(name).map(|value| match value {
+        "max" => CommitmentConfig::max(),
+        "recent" => CommitmentConfig::recent(),
+        "root" => CommitmentConfig::root(),
+        _ => CommitmentConfig::default(),
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

clap-utils/src/keypair.rs

@@ -47,13 +47,6 @@ pub fn parse_keypair_path(path: &str) -> KeypairUrl {
     }
 }
 
-pub fn check_for_usb<S>(mut items: impl Iterator<Item = S>) -> bool
-where
-    S: Into<String>,
-{
-    items.any(|arg| matches!(parse_keypair_path(&arg.into()), KeypairUrl::Usb(_)))
-}
-
 pub fn presigner_from_pubkey_sigs(
     pubkey: &Pubkey,
     signers: &[(Pubkey, Signature)],
@@ -263,20 +256,4 @@ mod tests {
             sanitize_seed_phrase(seed_phrase)
         );
     }
-
-    #[test]
-    fn test_check_for_usb() {
-        let args: Vec<&str> = vec![];
-        assert_eq!(check_for_usb(args.into_iter()), false);
-        let args = vec!["usb://"];
-        assert_eq!(check_for_usb(args.into_iter()), true);
-        let args = vec!["other"];
-        assert_eq!(check_for_usb(args.into_iter()), false);
-        let args = vec!["other", "usb://", "another"];
-        assert_eq!(check_for_usb(args.into_iter()), true);
-        let args = vec!["other", "another"];
-        assert_eq!(check_for_usb(args.into_iter()), false);
-        let args = vec!["usb://", "usb://"];
-        assert_eq!(check_for_usb(args.into_iter()), true);
-    }
 }

clap-utils/src/lib.rs

@@ -42,6 +42,7 @@ impl std::fmt::Debug for DisplayError {
     }
 }
 
+pub mod commitment;
 pub mod input_parsers;
 pub mod input_validators;
 pub mod keypair;

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.1.1"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,7 +11,10 @@ homepage = "https://solana.com/"
 [dependencies]
 dirs = "2.0.2"
 lazy_static = "1.4.0"
-serde = "1.0.105"
+serde = "1.0.104"
 serde_derive = "1.0.103"
 serde_yaml = "0.8.11"
 url = "2.1.1"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

cli-config/src/config.rs

@@ -60,17 +60,38 @@ impl Config {
         ws_url
             .set_scheme(if is_secure { "wss" } else { "ws" })
             .expect("unable to set scheme");
-        let ws_port = match json_rpc_url.port() {
-            Some(port) => port + 1,
-            None => {
-                if is_secure {
-                    8901
-                } else {
-                    8900
-                }
-            }
-        };
-        ws_url.set_port(Some(ws_port)).expect("unable to set port");
+        if let Some(port) = json_rpc_url.port() {
+            ws_url.set_port(Some(port + 1)).expect("unable to set port");
+        }
         ws_url.to_string()
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn compute_websocket_url() {
+        assert_eq!(
+            Config::compute_websocket_url(&"http://devnet.solana.com"),
+            "ws://devnet.solana.com/".to_string()
+        );
+
+        assert_eq!(
+            Config::compute_websocket_url(&"https://devnet.solana.com"),
+            "wss://devnet.solana.com/".to_string()
+        );
+
+        assert_eq!(
+            Config::compute_websocket_url(&"http://example.com:8899"),
+            "ws://example.com:8900/".to_string()
+        );
+        assert_eq!(
+            Config::compute_websocket_url(&"https://example.com:1234"),
+            "wss://example.com:1235/".to_string()
+        );
+
+        assert_eq!(Config::compute_websocket_url(&"garbage"), String::new());
+    }
+}

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.1.1"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,45 +11,49 @@ homepage = "https://solana.com/"
 [dependencies]
 bincode = "1.2.1"
 bs58 = "0.3.0"
-chrono = { version = "0.4.11", features = ["serde"] }
+chrono = { version = "0.4.10", features = ["serde"] }
 clap = "2.33.0"
 criterion-stats = "0.3.0"
 ctrlc = { version = "3.1.4", features = ["termination"] }
-console = "0.10.0"
+console = "0.9.2"
 dirs = "2.0.2"
 log = "0.4.8"
 indicatif = "0.14.0"
 humantime = "2.0.0"
 num-traits = "0.2"
 pretty-hex = "0.1.1"
-reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-serde = "1.0.105"
+reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
+serde = "1.0.104"
 serde_derive = "1.0.103"
-serde_json = "1.0.48"
-solana-budget-program = { path = "../programs/budget", version = "1.1.1" }
-solana-clap-utils = { path = "../clap-utils", version = "1.1.1" }
-solana-cli-config = { path = "../cli-config", version = "1.1.1" }
-solana-client = { path = "../client", version = "1.1.1" }
-solana-config-program = { path = "../programs/config", version = "1.1.1" }
-solana-faucet = { path = "../faucet", version = "1.1.1" }
-solana-logger = { path = "../logger", version = "1.1.1" }
-solana-net-utils = { path = "../net-utils", version = "1.1.1" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.1.1" }
-solana-runtime = { path = "../runtime", version = "1.1.1" }
-solana-sdk = { path = "../sdk", version = "1.1.1" }
-solana-stake-program = { path = "../programs/stake", version = "1.1.1" }
-solana-storage-program = { path = "../programs/storage", version = "1.1.1" }
-solana-vote-program = { path = "../programs/vote", version = "1.1.1" }
-solana-vote-signer = { path = "../vote-signer", version = "1.1.1" }
+serde_json = "1.0.46"
+solana-budget-program = { path = "../programs/budget", version = "1.0.22" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.22" }
+solana-cli-config = { path = "../cli-config", version = "1.0.22" }
+solana-client = { path = "../client", version = "1.0.22" }
+solana-config-program = { path = "../programs/config", version = "1.0.22" }
+solana-faucet = { path = "../faucet", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.0.22" }
+solana-runtime = { path = "../runtime", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
+solana-stake-program = { path = "../programs/stake", version = "1.0.22" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.22" }
+solana-transaction-status = { path = "../transaction-status", version = "1.0.22" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.22" }
+solana-vote-signer = { path = "../vote-signer", version = "1.0.22" }
 titlecase = "1.1.0"
-thiserror = "1.0.13"
+thiserror = "1.0.11"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.1.1" }
-solana-budget-program = { path = "../programs/budget", version = "1.1.1" }
+solana-core = { path = "../core", version = "1.0.22" }
+solana-budget-program = { path = "../programs/budget", version = "1.0.22" }
 tempfile = "3.1.0"
 
 [[bin]]
 name = "solana"
 path = "src/main.rs"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

cli/src/cli.rs

@@ -38,9 +38,9 @@ use solana_sdk::{
     message::Message,
     native_token::lamports_to_sol,
     program_utils::DecodeError,
-    pubkey::{Pubkey, MAX_SEED_LEN},
+    pubkey::Pubkey,
     signature::{Keypair, Signature, Signer, SignerError},
-    system_instruction::{self, SystemError},
+    system_instruction::{self, create_address_with_seed, SystemError, MAX_ADDRESS_SEED_LEN},
     system_program,
     transaction::{Transaction, TransactionError},
 };
@@ -173,8 +173,6 @@ pub enum CliCommand {
     Catchup {
         node_pubkey: Pubkey,
         node_json_rpc_url: Option<String>,
-        commitment_config: CommitmentConfig,
-        follow: bool,
     },
     ClusterVersion,
     CreateAddressWithSeed {
@@ -196,9 +194,6 @@ pub enum CliCommand {
     GetSlot {
         commitment_config: CommitmentConfig,
     },
-    TotalSupply {
-        commitment_config: CommitmentConfig,
-    },
     GetTransactionCount {
         commitment_config: CommitmentConfig,
     },
@@ -410,6 +405,7 @@ pub enum CliCommand {
         to: Pubkey,
         from: SignerIndex,
         sign_only: bool,
+        no_wait: bool,
         blockhash_query: BlockhashQuery,
         nonce_account: Option<Pubkey>,
         nonce_authority: SignerIndex,
@@ -542,7 +538,7 @@ impl CliConfig<'_> {
         if !self.signers.is_empty() {
             self.signers[0].try_pubkey()
         } else {
-            Err(SignerError::CustomError(
+            Err(SignerError::Custom(
                 "Default keypair must be set if pubkey arg not provided".to_string(),
             ))
         }
@@ -593,7 +589,6 @@ pub fn parse_command(
         }),
         ("epoch", Some(matches)) => parse_get_epoch(matches),
         ("slot", Some(matches)) => parse_get_slot(matches),
-        ("total-supply", Some(matches)) => parse_total_supply(matches),
         ("transaction-count", Some(matches)) => parse_get_transaction_count(matches),
         ("leader-schedule", Some(_matches)) => Ok(CliCommandInfo {
             command: CliCommand::LeaderSchedule,
@@ -908,6 +903,7 @@ pub fn parse_command(
             let lamports = lamports_of_sol(matches, "amount").unwrap();
             let to = pubkey_of_signer(matches, "to", wallet_manager)?.unwrap();
             let sign_only = matches.is_present(SIGN_ONLY_ARG.name);
+            let no_wait = matches.is_present("no_wait");
             let blockhash_query = BlockhashQuery::new_from_matches(matches);
             let nonce_account = pubkey_of_signer(matches, NONCE_ARG.name, wallet_manager)?;
             let (nonce_authority, nonce_authority_pubkey) =
@@ -933,6 +929,7 @@ pub fn parse_command(
                     lamports,
                     to,
                     sign_only,
+                    no_wait,
                     blockhash_query,
                     nonce_account,
                     nonce_authority: signer_info.index_of(nonce_authority_pubkey).unwrap(),
@@ -1071,7 +1068,7 @@ pub fn parse_create_address_with_seed(
 
     let seed = matches.value_of("seed").unwrap().to_string();
 
-    if seed.len() > MAX_SEED_LEN {
+    if seed.len() > MAX_ADDRESS_SEED_LEN {
         return Err(CliError::BadParameter(
             "Address seed must not be longer than 32 bytes".to_string(),
         ));
@@ -1098,7 +1095,7 @@ fn process_create_address_with_seed(
     } else {
         config.pubkey()?
     };
-    let address = Pubkey::create_with_seed(&from_pubkey, seed, program_id)?;
+    let address = create_address_with_seed(&from_pubkey, seed, program_id)?;
     Ok(address.to_string())
 }
 
@@ -1158,12 +1155,48 @@ fn process_balance(
     }
 }
 
-fn process_confirm(rpc_client: &RpcClient, signature: &Signature) -> ProcessResult {
-    match rpc_client.get_signature_status(&signature) {
+fn process_confirm(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    signature: &Signature,
+) -> ProcessResult {
+    match rpc_client.get_signature_status_with_commitment_and_history(
+        &signature,
+        CommitmentConfig::max(),
+        true,
+    ) {
         Ok(status) => {
             if let Some(result) = status {
                 match result {
-                    Ok(_) => Ok("Confirmed".to_string()),
+                    Ok(_) => {
+                        if config.verbose {
+                            match rpc_client.get_confirmed_transaction(
+                                signature,
+                                solana_transaction_status::TransactionEncoding::Binary,
+                            ) {
+                                Ok(confirmed_transaction) => {
+                                    println!("\nTransaction:");
+                                    crate::display::println_transaction(
+                                        &confirmed_transaction
+                                            .transaction
+                                            .transaction
+                                            .decode()
+                                            .expect("Successful decode"),
+                                        &confirmed_transaction.transaction.meta,
+                                        "  ",
+                                    );
+                                    println!();
+                                    Ok(format!("Confirmed in slot {}", confirmed_transaction.slot))
+                                }
+                                Err(err) => Ok(format!(
+                                    "Confirmed. Unable to get confirmed transaction details: {}",
+                                    err
+                                )),
+                            }
+                        } else {
+                            Ok("Confirmed".to_string())
+                        }
+                    }
                     Err(err) => Ok(format!("Transaction failed with error: {}", err)),
                 }
             } else {
@@ -1494,6 +1527,7 @@ fn process_transfer(
     to: &Pubkey,
     from: SignerIndex,
     sign_only: bool,
+    no_wait: bool,
     blockhash_query: &BlockhashQuery,
     nonce_account: Option<&Pubkey>,
     nonce_authority: SignerIndex,
@@ -1540,7 +1574,11 @@ fn process_transfer(
             &fee_calculator,
             &tx.message,
         )?;
-        let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
+        let result = if no_wait {
+            rpc_client.send_transaction(&tx)
+        } else {
+            rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers)
+        };
         log_instruction_custom_error::<SystemError>(result)
     }
 }
@@ -1595,15 +1633,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         CliCommand::Catchup {
             node_pubkey,
             node_json_rpc_url,
-            commitment_config,
-            follow,
-        } => process_catchup(
-            &rpc_client,
-            node_pubkey,
-            node_json_rpc_url,
-            *commitment_config,
-            *follow,
-        ),
+        } => process_catchup(&rpc_client, node_pubkey, node_json_rpc_url),
         CliCommand::ClusterVersion => process_cluster_version(&rpc_client),
         CliCommand::CreateAddressWithSeed {
             from_pubkey,
@@ -1622,9 +1652,6 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         CliCommand::GetSlot { commitment_config } => {
             process_get_slot(&rpc_client, *commitment_config)
         }
-        CliCommand::TotalSupply { commitment_config } => {
-            process_total_supply(&rpc_client, *commitment_config)
-        }
         CliCommand::GetTransactionCount { commitment_config } => {
             process_get_transaction_count(&rpc_client, *commitment_config)
         }
@@ -2043,7 +2070,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         // Cancel a contract by contract Pubkey
         CliCommand::Cancel(pubkey) => process_cancel(&rpc_client, config, &pubkey),
         // Confirm the last client transaction by signature
-        CliCommand::Confirm(signature) => process_confirm(&rpc_client, signature),
+        CliCommand::Confirm(signature) => process_confirm(&rpc_client, config, signature),
         // If client has positive balance, pay lamports to another address
         CliCommand::Pay(PayCommand {
             lamports,
@@ -2097,6 +2124,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             to,
             from,
             sign_only,
+            no_wait,
             ref blockhash_query,
             ref nonce_account,
             nonce_authority,
@@ -2108,6 +2136,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
             to,
             *from,
             *sign_only,
+            *no_wait,
             blockhash_query,
             nonce_account.as_ref(),
             *nonce_authority,
@@ -2190,7 +2219,7 @@ where
         Err(err) => {
             if let ClientErrorKind::TransactionError(TransactionError::InstructionError(
                 _,
-                InstructionError::CustomError(code),
+                InstructionError::Custom(code),
             )) = err.kind()
             {
                 if let Some(specific_error) = E::decode_custom_error_to_enum(*code) {
@@ -2514,6 +2543,12 @@ pub fn app<'ab, 'v>(name: &str, about: &'ab str, version: &'v str) -> App<'ab, '
                         .validator(is_valid_signer)
                         .help("Source account of funds (if different from client local account)"),
                 )
+                .arg(
+                    Arg::with_name("no_wait")
+                        .long("no-wait")
+                        .takes_value(false)
+                        .help("Return signature immediately after submitting the transaction, instead of waiting for confirmations"),
+                )
                 .offline_args()
                 .arg(nonce_arg())
                 .arg(nonce_authority_arg())
@@ -3393,7 +3428,7 @@ mod tests {
         };
         let address = process_command(&config);
         let expected_address =
-            Pubkey::create_with_seed(&from_pubkey, "seed", &solana_stake_program::id()).unwrap();
+            create_address_with_seed(&from_pubkey, "seed", &solana_stake_program::id()).unwrap();
         assert_eq!(address.unwrap(), expected_address.to_string());
 
         // Need airdrop cases
@@ -3574,6 +3609,33 @@ mod tests {
                     to: to_pubkey,
                     from: 0,
                     sign_only: false,
+                    no_wait: false,
+                    blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
+                    nonce_account: None,
+                    nonce_authority: 0,
+                    fee_payer: 0,
+                },
+                signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
+            }
+        );
+
+        // Test Transfer no-wait
+        let test_transfer = test_commands.clone().get_matches_from(vec![
+            "test",
+            "transfer",
+            "--no-wait",
+            &to_string,
+            "42",
+        ]);
+        assert_eq!(
+            parse_command(&test_transfer, &default_keypair_file, None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::Transfer {
+                    lamports: 42_000_000_000,
+                    to: to_pubkey,
+                    from: 0,
+                    sign_only: false,
+                    no_wait: true,
                     blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
                     nonce_account: None,
                     nonce_authority: 0,
@@ -3603,6 +3665,7 @@ mod tests {
                     to: to_pubkey,
                     from: 0,
                     sign_only: true,
+                    no_wait: false,
                     blockhash_query: BlockhashQuery::None(blockhash),
                     nonce_account: None,
                     nonce_authority: 0,
@@ -3637,6 +3700,7 @@ mod tests {
                     to: to_pubkey,
                     from: 0,
                     sign_only: false,
+                    no_wait: false,
                     blockhash_query: BlockhashQuery::FeeCalculator(
                         blockhash_query::Source::Cluster,
                         blockhash
@@ -3675,6 +3739,7 @@ mod tests {
                     to: to_pubkey,
                     from: 0,
                     sign_only: false,
+                    no_wait: false,
                     blockhash_query: BlockhashQuery::FeeCalculator(
                         blockhash_query::Source::NonceAccount(nonce_address),
                         blockhash

cli/src/cluster_query.rs

@@ -5,11 +5,15 @@ use crate::{
     },
     display::println_name_value,
 };
-use chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc};
 use clap::{value_t, value_t_or_exit, App, Arg, ArgMatches, SubCommand};
 use console::{style, Emoji};
 use indicatif::{ProgressBar, ProgressStyle};
-use solana_clap_utils::{input_parsers::*, input_validators::*, keypair::signer_from_path};
+use solana_clap_utils::{
+    commitment::{commitment_arg, COMMITMENT_ARG},
+    input_parsers::*,
+    input_validators::*,
+    keypair::signer_from_path,
+};
 use solana_client::{
     pubsub_client::{PubsubClient, SlotInfoMessage},
     rpc_client::RpcClient,
@@ -23,7 +27,6 @@ use solana_sdk::{
     epoch_schedule::Epoch,
     hash::Hash,
     message::Message,
-    native_token::lamports_to_sol,
     pubkey::Pubkey,
     signature::{Keypair, Signer},
     system_instruction,
@@ -69,20 +72,6 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .takes_value(true)
                         .validator(is_url)
                         .help("JSON RPC URL for validator, which is useful for validators with a private RPC service")
-                )
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Return information at maximum-lockout commitment level",
-                        ),
-                )
-                .arg(
-                    Arg::with_name("follow")
-                        .long("follow")
-                        .takes_value(false)
-                        .help("Continue reporting progress even after the validator has caught up"),
                 ),
         )
         .subcommand(
@@ -107,14 +96,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
             SubCommand::with_name("epoch-info")
             .about("Get information about the current epoch")
             .alias("get-epoch-info")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return information at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("genesis-hash")
@@ -124,48 +106,16 @@ impl ClusterQuerySubCommands for App<'_, '_> {
         .subcommand(
             SubCommand::with_name("slot").about("Get current slot")
             .alias("get-slot")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return slot at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("epoch").about("Get current epoch")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return epoch at maximum-lockout commitment level",
-                    ),
-            ),
-        )
-        .subcommand(
-            SubCommand::with_name("total-supply").about("Get total number of SOL")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return count at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("transaction-count").about("Get current transaction count")
             .alias("get-transaction-count")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return count at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("ping")
@@ -205,14 +155,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .default_value("15")
                         .help("Wait up to timeout seconds for transaction confirmation"),
                 )
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Wait until the transaction is confirmed at maximum-lockout commitment level",
-                        ),
-                ),
+                .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("live-slots")
@@ -263,20 +206,13 @@ impl ClusterQuerySubCommands for App<'_, '_> {
             SubCommand::with_name("validators")
                 .about("Show summary information about the current validators")
                 .alias("show-validators")
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Return information at maximum-lockout commitment level",
-                        ),
-                )
                 .arg(
                     Arg::with_name("lamports")
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
-                ),
+                )
+                .arg(commitment_arg()),
         )
     }
 }
@@ -287,18 +223,10 @@ pub fn parse_catchup(
 ) -> Result<CliCommandInfo, CliError> {
     let node_pubkey = pubkey_of_signer(matches, "node_pubkey", wallet_manager)?.unwrap();
     let node_json_rpc_url = value_t!(matches, "node_json_rpc_url", String).ok();
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
-    let follow = matches.is_present("follow");
     Ok(CliCommandInfo {
         command: CliCommand::Catchup {
             node_pubkey,
             node_json_rpc_url,
-            commitment_config,
-            follow,
         },
         signers: vec![],
     })
@@ -317,11 +245,7 @@ pub fn parse_cluster_ping(
         None
     };
     let timeout = Duration::from_secs(value_t_or_exit!(matches, "timeout", u64));
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::Ping {
             lamports,
@@ -348,11 +272,7 @@ pub fn parse_get_block_time(matches: &ArgMatches<'_>) -> Result<CliCommandInfo,
 }
 
 pub fn parse_get_epoch_info(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetEpochInfo { commitment_config },
         signers: vec![],
@@ -360,11 +280,7 @@ pub fn parse_get_epoch_info(matches: &ArgMatches<'_>) -> Result<CliCommandInfo,
 }
 
 pub fn parse_get_slot(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetSlot { commitment_config },
         signers: vec![],
@@ -372,35 +288,15 @@ pub fn parse_get_slot(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliErr
 }
 
 pub fn parse_get_epoch(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetEpoch { commitment_config },
         signers: vec![],
     })
 }
 
-pub fn parse_total_supply(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
-    Ok(CliCommandInfo {
-        command: CliCommand::TotalSupply { commitment_config },
-        signers: vec![],
-    })
-}
-
 pub fn parse_get_transaction_count(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetTransactionCount { commitment_config },
         signers: vec![],
@@ -426,11 +322,7 @@ pub fn parse_show_stakes(
 
 pub fn parse_show_validators(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let use_lamports_unit = matches.is_present("lamports");
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
 
     Ok(CliCommandInfo {
         command: CliCommand::ShowValidators {
@@ -454,37 +346,20 @@ pub fn process_catchup(
     rpc_client: &RpcClient,
     node_pubkey: &Pubkey,
     node_json_rpc_url: &Option<String>,
-    commitment_config: CommitmentConfig,
-    follow: bool,
 ) -> ProcessResult {
-    let sleep_interval = 5;
-
-    let progress_bar = new_spinner_progress_bar();
-    progress_bar.set_message("Connecting...");
+    let cluster_nodes = rpc_client.get_cluster_nodes()?;
 
     let node_client = if let Some(node_json_rpc_url) = node_json_rpc_url {
         RpcClient::new(node_json_rpc_url.to_string())
     } else {
-        let rpc_addr = loop {
-            let cluster_nodes = rpc_client.get_cluster_nodes()?;
-            if let Some(contact_info) = cluster_nodes
+        RpcClient::new_socket(
+            cluster_nodes
                 .iter()
                 .find(|contact_info| contact_info.pubkey == node_pubkey.to_string())
-            {
-                if let Some(rpc_addr) = contact_info.rpc {
-                    break rpc_addr;
-                }
-                progress_bar.set_message(&format!("RPC service not found for {}", node_pubkey));
-            } else {
-                progress_bar.set_message(&format!(
-                    "Contact information not found for {}",
-                    node_pubkey
-                ));
-            }
-            sleep(Duration::from_secs(sleep_interval as u64));
-        };
-
-        RpcClient::new_socket(rpc_addr)
+                .ok_or_else(|| format!("Contact information not found for {}", node_pubkey))?
+                .rpc
+                .ok_or_else(|| format!("RPC service not found for {}", node_pubkey))?,
+        )
     };
 
     let reported_node_pubkey = node_client.get_identity()?;
@@ -500,12 +375,16 @@ pub fn process_catchup(
         return Err("Both RPC URLs reference the same node, unable to monitor for catchup.  Try a different --url".into());
     }
 
+    let progress_bar = new_spinner_progress_bar();
+    progress_bar.set_message("Connecting...");
+
     let mut previous_rpc_slot = std::u64::MAX;
     let mut previous_slot_distance = 0;
+    let sleep_interval = 5;
     loop {
-        let rpc_slot = rpc_client.get_slot_with_commitment(commitment_config)?;
-        let node_slot = node_client.get_slot_with_commitment(commitment_config)?;
-        if !follow && node_slot > std::cmp::min(previous_rpc_slot, rpc_slot) {
+        let rpc_slot = rpc_client.get_slot_with_commitment(CommitmentConfig::recent())?;
+        let node_slot = node_client.get_slot_with_commitment(CommitmentConfig::recent())?;
+        if node_slot > std::cmp::min(previous_rpc_slot, rpc_slot) {
             progress_bar.finish_and_clear();
             return Ok(format!(
                 "{} has caught up (us:{} them:{})",
@@ -514,25 +393,35 @@ pub fn process_catchup(
         }
 
         let slot_distance = rpc_slot as i64 - node_slot as i64;
+        let slots_per_second =
+            (previous_slot_distance - slot_distance) as f64 / f64::from(sleep_interval);
+        let time_remaining = (slot_distance as f64 / slots_per_second).round();
+        let time_remaining = if !time_remaining.is_normal() || time_remaining <= 0.0 {
+            "".to_string()
+        } else {
+            format!(
+                ". Time remaining: {}",
+                humantime::format_duration(Duration::from_secs_f64(time_remaining))
+            )
+        };
+
         progress_bar.set_message(&format!(
-            "Validator is {} slots away (us:{} them:{}){}",
+            "{} slots behind (us:{} them:{}){}",
             slot_distance,
             node_slot,
             rpc_slot,
-            if slot_distance == 0 || previous_rpc_slot == std::u64::MAX {
+            if previous_rpc_slot == std::u64::MAX {
                 "".to_string()
             } else {
-                let slots_per_second =
-                    (previous_slot_distance - slot_distance) as f64 / f64::from(sleep_interval);
-
                 format!(
-                    " and {} at {:.1} slots/second",
+                    ", {} at {:.1} slots/second{}",
                     if slots_per_second < 0.0 {
                         "falling behind"
                     } else {
                         "gaining"
                     },
                     slots_per_second,
+                    time_remaining
                 )
             }
         ));
@@ -594,13 +483,7 @@ pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
 
 pub fn process_get_block_time(rpc_client: &RpcClient, slot: Slot) -> ProcessResult {
     let timestamp = rpc_client.get_block_time(slot)?;
-    let result = format!(
-        "{} (UnixTimestamp: {})",
-        DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(timestamp, 0), Utc)
-            .to_rfc3339_opts(SecondsFormat::Secs, true),
-        timestamp
-    );
-    Ok(result)
+    Ok(timestamp.to_string())
 }
 
 fn slot_to_human_time(slot: Slot) -> String {
@@ -853,14 +736,6 @@ pub fn process_show_block_production(
     Ok("".to_string())
 }
 
-pub fn process_total_supply(
-    rpc_client: &RpcClient,
-    commitment_config: CommitmentConfig,
-) -> ProcessResult {
-    let total_supply = rpc_client.total_supply_with_commitment(commitment_config.clone())?;
-    Ok(format!("{} SOL", lamports_to_sol(total_supply)))
-}
-
 pub fn process_get_transaction_count(
     rpc_client: &RpcClient,
     commitment_config: CommitmentConfig,
@@ -1398,19 +1273,6 @@ mod tests {
             }
         );
 
-        let test_total_supply = test_commands
-            .clone()
-            .get_matches_from(vec!["test", "total-supply"]);
-        assert_eq!(
-            parse_command(&test_total_supply, &default_keypair_file, None).unwrap(),
-            CliCommandInfo {
-                command: CliCommand::TotalSupply {
-                    commitment_config: CommitmentConfig::recent(),
-                },
-                signers: vec![],
-            }
-        );
-
         let test_transaction_count = test_commands
             .clone()
             .get_matches_from(vec!["test", "transaction-count"]);
@@ -1433,7 +1295,8 @@ mod tests {
             "2",
             "-t",
             "3",
-            "--confirmed",
+            "--commitment",
+            "max",
         ]);
         assert_eq!(
             parse_command(&test_ping, &default_keypair_file, None).unwrap(),
@@ -1443,7 +1306,7 @@ mod tests {
                     interval: Duration::from_secs(1),
                     count: Some(2),
                     timeout: Duration::from_secs(3),
-                    commitment_config: CommitmentConfig::default(),
+                    commitment_config: CommitmentConfig::max(),
                 },
                 signers: vec![default_keypair.into()],
             }

cli/src/display.rs

@@ -1,6 +1,11 @@
 use crate::cli::SettingType;
 use console::style;
-use solana_sdk::hash::Hash;
+use solana_sdk::{
+    hash::Hash, native_token::lamports_to_sol, program_utils::limited_deserialize,
+    transaction::Transaction,
+};
+use solana_transaction_status::RpcTransactionStatusMeta;
+use std::io;
 
 // Pretty print a "name value"
 pub fn println_name_value(name: &str, value: &str) {
@@ -49,3 +54,140 @@ pub fn println_signers(
     }
     println!();
 }
+
+pub fn write_transaction<W: io::Write>(
+    w: &mut W,
+    transaction: &Transaction,
+    transaction_status: &Option<RpcTransactionStatusMeta>,
+    prefix: &str,
+) -> io::Result<()> {
+    let message = &transaction.message;
+    writeln!(
+        w,
+        "{}Recent Blockhash: {:?}",
+        prefix, message.recent_blockhash
+    )?;
+    for (signature_index, signature) in transaction.signatures.iter().enumerate() {
+        writeln!(
+            w,
+            "{}Signature {}: {:?}",
+            prefix, signature_index, signature
+        )?;
+    }
+    writeln!(w, "{}{:?}", prefix, message.header)?;
+    for (account_index, account) in message.account_keys.iter().enumerate() {
+        writeln!(w, "{}Account {}: {:?}", prefix, account_index, account)?;
+    }
+    for (instruction_index, instruction) in message.instructions.iter().enumerate() {
+        let program_pubkey = message.account_keys[instruction.program_id_index as usize];
+        writeln!(w, "{}Instruction {}", prefix, instruction_index)?;
+        writeln!(
+            w,
+            "{}  Program: {} ({})",
+            prefix, program_pubkey, instruction.program_id_index
+        )?;
+        for (account_index, account) in instruction.accounts.iter().enumerate() {
+            let account_pubkey = message.account_keys[*account as usize];
+            writeln!(
+                w,
+                "{}  Account {}: {} ({})",
+                prefix, account_index, account_pubkey, account
+            )?;
+        }
+
+        let mut raw = true;
+        if program_pubkey == solana_vote_program::id() {
+            if let Ok(vote_instruction) = limited_deserialize::<
+                solana_vote_program::vote_instruction::VoteInstruction,
+            >(&instruction.data)
+            {
+                writeln!(w, "{}  {:?}", prefix, vote_instruction)?;
+                raw = false;
+            }
+        } else if program_pubkey == solana_stake_program::id() {
+            if let Ok(stake_instruction) = limited_deserialize::<
+                solana_stake_program::stake_instruction::StakeInstruction,
+            >(&instruction.data)
+            {
+                writeln!(w, "{}  {:?}", prefix, stake_instruction)?;
+                raw = false;
+            }
+        } else if program_pubkey == solana_sdk::system_program::id() {
+            if let Ok(system_instruction) = limited_deserialize::<
+                solana_sdk::system_instruction::SystemInstruction,
+            >(&instruction.data)
+            {
+                writeln!(w, "{}  {:?}", prefix, system_instruction)?;
+                raw = false;
+            }
+        }
+
+        if raw {
+            writeln!(w, "{}  Data: {:?}", prefix, instruction.data)?;
+        }
+    }
+
+    if let Some(transaction_status) = transaction_status {
+        writeln!(
+            w,
+            "{}Status: {}",
+            prefix,
+            match &transaction_status.status {
+                Ok(_) => "Ok".into(),
+                Err(err) => err.to_string(),
+            }
+        )?;
+        writeln!(
+            w,
+            "{}  Fee: {} SOL",
+            prefix,
+            lamports_to_sol(transaction_status.fee)
+        )?;
+        assert_eq!(
+            transaction_status.pre_balances.len(),
+            transaction_status.post_balances.len()
+        );
+        for (i, (pre, post)) in transaction_status
+            .pre_balances
+            .iter()
+            .zip(transaction_status.post_balances.iter())
+            .enumerate()
+        {
+            if pre == post {
+                writeln!(
+                    w,
+                    "{}  Account {} balance: {} SOL",
+                    prefix,
+                    i,
+                    lamports_to_sol(*pre)
+                )?;
+            } else {
+                writeln!(
+                    w,
+                    "{}  Account {} balance: {} SOL -> {} SOL",
+                    prefix,
+                    i,
+                    lamports_to_sol(*pre),
+                    lamports_to_sol(*post)
+                )?;
+            }
+        }
+    } else {
+        writeln!(w, "{}Status: Unavailable", prefix)?;
+    }
+
+    Ok(())
+}
+
+pub fn println_transaction(
+    transaction: &Transaction,
+    transaction_status: &Option<RpcTransactionStatusMeta>,
+    prefix: &str,
+) {
+    let mut w = Vec::new();
+    if write_transaction(&mut w, transaction, transaction_status, prefix).is_ok() {
+        if let Ok(s) = String::from_utf8(w) {
+            print!("{}", s);
+        }
+    }
+}

cli/src/main.rs

@@ -2,9 +2,7 @@ use clap::{crate_description, crate_name, AppSettings, Arg, ArgGroup, ArgMatches
 use console::style;
 
 use solana_clap_utils::{
-    input_validators::is_url,
-    keypair::{check_for_usb, SKIP_SEED_PHRASE_VALIDATION_ARG},
-    offline::SIGN_ONLY_ARG,
+    input_validators::is_url, keypair::SKIP_SEED_PHRASE_VALIDATION_ARG, offline::SIGN_ONLY_ARG,
     DisplayError,
 };
 use solana_cli::{
@@ -235,20 +233,12 @@ fn main() -> Result<(), Box<dyn error::Error>> {
     )
     .get_matches();
 
-    do_main(&matches, check_for_usb(std::env::args()))
-        .map_err(|err| DisplayError::new_as_boxed(err).into())
+    do_main(&matches).map_err(|err| DisplayError::new_as_boxed(err).into())
 }
 
-fn do_main(
-    matches: &ArgMatches<'_>,
-    need_wallet_manager: bool,
-) -> Result<(), Box<dyn error::Error>> {
+fn do_main(matches: &ArgMatches<'_>) -> Result<(), Box<dyn error::Error>> {
     if parse_settings(&matches)? {
-        let wallet_manager = if need_wallet_manager {
-            maybe_wallet_manager()?
-        } else {
-            None
-        };
+        let wallet_manager = maybe_wallet_manager()?;
 
         let (mut config, signers) = parse_args(&matches, wallet_manager)?;
         config.signers = signers.iter().map(|s| s.as_ref()).collect();

cli/src/nonce.rs

@@ -21,8 +21,9 @@ use solana_sdk::{
     },
     pubkey::Pubkey,
     system_instruction::{
-        advance_nonce_account, authorize_nonce_account, create_nonce_account,
-        create_nonce_account_with_seed, withdraw_nonce_account, NonceError, SystemError,
+        advance_nonce_account, authorize_nonce_account, create_address_with_seed,
+        create_nonce_account, create_nonce_account_with_seed, withdraw_nonce_account, NonceError,
+        SystemError,
     },
     system_program,
     transaction::Transaction,
@@ -473,7 +474,7 @@ pub fn process_create_nonce_account(
 ) -> ProcessResult {
     let nonce_account_pubkey = config.signers[nonce_account].pubkey();
     let nonce_account_address = if let Some(seed) = seed.clone() {
-        Pubkey::create_with_seed(&nonce_account_pubkey, &seed, &system_program::id())?
+        create_address_with_seed(&nonce_account_pubkey, &seed, &system_program::id())?
     } else {
         nonce_account_pubkey
     };

cli/src/stake.rs

@@ -17,7 +17,7 @@ use solana_sdk::{
     account_utils::StateMut,
     message::Message,
     pubkey::Pubkey,
-    system_instruction::SystemError,
+    system_instruction::{create_address_with_seed, SystemError},
     sysvar::{
         stake_history::{self, StakeHistory},
         Sysvar,
@@ -86,7 +86,7 @@ impl StakeSubCommands for App<'_, '_> {
                         .takes_value(true)
                         .validator(is_amount)
                         .required(true)
-                        .help("The amount to send to the stake account, in SOL")
+                        .help("The amount of send to the vote account, in SOL")
                 )
                 .arg(
                     Arg::with_name("custodian")
@@ -790,7 +790,7 @@ pub fn process_create_stake_account(
 ) -> ProcessResult {
     let stake_account = config.signers[stake_account];
     let stake_account_address = if let Some(seed) = seed {
-        Pubkey::create_with_seed(&stake_account.pubkey(), &seed, &solana_stake_program::id())?
+        create_address_with_seed(&stake_account.pubkey(), &seed, &solana_stake_program::id())?
     } else {
         stake_account.pubkey()
     };
@@ -1108,7 +1108,7 @@ pub fn process_split_stake(
     let stake_authority = config.signers[stake_authority];
 
     let split_stake_account_address = if let Some(seed) = split_stake_account_seed {
-        Pubkey::create_with_seed(
+        create_address_with_seed(
             &split_stake_account.pubkey(),
             &seed,
             &solana_stake_program::id(),

cli/src/vote.rs

@@ -4,12 +4,20 @@ use crate::cli::{
     ProcessResult, SignerIndex,
 };
 use clap::{value_t_or_exit, App, Arg, ArgMatches, SubCommand};
-use solana_clap_utils::{input_parsers::*, input_validators::*};
+use solana_clap_utils::{
+    commitment::{commitment_arg, COMMITMENT_ARG},
+    input_parsers::*,
+    input_validators::*,
+};
 use solana_client::rpc_client::RpcClient;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
-    account::Account, commitment_config::CommitmentConfig, message::Message, pubkey::Pubkey,
-    system_instruction::SystemError, transaction::Transaction,
+    account::Account,
+    commitment_config::CommitmentConfig,
+    message::Message,
+    pubkey::Pubkey,
+    system_instruction::{create_address_with_seed, SystemError},
+    transaction::Transaction,
 };
 use solana_vote_program::{
     vote_instruction::{self, withdraw, VoteError},
@@ -155,14 +163,6 @@ impl VoteSubCommands for App<'_, '_> {
             SubCommand::with_name("vote-account")
                 .about("Show the contents of a vote account")
                 .alias("show-vote-account")
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Return information at maximum-lockout commitment level",
-                        ),
-                )
                 .arg(
                     Arg::with_name("vote_account_pubkey")
                         .index(1)
@@ -177,7 +177,8 @@ impl VoteSubCommands for App<'_, '_> {
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
-                ),
+                )
+                .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("withdraw-from-vote-account")
@@ -318,11 +319,7 @@ pub fn parse_vote_get_account_command(
     let vote_account_pubkey =
         pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
     let use_lamports_unit = matches.is_present("lamports");
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::ShowVoteAccount {
             pubkey: vote_account_pubkey,
@@ -377,7 +374,7 @@ pub fn process_create_vote_account(
     let vote_account = config.signers[1];
     let vote_account_pubkey = vote_account.pubkey();
     let vote_account_address = if let Some(seed) = seed {
-        Pubkey::create_with_seed(&vote_account_pubkey, &seed, &solana_vote_program::id())?
+        create_address_with_seed(&vote_account_pubkey, &seed, &solana_vote_program::id())?
     } else {
         vote_account_pubkey
     };

cli/tests/nonce.rs

@@ -13,6 +13,7 @@ use solana_sdk::{
     hash::Hash,
     pubkey::Pubkey,
     signature::{keypair_from_seed, Keypair, Signer},
+    system_instruction::create_address_with_seed,
     system_program,
 };
 use std::{fs::remove_dir_all, sync::mpsc::channel, thread::sleep, time::Duration};
@@ -129,7 +130,7 @@ fn full_battery_tests(
     config_nonce.signers = vec![&nonce_keypair];
 
     let nonce_account = if let Some(seed) = seed.as_ref() {
-        Pubkey::create_with_seed(
+        create_address_with_seed(
             &config_nonce.signers[0].pubkey(),
             seed,
             &system_program::id(),
@@ -300,7 +301,7 @@ fn test_create_account_with_seed() {
     let authority_pubkey = offline_nonce_authority_signer.pubkey();
     let seed = authority_pubkey.to_string()[0..32].to_string();
     let nonce_address =
-        Pubkey::create_with_seed(&creator_pubkey, &seed, &system_program::id()).unwrap();
+        create_address_with_seed(&creator_pubkey, &seed, &system_program::id()).unwrap();
     check_balance(0, &rpc_client, &nonce_address);
 
     let mut creator_config = CliConfig::default();
@@ -337,6 +338,7 @@ fn test_create_account_with_seed() {
         to: to_address,
         from: 0,
         sign_only: true,
+        no_wait: false,
         blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_address),
         nonce_authority: 0,
@@ -357,6 +359,7 @@ fn test_create_account_with_seed() {
         to: to_address,
         from: 0,
         sign_only: false,
+        no_wait: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_address),
             sign_only.blockhash,

cli/tests/stake.rs

@@ -14,6 +14,7 @@ use solana_sdk::{
     nonce::State as NonceState,
     pubkey::Pubkey,
     signature::{keypair_from_seed, Keypair, Signer},
+    system_instruction::create_address_with_seed,
 };
 use solana_stake_program::{
     stake_instruction::LockupArgs,
@@ -159,7 +160,7 @@ fn test_seed_stake_delegation_and_deactivation() {
     .unwrap();
     check_balance(100_000, &rpc_client, &config_validator.signers[0].pubkey());
 
-    let stake_address = Pubkey::create_with_seed(
+    let stake_address = create_address_with_seed(
         &config_validator.signers[0].pubkey(),
         "hi there",
         &solana_stake_program::id(),
@@ -1522,7 +1523,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     };
     process_command(&config).unwrap();
     let seed_address =
-        Pubkey::create_with_seed(&stake_pubkey, seed, &solana_stake_program::id()).unwrap();
+        create_address_with_seed(&stake_pubkey, seed, &solana_stake_program::id()).unwrap();
     check_balance(50_000, &rpc_client, &seed_address);
 
     server.close().unwrap();

cli/tests/transfer.rs

@@ -68,6 +68,7 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        no_wait: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
@@ -95,6 +96,7 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: true,
+        no_wait: false,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
@@ -110,6 +112,7 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        no_wait: false,
         blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,
@@ -147,6 +150,7 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        no_wait: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
             nonce_hash,
@@ -187,6 +191,7 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: true,
+        no_wait: false,
         blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
@@ -202,6 +207,7 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        no_wait: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
             sign_only.blockhash,
@@ -269,6 +275,7 @@ fn test_transfer_multisession_signing() {
         to: to_pubkey,
         from: 1,
         sign_only: true,
+        no_wait: false,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
@@ -293,6 +300,7 @@ fn test_transfer_multisession_signing() {
         to: to_pubkey,
         from: 1,
         sign_only: true,
+        no_wait: false,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
@@ -314,6 +322,7 @@ fn test_transfer_multisession_signing() {
         to: to_pubkey,
         from: 1,
         sign_only: false,
+        no_wait: false,
         blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.1.1"
+version = "1.0.22"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,15 +14,15 @@ bs58 = "0.3.0"
 indicatif = "0.14.0"
 jsonrpc-core = "14.0.5"
 log = "0.4.8"
-rayon = "1.3.0"
-reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-serde = "1.0.105"
+rayon = "1.2.0"
+reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
+serde = "1.0.104"
 serde_derive = "1.0.103"
-serde_json = "1.0.48"
-solana-transaction-status = { path = "../transaction-status", version = "1.1.1" }
-solana-net-utils = { path = "../net-utils", version = "1.1.1" }
-solana-sdk = { path = "../sdk", version = "1.1.1" }
-solana-vote-program = { path = "../programs/vote", version = "1.1.1" }
+serde_json = "1.0.46"
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
+solana-transaction-status = { path = "../transaction-status", version = "1.0.22" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.22" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -31,4 +31,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "14.0.5"
 jsonrpc-http-server = "14.0.6"
-solana-logger = { path = "../logger", version = "1.1.1" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

client/src/mock_rpc_client_request.rs

@@ -40,7 +40,7 @@ impl GenericRpcClientRequest for MockRpcClientRequest {
     fn send(
         &self,
         request: &RpcRequest,
-        params: serde_json::Value,
+        _params: serde_json::Value,
         _retries: usize,
     ) -> Result<serde_json::Value> {
         if let Some(value) = self.mocks.write().unwrap().remove(request) {
@@ -50,17 +50,6 @@ impl GenericRpcClientRequest for MockRpcClientRequest {
             return Ok(Value::Null);
         }
         let val = match request {
-            RpcRequest::ConfirmTransaction => {
-                if let Some(params_array) = params.as_array() {
-                    if let Value::String(param_string) = &params_array[0] {
-                        Value::Bool(param_string == SIGNATURE)
-                    } else {
-                        Value::Null
-                    }
-                } else {
-                    Value::Null
-                }
-            }
             RpcRequest::GetBalance => serde_json::to_value(Response {
                 context: RpcResponseContext { slot: 1 },
                 value: Value::Number(Number::from(50)),
@@ -87,21 +76,6 @@ impl GenericRpcClientRequest for MockRpcClientRequest {
                 context: RpcResponseContext { slot: 1 },
                 value: serde_json::to_value(FeeRateGovernor::default()).unwrap(),
             })?,
-            RpcRequest::GetSignatureStatus => {
-                let response: Option<transaction::Result<()>> = if self.url == "account_in_use" {
-                    Some(Err(TransactionError::AccountInUse))
-                } else if self.url == "instruction_error" {
-                    Some(Err(TransactionError::InstructionError(
-                        0,
-                        InstructionError::UninitializedAccount,
-                    )))
-                } else if self.url == "sig_not_found" {
-                    None
-                } else {
-                    Some(Ok(()))
-                };
-                serde_json::to_value(response).unwrap()
-            }
             RpcRequest::GetSignatureStatuses => {
                 let status: transaction::Result<()> = if self.url == "account_in_use" {
                     Err(TransactionError::AccountInUse)
@@ -116,10 +90,12 @@ impl GenericRpcClientRequest for MockRpcClientRequest {
                 let status = if self.url == "sig_not_found" {
                     None
                 } else {
+                    let err = status.clone().err();
                     Some(TransactionStatus {
                         status,
                         slot: 1,
-                        confirmations: Some(0),
+                        confirmations: None,
+                        err,
                     })
                 };
                 serde_json::to_value(Response {

client/src/rpc_client.rs

@@ -4,7 +4,11 @@ use crate::{
     mock_rpc_client_request::{MockRpcClientRequest, Mocks},
     rpc_client_request::RpcClientRequest,
     rpc_request::{RpcError, RpcRequest},
-    rpc_response::*,
+    rpc_response::{
+        Response, RpcAccount, RpcBlockhashFeeCalculator, RpcContactInfo, RpcEpochInfo,
+        RpcFeeCalculator, RpcFeeRateGovernor, RpcIdentity, RpcKeyedAccount, RpcLeaderSchedule,
+        RpcResult, RpcVersionInfo, RpcVoteAccountStatus,
+    },
 };
 use bincode::serialize;
 use indicatif::{ProgressBar, ProgressStyle};
@@ -12,7 +16,10 @@ use log::*;
 use serde_json::{json, Value};
 use solana_sdk::{
     account::Account,
-    clock::{Slot, UnixTimestamp, DEFAULT_TICKS_PER_SECOND, DEFAULT_TICKS_PER_SLOT},
+    clock::{
+        Slot, UnixTimestamp, DEFAULT_TICKS_PER_SECOND, DEFAULT_TICKS_PER_SLOT,
+        MAX_HASH_AGE_IN_SECONDS,
+    },
     commitment_config::CommitmentConfig,
     epoch_schedule::EpochSchedule,
     fee_calculator::{FeeCalculator, FeeRateGovernor},
@@ -23,7 +30,9 @@ use solana_sdk::{
     signers::Signers,
     transaction::{self, Transaction, TransactionError},
 };
-use solana_transaction_status::{ConfirmedBlock, TransactionEncoding, TransactionStatus};
+use solana_transaction_status::{
+    ConfirmedBlock, ConfirmedTransaction, TransactionEncoding, TransactionStatus,
+};
 use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
 use std::{
     error,
@@ -77,17 +86,16 @@ impl RpcClient {
         signature: &Signature,
         commitment_config: CommitmentConfig,
     ) -> RpcResult<bool> {
-        let response = self
-            .client
-            .send(
-                &RpcRequest::ConfirmTransaction,
-                json!([signature.to_string(), commitment_config]),
-                0,
-            )
-            .map_err(|err| err.into_with_command("ConfirmTransaction"))?;
+        let Response { context, value } = self.get_signature_statuses(&[*signature])?;
 
-        serde_json::from_value::<Response<bool>>(response)
-            .map_err(|err| ClientError::new_with_command(err.into(), "ConfirmTransaction"))
+        Ok(Response {
+            context,
+            value: value[0]
+                .as_ref()
+                .filter(|result| result.satisfies_commitment(commitment_config))
+                .map(|result| result.status.is_ok())
+                .unwrap_or_default(),
+        })
     }
 
     pub fn send_transaction(&self, transaction: &Transaction) -> ClientResult<Signature> {
@@ -113,6 +121,18 @@ impl RpcClient {
         self.get_signature_status_with_commitment(signature, CommitmentConfig::default())
     }
 
+    pub fn get_signature_statuses(
+        &self,
+        signatures: &[Signature],
+    ) -> RpcResult<Vec<Option<TransactionStatus>>> {
+        let signatures: Vec<_> = signatures.iter().map(|s| s.to_string()).collect();
+        let signature_status =
+            self.client
+                .send(&RpcRequest::GetSignatureStatuses, json!([signatures]), 5)?;
+        Ok(serde_json::from_value(signature_status)
+            .map_err(|err| ClientError::new_with_command(err.into(), "GetSignatureStatuses"))?)
+    }
+
     pub fn get_signature_status_with_commitment(
         &self,
         signature: &Signature,
@@ -120,7 +140,7 @@ impl RpcClient {
     ) -> ClientResult<Option<transaction::Result<()>>> {
         let signature_status = self.client.send(
             &RpcRequest::GetSignatureStatuses,
-            json!([[signature.to_string()], commitment_config]),
+            json!([[signature.to_string()]]),
             5,
         )?;
         let result: Response<Vec<Option<TransactionStatus>>> =
@@ -128,6 +148,29 @@ impl RpcClient {
                 .map_err(|err| ClientError::new_with_command(err.into(), "GetSignatureStatuses"))?;
         Ok(result.value[0]
             .clone()
+            .filter(|result| result.satisfies_commitment(commitment_config))
+            .map(|status_meta| status_meta.status))
+    }
+
+    pub fn get_signature_status_with_commitment_and_history(
+        &self,
+        signature: &Signature,
+        commitment_config: CommitmentConfig,
+        search_transaction_history: bool,
+    ) -> ClientResult<Option<transaction::Result<()>>> {
+        let signature_status = self.client.send(
+            &RpcRequest::GetSignatureStatuses,
+            json!([[signature.to_string()], {
+                "searchTransactionHistory": search_transaction_history
+            }]),
+            5,
+        )?;
+        let result: Response<Vec<Option<TransactionStatus>>> =
+            serde_json::from_value(signature_status)
+                .map_err(|err| ClientError::new_with_command(err.into(), "GetSignatureStatuses"))?;
+        Ok(result.value[0]
+            .clone()
+            .filter(|result| result.satisfies_commitment(commitment_config))
             .map(|status_meta| status_meta.status))
     }
 
@@ -148,23 +191,6 @@ impl RpcClient {
             .map_err(|err| ClientError::new_with_command(err.into(), "GetSlot"))
     }
 
-    pub fn total_supply(&self) -> ClientResult<u64> {
-        self.total_supply_with_commitment(CommitmentConfig::default())
-    }
-
-    pub fn total_supply_with_commitment(
-        &self,
-        commitment_config: CommitmentConfig,
-    ) -> ClientResult<u64> {
-        let response = self
-            .client
-            .send(&RpcRequest::GetTotalSupply, json!([commitment_config]), 0)
-            .map_err(|err| err.into_with_command("GetTotalSupply"))?;
-
-        serde_json::from_value(response)
-            .map_err(|err| ClientError::new_with_command(err.into(), "GetTotalSupply"))
-    }
-
     pub fn get_vote_accounts(&self) -> ClientResult<RpcVoteAccountStatus> {
         self.get_vote_accounts_with_commitment(CommitmentConfig::default())
     }
@@ -228,6 +254,44 @@ impl RpcClient {
             .map_err(|err| ClientError::new_with_command(err.into(), "GetConfirmedBlocks"))
     }
 
+    pub fn get_confirmed_signatures_for_address(
+        &self,
+        address: &Pubkey,
+        start_slot: Slot,
+        end_slot: Slot,
+    ) -> ClientResult<Vec<Signature>> {
+        let response = self
+            .client
+            .send(
+                &RpcRequest::GetConfirmedSignaturesForAddress,
+                json!([address, start_slot, end_slot]),
+                0,
+            )
+            .map_err(|err| err.into_with_command("GetConfirmedSignaturesForAddress"))?;
+
+        serde_json::from_value(response).map_err(|err| {
+            ClientError::new_with_command(err.into(), "GetConfirmedSignaturesForAddress")
+        })
+    }
+
+    pub fn get_confirmed_transaction(
+        &self,
+        signature: &Signature,
+        encoding: TransactionEncoding,
+    ) -> ClientResult<ConfirmedTransaction> {
+        let response = self
+            .client
+            .send(
+                &RpcRequest::GetConfirmedTransaction,
+                json!([signature.to_string(), encoding]),
+                0,
+            )
+            .map_err(|err| err.into_with_command("GetConfirmedTransaction"))?;
+
+        serde_json::from_value(response)
+            .map_err(|err| ClientError::new_with_command(err.into(), "GetConfirmedTransaction"))
+    }
+
     pub fn get_block_time(&self, slot: Slot) -> ClientResult<UnixTimestamp> {
         let response = self
             .client
@@ -856,14 +920,13 @@ impl RpcClient {
         trace!("check_signature: {:?}", signature);
 
         for _ in 0..30 {
-            let response = self.client.send(
-                &RpcRequest::ConfirmTransaction,
-                json!([signature.to_string(), CommitmentConfig::recent()]),
-                0,
-            );
-
+            let response =
+                self.confirm_transaction_with_commitment(signature, CommitmentConfig::recent());
             match response {
-                Ok(Value::Bool(signature_status)) => {
+                Ok(Response {
+                    value: signature_status,
+                    ..
+                }) => {
                     if signature_status {
                         trace!("Response found signature");
                     } else {
@@ -872,12 +935,6 @@ impl RpcClient {
 
                     return signature_status;
                 }
-                Ok(other) => {
-                    debug!(
-                        "check_signature request failed, expected bool, got: {:?}",
-                        other
-                    );
-                }
                 Err(err) => {
                     debug!("check_signature request failed: {:?}", err);
                 }
@@ -950,7 +1007,7 @@ impl RpcClient {
             .client
             .send(
                 &RpcRequest::GetSignatureStatuses,
-                json!([[signature.to_string()], CommitmentConfig::recent().ok()]),
+                json!([[signature.to_string()]]),
                 1,
             )
             .map_err(|err| err.into_with_command("GetSignatureStatuses"))?;
@@ -1037,6 +1094,7 @@ impl RpcClient {
                 }
             }
         };
+        let now = Instant::now();
         loop {
             // Return when default (max) commitment is reached
             // Failed transactions have already been eliminated, `is_some` check is sufficient
@@ -1051,7 +1109,14 @@ impl RpcClient {
                 MAX_LOCKOUT_HISTORY + 1,
             ));
             sleep(Duration::from_millis(500));
-            confirmations = self.get_num_blocks_since_signature_confirmation(&signature)?;
+            confirmations = self
+                .get_num_blocks_since_signature_confirmation(&signature)
+                .unwrap_or(confirmations);
+            if now.elapsed().as_secs() >= MAX_HASH_AGE_IN_SECONDS as u64 {
+                return Err(
+                    RpcError::ForUser("transaction not finalized. This can happen when a transaction lands in an abandoned fork. Please retry.".to_string()).into(),
+                );
+            }
         }
     }
 

client/src/rpc_request.rs

@@ -3,7 +3,6 @@ use thiserror::Error;
 
 #[derive(Debug, PartialEq, Eq, Hash)]
 pub enum RpcRequest {
-    ConfirmTransaction,
     DeregisterNode,
     ValidatorExit,
     GetAccountInfo,
@@ -12,6 +11,8 @@ pub enum RpcRequest {
     GetClusterNodes,
     GetConfirmedBlock,
     GetConfirmedBlocks,
+    GetConfirmedSignaturesForAddress,
+    GetConfirmedTransaction,
     GetEpochInfo,
     GetEpochSchedule,
     GetGenesisHash,
@@ -22,7 +23,6 @@ pub enum RpcRequest {
     GetRecentBlockhash,
     GetFeeCalculatorForBlockhash,
     GetFeeRateGovernor,
-    GetSignatureStatus,
     GetSignatureStatuses,
     GetSlot,
     GetSlotLeader,
@@ -30,7 +30,6 @@ pub enum RpcRequest {
     GetStorageTurnRate,
     GetSlotsPerSegment,
     GetStoragePubkeysForSlot,
-    GetTotalSupply,
     GetTransactionCount,
     GetVersion,
     GetVoteAccounts,
@@ -46,7 +45,6 @@ impl RpcRequest {
     pub(crate) fn build_request_json(&self, id: u64, params: Value) -> Value {
         let jsonrpc = "2.0";
         let method = match self {
-            RpcRequest::ConfirmTransaction => "confirmTransaction",
             RpcRequest::DeregisterNode => "deregisterNode",
             RpcRequest::ValidatorExit => "validatorExit",
             RpcRequest::GetAccountInfo => "getAccountInfo",
@@ -55,6 +53,8 @@ impl RpcRequest {
             RpcRequest::GetClusterNodes => "getClusterNodes",
             RpcRequest::GetConfirmedBlock => "getConfirmedBlock",
             RpcRequest::GetConfirmedBlocks => "getConfirmedBlocks",
+            RpcRequest::GetConfirmedSignaturesForAddress => "getConfirmedSignaturesForAddress",
+            RpcRequest::GetConfirmedTransaction => "getConfirmedTransaction",
             RpcRequest::GetEpochInfo => "getEpochInfo",
             RpcRequest::GetEpochSchedule => "getEpochSchedule",
             RpcRequest::GetGenesisHash => "getGenesisHash",
@@ -65,7 +65,6 @@ impl RpcRequest {
             RpcRequest::GetRecentBlockhash => "getRecentBlockhash",
             RpcRequest::GetFeeCalculatorForBlockhash => "getFeeCalculatorForBlockhash",
             RpcRequest::GetFeeRateGovernor => "getFeeRateGovernor",
-            RpcRequest::GetSignatureStatus => "getSignatureStatus",
             RpcRequest::GetSignatureStatuses => "getSignatureStatuses",
             RpcRequest::GetSlot => "getSlot",
             RpcRequest::GetSlotLeader => "getSlotLeader",
@@ -73,7 +72,6 @@ impl RpcRequest {
             RpcRequest::GetStorageTurnRate => "getStorageTurnRate",
             RpcRequest::GetSlotsPerSegment => "getSlotsPerSegment",
             RpcRequest::GetStoragePubkeysForSlot => "getStoragePubkeysForSlot",
-            RpcRequest::GetTotalSupply => "getTotalSupply",
             RpcRequest::GetTransactionCount => "getTransactionCount",
             RpcRequest::GetVersion => "getVersion",
             RpcRequest::GetVoteAccounts => "getVoteAccounts",

client/src/rpc_response.rs

@@ -4,7 +4,7 @@ use solana_sdk::{
     clock::{Epoch, Slot},
     fee_calculator::{FeeCalculator, FeeRateGovernor},
     pubkey::Pubkey,
-    transaction::Result,
+    transaction::{Result, TransactionError},
 };
 use std::{collections::HashMap, net::SocketAddr, str::FromStr};
 
@@ -54,6 +54,12 @@ pub struct RpcKeyedAccount {
     pub account: RpcAccount,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSignatureResult {
+    pub err: Option<TransactionError>,
+}
+
 /// A duplicate representation of a Message for pretty JSON serialization
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
@@ -186,3 +192,19 @@ pub struct RpcStorageTurn {
     pub blockhash: String,
     pub slot: Slot,
 }
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcAccountBalance {
+    pub address: String,
+    pub lamports: u64,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSupply {
+    pub total: u64,
+    pub circulating: u64,
+    pub non_circulating: u64,
+    pub non_circulating_accounts: Vec<String>,
+}

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.1.1"
+version = "1.0.22"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -17,65 +17,66 @@ codecov = { repository = "solana-labs/solana", branch = "master", service = "git
 bincode = "1.2.1"
 bv = { version = "0.11.1", features = ["serde"] }
 bs58 = "0.3.0"
-byteorder = "1.3.4"
-chrono = { version = "0.4.11", features = ["serde"] }
+byteorder = "1.3.2"
+chrono = { version = "0.4.10", features = ["serde"] }
+compression = "0.1.5"
 core_affinity = "0.5.10"
-crossbeam-channel = "0.4"
+crossbeam-channel = "0.3"
 fs_extra = "1.1.0"
-flate2 = "1.0"
 indexmap = "1.3"
-itertools = "0.9.0"
+itertools = "0.8.2"
 jsonrpc-core = "14.0.5"
 jsonrpc-core-client = { version = "14.0.5", features = ["ws"] }
 jsonrpc-derive = "14.0.5"
 jsonrpc-http-server = "14.0.6"
 jsonrpc-pubsub = "14.0.6"
 jsonrpc-ws-server = "14.0.6"
+libc = "0.2.66"
 log = "0.4.8"
+nix = "0.17.0"
 num_cpus = "1.0.0"
 num-traits = "0.2"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
-rayon = "1.3.0"
-regex = "1.3.6"
-serde = "1.0.105"
+rand = "0.7.0"
+rand_chacha = "0.2.2"
+rayon = "1.2.0"
+regex = "1.3.4"
+serde = "1.0.104"
 serde_derive = "1.0.103"
-serde_json = "1.0.48"
-solana-budget-program = { path = "../programs/budget", version = "1.1.1" }
-solana-clap-utils = { path = "../clap-utils", version = "1.1.1" }
-solana-client = { path = "../client", version = "1.1.1" }
-solana-transaction-status = { path = "../transaction-status", version = "1.1.1" }
-solana-faucet = { path = "../faucet", version = "1.1.1" }
-ed25519-dalek = "=1.0.0-pre.1"
-solana-ledger = { path = "../ledger", version = "1.1.1" }
-solana-logger = { path = "../logger", version = "1.1.1" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.1.1" }
-solana-metrics = { path = "../metrics", version = "1.1.1" }
-solana-measure = { path = "../measure", version = "1.1.1" }
-solana-net-utils = { path = "../net-utils", version = "1.1.1" }
-solana-chacha-cuda = { path = "../chacha-cuda", version = "1.1.1" }
-solana-perf = { path = "../perf", version = "1.1.1" }
-solana-runtime = { path = "../runtime", version = "1.1.1" }
-solana-sdk = { path = "../sdk", version = "1.1.1" }
-solana-stake-program = { path = "../programs/stake", version = "1.1.1" }
-solana-storage-program = { path = "../programs/storage", version = "1.1.1" }
-solana-streamer = { path = "../streamer", version = "1.1.1" }
-solana-vote-program = { path = "../programs/vote", version = "1.1.1" }
-solana-vote-signer = { path = "../vote-signer", version = "1.1.1" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.1.1" }
+serde_json = "1.0.46"
+solana-budget-program = { path = "../programs/budget", version = "1.0.22" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.22" }
+solana-client = { path = "../client", version = "1.0.22" }
+solana-transaction-status = { path = "../transaction-status", version = "1.0.22" }
+solana-faucet = { path = "../faucet", version = "1.0.22" }
+ed25519-dalek = "=1.0.0-pre.3"
+solana-ledger = { path = "../ledger", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.0.22" }
+solana-metrics = { path = "../metrics", version = "1.0.22" }
+solana-measure = { path = "../measure", version = "1.0.22" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-chacha-cuda = { path = "../chacha-cuda", version = "1.0.22" }
+solana-perf = { path = "../perf", version = "1.0.22" }
+solana-runtime = { path = "../runtime", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
+solana-stake-program = { path = "../programs/stake", version = "1.0.22" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.22" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.22" }
+solana-vote-signer = { path = "../vote-signer", version = "1.0.22" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.0.22" }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = "0.1"
 tokio-codec = "0.1"
 tokio-fs = "0.1"
 tokio-io = "0.1"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.1.1" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.0.22" }
 trees = "0.2.1"
 
 [dev-dependencies]
 matches = "0.1.6"
-reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
-serial_test = "0.4.0"
+reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
+serial_test = "0.3.2"
 serial_test_derive = "0.4.0"
 systemstat = "0.1.5"
 
@@ -103,3 +104,6 @@ name = "cluster_info"
 [[bench]]
 name = "chacha"
 required-features = ["chacha"]
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

core/benches/banking_stage.rs

@@ -9,12 +9,12 @@ use rayon::prelude::*;
 use solana_core::banking_stage::{create_test_recorder, BankingStage};
 use solana_core::cluster_info::ClusterInfo;
 use solana_core::cluster_info::Node;
+use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+use solana_core::packet::to_packets_chunked;
 use solana_core::poh_recorder::WorkingBankEntry;
 use solana_ledger::blockstore_processor::process_entries;
 use solana_ledger::entry::{next_hash, Entry};
-use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
 use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
-use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_runtime::bank::Bank;
 use solana_sdk::genesis_config::GenesisConfig;

core/benches/retransmit_stage.rs

@@ -6,12 +6,12 @@ extern crate test;
 use log::*;
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::contact_info::ContactInfo;
+use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+use solana_core::packet::to_packets_chunked;
 use solana_core::retransmit_stage::retransmitter;
 use solana_ledger::bank_forks::BankForks;
-use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
 use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
 use solana_measure::measure::Measure;
-use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_runtime::bank::Bank;
 use solana_sdk::pubkey::Pubkey;

core/benches/sigverify_stage.rs

@@ -6,9 +6,9 @@ extern crate test;
 use crossbeam_channel::unbounded;
 use log::*;
 use rand::{thread_rng, Rng};
+use solana_core::packet::to_packets_chunked;
 use solana_core::sigverify::TransactionSigVerifier;
 use solana_core::sigverify_stage::SigVerifyStage;
-use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_sdk::hash::Hash;
 use solana_sdk::signature::{Keypair, Signer};

core/src/accounts_cleanup_service.rs

@@ -1,38 +0,0 @@
-// Service to clean up dead slots in accounts_db
-//
-// This can be expensive since we have to walk the append vecs being cleaned up.
-
-use solana_ledger::bank_forks::BankForks;
-use std::sync::{
-    atomic::{AtomicBool, Ordering},
-    Arc, RwLock,
-};
-use std::thread::{self, sleep, Builder, JoinHandle};
-use std::time::Duration;
-
-pub struct AccountsCleanupService {
-    t_cleanup: JoinHandle<()>,
-}
-
-impl AccountsCleanupService {
-    pub fn new(bank_forks: Arc<RwLock<BankForks>>, exit: &Arc<AtomicBool>) -> Self {
-        info!("AccountsCleanupService active");
-        let exit = exit.clone();
-        let t_cleanup = Builder::new()
-            .name("solana-accounts-cleanup".to_string())
-            .spawn(move || loop {
-                if exit.load(Ordering::Relaxed) {
-                    break;
-                }
-                let bank = bank_forks.read().unwrap().working_bank();
-                bank.clean_dead_slots();
-                sleep(Duration::from_millis(100));
-            })
-            .unwrap();
-        Self { t_cleanup }
-    }
-
-    pub fn join(self) -> thread::Result<()> {
-        self.t_cleanup.join()
-    }
-}

core/src/banking_stage.rs

@@ -3,6 +3,7 @@
 //! can do its processing in parallel with signature verification on the GPU.
 use crate::{
     cluster_info::ClusterInfo,
+    packet::{limited_deserialize, Packet, Packets, PACKETS_PER_BATCH},
     poh_recorder::{PohRecorder, PohRecorderError, WorkingBankEntry},
     poh_service::PohService,
 };
@@ -16,11 +17,7 @@ use solana_ledger::{
 };
 use solana_measure::{measure::Measure, thread_mem_usage};
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_info, inc_new_counter_warn};
-use solana_perf::{
-    cuda_runtime::PinnedVec,
-    packet::{limited_deserialize, Packet, Packets, PACKETS_PER_BATCH},
-    perf_libs,
-};
+use solana_perf::{cuda_runtime::PinnedVec, perf_libs};
 use solana_runtime::{
     accounts_db::ErrorCounters,
     bank::{Bank, TransactionBalancesSet, TransactionProcessResult},
@@ -1012,7 +1009,10 @@ pub fn create_test_recorder(
 mod tests {
     use super::*;
     use crate::{
-        cluster_info::Node, poh_recorder::WorkingBank,
+        cluster_info::Node,
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        packet::to_packets,
+        poh_recorder::WorkingBank,
         transaction_status_service::TransactionStatusService,
     };
     use crossbeam_channel::unbounded;
@@ -1020,10 +1020,8 @@ mod tests {
     use solana_ledger::{
         blockstore::entries_to_test_shreds,
         entry::{next_entry, Entry, EntrySlice},
-        genesis_utils::{create_genesis_config, GenesisConfigInfo},
         get_tmp_ledger_path,
     };
-    use solana_perf::packet::to_packets;
     use solana_runtime::bank::HashAgeKind;
     use solana_sdk::{
         instruction::InstructionError,
@@ -1982,13 +1980,23 @@ mod tests {
             {
                 if let EncodedTransaction::Json(transaction) = transaction {
                     if transaction.signatures[0] == success_signature.to_string() {
-                        assert_eq!(meta.unwrap().status, Ok(()));
+                        let meta = meta.unwrap();
+                        assert_eq!(meta.err, None);
+                        assert_eq!(meta.status, Ok(()));
                     } else if transaction.signatures[0] == ix_error_signature.to_string() {
+                        let meta = meta.unwrap();
                         assert_eq!(
-                            meta.unwrap().status,
+                            meta.err,
+                            Some(TransactionError::InstructionError(
+                                0,
+                                InstructionError::Custom(1)
+                            ))
+                        );
+                        assert_eq!(
+                            meta.status,
                             Err(TransactionError::InstructionError(
                                 0,
-                                InstructionError::CustomError(1)
+                                InstructionError::Custom(1)
                             ))
                         );
                     } else {

core/src/blockstream.rs

@@ -0,0 +1,283 @@
+//! The `blockstream` module provides a method for streaming entries out via a
+//! local unix socket, to provide client services such as a block explorer with
+//! real-time access to entries.
+
+use bincode::serialize;
+use chrono::{SecondsFormat, Utc};
+use serde_json::json;
+use solana_ledger::entry::Entry;
+use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
+use std::cell::RefCell;
+use std::io::Result;
+use std::path::{Path, PathBuf};
+
+pub trait EntryWriter: std::fmt::Debug {
+    fn write(&self, payload: String) -> Result<()>;
+}
+
+#[derive(Debug, Default)]
+pub struct EntryVec {
+    values: RefCell<Vec<String>>,
+}
+
+impl EntryWriter for EntryVec {
+    fn write(&self, payload: String) -> Result<()> {
+        self.values.borrow_mut().push(payload);
+        Ok(())
+    }
+}
+
+impl EntryVec {
+    pub fn new() -> Self {
+        EntryVec {
+            values: RefCell::new(Vec::new()),
+        }
+    }
+
+    pub fn entries(&self) -> Vec<String> {
+        self.values.borrow().clone()
+    }
+}
+
+#[derive(Debug)]
+pub struct EntrySocket {
+    unix_socket: PathBuf,
+}
+
+impl EntryWriter for EntrySocket {
+    #[cfg(not(windows))]
+    fn write(&self, payload: String) -> Result<()> {
+        use std::io::prelude::*;
+        use std::net::Shutdown;
+        use std::os::unix::net::UnixStream;
+
+        const MESSAGE_TERMINATOR: &str = "\n";
+
+        let mut socket = UnixStream::connect(&self.unix_socket)?;
+        socket.write_all(payload.as_bytes())?;
+        socket.write_all(MESSAGE_TERMINATOR.as_bytes())?;
+        socket.shutdown(Shutdown::Write)?;
+        Ok(())
+    }
+    #[cfg(windows)]
+    fn write(&self, _payload: String) -> Result<()> {
+        Err(std::io::Error::new(
+            std::io::ErrorKind::Other,
+            "EntryWriter::write() not implemented for windows",
+        ))
+    }
+}
+
+pub trait BlockstreamEvents {
+    fn emit_entry_event(
+        &self,
+        slot: Slot,
+        tick_height: u64,
+        leader_pubkey: &Pubkey,
+        entries: &Entry,
+    ) -> Result<()>;
+    fn emit_block_event(
+        &self,
+        slot: Slot,
+        tick_height: u64,
+        leader_pubkey: &Pubkey,
+        blockhash: Hash,
+    ) -> Result<()>;
+}
+
+#[derive(Debug)]
+pub struct Blockstream<T: EntryWriter> {
+    pub output: T,
+}
+
+impl<T> BlockstreamEvents for Blockstream<T>
+where
+    T: EntryWriter,
+{
+    fn emit_entry_event(
+        &self,
+        slot: Slot,
+        tick_height: u64,
+        leader_pubkey: &Pubkey,
+        entry: &Entry,
+    ) -> Result<()> {
+        let transactions: Vec<Vec<u8>> = serialize_transactions(entry);
+        let stream_entry = json!({
+            "num_hashes": entry.num_hashes,
+            "hash": entry.hash,
+            "transactions": transactions
+        });
+        let json_entry = serde_json::to_string(&stream_entry)?;
+        let payload = format!(
+            r#"{{"dt":"{}","t":"entry","s":{},"h":{},"l":"{:?}","entry":{}}}"#,
+            Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
+            slot,
+            tick_height,
+            leader_pubkey,
+            json_entry,
+        );
+        self.output.write(payload)?;
+        Ok(())
+    }
+
+    fn emit_block_event(
+        &self,
+        slot: Slot,
+        tick_height: u64,
+        leader_pubkey: &Pubkey,
+        blockhash: Hash,
+    ) -> Result<()> {
+        let payload = format!(
+            r#"{{"dt":"{}","t":"block","s":{},"h":{},"l":"{:?}","hash":"{:?}"}}"#,
+            Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
+            slot,
+            tick_height,
+            leader_pubkey,
+            blockhash,
+        );
+        self.output.write(payload)?;
+        Ok(())
+    }
+}
+
+pub type SocketBlockstream = Blockstream<EntrySocket>;
+
+impl SocketBlockstream {
+    pub fn new(unix_socket: &Path) -> Self {
+        Blockstream {
+            output: EntrySocket {
+                unix_socket: unix_socket.to_path_buf(),
+            },
+        }
+    }
+}
+
+pub type MockBlockstream = Blockstream<EntryVec>;
+
+impl MockBlockstream {
+    pub fn new(_: &Path) -> Self {
+        Blockstream {
+            output: EntryVec::new(),
+        }
+    }
+
+    pub fn entries(&self) -> Vec<String> {
+        self.output.entries()
+    }
+}
+
+fn serialize_transactions(entry: &Entry) -> Vec<Vec<u8>> {
+    entry
+        .transactions
+        .iter()
+        .map(|tx| serialize(&tx).unwrap())
+        .collect()
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use chrono::{DateTime, FixedOffset};
+    use serde_json::Value;
+    use solana_sdk::hash::Hash;
+    use solana_sdk::signature::{Keypair, Signer};
+    use solana_sdk::system_transaction;
+    use std::collections::HashSet;
+    use std::path::PathBuf;
+
+    #[test]
+    fn test_serialize_transactions() {
+        let entry = Entry::new(&Hash::default(), 1, vec![]);
+        let empty_vec: Vec<Vec<u8>> = vec![];
+        assert_eq!(serialize_transactions(&entry), empty_vec);
+
+        let keypair0 = Keypair::new();
+        let keypair1 = Keypair::new();
+        let tx0 = system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default());
+        let tx1 = system_transaction::transfer(&keypair1, &keypair0.pubkey(), 2, Hash::default());
+        let serialized_tx0 = serialize(&tx0).unwrap();
+        let serialized_tx1 = serialize(&tx1).unwrap();
+        let entry = Entry::new(&Hash::default(), 1, vec![tx0, tx1]);
+        assert_eq!(
+            serialize_transactions(&entry),
+            vec![serialized_tx0, serialized_tx1]
+        );
+    }
+
+    #[test]
+    fn test_blockstream() -> () {
+        let blockstream = MockBlockstream::new(&PathBuf::from("test_stream"));
+        let ticks_per_slot = 5;
+
+        let mut blockhash = Hash::default();
+        let mut entries = Vec::new();
+        let mut expected_entries = Vec::new();
+
+        let tick_height_initial = 1;
+        let tick_height_final = tick_height_initial + ticks_per_slot + 2;
+        let mut curr_slot = 0;
+        let leader_pubkey = Pubkey::new_rand();
+
+        for tick_height in tick_height_initial..=tick_height_final {
+            if tick_height == 5 {
+                blockstream
+                    .emit_block_event(curr_slot, tick_height, &leader_pubkey, blockhash)
+                    .unwrap();
+                curr_slot += 1;
+            }
+            let entry = Entry::new(&mut blockhash, 1, vec![]); // just ticks
+            blockhash = entry.hash;
+            blockstream
+                .emit_entry_event(curr_slot, tick_height, &leader_pubkey, &entry)
+                .unwrap();
+            expected_entries.push(entry.clone());
+            entries.push(entry);
+        }
+
+        assert_eq!(
+            blockstream.entries().len() as u64,
+            // one entry per tick (1..=N+2) is +3, plus one block
+            ticks_per_slot + 3 + 1
+        );
+
+        let mut j = 0;
+        let mut matched_entries = 0;
+        let mut matched_slots = HashSet::new();
+        let mut matched_blocks = HashSet::new();
+
+        for item in blockstream.entries() {
+            let json: Value = serde_json::from_str(&item).unwrap();
+            let dt_str = json["dt"].as_str().unwrap();
+
+            // Ensure `ts` field parses as valid DateTime
+            let _dt: DateTime<FixedOffset> = DateTime::parse_from_rfc3339(dt_str).unwrap();
+
+            let item_type = json["t"].as_str().unwrap();
+            match item_type {
+                "block" => {
+                    let hash = json["hash"].to_string();
+                    matched_blocks.insert(hash);
+                }
+
+                "entry" => {
+                    let slot = json["s"].as_u64().unwrap();
+                    matched_slots.insert(slot);
+                    let entry_obj = json["entry"].clone();
+                    let entry: Entry = serde_json::from_value(entry_obj).unwrap();
+
+                    assert_eq!(entry, expected_entries[j]);
+                    matched_entries += 1;
+                    j += 1;
+                }
+
+                _ => {
+                    assert!(false, "unknown item type {}", item);
+                }
+            }
+        }
+
+        assert_eq!(matched_entries, expected_entries.len());
+        assert_eq!(matched_slots.len(), 2);
+        assert_eq!(matched_blocks.len(), 1);
+    }
+}

core/src/blockstream_service.rs

@@ -0,0 +1,228 @@
+//! The `blockstream_service` implements optional streaming of entries and block metadata
+//! using the `blockstream` module, providing client services such as a block explorer with
+//! real-time access to entries.
+
+use crate::blockstream::BlockstreamEvents;
+#[cfg(test)]
+use crate::blockstream::MockBlockstream as Blockstream;
+#[cfg(not(test))]
+use crate::blockstream::SocketBlockstream as Blockstream;
+use crate::result::{Error, Result};
+use solana_ledger::blockstore::Blockstore;
+use solana_sdk::pubkey::Pubkey;
+use std::path::Path;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{Receiver, RecvTimeoutError};
+use std::sync::Arc;
+use std::thread::{self, Builder, JoinHandle};
+use std::time::Duration;
+
+pub struct BlockstreamService {
+    t_blockstream: JoinHandle<()>,
+}
+
+impl BlockstreamService {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(
+        slot_full_receiver: Receiver<(u64, Pubkey)>,
+        blockstore: Arc<Blockstore>,
+        unix_socket: &Path,
+        exit: &Arc<AtomicBool>,
+    ) -> Self {
+        let mut blockstream = Blockstream::new(unix_socket);
+        let exit = exit.clone();
+        let t_blockstream = Builder::new()
+            .name("solana-blockstream".to_string())
+            .spawn(move || loop {
+                if exit.load(Ordering::Relaxed) {
+                    break;
+                }
+                if let Err(e) =
+                    Self::process_entries(&slot_full_receiver, &blockstore, &mut blockstream)
+                {
+                    match e {
+                        Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
+                        Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
+                        _ => info!("Error from process_entries: {:?}", e),
+                    }
+                }
+            })
+            .unwrap();
+        Self { t_blockstream }
+    }
+    fn process_entries(
+        slot_full_receiver: &Receiver<(u64, Pubkey)>,
+        blockstore: &Arc<Blockstore>,
+        blockstream: &mut Blockstream,
+    ) -> Result<()> {
+        let timeout = Duration::new(1, 0);
+        let (slot, slot_leader) = slot_full_receiver.recv_timeout(timeout)?;
+
+        // Slot might not exist due to LedgerCleanupService, check first
+        let blockstore_meta = blockstore.meta(slot).unwrap();
+        if let Some(blockstore_meta) = blockstore_meta {
+            // Return error to main loop. Thread won't exit, will just log the error
+            let entries = blockstore.get_slot_entries(slot, 0, None)?;
+            let _parent_slot = if slot == 0 {
+                None
+            } else {
+                Some(blockstore_meta.parent_slot)
+            };
+            let ticks_per_slot = entries.iter().filter(|entry| entry.is_tick()).count() as u64;
+            let mut tick_height = ticks_per_slot * slot;
+
+            for (i, entry) in entries.iter().enumerate() {
+                if entry.is_tick() {
+                    tick_height += 1;
+                }
+                blockstream
+                    .emit_entry_event(slot, tick_height, &slot_leader, &entry)
+                    .unwrap_or_else(|e| {
+                        debug!("Blockstream error: {:?}, {:?}", e, blockstream.output);
+                    });
+                if i == entries.len() - 1 {
+                    blockstream
+                        .emit_block_event(slot, tick_height, &slot_leader, entry.hash)
+                        .unwrap_or_else(|e| {
+                            debug!("Blockstream error: {:?}, {:?}", e, blockstream.output);
+                        });
+                }
+            }
+        }
+        Ok(())
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.t_blockstream.join()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use bincode::{deserialize, serialize};
+    use chrono::{DateTime, FixedOffset};
+    use serde_json::Value;
+    use solana_ledger::create_new_tmp_ledger;
+    use solana_ledger::entry::{create_ticks, Entry};
+    use solana_sdk::hash::Hash;
+    use solana_sdk::signature::{Keypair, Signer};
+    use solana_sdk::system_transaction;
+    use std::path::PathBuf;
+    use std::sync::mpsc::channel;
+
+    #[test]
+    fn test_blockstream_service_process_entries() {
+        let ticks_per_slot = 5;
+        let leader_pubkey = Pubkey::new_rand();
+
+        // Set up genesis config and blockstore
+        let GenesisConfigInfo {
+            mut genesis_config, ..
+        } = create_genesis_config(1000);
+        genesis_config.ticks_per_slot = ticks_per_slot;
+
+        let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
+        let blockstore = Blockstore::open(&ledger_path).unwrap();
+
+        // Set up blockstream
+        let mut blockstream = Blockstream::new(&PathBuf::from("test_stream"));
+
+        // Set up dummy channel to receive a full-slot notification
+        let (slot_full_sender, slot_full_receiver) = channel();
+
+        // Create entries - 4 ticks + 1 populated entry + 1 tick
+        let mut entries = create_ticks(4, 0, Hash::default());
+
+        let keypair = Keypair::new();
+        let mut blockhash = entries[3].hash;
+        let tx = system_transaction::transfer(&keypair, &keypair.pubkey(), 1, Hash::default());
+        let entry = Entry::new(&mut blockhash, 1, vec![tx]);
+        blockhash = entry.hash;
+        entries.push(entry);
+        let final_tick = create_ticks(1, 0, blockhash);
+        entries.extend_from_slice(&final_tick);
+
+        let expected_entries = entries.clone();
+        let expected_tick_heights = [6, 7, 8, 9, 9, 10];
+
+        blockstore
+            .write_entries(
+                1,
+                0,
+                0,
+                ticks_per_slot,
+                None,
+                true,
+                &Arc::new(Keypair::new()),
+                entries,
+                0,
+            )
+            .unwrap();
+
+        slot_full_sender.send((1, leader_pubkey)).unwrap();
+        BlockstreamService::process_entries(
+            &slot_full_receiver,
+            &Arc::new(blockstore),
+            &mut blockstream,
+        )
+        .unwrap();
+        assert_eq!(blockstream.entries().len(), 7);
+
+        let (entry_events, block_events): (Vec<Value>, Vec<Value>) = blockstream
+            .entries()
+            .iter()
+            .map(|item| {
+                let json: Value = serde_json::from_str(&item).unwrap();
+                let dt_str = json["dt"].as_str().unwrap();
+                // Ensure `ts` field parses as valid DateTime
+                let _dt: DateTime<FixedOffset> = DateTime::parse_from_rfc3339(dt_str).unwrap();
+                json
+            })
+            .partition(|json| {
+                let item_type = json["t"].as_str().unwrap();
+                item_type == "entry"
+            });
+        for (i, json) in entry_events.iter().enumerate() {
+            let height = json["h"].as_u64().unwrap();
+            assert_eq!(height, expected_tick_heights[i]);
+            let entry_obj = json["entry"].clone();
+            let tx = entry_obj["transactions"].as_array().unwrap();
+            let entry: Entry;
+            if tx.len() == 0 {
+                entry = serde_json::from_value(entry_obj).unwrap();
+            } else {
+                let entry_json = entry_obj.as_object().unwrap();
+                entry = Entry {
+                    num_hashes: entry_json.get("num_hashes").unwrap().as_u64().unwrap(),
+                    hash: serde_json::from_value(entry_json.get("hash").unwrap().clone()).unwrap(),
+                    transactions: entry_json
+                        .get("transactions")
+                        .unwrap()
+                        .as_array()
+                        .unwrap()
+                        .into_iter()
+                        .enumerate()
+                        .map(|(j, tx)| {
+                            let tx_vec: Vec<u8> = serde_json::from_value(tx.clone()).unwrap();
+                            // Check explicitly that transaction matches bincode-serialized format
+                            assert_eq!(
+                                tx_vec,
+                                serialize(&expected_entries[i].transactions[j]).unwrap()
+                            );
+                            deserialize(&tx_vec).unwrap()
+                        })
+                        .collect(),
+                };
+            }
+            assert_eq!(entry, expected_entries[i]);
+        }
+        for json in block_events {
+            let slot = json["s"].as_u64().unwrap();
+            assert_eq!(1, slot);
+            let height = json["h"].as_u64().unwrap();
+            assert_eq!(2 * ticks_per_slot, height);
+        }
+    }
+}

core/src/broadcast_stage.rs

@@ -1,41 +1,30 @@
 //! A stage to broadcast data from a leader node to validators
-use self::{
-    broadcast_fake_shreds_run::BroadcastFakeShredsRun,
-    fail_entry_verification_broadcast_run::FailEntryVerificationBroadcastRun,
-    standard_broadcast_run::StandardBroadcastRun,
-};
-use crate::{
-    cluster_info::{ClusterInfo, ClusterInfoError},
-    poh_recorder::WorkingBankEntry,
-    result::{Error, Result},
-};
-use crossbeam_channel::{
-    Receiver as CrossbeamReceiver, RecvTimeoutError as CrossbeamRecvTimeoutError,
-    Sender as CrossbeamSender,
-};
-use solana_ledger::{blockstore::Blockstore, shred::Shred, staking_utils};
+use self::broadcast_fake_shreds_run::BroadcastFakeShredsRun;
+use self::fail_entry_verification_broadcast_run::FailEntryVerificationBroadcastRun;
+use self::standard_broadcast_run::StandardBroadcastRun;
+use crate::cluster_info::{ClusterInfo, ClusterInfoError};
+use crate::poh_recorder::WorkingBankEntry;
+use crate::result::{Error, Result};
+use solana_ledger::blockstore::Blockstore;
+use solana_ledger::shred::Shred;
+use solana_ledger::staking_utils;
 use solana_metrics::{inc_new_counter_error, inc_new_counter_info};
-use solana_runtime::bank::Bank;
-use solana_sdk::{clock::Slot, pubkey::Pubkey};
-use std::{
-    collections::HashMap,
-    net::UdpSocket,
-    sync::atomic::{AtomicBool, Ordering},
-    sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender},
-    sync::{Arc, Mutex, RwLock},
-    thread::{self, Builder, JoinHandle},
-    time::{Duration, Instant},
-};
+use solana_sdk::pubkey::Pubkey;
+use std::collections::HashMap;
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender};
+use std::sync::{Arc, Mutex, RwLock};
+use std::thread::{self, Builder, JoinHandle};
+use std::time::Instant;
+
+pub const NUM_INSERT_THREADS: usize = 2;
 
 mod broadcast_fake_shreds_run;
 pub(crate) mod broadcast_utils;
 mod fail_entry_verification_broadcast_run;
 mod standard_broadcast_run;
 
-pub const NUM_INSERT_THREADS: usize = 2;
-pub type RetransmitSlotsSender = CrossbeamSender<HashMap<Slot, Arc<Bank>>>;
-pub type RetransmitSlotsReceiver = CrossbeamReceiver<HashMap<Slot, Arc<Bank>>>;
-
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub enum BroadcastStageReturnType {
     ChannelDisconnected,
@@ -54,7 +43,6 @@ impl BroadcastStageType {
         sock: Vec<UdpSocket>,
         cluster_info: Arc<RwLock<ClusterInfo>>,
         receiver: Receiver<WorkingBankEntry>,
-        retransmit_slots_receiver: RetransmitSlotsReceiver,
         exit_sender: &Arc<AtomicBool>,
         blockstore: &Arc<Blockstore>,
         shred_version: u16,
@@ -65,7 +53,6 @@ impl BroadcastStageType {
                 sock,
                 cluster_info,
                 receiver,
-                retransmit_slots_receiver,
                 exit_sender,
                 blockstore,
                 StandardBroadcastRun::new(keypair, shred_version),
@@ -75,7 +62,6 @@ impl BroadcastStageType {
                 sock,
                 cluster_info,
                 receiver,
-                retransmit_slots_receiver,
                 exit_sender,
                 blockstore,
                 FailEntryVerificationBroadcastRun::new(keypair, shred_version),
@@ -85,7 +71,6 @@ impl BroadcastStageType {
                 sock,
                 cluster_info,
                 receiver,
-                retransmit_slots_receiver,
                 exit_sender,
                 blockstore,
                 BroadcastFakeShredsRun::new(keypair, 0, shred_version),
@@ -94,7 +79,7 @@ impl BroadcastStageType {
     }
 }
 
-pub type TransmitShreds = (Option<Arc<HashMap<Pubkey, u64>>>, Arc<Vec<Shred>>);
+type TransmitShreds = (Option<Arc<HashMap<Pubkey, u64>>>, Arc<Vec<Shred>>);
 trait BroadcastRun {
     fn run(
         &mut self,
@@ -150,27 +135,25 @@ impl BroadcastStage {
         loop {
             let res =
                 broadcast_stage_run.run(blockstore, receiver, socket_sender, blockstore_sender);
-            let res = Self::handle_error(res, "run");
+            let res = Self::handle_error(res);
             if let Some(res) = res {
                 return res;
             }
         }
     }
-    fn handle_error(r: Result<()>, name: &str) -> Option<BroadcastStageReturnType> {
+    fn handle_error(r: Result<()>) -> Option<BroadcastStageReturnType> {
         if let Err(e) = r {
             match e {
                 Error::RecvTimeoutError(RecvTimeoutError::Disconnected)
                 | Error::SendError
-                | Error::RecvError(RecvError)
-                | Error::CrossbeamRecvTimeoutError(CrossbeamRecvTimeoutError::Disconnected) => {
+                | Error::RecvError(RecvError) => {
                     return Some(BroadcastStageReturnType::ChannelDisconnected);
                 }
-                Error::RecvTimeoutError(RecvTimeoutError::Timeout)
-                | Error::CrossbeamRecvTimeoutError(CrossbeamRecvTimeoutError::Timeout) => (),
+                Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
                 Error::ClusterInfoError(ClusterInfoError::NoPeers) => (), // TODO: Why are the unit-tests throwing hundreds of these?
                 _ => {
                     inc_new_counter_error!("streamer-broadcaster-error", 1, 1);
-                    error!("{} broadcaster error: {:?}", name, e);
+                    error!("broadcaster error: {:?}", e);
                 }
             }
         }
@@ -197,7 +180,6 @@ impl BroadcastStage {
         socks: Vec<UdpSocket>,
         cluster_info: Arc<RwLock<ClusterInfo>>,
         receiver: Receiver<WorkingBankEntry>,
-        retransmit_slots_receiver: RetransmitSlotsReceiver,
         exit_sender: &Arc<AtomicBool>,
         blockstore: &Arc<Blockstore>,
         broadcast_stage_run: impl BroadcastRun + Send + 'static + Clone,
@@ -207,8 +189,6 @@ impl BroadcastStage {
         let (socket_sender, socket_receiver) = channel();
         let (blockstore_sender, blockstore_receiver) = channel();
         let bs_run = broadcast_stage_run.clone();
-
-        let socket_sender_ = socket_sender.clone();
         let thread_hdl = Builder::new()
             .name("solana-broadcaster".to_string())
             .spawn(move || {
@@ -216,7 +196,7 @@ impl BroadcastStage {
                 Self::run(
                     &btree,
                     &receiver,
-                    &socket_sender_,
+                    &socket_sender,
                     &blockstore_sender,
                     bs_run,
                 )
@@ -232,7 +212,7 @@ impl BroadcastStage {
                 .name("solana-broadcaster-transmit".to_string())
                 .spawn(move || loop {
                     let res = bs_transmit.transmit(&socket_receiver, &cluster_info, &sock);
-                    let res = Self::handle_error(res, "solana-broadcaster-transmit");
+                    let res = Self::handle_error(res);
                     if let Some(res) = res {
                         return res;
                     }
@@ -249,7 +229,7 @@ impl BroadcastStage {
                 .name("solana-broadcaster-record".to_string())
                 .spawn(move || loop {
                     let res = bs_record.record(&blockstore_receiver, &btree);
-                    let res = Self::handle_error(res, "solana-broadcaster-record");
+                    let res = Self::handle_error(res);
                     if let Some(res) = res {
                         return res;
                     }
@@ -258,68 +238,9 @@ impl BroadcastStage {
             thread_hdls.push(t);
         }
 
-        let blockstore = blockstore.clone();
-        let retransmit_thread = Builder::new()
-            .name("solana-broadcaster-retransmit".to_string())
-            .spawn(move || loop {
-                if let Some(res) = Self::handle_error(
-                    Self::check_retransmit_signals(
-                        &blockstore,
-                        &retransmit_slots_receiver,
-                        &socket_sender,
-                    ),
-                    "solana-broadcaster-retransmit-check_retransmit_signals",
-                ) {
-                    return res;
-                }
-            })
-            .unwrap();
-
-        thread_hdls.push(retransmit_thread);
         Self { thread_hdls }
     }
 
-    fn check_retransmit_signals(
-        blockstore: &Blockstore,
-        retransmit_slots_receiver: &RetransmitSlotsReceiver,
-        socket_sender: &Sender<TransmitShreds>,
-    ) -> Result<()> {
-        let timer = Duration::from_millis(100);
-
-        // Check for a retransmit signal
-        let mut retransmit_slots = retransmit_slots_receiver.recv_timeout(timer)?;
-        while let Ok(new_retransmit_slots) = retransmit_slots_receiver.try_recv() {
-            retransmit_slots.extend(new_retransmit_slots);
-        }
-
-        for (_, bank) in retransmit_slots.iter() {
-            let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
-            let stakes = staking_utils::staked_nodes_at_epoch(&bank, bank_epoch);
-            let stakes = stakes.map(Arc::new);
-            let data_shreds = Arc::new(
-                blockstore
-                    .get_data_shreds_for_slot(bank.slot(), 0)
-                    .expect("My own shreds must be reconstructable"),
-            );
-
-            if !data_shreds.is_empty() {
-                socket_sender.send((stakes.clone(), data_shreds))?;
-            }
-
-            let coding_shreds = Arc::new(
-                blockstore
-                    .get_coding_shreds_for_slot(bank.slot(), 0)
-                    .expect("My own shreds must be reconstructable"),
-            );
-
-            if !coding_shreds.is_empty() {
-                socket_sender.send((stakes.clone(), coding_shreds))?;
-            }
-        }
-
-        Ok(())
-    }
-
     pub fn join(self) -> thread::Result<BroadcastStageReturnType> {
         for thread_hdl in self.thread_hdls.into_iter() {
             let _ = thread_hdl.join();
@@ -329,137 +250,22 @@ impl BroadcastStage {
 }
 
 #[cfg(test)]
-pub mod test {
+mod test {
     use super::*;
     use crate::cluster_info::{ClusterInfo, Node};
-    use crossbeam_channel::unbounded;
-    use solana_ledger::{
-        blockstore::{make_slot_entries, Blockstore},
-        entry::create_ticks,
-        genesis_utils::{create_genesis_config, GenesisConfigInfo},
-        get_tmp_ledger_path,
-        shred::{max_ticks_per_n_shreds, Shredder, RECOMMENDED_FEC_RATE},
-    };
+    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use solana_ledger::entry::create_ticks;
+    use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
     use solana_runtime::bank::Bank;
-    use solana_sdk::{
-        hash::Hash,
-        pubkey::Pubkey,
-        signature::{Keypair, Signer},
-    };
-    use std::{
-        path::Path,
-        sync::atomic::AtomicBool,
-        sync::mpsc::channel,
-        sync::{Arc, RwLock},
-        thread::sleep,
-    };
-
-    pub fn make_transmit_shreds(
-        slot: Slot,
-        num: u64,
-        stakes: Option<Arc<HashMap<Pubkey, u64>>>,
-    ) -> (
-        Vec<Shred>,
-        Vec<Shred>,
-        Vec<TransmitShreds>,
-        Vec<TransmitShreds>,
-    ) {
-        let num_entries = max_ticks_per_n_shreds(num);
-        let (data_shreds, _) = make_slot_entries(slot, 0, num_entries);
-        let keypair = Arc::new(Keypair::new());
-        let shredder = Shredder::new(slot, 0, RECOMMENDED_FEC_RATE, keypair, 0, 0)
-            .expect("Expected to create a new shredder");
-
-        let coding_shreds = shredder.data_shreds_to_coding_shreds(&data_shreds[0..]);
-        (
-            data_shreds.clone(),
-            coding_shreds.clone(),
-            data_shreds
-                .into_iter()
-                .map(|s| (stakes.clone(), Arc::new(vec![s])))
-                .collect(),
-            coding_shreds
-                .into_iter()
-                .map(|s| (stakes.clone(), Arc::new(vec![s])))
-                .collect(),
-        )
-    }
-
-    fn check_all_shreds_received(
-        transmit_receiver: &Receiver<TransmitShreds>,
-        mut data_index: u64,
-        mut coding_index: u64,
-        num_expected_data_shreds: u64,
-        num_expected_coding_shreds: u64,
-    ) {
-        while let Ok(new_retransmit_slots) = transmit_receiver.try_recv() {
-            if new_retransmit_slots.1[0].is_data() {
-                for data_shred in new_retransmit_slots.1.iter() {
-                    assert_eq!(data_shred.index() as u64, data_index);
-                    data_index += 1;
-                }
-            } else {
-                assert_eq!(new_retransmit_slots.1[0].index() as u64, coding_index);
-                for coding_shred in new_retransmit_slots.1.iter() {
-                    assert_eq!(coding_shred.index() as u64, coding_index);
-                    coding_index += 1;
-                }
-            }
-        }
-
-        assert_eq!(num_expected_data_shreds, data_index);
-        assert_eq!(num_expected_coding_shreds, coding_index);
-    }
-
-    #[test]
-    fn test_duplicate_retransmit_signal() {
-        // Setup
-        let ledger_path = get_tmp_ledger_path!();
-        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
-        let (transmit_sender, transmit_receiver) = channel();
-        let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
-        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(100_000);
-        let bank0 = Arc::new(Bank::new(&genesis_config));
-
-        // Make some shreds
-        let updated_slot = 0;
-        let (all_data_shreds, all_coding_shreds, _, _all_coding_transmit_shreds) =
-            make_transmit_shreds(updated_slot, 10, None);
-        let num_data_shreds = all_data_shreds.len();
-        let num_coding_shreds = all_coding_shreds.len();
-        assert!(num_data_shreds >= 10);
-
-        // Insert all the shreds
-        blockstore
-            .insert_shreds(all_data_shreds, None, true)
-            .unwrap();
-        blockstore
-            .insert_shreds(all_coding_shreds, None, true)
-            .unwrap();
-
-        // Insert duplicate retransmit signal, blocks should
-        // only be retransmitted once
-        retransmit_slots_sender
-            .send(vec![(updated_slot, bank0.clone())].into_iter().collect())
-            .unwrap();
-        retransmit_slots_sender
-            .send(vec![(updated_slot, bank0.clone())].into_iter().collect())
-            .unwrap();
-        BroadcastStage::check_retransmit_signals(
-            &blockstore,
-            &retransmit_slots_receiver,
-            &transmit_sender,
-        )
-        .unwrap();
-        // Check all the data shreds were received only once
-        check_all_shreds_received(
-            &transmit_receiver,
-            0,
-            0,
-            num_data_shreds as u64,
-            num_coding_shreds as u64,
-        );
-    }
+    use solana_sdk::hash::Hash;
+    use solana_sdk::pubkey::Pubkey;
+    use solana_sdk::signature::{Keypair, Signer};
+    use std::path::Path;
+    use std::sync::atomic::AtomicBool;
+    use std::sync::mpsc::channel;
+    use std::sync::{Arc, RwLock};
+    use std::thread::sleep;
+    use std::time::Duration;
 
     struct MockBroadcastStage {
         blockstore: Arc<Blockstore>,
@@ -471,7 +277,6 @@ pub mod test {
         leader_pubkey: &Pubkey,
         ledger_path: &Path,
         entry_receiver: Receiver<WorkingBankEntry>,
-        retransmit_slots_receiver: RetransmitSlotsReceiver,
     ) -> MockBroadcastStage {
         // Make the database ledger
         let blockstore = Arc::new(Blockstore::open(ledger_path).unwrap());
@@ -499,7 +304,6 @@ pub mod test {
             leader_info.sockets.broadcast,
             cluster_info,
             entry_receiver,
-            retransmit_slots_receiver,
             &exit_sender,
             &blockstore,
             StandardBroadcastRun::new(leader_keypair, 0),
@@ -522,12 +326,10 @@ pub mod test {
             let leader_keypair = Keypair::new();
 
             let (entry_sender, entry_receiver) = channel();
-            let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
             let broadcast_service = setup_dummy_broadcast_service(
                 &leader_keypair.pubkey(),
                 &ledger_path,
                 entry_receiver,
-                retransmit_slots_receiver,
             );
             let start_tick_height;
             let max_tick_height;
@@ -546,7 +348,6 @@ pub mod test {
                         .expect("Expect successful send to broadcast service");
                 }
             }
-
             sleep(Duration::from_millis(2000));
 
             trace!(
@@ -563,7 +364,6 @@ pub mod test {
             assert_eq!(entries.len(), max_tick_height as usize);
 
             drop(entry_sender);
-            drop(retransmit_slots_sender);
             broadcast_service
                 .broadcast_service
                 .join()

core/src/broadcast_stage/broadcast_utils.rs

@@ -76,7 +76,7 @@ pub(super) fn recv_slot_entries(receiver: &Receiver<WorkingBankEntry>) -> Result
 #[cfg(test)]
 mod tests {
     use super::*;
-    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_sdk::genesis_config::GenesisConfig;
     use solana_sdk::pubkey::Pubkey;
     use solana_sdk::system_transaction;

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -1,11 +1,11 @@
 use super::broadcast_utils::{self, ReceiveResults};
 use super::*;
 use crate::broadcast_stage::broadcast_utils::UnfinishedSlotInfo;
-use solana_ledger::{
-    entry::Entry,
-    shred::{Shred, Shredder, RECOMMENDED_FEC_RATE, SHRED_TICK_REFERENCE_MASK},
-};
-use solana_sdk::{pubkey::Pubkey, signature::Keypair, timing::duration_as_us};
+use solana_ledger::entry::Entry;
+use solana_ledger::shred::{Shred, Shredder, RECOMMENDED_FEC_RATE, SHRED_TICK_REFERENCE_MASK};
+use solana_sdk::pubkey::Pubkey;
+use solana_sdk::signature::Keypair;
+use solana_sdk::timing::duration_as_us;
 use std::collections::HashMap;
 use std::time::Duration;
 
@@ -212,8 +212,7 @@ impl StandardBroadcastRun {
         blockstore_sender.send(data_shreds.clone())?;
         let coding_shreds = shredder.data_shreds_to_coding_shreds(&data_shreds[0..last_data_shred]);
         let coding_shreds = Arc::new(coding_shreds);
-        socket_sender.send((stakes, coding_shreds.clone()))?;
-        blockstore_sender.send(coding_shreds)?;
+        socket_sender.send((stakes, coding_shreds))?;
         self.update_broadcast_stats(BroadcastStats {
             shredding_elapsed: duration_as_us(&to_shreds_elapsed),
             receive_elapsed: duration_as_us(&receive_elapsed),
@@ -354,13 +353,14 @@ impl BroadcastRun for StandardBroadcastRun {
 mod test {
     use super::*;
     use crate::cluster_info::{ClusterInfo, Node};
-    use solana_ledger::genesis_utils::create_genesis_config;
+    use crate::genesis_utils::create_genesis_config;
     use solana_ledger::{
         blockstore::Blockstore, entry::create_ticks, get_tmp_ledger_path,
         shred::max_ticks_per_n_shreds,
     };
     use solana_runtime::bank::Bank;
     use solana_sdk::{
+        clock::Slot,
         genesis_config::GenesisConfig,
         signature::{Keypair, Signer},
     };

core/src/cluster_info.rs

@@ -12,18 +12,27 @@
 //! * layer 2 - Everyone else, if layer 1 is `2^10`, layer 2 should be able to fit `2^20` number of nodes.
 //!
 //! Bank needs to provide an interface for us to query the stake weight
+use crate::packet::limited_deserialize;
+use crate::streamer::{PacketReceiver, PacketSender};
 use crate::{
     contact_info::ContactInfo,
     crds_gossip::CrdsGossip,
     crds_gossip_error::CrdsGossipError,
     crds_gossip_pull::{CrdsFilter, CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS},
     crds_value::{
-        self, CrdsData, CrdsValue, CrdsValueLabel, EpochSlotsIndex, LowestSlot, SnapshotHash, Vote,
+        self, CrdsData, CrdsValue, CrdsValueLabel, EpochSlots, SnapshotHash, Vote, MAX_WALLCLOCK,
     },
-    epoch_slots::EpochSlots,
+    packet::{Packet, PACKET_DATA_SIZE},
     result::{Error, Result},
+    sendmmsg::{multicast, send_mmsg},
     weighted_shuffle::{weighted_best, weighted_shuffle},
 };
+
+use rand::distributions::{Distribution, WeightedIndex};
+use rand::SeedableRng;
+use rand_chacha::ChaChaRng;
+use solana_sdk::sanitize::{Sanitize, SanitizeError};
+
 use bincode::{serialize, serialized_size};
 use core::cmp;
 use itertools::Itertools;
@@ -31,32 +40,28 @@ use rayon::iter::IntoParallelIterator;
 use rayon::iter::ParallelIterator;
 use rayon::ThreadPool;
 use solana_ledger::{bank_forks::BankForks, staking_utils};
+use solana_measure::measure::Measure;
 use solana_measure::thread_mem_usage;
 use solana_metrics::{datapoint_debug, inc_new_counter_debug, inc_new_counter_error};
 use solana_net_utils::{
     bind_common, bind_common_in_range, bind_in_range, find_available_port_in_range,
     multi_bind_in_range, PortRange,
 };
-use solana_perf::packet::{
-    limited_deserialize, to_packets_with_destination, Packet, Packets, PacketsRecycler,
-    PACKET_DATA_SIZE,
-};
+use solana_perf::packet::{to_packets_with_destination, Packets, PacketsRecycler};
 use solana_rayon_threadlimit::get_thread_count;
 use solana_sdk::hash::Hash;
 use solana_sdk::timing::duration_as_s;
 use solana_sdk::{
-    clock::{Slot, DEFAULT_MS_PER_SLOT, DEFAULT_SLOTS_PER_EPOCH},
+    clock::{Slot, DEFAULT_MS_PER_SLOT},
     pubkey::Pubkey,
     signature::{Keypair, Signable, Signature, Signer},
     timing::{duration_as_ms, timestamp},
     transaction::Transaction,
 };
-use solana_streamer::sendmmsg::{multicast, send_mmsg};
-use solana_streamer::streamer::{PacketReceiver, PacketSender};
 use std::{
     borrow::Cow,
     cmp::min,
-    collections::{HashMap, HashSet},
+    collections::{BTreeSet, HashMap, HashSet},
     fmt,
     net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, UdpSocket},
     sync::atomic::{AtomicBool, Ordering},
@@ -72,8 +77,7 @@ pub const DATA_PLANE_FANOUT: usize = 200;
 /// milliseconds we sleep for between gossip requests
 pub const GOSSIP_SLEEP_MILLIS: u64 = 100;
 /// The maximum size of a bloom filter
-pub const MAX_BLOOM_SIZE: usize = MAX_CRDS_OBJECT_SIZE;
-pub const MAX_CRDS_OBJECT_SIZE: usize = 928;
+pub const MAX_BLOOM_SIZE: usize = 1018;
 /// The maximum size of a protocol payload
 const MAX_PROTOCOL_PAYLOAD_SIZE: u64 = PACKET_DATA_SIZE as u64 - MAX_PROTOCOL_HEADER_SIZE;
 /// The largest protocol header size
@@ -94,6 +98,12 @@ pub enum ClusterInfoError {
     BadGossipAddress,
 }
 #[derive(Clone)]
+pub struct DataBudget {
+    bytes: usize, // amount of bytes we have in the budget to send
+    last_timestamp_ms: u64, // Last time that we upped the bytes count,
+                  // used to detect when to up the bytes budget again
+}
+#[derive(Clone)]
 pub struct ClusterInfo {
     /// The network
     pub gossip: CrdsGossip,
@@ -102,6 +112,8 @@ pub struct ClusterInfo {
     /// The network entrypoint
     entrypoint: Option<ContactInfo>,
     last_datapoint_submit: Instant,
+
+    outbound_budget: DataBudget,
 }
 
 #[derive(Default, Clone)]
@@ -142,6 +154,15 @@ pub struct PruneData {
     pub wallclock: u64,
 }
 
+impl Sanitize for PruneData {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        Ok(())
+    }
+}
+
 impl Signable for PruneData {
     fn pubkey(&self) -> Pubkey {
         self.pubkey
@@ -187,6 +208,14 @@ pub fn make_accounts_hashes_message(
     Some(CrdsValue::new_signed(message, keypair))
 }
 
+fn distance(a: u64, b: u64) -> u64 {
+    if a > b {
+        a - b
+    } else {
+        b - a
+    }
+}
+
 // TODO These messages should go through the gpu pipeline for spam filtering
 #[derive(Serialize, Deserialize, Debug)]
 #[allow(clippy::large_enum_variant)]
@@ -198,6 +227,31 @@ enum Protocol {
     PruneMessage(Pubkey, PruneData),
 }
 
+impl Sanitize for Protocol {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        match self {
+            Protocol::PullRequest(filter, val) => {
+                filter.sanitize()?;
+                val.sanitize()
+            }
+            Protocol::PullResponse(_, val) => val.sanitize(),
+            Protocol::PushMessage(_, val) => val.sanitize(),
+            Protocol::PruneMessage(_, val) => val.sanitize(),
+        }
+    }
+}
+
+// Rating for pull requests
+// A response table is generated as a
+// 2-d table arranged by target nodes and a
+// list of responses for that node,
+// to/responses_index is a location in that table.
+struct ResponseScore {
+    to: usize,              // to, index of who the response is to
+    responses_index: usize, // index into the list of responses for a given to
+    score: u64,             // Relative score of the response
+}
+
 impl ClusterInfo {
     /// Without a valid keypair gossip will not function. Only useful for tests.
     pub fn new_with_invalid_keypair(contact_info: ContactInfo) -> Self {
@@ -210,9 +264,13 @@ impl ClusterInfo {
             keypair,
             entrypoint: None,
             last_datapoint_submit: Instant::now(),
+            outbound_budget: DataBudget {
+                bytes: 0,
+                last_timestamp_ms: 0,
+            },
         };
-        let id = contact_info.id;
-        me.gossip.set_self(&id);
+        me.gossip.set_self(&contact_info.id);
+        me.gossip.set_shred_version(contact_info.shred_version);
         me.insert_self(contact_info);
         me.push_self(&HashMap::new());
         me
@@ -261,65 +319,63 @@ impl ClusterInfo {
         self.lookup(&self.id()).cloned().unwrap()
     }
 
-    pub fn lookup_epoch_slots(&self, ix: EpochSlotsIndex) -> EpochSlots {
-        let entry = CrdsValueLabel::EpochSlots(ix, self.id());
-        self.gossip
-            .crds
-            .lookup(&entry)
-            .and_then(CrdsValue::epoch_slots)
-            .cloned()
-            .unwrap_or_else(|| EpochSlots::new(self.id(), timestamp()))
-    }
-
     pub fn contact_info_trace(&self) -> String {
         let now = timestamp();
         let mut spy_nodes = 0;
         let mut archivers = 0;
+        let mut different_shred_nodes = 0;
         let my_pubkey = self.my_data().id;
+        let my_shred_version = self.my_data().shred_version;
         let nodes: Vec<_> = self
             .all_peers()
             .into_iter()
-            .map(|(node, last_updated)| {
+            .filter_map(|(node, last_updated)| {
                 if Self::is_spy_node(&node) {
                     spy_nodes += 1;
                 } else if Self::is_archiver(&node) {
                     archivers += 1;
                 }
-                fn addr_to_string(default_ip: &IpAddr, addr: &SocketAddr) -> String {
-                    if ContactInfo::is_valid_address(addr) {
-                        if &addr.ip() == default_ip {
-                            addr.port().to_string()
-                        } else {
-                            addr.to_string()
-                        }
-                    } else {
-                        "none".to_string()
-                    }
-                }
 
-                let ip_addr = node.gossip.ip();
-                format!(
-                    "{:15} {:2}| {:5} | {:44} | {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {}\n",
-                    if ContactInfo::is_valid_address(&node.gossip) {
-                        ip_addr.to_string()
-                    } else {
-                        "none".to_string()
-                    },
-                    if node.id == my_pubkey { "me" } else { "" }.to_string(),
-                    now.saturating_sub(last_updated),
-                    node.id.to_string(),
-                    addr_to_string(&ip_addr, &node.gossip),
-                    addr_to_string(&ip_addr, &node.tpu),
-                    addr_to_string(&ip_addr, &node.tpu_forwards),
-                    addr_to_string(&ip_addr, &node.tvu),
-                    addr_to_string(&ip_addr, &node.tvu_forwards),
-                    addr_to_string(&ip_addr, &node.repair),
-                    addr_to_string(&ip_addr, &node.serve_repair),
-                    addr_to_string(&ip_addr, &node.storage_addr),
-                    addr_to_string(&ip_addr, &node.rpc),
-                    addr_to_string(&ip_addr, &node.rpc_pubsub),
-                    node.shred_version,
-                )
+                if my_shred_version != 0 && (node.shred_version != 0 && node.shred_version != my_shred_version) {
+                    different_shred_nodes += 1;
+                    None
+                } else {
+                    fn addr_to_string(default_ip: &IpAddr, addr: &SocketAddr) -> String {
+                        if ContactInfo::is_valid_address(addr) {
+                            if &addr.ip() == default_ip {
+                                addr.port().to_string()
+                            } else {
+                                addr.to_string()
+                            }
+                        } else {
+                            "none".to_string()
+                        }
+                    }
+
+                    let ip_addr = node.gossip.ip();
+                    Some(format!(
+                        "{:15} {:2}| {:5} | {:44} | {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {}\n",
+                        if ContactInfo::is_valid_address(&node.gossip) {
+                            ip_addr.to_string()
+                        } else {
+                            "none".to_string()
+                        },
+                        if node.id == my_pubkey { "me" } else { "" }.to_string(),
+                        now.saturating_sub(last_updated),
+                        node.id.to_string(),
+                        addr_to_string(&ip_addr, &node.gossip),
+                        addr_to_string(&ip_addr, &node.tpu),
+                        addr_to_string(&ip_addr, &node.tpu_forwards),
+                        addr_to_string(&ip_addr, &node.tvu),
+                        addr_to_string(&ip_addr, &node.tvu_forwards),
+                        addr_to_string(&ip_addr, &node.repair),
+                        addr_to_string(&ip_addr, &node.serve_repair),
+                        addr_to_string(&ip_addr, &node.storage_addr),
+                        addr_to_string(&ip_addr, &node.rpc),
+                        addr_to_string(&ip_addr, &node.rpc_pubsub),
+                        node.shred_version,
+                    ))
+                }
             })
             .collect();
 
@@ -329,7 +385,7 @@ impl ClusterInfo {
              ------------------+-------+----------------------------------------------+\
              ------+------+------+------+------+------+------+------+------+------+--------\n\
              {}\
-             Nodes: {}{}{}",
+             Nodes: {}{}{}{}",
             nodes.join(""),
             nodes.len() - spy_nodes - archivers,
             if archivers > 0 {
@@ -341,84 +397,33 @@ impl ClusterInfo {
                 format!("\nSpies: {}", spy_nodes)
             } else {
                 "".to_string()
+            },
+            if spy_nodes > 0 {
+                format!(
+                    "\nNodes with different shred version: {}",
+                    different_shred_nodes
+                )
+            } else {
+                "".to_string()
             }
         )
     }
 
-    pub fn push_lowest_slot(&mut self, id: Pubkey, min: Slot) {
+    pub fn push_epoch_slots(
+        &mut self,
+        id: Pubkey,
+        _root: Slot,
+        min: Slot,
+        _slots: BTreeSet<Slot>,
+        _incomplete_slots: &BTreeSet<Slot>,
+    ) {
         let now = timestamp();
-        let last = self
-            .gossip
-            .crds
-            .lookup(&CrdsValueLabel::LowestSlot(self.id()))
-            .and_then(|x| x.lowest_slot())
-            .map(|x| x.lowest)
-            .unwrap_or(0);
-        if min > last {
-            let entry = CrdsValue::new_signed(
-                CrdsData::LowestSlot(0, LowestSlot::new(id, min, now)),
-                &self.keypair,
-            );
-            self.gossip
-                .process_push_message(&self.id(), vec![entry], now);
-        }
-    }
-
-    pub fn push_epoch_slots(&mut self, update: &[Slot]) {
-        let mut num = 0;
-        let mut current_slots: Vec<_> = (0..crds_value::MAX_EPOCH_SLOTS)
-            .filter_map(|ix| {
-                Some((
-                    self.gossip
-                        .crds
-                        .lookup(&CrdsValueLabel::EpochSlots(ix, self.id()))
-                        .and_then(CrdsValue::epoch_slots)
-                        .and_then(|x| Some((x.wallclock, x.first_slot()?)))?,
-                    ix,
-                ))
-            })
-            .collect();
-        current_slots.sort();
-        let min_slot: Slot = current_slots
-            .iter()
-            .map(|((_, s), _)| *s)
-            .min()
-            .unwrap_or(0);
-        let max_slot: Slot = update.iter().max().cloned().unwrap_or(0);
-        let total_slots = max_slot as isize - min_slot as isize;
-        // WARN if CRDS is not storing at least a full epoch worth of slots
-        if DEFAULT_SLOTS_PER_EPOCH as isize > total_slots
-            && crds_value::MAX_EPOCH_SLOTS as usize <= current_slots.len()
-        {
-            inc_new_counter_warn!("cluster_info-epoch_slots-filled", 1);
-            warn!(
-                "EPOCH_SLOTS are filling up FAST {}/{}",
-                total_slots,
-                current_slots.len()
-            );
-        }
-        let mut reset = false;
-        let mut epoch_slot_index = current_slots.last().map(|(_, x)| *x).unwrap_or(0);
-        while num < update.len() {
-            let ix = (epoch_slot_index % crds_value::MAX_EPOCH_SLOTS) as u8;
-            let now = timestamp();
-            let mut slots = if !reset {
-                self.lookup_epoch_slots(ix)
-            } else {
-                EpochSlots::new(self.id(), now)
-            };
-            let n = slots.fill(&update[num..], now);
-            if n > 0 {
-                let entry = CrdsValue::new_signed(CrdsData::EpochSlots(ix, slots), &self.keypair);
-                self.gossip
-                    .process_push_message(&self.id(), vec![entry], now);
-            }
-            num += n;
-            if num < update.len() {
-                epoch_slot_index += 1;
-                reset = true;
-            }
-        }
+        let entry = CrdsValue::new_signed(
+            CrdsData::EpochSlots(0, EpochSlots::new(id, min, now)),
+            &self.keypair,
+        );
+        self.gossip
+            .process_push_message(&self.id(), vec![entry], now);
     }
 
     pub fn push_message(&mut self, message: CrdsValue) {
@@ -474,23 +479,23 @@ impl ClusterInfo {
     /// since. This allows the bank to query for new votes only.
     ///
     /// * return - The votes, and the max timestamp from the new set.
-    pub fn get_votes(&self, since: u64) -> (Vec<CrdsValueLabel>, Vec<Transaction>, u64) {
-        let mut max_ts = since;
-        let (labels, txs): (Vec<CrdsValueLabel>, Vec<Transaction>) = self
+    pub fn get_votes(&self, since: u64) -> (Vec<Transaction>, u64) {
+        let votes: Vec<_> = self
             .gossip
             .crds
             .table
-            .iter()
-            .filter(|(_, x)| x.insert_timestamp > since)
-            .filter_map(|(label, x)| {
-                max_ts = std::cmp::max(x.insert_timestamp, max_ts);
+            .values()
+            .filter(|x| x.insert_timestamp > since)
+            .filter_map(|x| {
                 x.value
                     .vote()
-                    .map(|v| (label.clone(), v.transaction.clone()))
+                    .map(|v| (x.insert_timestamp, v.transaction.clone()))
             })
-            .unzip();
+            .collect();
+        let max_ts = votes.iter().map(|x| x.0).max().unwrap_or(since);
+        let txs: Vec<Transaction> = votes.into_iter().map(|x| x.1).collect();
         inc_new_counter_info!("cluster_info-get_votes-count", txs.len());
-        (labels, txs, max_ts)
+        (txs, max_ts)
     }
 
     pub fn get_snapshot_hash(&self, slot: Slot) -> Vec<(Pubkey, Hash)> {
@@ -526,39 +531,21 @@ impl ClusterInfo {
             .map(|x| &x.value.snapshot_hash().unwrap().hashes)
     }
 
-    pub fn get_lowest_slot_for_node(
+    pub fn get_epoch_state_for_node(
         &self,
         pubkey: &Pubkey,
         since: Option<u64>,
-    ) -> Option<(&LowestSlot, u64)> {
+    ) -> Option<(&EpochSlots, u64)> {
         self.gossip
             .crds
             .table
-            .get(&CrdsValueLabel::LowestSlot(*pubkey))
+            .get(&CrdsValueLabel::EpochSlots(*pubkey))
             .filter(|x| {
                 since
                     .map(|since| x.insert_timestamp > since)
                     .unwrap_or(true)
             })
-            .map(|x| (x.value.lowest_slot().unwrap(), x.insert_timestamp))
-    }
-
-    pub fn get_epoch_slots_since(&self, since: Option<u64>) -> (Vec<EpochSlots>, Option<u64>) {
-        let vals: Vec<_> = self
-            .gossip
-            .crds
-            .table
-            .values()
-            .filter(|x| {
-                since
-                    .map(|since| x.insert_timestamp > since)
-                    .unwrap_or(true)
-            })
-            .filter_map(|x| Some((x.value.epoch_slots()?, x.insert_timestamp)))
-            .collect();
-        let max = vals.iter().map(|x| x.1).max().or(since);
-        let vec = vals.into_iter().map(|x| x.0).cloned().collect();
-        (vec, max)
+            .map(|x| (x.value.epoch_slots().unwrap(), x.insert_timestamp))
     }
 
     pub fn get_contact_info_for_node(&self, pubkey: &Pubkey) -> Option<&ContactInfo> {
@@ -702,8 +689,8 @@ impl ClusterInfo {
                     && x.shred_version == me.shred_version
                     && ContactInfo::is_valid_address(&x.serve_repair)
                     && {
-                        self.get_lowest_slot_for_node(&x.id, None)
-                            .map(|(lowest_slot, _)| lowest_slot.lowest <= slot)
+                        self.get_epoch_state_for_node(&x.id, None)
+                            .map(|(epoch_slots, _)| epoch_slots.lowest <= slot)
                             .unwrap_or_else(|| /* fallback to legacy behavior */ true)
                     }
             })
@@ -991,7 +978,7 @@ impl ClusterInfo {
         let mut num_live_peers = 1i64;
         peers.iter().for_each(|p| {
             // A peer is considered live if they generated their contact info recently
-            if timestamp() - p.wallclock <= CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS {
+            if distance(timestamp(), p.wallclock) <= CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS {
                 num_live_peers += 1;
             }
         });
@@ -1270,6 +1257,10 @@ impl ClusterInfo {
                                         entrypoint.shred_version, entrypoint.id
                                     );
                                     self_info.shred_version = entrypoint.shred_version;
+                                    obj.write()
+                                        .unwrap()
+                                        .gossip
+                                        .set_shred_version(entrypoint.shred_version);
                                     obj.write().unwrap().insert_self(self_info);
                                     adopt_shred_version = false;
                                 }
@@ -1309,6 +1300,7 @@ impl ClusterInfo {
             let from_addr = packet.meta.addr();
             limited_deserialize(&packet.data[..packet.meta.size])
                 .into_iter()
+                .filter(|r: &Protocol| r.sanitize().is_ok())
                 .for_each(|request| match request {
                     Protocol::PullRequest(filter, caller) => {
                         let start = allocated.get();
@@ -1407,20 +1399,43 @@ impl ClusterInfo {
                 })
         });
         // process the collected pulls together
-        let rsp = Self::handle_pull_requests(me, recycler, gossip_pull_data);
+        let rsp = Self::handle_pull_requests(me, recycler, gossip_pull_data, stakes);
         if let Some(rsp) = rsp {
             let _ignore_disconnect = response_sender.send(rsp);
         }
     }
 
+    // Pull requests take an incoming bloom filter of contained entries from a node
+    // and tries to send back to them the values it detects are missing.
     fn handle_pull_requests(
         me: &Arc<RwLock<Self>>,
         recycler: &PacketsRecycler,
         requests: Vec<PullData>,
+        stakes: &HashMap<Pubkey, u64>,
     ) -> Option<Packets> {
         // split the requests into addrs and filters
         let mut caller_and_filters = vec![];
         let mut addrs = vec![];
+        let mut time = Measure::start("handle_pull_requests");
+        {
+            let mut cluster_info = me.write().unwrap();
+
+            let now = timestamp();
+            const INTERVAL_MS: u64 = 100;
+            // allow 50kBps per staked validator, epoch slots + votes ~= 1.5kB/slot ~= 4kB/s
+            const BYTES_PER_INTERVAL: usize = 5000;
+            const MAX_BUDGET_MULTIPLE: usize = 5; // allow budget build-up to 5x the interval default
+
+            if now - cluster_info.outbound_budget.last_timestamp_ms > INTERVAL_MS {
+                let len = std::cmp::max(stakes.len(), 2);
+                cluster_info.outbound_budget.bytes += len * BYTES_PER_INTERVAL;
+                cluster_info.outbound_budget.bytes = std::cmp::min(
+                    cluster_info.outbound_budget.bytes,
+                    MAX_BUDGET_MULTIPLE * len * BYTES_PER_INTERVAL,
+                );
+                cluster_info.outbound_budget.last_timestamp_ms = now;
+            }
+        }
         for pull_data in requests {
             caller_and_filters.push((pull_data.caller, pull_data.filter));
             addrs.push(pull_data.from_addr);
@@ -1432,30 +1447,101 @@ impl ClusterInfo {
             .unwrap()
             .gossip
             .process_pull_requests(caller_and_filters, now);
-        let mut packets = Packets::new_with_recycler(recycler.clone(), 64, "handle_pull_requests");
-        pull_responses
+
+        // Filter bad to addresses
+        let pull_responses: Vec<_> = pull_responses
             .into_iter()
             .zip(addrs.into_iter())
-            .for_each(|(response, from_addr)| {
-                if !from_addr.ip().is_unspecified() && from_addr.port() != 0 {
-                    let len = response.len();
-                    trace!("get updates since response {}", len);
-                    inc_new_counter_debug!("cluster_info-pull_request-rsp", len);
-                    Self::split_gossip_messages(response)
-                        .into_iter()
-                        .for_each(|payload| {
-                            let protocol = Protocol::PullResponse(self_id, payload);
-                            // The remote node may not know its public IP:PORT. Instead of responding to the caller's
-                            // gossip addr, respond to the origin addr. The last origin addr is picked from the list of
-                            // addrs.
-                            packets
-                                .packets
-                                .push(Packet::from_data(&from_addr, protocol))
-                        })
+            .filter_map(|(responses, from_addr)| {
+                if !from_addr.ip().is_unspecified()
+                    && from_addr.port() != 0
+                    && !responses.is_empty()
+                {
+                    Some((responses, from_addr))
                 } else {
-                    trace!("Dropping Gossip pull response, as destination is unknown");
+                    None
                 }
-            });
+            })
+            .collect();
+
+        if pull_responses.is_empty() {
+            return None;
+        }
+
+        let mut stats: Vec<_> = pull_responses
+            .iter()
+            .enumerate()
+            .map(|(i, (responses, _from_addr))| {
+                let score: u64 = if stakes.get(&responses[0].pubkey()).is_some() {
+                    2
+                } else {
+                    1
+                };
+                responses
+                    .iter()
+                    .enumerate()
+                    .map(|(j, _response)| ResponseScore {
+                        to: i,
+                        responses_index: j,
+                        score,
+                    })
+                    .collect::<Vec<ResponseScore>>()
+            })
+            .flatten()
+            .collect();
+
+        stats.sort_by(|a, b| a.score.cmp(&b.score));
+        let weights: Vec<_> = stats.iter().map(|stat| stat.score).collect();
+
+        let seed = [48u8; 32];
+        let rng = &mut ChaChaRng::from_seed(seed);
+        let weighted_index = WeightedIndex::new(weights).unwrap();
+
+        let mut packets = Packets::new_with_recycler(recycler.clone(), 64, "handle_pull_requests");
+        let mut total_bytes = 0;
+        let outbound_budget = me.read().unwrap().outbound_budget.bytes;
+        let mut sent = HashSet::new();
+        while sent.len() < stats.len() {
+            let index = weighted_index.sample(rng);
+            if sent.contains(&index) {
+                continue;
+            }
+            sent.insert(index);
+            let stat = &stats[index];
+            let from_addr = pull_responses[stat.to].1;
+            let response = pull_responses[stat.to].0[stat.responses_index].clone();
+            let protocol = Protocol::PullResponse(self_id, vec![response]);
+            packets
+                .packets
+                .push(Packet::from_data(&from_addr, protocol));
+            let len = packets.packets.len();
+            total_bytes += packets.packets[len - 1].meta.size;
+
+            if total_bytes > outbound_budget {
+                inc_new_counter_info!("gossip_pull_request-no_budget", 1);
+                break;
+            }
+        }
+        {
+            let mut cluster_info = me.write().unwrap();
+            cluster_info.outbound_budget.bytes = cluster_info
+                .outbound_budget
+                .bytes
+                .saturating_sub(total_bytes);
+        }
+        time.stop();
+        inc_new_counter_info!("gossip_pull_request-sent_requests", sent.len());
+        inc_new_counter_info!(
+            "gossip_pull_request-dropped_requests",
+            stats.len() - sent.len()
+        );
+        debug!(
+            "handle_pull_requests: {} sent: {} total: {} total_bytes: {}",
+            time,
+            sent.len(),
+            stats.len(),
+            total_bytes
+        );
         if packets.is_empty() {
             return None;
         }
@@ -1565,7 +1651,10 @@ impl ClusterInfo {
             requests.push(more_reqs)
         }
         if num_requests >= MAX_GOSSIP_TRAFFIC {
-            warn!("Too much gossip traffic, ignoring some messages");
+            warn!(
+                "Too much gossip traffic, ignoring some messages (requests={}, max requests={})",
+                num_requests, MAX_GOSSIP_TRAFFIC
+            );
         }
         let epoch_ms;
         let stakes: HashMap<_, _> = match bank_forks {
@@ -1938,7 +2027,7 @@ impl Node {
 
 fn report_time_spent(label: &str, time: &Duration, extra: &str) {
     let count = duration_as_ms(time);
-    if count > 5 {
+    if count > 100 {
         info!("{} took: {} ms {}", label, count, extra);
     }
 }
@@ -2274,62 +2363,30 @@ mod tests {
     #[test]
     fn test_push_vote() {
         let keys = Keypair::new();
+        let now = timestamp();
         let contact_info = ContactInfo::new_localhost(&keys.pubkey(), 0);
         let mut cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
 
         // make sure empty crds is handled correctly
-        let now = timestamp();
-        let (_, votes, max_ts) = cluster_info.get_votes(now);
+        let (votes, max_ts) = cluster_info.get_votes(now);
         assert_eq!(votes, vec![]);
         assert_eq!(max_ts, now);
 
         // add a vote
         let tx = test_tx();
-        let index = 1;
-        cluster_info.push_vote(index, tx.clone());
+        cluster_info.push_vote(0, tx.clone());
 
         // -1 to make sure that the clock is strictly lower then when insert occurred
-        let (labels, votes, max_ts) = cluster_info.get_votes(now - 1);
+        let (votes, max_ts) = cluster_info.get_votes(now - 1);
         assert_eq!(votes, vec![tx]);
-        assert_eq!(labels.len(), 1);
-        match labels[0] {
-            CrdsValueLabel::Vote(_, pubkey) => {
-                assert_eq!(pubkey, keys.pubkey());
-            }
-
-            _ => panic!("Bad match"),
-        }
         assert!(max_ts >= now - 1);
 
         // make sure timestamp filter works
-        let (_, votes, new_max_ts) = cluster_info.get_votes(max_ts);
+        let (votes, new_max_ts) = cluster_info.get_votes(max_ts);
         assert_eq!(votes, vec![]);
         assert_eq!(max_ts, new_max_ts);
     }
 
-    #[test]
-    fn test_push_epoch_slots() {
-        let keys = Keypair::new();
-        let contact_info = ContactInfo::new_localhost(&keys.pubkey(), 0);
-        let mut cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
-        let (slots, since) = cluster_info.get_epoch_slots_since(None);
-        assert!(slots.is_empty());
-        assert!(since.is_none());
-        cluster_info.push_epoch_slots(&[0]);
-
-        let (slots, since) = cluster_info.get_epoch_slots_since(Some(std::u64::MAX));
-        assert!(slots.is_empty());
-        assert_eq!(since, Some(std::u64::MAX));
-
-        let (slots, since) = cluster_info.get_epoch_slots_since(None);
-        assert_eq!(slots.len(), 1);
-        assert!(since.is_some());
-
-        let (slots, since2) = cluster_info.get_epoch_slots_since(since.clone());
-        assert!(slots.is_empty());
-        assert_eq!(since2, since);
-    }
-
     #[test]
     fn test_add_entrypoint() {
         let node_keypair = Arc::new(Keypair::new());
@@ -2383,9 +2440,13 @@ mod tests {
 
     #[test]
     fn test_split_messages_large() {
-        let value = CrdsValue::new_unsigned(CrdsData::LowestSlot(
+        let mut btree_slots = BTreeSet::new();
+        for i in 0..128 {
+            btree_slots.insert(i);
+        }
+        let value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
             0,
-            LowestSlot::new(Pubkey::default(), 0, 0),
+            EpochSlots::new(Pubkey::default(), 0, 0),
         ));
         test_split_messages(value);
     }
@@ -2547,9 +2608,9 @@ mod tests {
             let other_node_pubkey = Pubkey::new_rand();
             let other_node = ContactInfo::new_localhost(&other_node_pubkey, timestamp());
             cluster_info.insert_info(other_node.clone());
-            let value = CrdsValue::new_unsigned(CrdsData::LowestSlot(
+            let value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
                 0,
-                LowestSlot::new(other_node_pubkey, peer_lowest, timestamp()),
+                EpochSlots::new(other_node_pubkey, peer_lowest, timestamp()),
             ));
             let _ = cluster_info.gossip.crds.insert(value, timestamp());
         }
@@ -2559,8 +2620,7 @@ mod tests {
 
     #[test]
     fn test_max_bloom_size() {
-        // check that the constant fits into the dynamic size
-        assert!(MAX_BLOOM_SIZE <= max_bloom_size());
+        assert_eq!(MAX_BLOOM_SIZE, max_bloom_size());
     }
 
     #[test]
@@ -2599,6 +2659,14 @@ mod tests {
         assert_eq!(MAX_PROTOCOL_HEADER_SIZE, max_protocol_size);
     }
 
+    #[test]
+    fn test_protocol_sanitize() {
+        let mut pd = PruneData::default();
+        pd.wallclock = MAX_WALLCLOCK;
+        let msg = Protocol::PruneMessage(Pubkey::default(), pd);
+        assert_eq!(msg.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+    }
+
     // computes the maximum size for pull request blooms
     fn max_bloom_size() -> usize {
         let filter_size = serialized_size(&CrdsFilter::default())
@@ -2611,26 +2679,4 @@ mod tests {
             serialized_size(&protocol).expect("unable to serialize gossip protocol") as usize;
         PACKET_DATA_SIZE - (protocol_size - filter_size)
     }
-
-    #[test]
-    fn test_push_epoch_slots_large() {
-        use rand::Rng;
-        let node_keypair = Arc::new(Keypair::new());
-        let mut cluster_info = ClusterInfo::new(
-            ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()),
-            node_keypair,
-        );
-        let mut range: Vec<Slot> = vec![];
-        //random should be hard to compress
-        for _ in 0..32000 {
-            let last = *range.last().unwrap_or(&0);
-            range.push(last + rand::thread_rng().gen_range(1, 32));
-        }
-        cluster_info.push_epoch_slots(&range[..16000]);
-        cluster_info.push_epoch_slots(&range[16000..]);
-        let (slots, since) = cluster_info.get_epoch_slots_since(None);
-        let slots: Vec<_> = slots.iter().flat_map(|x| x.to_slots(0)).collect();
-        assert_eq!(slots, range);
-        assert!(since.is_some());
-    }
 }

core/src/cluster_slots.rs

@@ -1,348 +0,0 @@
-use crate::{
-    cluster_info::ClusterInfo, contact_info::ContactInfo, epoch_slots::EpochSlots,
-    serve_repair::RepairType,
-};
-use solana_ledger::bank_forks::BankForks;
-use solana_runtime::epoch_stakes::NodeIdToVoteAccounts;
-use solana_sdk::{clock::Slot, pubkey::Pubkey};
-use std::{
-    collections::{HashMap, HashSet},
-    sync::{Arc, RwLock},
-};
-
-pub type SlotPubkeys = HashMap<Arc<Pubkey>, u64>;
-pub type ClusterSlotsMap = RwLock<HashMap<Slot, Arc<RwLock<SlotPubkeys>>>>;
-
-#[derive(Default)]
-pub struct ClusterSlots {
-    cluster_slots: ClusterSlotsMap,
-    keys: RwLock<HashSet<Arc<Pubkey>>>,
-    since: RwLock<Option<u64>>,
-    validator_stakes: RwLock<Arc<NodeIdToVoteAccounts>>,
-    epoch: RwLock<Option<u64>>,
-    self_id: RwLock<Pubkey>,
-}
-
-impl ClusterSlots {
-    pub fn lookup(&self, slot: Slot) -> Option<Arc<RwLock<SlotPubkeys>>> {
-        self.cluster_slots.read().unwrap().get(&slot).cloned()
-    }
-    pub fn update(
-        &self,
-        root: Slot,
-        cluster_info: &RwLock<ClusterInfo>,
-        bank_forks: &RwLock<BankForks>,
-    ) {
-        self.update_peers(cluster_info, bank_forks);
-        let since = *self.since.read().unwrap();
-        let epoch_slots = cluster_info.read().unwrap().get_epoch_slots_since(since);
-        self.update_internal(root, epoch_slots);
-    }
-    fn update_internal(&self, root: Slot, epoch_slots: (Vec<EpochSlots>, Option<u64>)) {
-        let (epoch_slots_list, since) = epoch_slots;
-        for epoch_slots in epoch_slots_list {
-            let slots = epoch_slots.to_slots(root);
-            for slot in &slots {
-                if *slot <= root {
-                    continue;
-                }
-                let pubkey = Arc::new(epoch_slots.from);
-                let exists = self.keys.read().unwrap().get(&pubkey).is_some();
-                if !exists {
-                    self.keys.write().unwrap().insert(pubkey.clone());
-                }
-                let from = self.keys.read().unwrap().get(&pubkey).unwrap().clone();
-                let balance = self
-                    .validator_stakes
-                    .read()
-                    .unwrap()
-                    .get(&from)
-                    .map(|v| v.total_stake)
-                    .unwrap_or(0);
-
-                let mut slot_pubkeys = self.cluster_slots.read().unwrap().get(slot).cloned();
-                if slot_pubkeys.is_none() {
-                    let new_slot_pubkeys = Arc::new(RwLock::new(HashMap::default()));
-                    self.cluster_slots
-                        .write()
-                        .unwrap()
-                        .insert(*slot, new_slot_pubkeys.clone());
-                    slot_pubkeys = Some(new_slot_pubkeys);
-                }
-
-                slot_pubkeys
-                    .unwrap()
-                    .write()
-                    .unwrap()
-                    .insert(from.clone(), balance);
-            }
-        }
-        self.cluster_slots.write().unwrap().retain(|x, _| *x > root);
-        self.keys
-            .write()
-            .unwrap()
-            .retain(|x| Arc::strong_count(x) > 1);
-        *self.since.write().unwrap() = since;
-    }
-    pub fn collect(&self, id: &Pubkey) -> HashSet<Slot> {
-        self.cluster_slots
-            .read()
-            .unwrap()
-            .iter()
-            .filter(|(_, keys)| keys.read().unwrap().get(id).is_some())
-            .map(|(slot, _)| slot)
-            .cloned()
-            .collect()
-    }
-
-    fn update_peers(&self, cluster_info: &RwLock<ClusterInfo>, bank_forks: &RwLock<BankForks>) {
-        let root_bank = bank_forks.read().unwrap().root_bank().clone();
-        let root_epoch = root_bank.epoch();
-        let my_epoch = *self.epoch.read().unwrap();
-
-        if Some(root_epoch) != my_epoch {
-            let validator_stakes = root_bank
-                .epoch_stakes(root_epoch)
-                .expect(
-                    "Bank must have epoch stakes
-                for its own epoch",
-                )
-                .node_id_to_vote_accounts()
-                .clone();
-
-            *self.validator_stakes.write().unwrap() = validator_stakes;
-            let id = cluster_info.read().unwrap().id();
-            *self.self_id.write().unwrap() = id;
-            *self.epoch.write().unwrap() = Some(root_epoch);
-        }
-    }
-
-    pub fn compute_weights(&self, slot: Slot, repair_peers: &[ContactInfo]) -> Vec<(u64, usize)> {
-        let slot_peers = self.lookup(slot);
-        repair_peers
-            .iter()
-            .enumerate()
-            .map(|(i, x)| {
-                let peer_stake = slot_peers
-                    .as_ref()
-                    .and_then(|v| v.read().unwrap().get(&x.id).cloned())
-                    .unwrap_or(0);
-                (
-                    1 + peer_stake
-                        + self
-                            .validator_stakes
-                            .read()
-                            .unwrap()
-                            .get(&x.id)
-                            .map(|v| v.total_stake)
-                            .unwrap_or(0),
-                    i,
-                )
-            })
-            .collect()
-    }
-
-    pub fn generate_repairs_for_missing_slots(
-        &self,
-        self_id: &Pubkey,
-        root: Slot,
-    ) -> Vec<RepairType> {
-        let my_slots = self.collect(self_id);
-        self.cluster_slots
-            .read()
-            .unwrap()
-            .keys()
-            .filter(|x| **x > root)
-            .filter(|x| !my_slots.contains(*x))
-            .map(|x| RepairType::HighestShred(*x, 0))
-            .collect()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use solana_runtime::epoch_stakes::NodeVoteAccounts;
-
-    #[test]
-    fn test_default() {
-        let cs = ClusterSlots::default();
-        assert!(cs.cluster_slots.read().unwrap().is_empty());
-        assert!(cs.since.read().unwrap().is_none());
-    }
-
-    #[test]
-    fn test_update_noop() {
-        let cs = ClusterSlots::default();
-        cs.update_internal(0, (vec![], None));
-        assert!(cs.cluster_slots.read().unwrap().is_empty());
-        assert!(cs.since.read().unwrap().is_none());
-    }
-
-    #[test]
-    fn test_update_empty() {
-        let cs = ClusterSlots::default();
-        let epoch_slot = EpochSlots::default();
-        cs.update_internal(0, (vec![epoch_slot], Some(0)));
-        assert_eq!(*cs.since.read().unwrap(), Some(0));
-        assert!(cs.lookup(0).is_none());
-    }
-
-    #[test]
-    fn test_update_rooted() {
-        //root is 0, so it should clear out the slot
-        let cs = ClusterSlots::default();
-        let mut epoch_slot = EpochSlots::default();
-        epoch_slot.fill(&[0], 0);
-        cs.update_internal(0, (vec![epoch_slot], Some(0)));
-        assert_eq!(*cs.since.read().unwrap(), Some(0));
-        assert!(cs.lookup(0).is_none());
-    }
-
-    #[test]
-    fn test_update_new_slot() {
-        let cs = ClusterSlots::default();
-        let mut epoch_slot = EpochSlots::default();
-        epoch_slot.fill(&[1], 0);
-        cs.update_internal(0, (vec![epoch_slot], Some(0)));
-        assert_eq!(*cs.since.read().unwrap(), Some(0));
-        assert!(cs.lookup(0).is_none());
-        assert!(cs.lookup(1).is_some());
-        assert_eq!(
-            cs.lookup(1)
-                .unwrap()
-                .read()
-                .unwrap()
-                .get(&Pubkey::default()),
-            Some(&0)
-        );
-    }
-
-    #[test]
-    fn test_compute_weights() {
-        let cs = ClusterSlots::default();
-        let ci = ContactInfo::default();
-        assert_eq!(cs.compute_weights(0, &[ci]), vec![(1, 0)]);
-    }
-
-    #[test]
-    fn test_best_peer_2() {
-        let cs = ClusterSlots::default();
-        let mut c1 = ContactInfo::default();
-        let mut c2 = ContactInfo::default();
-        let mut map = HashMap::new();
-        let k1 = Pubkey::new_rand();
-        let k2 = Pubkey::new_rand();
-        map.insert(Arc::new(k1.clone()), std::u64::MAX / 2);
-        map.insert(Arc::new(k2.clone()), 0);
-        cs.cluster_slots
-            .write()
-            .unwrap()
-            .insert(0, Arc::new(RwLock::new(map)));
-        c1.id = k1;
-        c2.id = k2;
-        assert_eq!(
-            cs.compute_weights(0, &[c1, c2]),
-            vec![(std::u64::MAX / 2 + 1, 0), (1, 1)]
-        );
-    }
-
-    #[test]
-    fn test_best_peer_3() {
-        let cs = ClusterSlots::default();
-        let mut c1 = ContactInfo::default();
-        let mut c2 = ContactInfo::default();
-        let mut map = HashMap::new();
-        let k1 = Pubkey::new_rand();
-        let k2 = Pubkey::new_rand();
-        map.insert(Arc::new(k2.clone()), 0);
-        cs.cluster_slots
-            .write()
-            .unwrap()
-            .insert(0, Arc::new(RwLock::new(map)));
-        //make sure default weights are used as well
-        let validator_stakes: HashMap<_, _> = vec![(
-            *Arc::new(k1.clone()),
-            NodeVoteAccounts {
-                total_stake: std::u64::MAX / 2,
-                vote_accounts: vec![Pubkey::default()],
-            },
-        )]
-        .into_iter()
-        .collect();
-        *cs.validator_stakes.write().unwrap() = Arc::new(validator_stakes);
-        c1.id = k1;
-        c2.id = k2;
-        assert_eq!(
-            cs.compute_weights(0, &[c1, c2]),
-            vec![(std::u64::MAX / 2 + 1, 0), (1, 1)]
-        );
-    }
-
-    #[test]
-    fn test_update_new_staked_slot() {
-        let cs = ClusterSlots::default();
-        let mut epoch_slot = EpochSlots::default();
-        epoch_slot.fill(&[1], 0);
-
-        let map = Arc::new(
-            vec![(
-                Pubkey::default(),
-                NodeVoteAccounts {
-                    total_stake: 1,
-                    vote_accounts: vec![Pubkey::default()],
-                },
-            )]
-            .into_iter()
-            .collect(),
-        );
-
-        *cs.validator_stakes.write().unwrap() = map;
-        cs.update_internal(0, (vec![epoch_slot], None));
-        assert!(cs.lookup(1).is_some());
-        assert_eq!(
-            cs.lookup(1)
-                .unwrap()
-                .read()
-                .unwrap()
-                .get(&Pubkey::default()),
-            Some(&1)
-        );
-    }
-
-    #[test]
-    fn test_generate_repairs() {
-        let cs = ClusterSlots::default();
-        let mut epoch_slot = EpochSlots::default();
-        epoch_slot.fill(&[1], 0);
-        cs.update_internal(0, (vec![epoch_slot], None));
-        let self_id = Pubkey::new_rand();
-        assert_eq!(
-            cs.generate_repairs_for_missing_slots(&self_id, 0),
-            vec![RepairType::HighestShred(1, 0)]
-        )
-    }
-
-    #[test]
-    fn test_collect_my_slots() {
-        let cs = ClusterSlots::default();
-        let mut epoch_slot = EpochSlots::default();
-        epoch_slot.fill(&[1], 0);
-        let self_id = epoch_slot.from;
-        cs.update_internal(0, (vec![epoch_slot], None));
-        let slots: Vec<Slot> = cs.collect(&self_id).into_iter().collect();
-        assert_eq!(slots, vec![1]);
-    }
-
-    #[test]
-    fn test_generate_repairs_existing() {
-        let cs = ClusterSlots::default();
-        let mut epoch_slot = EpochSlots::default();
-        epoch_slot.fill(&[1], 0);
-        let self_id = epoch_slot.from;
-        cs.update_internal(0, (vec![epoch_slot], None));
-        assert!(cs
-            .generate_repairs_for_missing_slots(&self_id, 0)
-            .is_empty());
-    }
-}

core/src/commitment.rs

@@ -1,4 +1,5 @@
 use crate::consensus::VOTE_THRESHOLD_SIZE;
+use solana_ledger::blockstore::Blockstore;
 use solana_measure::measure::Measure;
 use solana_metrics::inc_new_counter_info;
 use solana_runtime::bank::Bank;
@@ -13,9 +14,11 @@ use std::{
     time::Duration,
 };
 
+pub type BlockCommitmentArray = [u64; MAX_LOCKOUT_HISTORY + 1];
+
 #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
 pub struct BlockCommitment {
-    pub commitment: [u64; MAX_LOCKOUT_HISTORY],
+    pub commitment: BlockCommitmentArray,
 }
 
 impl BlockCommitment {
@@ -28,17 +31,27 @@ impl BlockCommitment {
         assert!(confirmation_count > 0 && confirmation_count <= MAX_LOCKOUT_HISTORY);
         self.commitment[confirmation_count - 1]
     }
+
+    pub fn increase_rooted_stake(&mut self, stake: u64) {
+        self.commitment[MAX_LOCKOUT_HISTORY] += stake;
+    }
+
+    pub fn get_rooted_stake(&self) -> u64 {
+        self.commitment[MAX_LOCKOUT_HISTORY]
+    }
+
     #[cfg(test)]
-    pub(crate) fn new(commitment: [u64; MAX_LOCKOUT_HISTORY]) -> Self {
+    pub(crate) fn new(commitment: BlockCommitmentArray) -> Self {
         Self { commitment }
     }
 }
 
-#[derive(Default)]
 pub struct BlockCommitmentCache {
     block_commitment: HashMap<Slot, BlockCommitment>,
+    largest_confirmed_root: Slot,
     total_stake: u64,
     bank: Arc<Bank>,
+    blockstore: Arc<Blockstore>,
     root: Slot,
 }
 
@@ -59,22 +72,41 @@ impl std::fmt::Debug for BlockCommitmentCache {
 impl BlockCommitmentCache {
     pub fn new(
         block_commitment: HashMap<Slot, BlockCommitment>,
+        largest_confirmed_root: Slot,
         total_stake: u64,
         bank: Arc<Bank>,
+        blockstore: Arc<Blockstore>,
         root: Slot,
     ) -> Self {
         Self {
             block_commitment,
+            largest_confirmed_root,
             total_stake,
             bank,
+            blockstore,
             root,
         }
     }
 
+    pub fn default_with_blockstore(blockstore: Arc<Blockstore>) -> Self {
+        Self {
+            block_commitment: HashMap::default(),
+            largest_confirmed_root: Slot::default(),
+            total_stake: u64::default(),
+            bank: Arc::new(Bank::default()),
+            blockstore,
+            root: Slot::default(),
+        }
+    }
+
     pub fn get_block_commitment(&self, slot: Slot) -> Option<&BlockCommitment> {
         self.block_commitment.get(&slot)
     }
 
+    pub fn largest_confirmed_root(&self) -> Slot {
+        self.largest_confirmed_root
+    }
+
     pub fn total_stake(&self) -> u64 {
         self.total_stake
     }
@@ -110,16 +142,30 @@ impl BlockCommitmentCache {
             0
         })
     }
+
+    pub fn is_confirmed_rooted(&self, slot: Slot) -> bool {
+        slot <= self.largest_confirmed_root()
+            && (self.blockstore.is_root(slot) || self.bank.status_cache_ancestors().contains(&slot))
+    }
+
     #[cfg(test)]
-    pub fn new_for_tests() -> Self {
+    pub fn new_for_tests_with_blockstore(blockstore: Arc<Blockstore>) -> Self {
         let mut block_commitment: HashMap<Slot, BlockCommitment> = HashMap::new();
         block_commitment.insert(0, BlockCommitment::default());
         Self {
             block_commitment,
+            blockstore,
             total_stake: 42,
-            ..Self::default()
+            largest_confirmed_root: Slot::default(),
+            bank: Arc::new(Bank::default()),
+            root: Slot::default(),
         }
     }
+
+    #[cfg(test)]
+    pub(crate) fn set_get_largest_confirmed_root(&mut self, root: Slot) {
+        self.largest_confirmed_root = root;
+    }
 }
 
 pub struct CommitmentAggregationData {
@@ -138,6 +184,18 @@ impl CommitmentAggregationData {
     }
 }
 
+fn get_largest_confirmed_root(mut rooted_stake: Vec<(Slot, u64)>, total_stake: u64) -> Slot {
+    rooted_stake.sort_by(|a, b| a.0.cmp(&b.0).reverse());
+    let mut stake_sum = 0;
+    for (root, stake) in rooted_stake {
+        stake_sum += stake;
+        if (stake_sum as f64 / total_stake as f64) > VOTE_THRESHOLD_SIZE {
+            return root;
+        }
+    }
+    0
+}
+
 pub struct AggregateCommitmentService {
     t_commitment: JoinHandle<()>,
 }
@@ -195,12 +253,18 @@ impl AggregateCommitmentService {
             }
 
             let mut aggregate_commitment_time = Measure::start("aggregate-commitment-ms");
-            let block_commitment = Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
+            let (block_commitment, rooted_stake) =
+                Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
+
+            let largest_confirmed_root =
+                get_largest_confirmed_root(rooted_stake, aggregation_data.total_staked);
 
             let mut new_block_commitment = BlockCommitmentCache::new(
                 block_commitment,
+                largest_confirmed_root,
                 aggregation_data.total_staked,
                 aggregation_data.bank,
+                block_commitment_cache.read().unwrap().blockstore.clone(),
                 aggregation_data.root,
             );
 
@@ -215,7 +279,10 @@ impl AggregateCommitmentService {
         }
     }
 
-    pub fn aggregate_commitment(ancestors: &[Slot], bank: &Bank) -> HashMap<Slot, BlockCommitment> {
+    pub fn aggregate_commitment(
+        ancestors: &[Slot],
+        bank: &Bank,
+    ) -> (HashMap<Slot, BlockCommitment>, Vec<(Slot, u64)>) {
         assert!(!ancestors.is_empty());
 
         // Check ancestors is sorted
@@ -224,6 +291,7 @@ impl AggregateCommitmentService {
         }
 
         let mut commitment = HashMap::new();
+        let mut rooted_stake: Vec<(Slot, u64)> = Vec::new();
         for (_, (lamports, account)) in bank.vote_accounts().into_iter() {
             if lamports == 0 {
                 continue;
@@ -236,17 +304,19 @@ impl AggregateCommitmentService {
             let vote_state = vote_state.unwrap();
             Self::aggregate_commitment_for_vote_account(
                 &mut commitment,
+                &mut rooted_stake,
                 &vote_state,
                 ancestors,
                 lamports,
             );
         }
 
-        commitment
+        (commitment, rooted_stake)
     }
 
     fn aggregate_commitment_for_vote_account(
         commitment: &mut HashMap<Slot, BlockCommitment>,
+        rooted_stake: &mut Vec<(Slot, u64)>,
         vote_state: &VoteState,
         ancestors: &[Slot],
         lamports: u64,
@@ -259,12 +329,13 @@ impl AggregateCommitmentService {
                     commitment
                         .entry(*a)
                         .or_insert_with(BlockCommitment::default)
-                        .increase_confirmation_stake(MAX_LOCKOUT_HISTORY, lamports);
+                        .increase_rooted_stake(lamports);
                 } else {
                     ancestors_index = i;
                     break;
                 }
             }
+            rooted_stake.push((root, lamports));
         }
 
         for vote in &vote_state.votes {
@@ -290,7 +361,8 @@ impl AggregateCommitmentService {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use solana_ledger::get_tmp_ledger_path;
     use solana_sdk::pubkey::Pubkey;
     use solana_stake_program::stake_state;
     use solana_vote_program::vote_state::{self, VoteStateVersions};
@@ -308,6 +380,8 @@ mod tests {
     #[test]
     fn test_get_confirmations() {
         let bank = Arc::new(Bank::default());
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         // Build BlockCommitmentCache with votes at depths 0 and 1 for 2 slots
         let mut cache0 = BlockCommitment::default();
         cache0.increase_confirmation_stake(1, 5);
@@ -325,7 +399,8 @@ mod tests {
         block_commitment.entry(0).or_insert(cache0.clone());
         block_commitment.entry(1).or_insert(cache1.clone());
         block_commitment.entry(2).or_insert(cache2.clone());
-        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 50, bank, 0);
+        let block_commitment_cache =
+            BlockCommitmentCache::new(block_commitment, 0, 50, bank, blockstore, 0);
 
         assert_eq!(block_commitment_cache.get_confirmation_count(0), Some(2));
         assert_eq!(block_commitment_cache.get_confirmation_count(1), Some(1));
@@ -333,17 +408,68 @@ mod tests {
         assert_eq!(block_commitment_cache.get_confirmation_count(3), None,);
     }
 
+    #[test]
+    fn test_is_confirmed_rooted() {
+        let bank = Arc::new(Bank::default());
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        blockstore.set_roots(&[0, 1]).unwrap();
+        // Build BlockCommitmentCache with rooted slots
+        let mut cache0 = BlockCommitment::default();
+        cache0.increase_rooted_stake(50);
+        let mut cache1 = BlockCommitment::default();
+        cache1.increase_rooted_stake(40);
+        let mut cache2 = BlockCommitment::default();
+        cache2.increase_rooted_stake(20);
+
+        let mut block_commitment = HashMap::new();
+        block_commitment.entry(1).or_insert(cache0.clone());
+        block_commitment.entry(2).or_insert(cache1.clone());
+        block_commitment.entry(3).or_insert(cache2.clone());
+        let largest_confirmed_root = 1;
+        let block_commitment_cache = BlockCommitmentCache::new(
+            block_commitment,
+            largest_confirmed_root,
+            50,
+            bank,
+            blockstore,
+            0,
+        );
+
+        assert!(block_commitment_cache.is_confirmed_rooted(0));
+        assert!(block_commitment_cache.is_confirmed_rooted(1));
+        assert!(!block_commitment_cache.is_confirmed_rooted(2));
+        assert!(!block_commitment_cache.is_confirmed_rooted(3));
+    }
+
+    #[test]
+    fn test_get_largest_confirmed_root() {
+        assert_eq!(get_largest_confirmed_root(vec![], 10), 0);
+        let mut rooted_stake = vec![];
+        rooted_stake.push((0, 5));
+        rooted_stake.push((1, 5));
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 10), 0);
+        let mut rooted_stake = vec![];
+        rooted_stake.push((1, 5));
+        rooted_stake.push((0, 10));
+        rooted_stake.push((2, 5));
+        rooted_stake.push((1, 4));
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 10), 1);
+    }
+
     #[test]
     fn test_aggregate_commitment_for_vote_account_1() {
         let ancestors = vec![3, 4, 5, 7, 9, 11];
         let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
         let lamports = 5;
         let mut vote_state = VoteState::default();
 
-        let root = ancestors.last().unwrap();
-        vote_state.root_slot = Some(*root);
+        let root = ancestors.last().unwrap().clone();
+        vote_state.root_slot = Some(root);
         AggregateCommitmentService::aggregate_commitment_for_vote_account(
             &mut commitment,
+            &mut rooted_stake,
             &vote_state,
             &ancestors,
             lamports,
@@ -351,15 +477,17 @@ mod tests {
 
         for a in ancestors {
             let mut expected = BlockCommitment::default();
-            expected.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, lamports);
+            expected.increase_rooted_stake(lamports);
             assert_eq!(*commitment.get(&a).unwrap(), expected);
         }
+        assert_eq!(rooted_stake[0], (root, lamports));
     }
 
     #[test]
     fn test_aggregate_commitment_for_vote_account_2() {
         let ancestors = vec![3, 4, 5, 7, 9, 11];
         let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
         let lamports = 5;
         let mut vote_state = VoteState::default();
 
@@ -368,6 +496,7 @@ mod tests {
         vote_state.process_slot_vote_unchecked(*ancestors.last().unwrap());
         AggregateCommitmentService::aggregate_commitment_for_vote_account(
             &mut commitment,
+            &mut rooted_stake,
             &vote_state,
             &ancestors,
             lamports,
@@ -376,7 +505,7 @@ mod tests {
         for a in ancestors {
             if a <= root {
                 let mut expected = BlockCommitment::default();
-                expected.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, lamports);
+                expected.increase_rooted_stake(lamports);
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             } else {
                 let mut expected = BlockCommitment::default();
@@ -384,12 +513,14 @@ mod tests {
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             }
         }
+        assert_eq!(rooted_stake[0], (root, lamports));
     }
 
     #[test]
     fn test_aggregate_commitment_for_vote_account_3() {
         let ancestors = vec![3, 4, 5, 7, 9, 10, 11];
         let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
         let lamports = 5;
         let mut vote_state = VoteState::default();
 
@@ -400,6 +531,7 @@ mod tests {
         vote_state.process_slot_vote_unchecked(ancestors[6]);
         AggregateCommitmentService::aggregate_commitment_for_vote_account(
             &mut commitment,
+            &mut rooted_stake,
             &vote_state,
             &ancestors,
             lamports,
@@ -408,7 +540,7 @@ mod tests {
         for (i, a) in ancestors.iter().enumerate() {
             if *a <= root {
                 let mut expected = BlockCommitment::default();
-                expected.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, lamports);
+                expected.increase_rooted_stake(lamports);
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             } else if i <= 4 {
                 let mut expected = BlockCommitment::default();
@@ -420,6 +552,7 @@ mod tests {
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             }
         }
+        assert_eq!(rooted_stake[0], (root, lamports));
     }
 
     #[test]
@@ -429,6 +562,8 @@ mod tests {
             mut genesis_config, ..
         } = create_genesis_config(10_000);
 
+        let rooted_stake_amount = 40;
+
         let sk1 = Pubkey::new_rand();
         let pk1 = Pubkey::new_rand();
         let mut vote_account1 = vote_state::create_account(&pk1, &Pubkey::new_rand(), 0, 100);
@@ -439,12 +574,36 @@ mod tests {
         let mut vote_account2 = vote_state::create_account(&pk2, &Pubkey::new_rand(), 0, 50);
         let stake_account2 =
             stake_state::create_account(&sk2, &pk2, &vote_account2, &genesis_config.rent, 50);
+        let sk3 = Pubkey::new_rand();
+        let pk3 = Pubkey::new_rand();
+        let mut vote_account3 = vote_state::create_account(&pk3, &Pubkey::new_rand(), 0, 1);
+        let stake_account3 = stake_state::create_account(
+            &sk3,
+            &pk3,
+            &vote_account3,
+            &genesis_config.rent,
+            rooted_stake_amount,
+        );
+        let sk4 = Pubkey::new_rand();
+        let pk4 = Pubkey::new_rand();
+        let mut vote_account4 = vote_state::create_account(&pk4, &Pubkey::new_rand(), 0, 1);
+        let stake_account4 = stake_state::create_account(
+            &sk4,
+            &pk4,
+            &vote_account4,
+            &genesis_config.rent,
+            rooted_stake_amount,
+        );
 
         genesis_config.accounts.extend(vec![
             (pk1, vote_account1.clone()),
             (sk1, stake_account1),
             (pk2, vote_account2.clone()),
             (sk2, stake_account2),
+            (pk3, vote_account3.clone()),
+            (sk3, stake_account3),
+            (pk4, vote_account4.clone()),
+            (sk4, stake_account4),
         ]);
 
         // Create bank
@@ -464,7 +623,20 @@ mod tests {
         VoteState::to(&versioned, &mut vote_account2).unwrap();
         bank.store_account(&pk2, &vote_account2);
 
-        let commitment = AggregateCommitmentService::aggregate_commitment(&ancestors, &bank);
+        let mut vote_state3 = VoteState::from(&vote_account3).unwrap();
+        vote_state3.root_slot = Some(1);
+        let versioned = VoteStateVersions::Current(Box::new(vote_state3));
+        VoteState::to(&versioned, &mut vote_account3).unwrap();
+        bank.store_account(&pk3, &vote_account3);
+
+        let mut vote_state4 = VoteState::from(&vote_account4).unwrap();
+        vote_state4.root_slot = Some(2);
+        let versioned = VoteStateVersions::Current(Box::new(vote_state4));
+        VoteState::to(&versioned, &mut vote_account4).unwrap();
+        bank.store_account(&pk4, &vote_account4);
+
+        let (commitment, rooted_stake) =
+            AggregateCommitmentService::aggregate_commitment(&ancestors, &bank);
 
         for a in ancestors {
             if a <= 3 {
@@ -488,5 +660,7 @@ mod tests {
                 assert!(commitment.get(&a).is_none());
             }
         }
+        assert_eq!(rooted_stake.len(), 2);
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 100), 1)
     }
 }

core/src/consensus.rs

@@ -1,4 +1,3 @@
-use crate::progress_map::ProgressMap;
 use chrono::prelude::*;
 use solana_ledger::bank_forks::BankForks;
 use solana_runtime::bank::Bank;
@@ -356,18 +355,6 @@ impl Tower {
         }
     }
 
-    pub(crate) fn check_switch_threshold(
-        &self,
-        _slot: Slot,
-        _ancestors: &HashMap<Slot, HashSet<u64>>,
-        _descendants: &HashMap<Slot, HashSet<u64>>,
-        _progress: &ProgressMap,
-        _total_epoch_stake: u64,
-        _epoch_vote_accounts: &HashMap<Pubkey, (u64, Account)>,
-    ) -> bool {
-        true
-    }
-
     /// Update lockouts for all the ancestors
     fn update_ancestor_lockouts(
         stake_lockouts: &mut HashMap<Slot, StakeLockout>,
@@ -481,12 +468,7 @@ impl Tower {
 #[cfg(test)]
 pub mod test {
     use super::*;
-    use crate::{
-        cluster_info_vote_listener::VoteTracker,
-        cluster_slots::ClusterSlots,
-        progress_map::ForkProgress,
-        replay_stage::{HeaviestForkFailures, ReplayStage},
-    };
+    use crate::replay_stage::{ForkProgress, ReplayStage};
     use solana_ledger::bank_forks::BankForks;
     use solana_runtime::{
         bank::Bank,
@@ -529,9 +511,9 @@ pub mod test {
             cluster_votes: &mut HashMap<Pubkey, Vec<u64>>,
             validator_keypairs: &HashMap<Pubkey, ValidatorVoteKeypairs>,
             my_keypairs: &ValidatorVoteKeypairs,
-            progress: &mut ProgressMap,
+            progress: &mut HashMap<u64, ForkProgress>,
             tower: &mut Tower,
-        ) -> Vec<HeaviestForkFailures> {
+        ) -> Vec<VoteFailures> {
             let node = self
                 .find_node_and_update_simulation(vote_slot)
                 .expect("Vote to simulate must be for a slot in the tree");
@@ -568,7 +550,7 @@ pub mod test {
                 info!("parent of {} is {}", missing_slot, parent_bank.slot(),);
                 progress
                     .entry(missing_slot)
-                    .or_insert_with(|| ForkProgress::new(parent_bank.last_blockhash(), None, None));
+                    .or_insert_with(|| ForkProgress::new(parent_bank.last_blockhash()));
 
                 // Create the missing bank
                 let new_bank =
@@ -613,10 +595,6 @@ pub mod test {
                 &mut frozen_banks,
                 tower,
                 progress,
-                &VoteTracker::default(),
-                &ClusterSlots::default(),
-                bank_forks,
-                &mut HashSet::new(),
             );
 
             let bank = bank_forks
@@ -633,23 +611,17 @@ pub mod test {
             info!("lockouts: {:?}", fork_progress.fork_stats.stake_lockouts);
             let mut failures = vec![];
             if fork_progress.fork_stats.is_locked_out {
-                failures.push(HeaviestForkFailures::LockedOut(vote_slot));
+                failures.push(VoteFailures::LockedOut(vote_slot));
             }
             if !fork_progress.fork_stats.vote_threshold {
-                failures.push(HeaviestForkFailures::FailedThreshold(vote_slot));
+                failures.push(VoteFailures::FailedThreshold(vote_slot));
             }
             if !failures.is_empty() {
                 return failures;
             }
             let vote = tower.new_vote_from_bank(&bank, &my_vote_pubkey).0;
             if let Some(new_root) = tower.record_bank_vote(vote) {
-                ReplayStage::handle_new_root(
-                    new_root,
-                    bank_forks,
-                    progress,
-                    &None,
-                    &mut HashSet::new(),
-                );
+                ReplayStage::handle_new_root(new_root, bank_forks, progress, &None, None);
             }
 
             // Mark the vote for this bank under this node's pubkey so it will be
@@ -699,17 +671,22 @@ pub mod test {
         }
     }
 
+    #[derive(PartialEq, Debug)]
+    pub(crate) enum VoteFailures {
+        LockedOut(u64),
+        FailedThreshold(u64),
+    }
+
     // Setup BankForks with bank 0 and all the validator accounts
     pub(crate) fn initialize_state(
         validator_keypairs_map: &HashMap<Pubkey, ValidatorVoteKeypairs>,
-        stake: u64,
-    ) -> (BankForks, ProgressMap) {
+    ) -> (BankForks, HashMap<u64, ForkProgress>) {
         let validator_keypairs: Vec<_> = validator_keypairs_map.values().collect();
         let GenesisConfigInfo {
             genesis_config,
             mint_keypair,
             voting_keypair: _,
-        } = create_genesis_config_with_vote_accounts(1_000_000_000, &validator_keypairs, stake);
+        } = create_genesis_config_with_vote_accounts(1_000_000_000, &validator_keypairs);
 
         let bank0 = Bank::new(&genesis_config);
 
@@ -718,8 +695,8 @@ pub mod test {
         }
 
         bank0.freeze();
-        let mut progress = ProgressMap::default();
-        progress.insert(0, ForkProgress::new(bank0.last_blockhash(), None, None));
+        let mut progress = HashMap::new();
+        progress.insert(0, ForkProgress::new(bank0.last_blockhash()));
         (BankForks::new(0, bank0), progress)
     }
 
@@ -751,7 +728,7 @@ pub mod test {
         bank_forks: &RwLock<BankForks>,
         cluster_votes: &mut HashMap<Pubkey, Vec<u64>>,
         keypairs: &HashMap<Pubkey, ValidatorVoteKeypairs>,
-        progress: &mut ProgressMap,
+        progress: &mut HashMap<u64, ForkProgress>,
     ) -> bool {
         // Check that within some reasonable time, validator can make a new
         // root on this fork
@@ -796,7 +773,7 @@ pub mod test {
         );
 
         // Initialize BankForks
-        let (bank_forks, mut progress) = initialize_state(&keypairs, 10_000);
+        let (bank_forks, mut progress) = initialize_state(&keypairs);
         let bank_forks = RwLock::new(bank_forks);
 
         // Create the tree of banks
@@ -876,7 +853,7 @@ pub mod test {
         votes.extend((45..=50).into_iter());
 
         let mut cluster_votes: HashMap<Pubkey, Vec<Slot>> = HashMap::new();
-        let (bank_forks, mut progress) = initialize_state(&keypairs, 10_000);
+        let (bank_forks, mut progress) = initialize_state(&keypairs);
         let bank_forks = RwLock::new(bank_forks);
 
         // Simulate the votes. Should fail on trying to come back to the main fork

core/src/contact_info.rs

@@ -1,6 +1,8 @@
+use crate::crds_value::MAX_WALLCLOCK;
 use solana_sdk::pubkey::Pubkey;
 #[cfg(test)]
 use solana_sdk::rpc_port;
+use solana_sdk::sanitize::{Sanitize, SanitizeError};
 #[cfg(test)]
 use solana_sdk::signature::{Keypair, Signer};
 use solana_sdk::timing::timestamp;
@@ -37,6 +39,15 @@ pub struct ContactInfo {
     pub shred_version: u16,
 }
 
+impl Sanitize for ContactInfo {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        Ok(())
+    }
+}
+
 impl Ord for ContactInfo {
     fn cmp(&self, other: &Self) -> Ordering {
         self.id.cmp(&other.id)
@@ -314,4 +325,12 @@ mod tests {
         ci.rpc = socketaddr!("127.0.0.1:234");
         assert!(ci.valid_client_facing_addr().is_some());
     }
+
+    #[test]
+    fn test_sanitize() {
+        let mut ci = ContactInfo::default();
+        assert_eq!(ci.sanitize(), Ok(()));
+        ci.wallclock = MAX_WALLCLOCK;
+        assert_eq!(ci.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+    }
 }

core/src/crds_gossip.rs

@@ -20,6 +20,7 @@ pub const CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS: usize = 500;
 pub struct CrdsGossip {
     pub crds: Crds,
     pub id: Pubkey,
+    pub shred_version: u16,
     pub push: CrdsGossipPush,
     pub pull: CrdsGossipPull,
 }
@@ -29,6 +30,7 @@ impl Default for CrdsGossip {
         CrdsGossip {
             crds: Crds::default(),
             id: Pubkey::default(),
+            shred_version: 0,
             push: CrdsGossipPush::default(),
             pull: CrdsGossipPull::default(),
         }
@@ -39,6 +41,9 @@ impl CrdsGossip {
     pub fn set_self(&mut self, id: &Pubkey) {
         self.id = *id;
     }
+    pub fn set_shred_version(&mut self, shred_version: u16) {
+        self.shred_version = shred_version;
+    }
 
     /// process a push message to the network
     pub fn process_push_message(
@@ -122,6 +127,7 @@ impl CrdsGossip {
             &self.crds,
             stakes,
             &self.id,
+            self.shred_version,
             self.pull.pull_request_time.len(),
             CRDS_GOSSIP_NUM_ACTIVE,
         )
@@ -134,8 +140,14 @@ impl CrdsGossip {
         stakes: &HashMap<Pubkey, u64>,
         bloom_size: usize,
     ) -> Result<(Pubkey, Vec<CrdsFilter>, CrdsValue), CrdsGossipError> {
-        self.pull
-            .new_pull_request(&self.crds, &self.id, now, stakes, bloom_size)
+        self.pull.new_pull_request(
+            &self.crds,
+            &self.id,
+            self.shred_version,
+            now,
+            stakes,
+            bloom_size,
+        )
     }
 
     /// time when a request to `from` was initiated

core/src/crds_gossip_pull.rs

@@ -14,7 +14,6 @@ use crate::crds::Crds;
 use crate::crds_gossip::{get_stake, get_weight, CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS};
 use crate::crds_gossip_error::CrdsGossipError;
 use crate::crds_value::{CrdsValue, CrdsValueLabel};
-use rand;
 use rand::distributions::{Distribution, WeightedIndex};
 use rand::Rng;
 use solana_runtime::bloom::Bloom;
@@ -37,6 +36,13 @@ pub struct CrdsFilter {
     mask_bits: u32,
 }
 
+impl solana_sdk::sanitize::Sanitize for CrdsFilter {
+    fn sanitize(&self) -> std::result::Result<(), solana_sdk::sanitize::SanitizeError> {
+        self.filter.sanitize()?;
+        Ok(())
+    }
+}
+
 impl CrdsFilter {
     pub fn new_rand(num_items: usize, max_bytes: usize) -> Self {
         let max_bits = (max_bytes * 8) as f64;
@@ -138,11 +144,12 @@ impl CrdsGossipPull {
         &self,
         crds: &Crds,
         self_id: &Pubkey,
+        self_shred_version: u16,
         now: u64,
         stakes: &HashMap<Pubkey, u64>,
         bloom_size: usize,
     ) -> Result<(Pubkey, Vec<CrdsFilter>, CrdsValue), CrdsGossipError> {
-        let options = self.pull_options(crds, &self_id, now, stakes);
+        let options = self.pull_options(crds, &self_id, self_shred_version, now, stakes);
         if options.is_empty() {
             return Err(CrdsGossipError::NoPeers);
         }
@@ -159,13 +166,20 @@ impl CrdsGossipPull {
         &self,
         crds: &'a Crds,
         self_id: &Pubkey,
+        self_shred_version: u16,
         now: u64,
         stakes: &HashMap<Pubkey, u64>,
     ) -> Vec<(f32, &'a ContactInfo)> {
         crds.table
             .values()
             .filter_map(|v| v.value.contact_info())
-            .filter(|v| v.id != *self_id && ContactInfo::is_valid_address(&v.gossip))
+            .filter(|v| {
+                v.id != *self_id
+                    && ContactInfo::is_valid_address(&v.gossip)
+                    && (self_shred_version == 0
+                        || v.shred_version == 0
+                        || self_shred_version == v.shred_version)
+            })
             .map(|item| {
                 let max_weight = f32::from(u16::max_value()) - 1.0;
                 let req_time: u64 = *self.pull_request_time.get(&item.id).unwrap_or(&0);
@@ -396,7 +410,7 @@ mod test {
             stakes.insert(id, i * 100);
         }
         let now = 1024;
-        let mut options = node.pull_options(&crds, &me.label().pubkey(), now, &stakes);
+        let mut options = node.pull_options(&crds, &me.label().pubkey(), 0, now, &stakes);
         assert!(!options.is_empty());
         options.sort_by(|(weight_l, _), (weight_r, _)| weight_r.partial_cmp(weight_l).unwrap());
         // check that the highest stake holder is also the heaviest weighted.
@@ -406,6 +420,66 @@ mod test {
         );
     }
 
+    #[test]
+    fn test_no_pulls_from_different_shred_versions() {
+        let mut crds = Crds::default();
+        let stakes = HashMap::new();
+        let node = CrdsGossipPull::default();
+
+        let gossip = socketaddr!("127.0.0.1:1234");
+
+        let me = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let spy = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 0,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_123 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_456 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 456,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+
+        crds.insert(me.clone(), 0).unwrap();
+        crds.insert(spy.clone(), 0).unwrap();
+        crds.insert(node_123.clone(), 0).unwrap();
+        crds.insert(node_456.clone(), 0).unwrap();
+
+        // shred version 123 should ignore 456 nodes
+        let options = node
+            .pull_options(&crds, &me.label().pubkey(), 123, 0, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 2);
+        assert!(options.contains(&spy.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+
+        // spy nodes will see all
+        let options = node
+            .pull_options(&crds, &spy.label().pubkey(), 0, 0, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 3);
+        assert!(options.contains(&me.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+        assert!(options.contains(&node_456.pubkey()));
+    }
+
     #[test]
     fn test_new_pull_request() {
         let mut crds = Crds::default();
@@ -416,13 +490,13 @@ mod test {
         let id = entry.label().pubkey();
         let node = CrdsGossipPull::default();
         assert_eq!(
-            node.new_pull_request(&crds, &id, 0, &HashMap::new(), PACKET_DATA_SIZE),
+            node.new_pull_request(&crds, &id, 0, 0, &HashMap::new(), PACKET_DATA_SIZE),
             Err(CrdsGossipError::NoPeers)
         );
 
         crds.insert(entry.clone(), 0).unwrap();
         assert_eq!(
-            node.new_pull_request(&crds, &id, 0, &HashMap::new(), PACKET_DATA_SIZE),
+            node.new_pull_request(&crds, &id, 0, 0, &HashMap::new(), PACKET_DATA_SIZE),
             Err(CrdsGossipError::NoPeers)
         );
 
@@ -431,7 +505,7 @@ mod test {
             0,
         )));
         crds.insert(new.clone(), 0).unwrap();
-        let req = node.new_pull_request(&crds, &id, 0, &HashMap::new(), PACKET_DATA_SIZE);
+        let req = node.new_pull_request(&crds, &id, 0, 0, &HashMap::new(), PACKET_DATA_SIZE);
         let (to, _, self_info) = req.unwrap();
         assert_eq!(to, new.label().pubkey());
         assert_eq!(self_info, entry);
@@ -466,6 +540,7 @@ mod test {
             let req = node.new_pull_request(
                 &crds,
                 &node_pubkey,
+                0,
                 u64::max_value(),
                 &HashMap::new(),
                 PACKET_DATA_SIZE,
@@ -495,6 +570,7 @@ mod test {
             &node_crds,
             &node_pubkey,
             0,
+            0,
             &HashMap::new(),
             PACKET_DATA_SIZE,
         );
@@ -567,6 +643,7 @@ mod test {
                 &node_crds,
                 &node_pubkey,
                 0,
+                0,
                 &HashMap::new(),
                 PACKET_DATA_SIZE,
             );

core/src/crds_gossip_push.rs

@@ -236,13 +236,14 @@ impl CrdsGossipPush {
         crds: &Crds,
         stakes: &HashMap<Pubkey, u64>,
         self_id: &Pubkey,
+        self_shred_version: u16,
         network_size: usize,
         ratio: usize,
     ) {
         let need = Self::compute_need(self.num_active, self.active_set.len(), ratio);
         let mut new_items = HashMap::new();
 
-        let options: Vec<_> = self.push_options(crds, &self_id, stakes);
+        let options: Vec<_> = self.push_options(crds, &self_id, self_shred_version, stakes);
         if options.is_empty() {
             return;
         }
@@ -288,13 +289,20 @@ impl CrdsGossipPush {
         &self,
         crds: &'a Crds,
         self_id: &Pubkey,
+        self_shred_version: u16,
         stakes: &HashMap<Pubkey, u64>,
     ) -> Vec<(f32, &'a ContactInfo)> {
         crds.table
             .values()
             .filter(|v| v.value.contact_info().is_some())
             .map(|v| (v.value.contact_info().unwrap(), v))
-            .filter(|(info, _)| info.id != *self_id && ContactInfo::is_valid_address(&info.gossip))
+            .filter(|(info, _)| {
+                info.id != *self_id
+                    && ContactInfo::is_valid_address(&info.gossip)
+                    && (self_shred_version == 0
+                        || info.shred_version == 0
+                        || self_shred_version == info.shred_version)
+            })
             .map(|(info, value)| {
                 let max_weight = f32::from(u16::max_value()) - 1.0;
                 let last_updated: u64 = value.local_timestamp;
@@ -510,7 +518,7 @@ mod test {
         )));
 
         assert_eq!(crds.insert(value1.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         assert!(push.active_set.get(&value1.label().pubkey()).is_some());
         let value2 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
@@ -520,7 +528,7 @@ mod test {
         assert!(push.active_set.get(&value2.label().pubkey()).is_none());
         assert_eq!(crds.insert(value2.clone(), 0), Ok(None));
         for _ in 0..30 {
-            push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+            push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
             if push.active_set.get(&value2.label().pubkey()).is_some() {
                 break;
             }
@@ -533,7 +541,7 @@ mod test {
             ));
             assert_eq!(crds.insert(value2.clone(), 0), Ok(None));
         }
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
         assert_eq!(push.active_set.len(), push.num_active);
     }
     #[test]
@@ -551,7 +559,7 @@ mod test {
             crds.insert(peer.clone(), time).unwrap();
             stakes.insert(id, i * 100);
         }
-        let mut options = push.push_options(&crds, &Pubkey::default(), &stakes);
+        let mut options = push.push_options(&crds, &Pubkey::default(), 0, &stakes);
         assert!(!options.is_empty());
         options.sort_by(|(weight_l, _), (weight_r, _)| weight_r.partial_cmp(weight_l).unwrap());
         // check that the highest stake holder is also the heaviest weighted.
@@ -560,6 +568,66 @@ mod test {
             10_000_u64
         );
     }
+
+    #[test]
+    fn test_no_pushes_to_from_different_shred_versions() {
+        let mut crds = Crds::default();
+        let stakes = HashMap::new();
+        let node = CrdsGossipPush::default();
+
+        let gossip = socketaddr!("127.0.0.1:1234");
+
+        let me = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let spy = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 0,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_123 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_456 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 456,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+
+        crds.insert(me.clone(), 0).unwrap();
+        crds.insert(spy.clone(), 0).unwrap();
+        crds.insert(node_123.clone(), 0).unwrap();
+        crds.insert(node_456.clone(), 0).unwrap();
+
+        // shred version 123 should ignore 456 nodes
+        let options = node
+            .push_options(&crds, &me.label().pubkey(), 123, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 2);
+        assert!(options.contains(&spy.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+
+        // spy nodes will see all
+        let options = node
+            .push_options(&crds, &spy.label().pubkey(), 0, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 3);
+        assert!(options.contains(&me.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+        assert!(options.contains(&node_456.pubkey()));
+    }
     #[test]
     fn test_new_push_messages() {
         let mut crds = Crds::default();
@@ -569,7 +637,7 @@ mod test {
             0,
         )));
         assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         let new_msg = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
             &Pubkey::new_rand(),
@@ -606,7 +674,7 @@ mod test {
             push.process_push_message(&mut crds, &Pubkey::default(), peer_3.clone(), 0),
             Ok(None)
         );
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         // push 3's contact info to 1 and 2 and 3
         let new_msg = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
@@ -628,7 +696,7 @@ mod test {
             0,
         )));
         assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         let new_msg = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
             &Pubkey::new_rand(),
@@ -651,7 +719,7 @@ mod test {
             0,
         )));
         assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         let mut ci = ContactInfo::new_localhost(&Pubkey::new_rand(), 0);
         ci.wallclock = 1;

core/src/crds_value.rs

@@ -1,7 +1,6 @@
 use crate::contact_info::ContactInfo;
-use crate::deprecated;
-use crate::epoch_slots::EpochSlots;
 use bincode::{serialize, serialized_size};
+use solana_sdk::sanitize::{Sanitize, SanitizeError};
 use solana_sdk::timing::timestamp;
 use solana_sdk::{
     clock::Slot,
@@ -16,11 +15,14 @@ use std::{
     fmt,
 };
 
+pub const MAX_WALLCLOCK: u64 = 1_000_000_000_000_000;
+pub const MAX_SLOT: u64 = 1_000_000_000_000_000;
+
 pub type VoteIndex = u8;
 pub const MAX_VOTES: VoteIndex = 32;
 
-pub type EpochSlotsIndex = u8;
-pub const MAX_EPOCH_SLOTS: EpochSlotsIndex = 255;
+pub type EpochSlotIndex = u8;
+pub const MAX_EPOCH_SLOTS: EpochSlotIndex = 1;
 
 /// CrdsValue that is replicated across the cluster
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -29,6 +31,13 @@ pub struct CrdsValue {
     pub data: CrdsData,
 }
 
+impl Sanitize for CrdsValue {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        self.signature.sanitize()?;
+        self.data.sanitize()
+    }
+}
+
 impl Signable for CrdsValue {
     fn pubkey(&self) -> Pubkey {
         self.pubkey()
@@ -47,30 +56,74 @@ impl Signable for CrdsValue {
     }
 
     fn verify(&self) -> bool {
-        let sig_check = self
-            .get_signature()
-            .verify(&self.pubkey().as_ref(), self.signable_data().borrow());
-        let data_check = match &self.data {
-            CrdsData::Vote(ix, _) => *ix < MAX_VOTES,
-            CrdsData::EpochSlots(ix, _) => *ix < MAX_EPOCH_SLOTS,
-            _ => true,
-        };
-        sig_check && data_check
+        self.get_signature()
+            .verify(&self.pubkey().as_ref(), self.signable_data().borrow())
     }
 }
 
 /// CrdsData that defines the different types of items CrdsValues can hold
 /// * Merge Strategy - Latest wallclock is picked
-/// * LowestSlot index is deprecated
 #[allow(clippy::large_enum_variant)]
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub enum CrdsData {
     ContactInfo(ContactInfo),
     Vote(VoteIndex, Vote),
-    LowestSlot(u8, LowestSlot),
+    EpochSlots(EpochSlotIndex, EpochSlots),
     SnapshotHashes(SnapshotHash),
     AccountsHashes(SnapshotHash),
-    EpochSlots(EpochSlotsIndex, EpochSlots),
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+pub enum CompressionType {
+    Uncompressed,
+    GZip,
+    BZip2,
+}
+
+impl Default for CompressionType {
+    fn default() -> Self {
+        Self::Uncompressed
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
+pub struct EpochIncompleteSlots {
+    pub first: Slot,
+    pub compression: CompressionType,
+    pub compressed_list: Vec<u8>,
+}
+
+impl Sanitize for EpochIncompleteSlots {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.first >= MAX_SLOT {
+            return Err(SanitizeError::InvalidValue);
+        }
+        //rest of the data doesn't matter since we no longer decompress
+        //these values
+        Ok(())
+    }
+}
+
+impl Sanitize for CrdsData {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        match self {
+            CrdsData::ContactInfo(val) => val.sanitize(),
+            CrdsData::Vote(ix, val) => {
+                if *ix >= MAX_VOTES {
+                    return Err(SanitizeError::ValueOutOfBounds);
+                }
+                val.sanitize()
+            }
+            CrdsData::SnapshotHashes(val) => val.sanitize(),
+            CrdsData::AccountsHashes(val) => val.sanitize(),
+            CrdsData::EpochSlots(ix, val) => {
+                if *ix as usize >= MAX_EPOCH_SLOTS as usize {
+                    return Err(SanitizeError::ValueOutOfBounds);
+                }
+                val.sanitize()
+            }
+        }
+    }
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -80,6 +133,20 @@ pub struct SnapshotHash {
     pub wallclock: u64,
 }
 
+impl Sanitize for SnapshotHash {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        for (slot, _) in &self.hashes {
+            if *slot >= MAX_SLOT {
+                return Err(SanitizeError::ValueOutOfBounds);
+            }
+        }
+        self.from.sanitize()
+    }
+}
+
 impl SnapshotHash {
     pub fn new(from: Pubkey, hashes: Vec<(Slot, Hash)>) -> Self {
         Self {
@@ -89,17 +156,18 @@ impl SnapshotHash {
         }
     }
 }
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-pub struct LowestSlot {
+pub struct EpochSlots {
     pub from: Pubkey,
-    root: Slot, //deprecated
+    root: Slot,
     pub lowest: Slot,
-    slots: BTreeSet<Slot>,                        //deprecated
-    stash: Vec<deprecated::EpochIncompleteSlots>, //deprecated
+    slots: BTreeSet<Slot>,
+    stash: Vec<EpochIncompleteSlots>,
     pub wallclock: u64,
 }
 
-impl LowestSlot {
+impl EpochSlots {
     pub fn new(from: Pubkey, lowest: Slot, wallclock: u64) -> Self {
         Self {
             from,
@@ -112,6 +180,27 @@ impl LowestSlot {
     }
 }
 
+impl Sanitize for EpochSlots {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        if self.lowest >= MAX_SLOT {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        if self.root >= MAX_SLOT {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        for slot in &self.slots {
+            if *slot >= MAX_SLOT {
+                return Err(SanitizeError::ValueOutOfBounds);
+            }
+        }
+        self.stash.sanitize()?;
+        self.from.sanitize()
+    }
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub struct Vote {
     pub from: Pubkey,
@@ -119,6 +208,16 @@ pub struct Vote {
     pub wallclock: u64,
 }
 
+impl Sanitize for Vote {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        self.from.sanitize()?;
+        self.transaction.sanitize()
+    }
+}
+
 impl Vote {
     pub fn new(from: &Pubkey, transaction: Transaction, wallclock: u64) -> Self {
         Self {
@@ -135,9 +234,8 @@ impl Vote {
 pub enum CrdsValueLabel {
     ContactInfo(Pubkey),
     Vote(VoteIndex, Pubkey),
-    LowestSlot(Pubkey),
+    EpochSlots(Pubkey),
     SnapshotHashes(Pubkey),
-    EpochSlots(EpochSlotsIndex, Pubkey),
     AccountsHashes(Pubkey),
 }
 
@@ -146,9 +244,8 @@ impl fmt::Display for CrdsValueLabel {
         match self {
             CrdsValueLabel::ContactInfo(_) => write!(f, "ContactInfo({})", self.pubkey()),
             CrdsValueLabel::Vote(ix, _) => write!(f, "Vote({}, {})", ix, self.pubkey()),
-            CrdsValueLabel::LowestSlot(_) => write!(f, "LowestSlot({})", self.pubkey()),
-            CrdsValueLabel::SnapshotHashes(_) => write!(f, "SnapshotHash({})", self.pubkey()),
-            CrdsValueLabel::EpochSlots(ix, _) => write!(f, "EpochSlots({}, {})", ix, self.pubkey()),
+            CrdsValueLabel::EpochSlots(_) => write!(f, "EpochSlots({})", self.pubkey()),
+            CrdsValueLabel::SnapshotHashes(_) => write!(f, "SnapshotHashes({})", self.pubkey()),
             CrdsValueLabel::AccountsHashes(_) => write!(f, "AccountsHashes({})", self.pubkey()),
         }
     }
@@ -159,9 +256,8 @@ impl CrdsValueLabel {
         match self {
             CrdsValueLabel::ContactInfo(p) => *p,
             CrdsValueLabel::Vote(_, p) => *p,
-            CrdsValueLabel::LowestSlot(p) => *p,
+            CrdsValueLabel::EpochSlots(p) => *p,
             CrdsValueLabel::SnapshotHashes(p) => *p,
-            CrdsValueLabel::EpochSlots(_, p) => *p,
             CrdsValueLabel::AccountsHashes(p) => *p,
         }
     }
@@ -187,30 +283,27 @@ impl CrdsValue {
         match &self.data {
             CrdsData::ContactInfo(contact_info) => contact_info.wallclock,
             CrdsData::Vote(_, vote) => vote.wallclock,
-            CrdsData::LowestSlot(_, obj) => obj.wallclock,
+            CrdsData::EpochSlots(_, vote) => vote.wallclock,
             CrdsData::SnapshotHashes(hash) => hash.wallclock,
             CrdsData::AccountsHashes(hash) => hash.wallclock,
-            CrdsData::EpochSlots(_, p) => p.wallclock,
         }
     }
     pub fn pubkey(&self) -> Pubkey {
         match &self.data {
             CrdsData::ContactInfo(contact_info) => contact_info.id,
             CrdsData::Vote(_, vote) => vote.from,
-            CrdsData::LowestSlot(_, slots) => slots.from,
+            CrdsData::EpochSlots(_, slots) => slots.from,
             CrdsData::SnapshotHashes(hash) => hash.from,
             CrdsData::AccountsHashes(hash) => hash.from,
-            CrdsData::EpochSlots(_, p) => p.from,
         }
     }
     pub fn label(&self) -> CrdsValueLabel {
         match &self.data {
             CrdsData::ContactInfo(_) => CrdsValueLabel::ContactInfo(self.pubkey()),
             CrdsData::Vote(ix, _) => CrdsValueLabel::Vote(*ix, self.pubkey()),
-            CrdsData::LowestSlot(_, _) => CrdsValueLabel::LowestSlot(self.pubkey()),
+            CrdsData::EpochSlots(_, _) => CrdsValueLabel::EpochSlots(self.pubkey()),
             CrdsData::SnapshotHashes(_) => CrdsValueLabel::SnapshotHashes(self.pubkey()),
             CrdsData::AccountsHashes(_) => CrdsValueLabel::AccountsHashes(self.pubkey()),
-            CrdsData::EpochSlots(ix, _) => CrdsValueLabel::EpochSlots(*ix, self.pubkey()),
         }
     }
     pub fn contact_info(&self) -> Option<&ContactInfo> {
@@ -233,9 +326,9 @@ impl CrdsValue {
         }
     }
 
-    pub fn lowest_slot(&self) -> Option<&LowestSlot> {
+    pub fn epoch_slots(&self) -> Option<&EpochSlots> {
         match &self.data {
-            CrdsData::LowestSlot(_, slots) => Some(slots),
+            CrdsData::EpochSlots(_, slots) => Some(slots),
             _ => None,
         }
     }
@@ -254,23 +347,15 @@ impl CrdsValue {
         }
     }
 
-    pub fn epoch_slots(&self) -> Option<&EpochSlots> {
-        match &self.data {
-            CrdsData::EpochSlots(_, slots) => Some(slots),
-            _ => None,
-        }
-    }
-
     /// Return all the possible labels for a record identified by Pubkey.
     pub fn record_labels(key: &Pubkey) -> Vec<CrdsValueLabel> {
         let mut labels = vec![
             CrdsValueLabel::ContactInfo(*key),
-            CrdsValueLabel::LowestSlot(*key),
+            CrdsValueLabel::EpochSlots(*key),
             CrdsValueLabel::SnapshotHashes(*key),
             CrdsValueLabel::AccountsHashes(*key),
         ];
         labels.extend((0..MAX_VOTES).map(|ix| CrdsValueLabel::Vote(ix, *key)));
-        labels.extend((0..MAX_EPOCH_SLOTS).map(|ix| CrdsValueLabel::EpochSlots(ix, *key)));
         labels
     }
 
@@ -318,18 +403,15 @@ mod test {
 
     #[test]
     fn test_labels() {
-        let mut hits = [false; 4 + MAX_VOTES as usize + MAX_EPOCH_SLOTS as usize];
+        let mut hits = [false; 4 + MAX_VOTES as usize];
         // this method should cover all the possible labels
         for v in &CrdsValue::record_labels(&Pubkey::default()) {
             match v {
                 CrdsValueLabel::ContactInfo(_) => hits[0] = true,
-                CrdsValueLabel::LowestSlot(_) => hits[1] = true,
+                CrdsValueLabel::EpochSlots(_) => hits[1] = true,
                 CrdsValueLabel::SnapshotHashes(_) => hits[2] = true,
                 CrdsValueLabel::AccountsHashes(_) => hits[3] = true,
                 CrdsValueLabel::Vote(ix, _) => hits[*ix as usize + 4] = true,
-                CrdsValueLabel::EpochSlots(ix, _) => {
-                    hits[*ix as usize + MAX_VOTES as usize + 4] = true
-                }
             }
         }
         assert!(hits.iter().all(|x| *x));
@@ -349,13 +431,13 @@ mod test {
         let key = v.clone().vote().unwrap().from;
         assert_eq!(v.label(), CrdsValueLabel::Vote(0, key));
 
-        let v = CrdsValue::new_unsigned(CrdsData::LowestSlot(
+        let v = CrdsValue::new_unsigned(CrdsData::EpochSlots(
             0,
-            LowestSlot::new(Pubkey::default(), 0, 0),
+            EpochSlots::new(Pubkey::default(), 0, 0),
         ));
         assert_eq!(v.wallclock(), 0);
-        let key = v.clone().lowest_slot().unwrap().from;
-        assert_eq!(v.label(), CrdsValueLabel::LowestSlot(key));
+        let key = v.clone().epoch_slots().unwrap().from;
+        assert_eq!(v.label(), CrdsValueLabel::EpochSlots(key));
     }
 
     #[test]
@@ -372,9 +454,9 @@ mod test {
             Vote::new(&keypair.pubkey(), test_tx(), timestamp()),
         ));
         verify_signatures(&mut v, &keypair, &wrong_keypair);
-        v = CrdsValue::new_unsigned(CrdsData::LowestSlot(
+        v = CrdsValue::new_unsigned(CrdsData::EpochSlots(
             0,
-            LowestSlot::new(keypair.pubkey(), 0, timestamp()),
+            EpochSlots::new(keypair.pubkey(), 0, timestamp()),
         ));
         verify_signatures(&mut v, &keypair, &wrong_keypair);
     }
@@ -389,20 +471,20 @@ mod test {
             ),
             &keypair,
         );
-        assert!(!vote.verify());
+        assert!(vote.sanitize().is_err());
     }
 
     #[test]
     fn test_max_epoch_slots_index() {
         let keypair = Keypair::new();
         let item = CrdsValue::new_signed(
-            CrdsData::EpochSlots(
-                MAX_EPOCH_SLOTS,
-                EpochSlots::new(keypair.pubkey(), timestamp()),
+            CrdsData::Vote(
+                MAX_VOTES,
+                Vote::new(&keypair.pubkey(), test_tx(), timestamp()),
             ),
             &keypair,
         );
-        assert!(!item.verify());
+        assert_eq!(item.sanitize(), Err(SanitizeError::ValueOutOfBounds));
     }
     #[test]
     fn test_compute_vote_index_empty() {

core/src/deprecated.rs

@@ -1,21 +0,0 @@
-use solana_sdk::clock::Slot;
-
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-enum CompressionType {
-    Uncompressed,
-    GZip,
-    BZip2,
-}
-
-impl Default for CompressionType {
-    fn default() -> Self {
-        Self::Uncompressed
-    }
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
-pub(crate) struct EpochIncompleteSlots {
-    first: Slot,
-    compression: CompressionType,
-    compressed_list: Vec<u8>,
-}

core/src/epoch_slots.rs

@@ -1,401 +0,0 @@
-use crate::cluster_info::MAX_CRDS_OBJECT_SIZE;
-use bincode::serialized_size;
-use bv::BitVec;
-use flate2::{Compress, Compression, Decompress, FlushCompress, FlushDecompress};
-use solana_sdk::clock::Slot;
-use solana_sdk::pubkey::Pubkey;
-
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-pub struct Uncompressed {
-    pub first_slot: Slot,
-    pub num: usize,
-    pub slots: BitVec<u8>,
-}
-
-#[derive(Deserialize, Serialize, Clone, Debug, PartialEq)]
-pub struct Flate2 {
-    pub first_slot: Slot,
-    pub num: usize,
-    pub compressed: Vec<u8>,
-}
-
-#[derive(Debug, PartialEq)]
-pub enum Error {
-    CompressError,
-    DecompressError,
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
-
-impl std::convert::From<flate2::CompressError> for Error {
-    fn from(_e: flate2::CompressError) -> Error {
-        Error::CompressError
-    }
-}
-impl std::convert::From<flate2::DecompressError> for Error {
-    fn from(_e: flate2::DecompressError) -> Error {
-        Error::DecompressError
-    }
-}
-
-impl Flate2 {
-    fn deflate(mut unc: Uncompressed) -> Result<Self> {
-        let mut compressed = Vec::with_capacity(unc.slots.block_capacity());
-        let mut compressor = Compress::new(Compression::best(), false);
-        let first_slot = unc.first_slot;
-        let num = unc.num;
-        unc.slots.shrink_to_fit();
-        let bits = unc.slots.into_boxed_slice();
-        compressor.compress_vec(&bits, &mut compressed, FlushCompress::Finish)?;
-        let rv = Self {
-            first_slot,
-            num,
-            compressed,
-        };
-        let _ = rv.inflate()?;
-        Ok(rv)
-    }
-    pub fn inflate(&self) -> Result<Uncompressed> {
-        //add some head room for the decompressor which might spill more bits
-        let mut uncompressed = Vec::with_capacity(32 + (self.num + 4) / 8);
-        let mut decompress = Decompress::new(false);
-        decompress.decompress_vec(&self.compressed, &mut uncompressed, FlushDecompress::Finish)?;
-        Ok(Uncompressed {
-            first_slot: self.first_slot,
-            num: self.num,
-            slots: BitVec::from_bits(&uncompressed),
-        })
-    }
-}
-
-impl Uncompressed {
-    pub fn new(max_size: usize) -> Self {
-        Self {
-            num: 0,
-            first_slot: 0,
-            slots: BitVec::new_fill(false, 8 * max_size as u64),
-        }
-    }
-    pub fn to_slots(&self, min_slot: Slot) -> Vec<Slot> {
-        let mut rv = vec![];
-        let start = if min_slot < self.first_slot {
-            0 as usize
-        } else {
-            (min_slot - self.first_slot) as usize
-        };
-        for i in start..self.num {
-            if self.slots.get(i as u64) {
-                rv.push(self.first_slot + i as Slot);
-            }
-        }
-        rv
-    }
-    pub fn add(&mut self, slots: &[Slot]) -> usize {
-        for (i, s) in slots.iter().enumerate() {
-            if self.num == 0 {
-                self.first_slot = *s;
-            }
-            if *s < self.first_slot {
-                return i;
-            }
-            if *s - self.first_slot >= self.slots.capacity() {
-                return i;
-            }
-            self.slots.set(*s - self.first_slot, true);
-            self.num = std::cmp::max(self.num, 1 + (*s - self.first_slot) as usize);
-        }
-        slots.len()
-    }
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-pub enum CompressedSlots {
-    Flate2(Flate2),
-    Uncompressed(Uncompressed),
-}
-
-impl Default for CompressedSlots {
-    fn default() -> Self {
-        CompressedSlots::new(0)
-    }
-}
-
-impl CompressedSlots {
-    fn new(max_size: usize) -> Self {
-        CompressedSlots::Uncompressed(Uncompressed::new(max_size))
-    }
-
-    pub fn first_slot(&self) -> Slot {
-        match self {
-            CompressedSlots::Uncompressed(a) => a.first_slot,
-            CompressedSlots::Flate2(b) => b.first_slot,
-        }
-    }
-
-    pub fn num_slots(&self) -> usize {
-        match self {
-            CompressedSlots::Uncompressed(a) => a.num,
-            CompressedSlots::Flate2(b) => b.num,
-        }
-    }
-
-    pub fn add(&mut self, slots: &[Slot]) -> usize {
-        match self {
-            CompressedSlots::Uncompressed(vals) => vals.add(slots),
-            CompressedSlots::Flate2(_) => 0,
-        }
-    }
-    pub fn to_slots(&self, min_slot: Slot) -> Result<Vec<Slot>> {
-        match self {
-            CompressedSlots::Uncompressed(vals) => Ok(vals.to_slots(min_slot)),
-            CompressedSlots::Flate2(vals) => {
-                let unc = vals.inflate()?;
-                Ok(unc.to_slots(min_slot))
-            }
-        }
-    }
-    pub fn deflate(&mut self) -> Result<()> {
-        match self {
-            CompressedSlots::Uncompressed(vals) => {
-                let unc = vals.clone();
-                let compressed = Flate2::deflate(unc)?;
-                let mut new = CompressedSlots::Flate2(compressed);
-                std::mem::swap(self, &mut new);
-                Ok(())
-            }
-            CompressedSlots::Flate2(_) => Ok(()),
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
-pub struct EpochSlots {
-    pub from: Pubkey,
-    pub slots: Vec<CompressedSlots>,
-    pub wallclock: u64,
-}
-
-impl EpochSlots {
-    pub fn new(from: Pubkey, now: u64) -> Self {
-        Self {
-            from,
-            wallclock: now,
-            slots: vec![],
-        }
-    }
-    pub fn fill(&mut self, slots: &[Slot], now: u64) -> usize {
-        let mut num = 0;
-        self.wallclock = std::cmp::max(now, self.wallclock + 1);
-        while num < slots.len() {
-            num += self.add(&slots[num..]);
-            if num < slots.len() {
-                if self.deflate().is_err() {
-                    return num;
-                }
-                let space = self.max_compressed_slot_size();
-                if space > 0 {
-                    let cslot = CompressedSlots::new(space as usize);
-                    self.slots.push(cslot);
-                } else {
-                    return num;
-                }
-            }
-        }
-        num
-    }
-    pub fn add(&mut self, slots: &[Slot]) -> usize {
-        let mut num = 0;
-        for s in &mut self.slots {
-            num += s.add(&slots[num..]);
-            if num >= slots.len() {
-                break;
-            }
-        }
-        num
-    }
-    pub fn deflate(&mut self) -> Result<()> {
-        for s in self.slots.iter_mut() {
-            s.deflate()?;
-        }
-        Ok(())
-    }
-    pub fn max_compressed_slot_size(&self) -> isize {
-        let len_header = serialized_size(self).unwrap();
-        let len_slot = serialized_size(&CompressedSlots::default()).unwrap();
-        MAX_CRDS_OBJECT_SIZE as isize - (len_header + len_slot) as isize
-    }
-
-    pub fn first_slot(&self) -> Option<Slot> {
-        self.slots.iter().map(|s| s.first_slot()).min()
-    }
-
-    pub fn to_slots(&self, min_slot: Slot) -> Vec<Slot> {
-        self.slots
-            .iter()
-            .filter(|s| min_slot < s.first_slot() + s.num_slots() as u64)
-            .filter_map(|s| s.to_slots(min_slot).ok())
-            .flatten()
-            .collect()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    #[test]
-    fn test_epoch_slots_max_size() {
-        let epoch_slots = EpochSlots::default();
-        assert!(epoch_slots.max_compressed_slot_size() > 0);
-    }
-
-    #[test]
-    fn test_epoch_slots_uncompressed_add_1() {
-        let mut slots = Uncompressed::new(1);
-        assert_eq!(slots.slots.capacity(), 8);
-        assert_eq!(slots.add(&[1]), 1);
-        assert_eq!(slots.to_slots(1), vec![1]);
-        assert!(slots.to_slots(2).is_empty());
-    }
-    #[test]
-    fn test_epoch_slots_uncompressed_add_2() {
-        let mut slots = Uncompressed::new(1);
-        assert_eq!(slots.add(&[1, 2]), 2);
-        assert_eq!(slots.to_slots(1), vec![1, 2]);
-    }
-    #[test]
-    fn test_epoch_slots_uncompressed_add_3a() {
-        let mut slots = Uncompressed::new(1);
-        assert_eq!(slots.add(&[1, 3, 2]), 3);
-        assert_eq!(slots.to_slots(1), vec![1, 2, 3]);
-    }
-
-    #[test]
-    fn test_epoch_slots_uncompressed_add_3b() {
-        let mut slots = Uncompressed::new(1);
-        assert_eq!(slots.add(&[1, 10, 2]), 1);
-        assert_eq!(slots.to_slots(1), vec![1]);
-    }
-
-    #[test]
-    fn test_epoch_slots_uncompressed_add_3c() {
-        let mut slots = Uncompressed::new(2);
-        assert_eq!(slots.add(&[1, 10, 2]), 3);
-        assert_eq!(slots.to_slots(1), vec![1, 2, 10]);
-        assert_eq!(slots.to_slots(2), vec![2, 10]);
-        assert_eq!(slots.to_slots(3), vec![10]);
-        assert_eq!(slots.to_slots(11).is_empty(), true);
-    }
-    #[test]
-    fn test_epoch_slots_compressed() {
-        let mut slots = Uncompressed::new(100);
-        slots.add(&[1, 701, 2]);
-        assert_eq!(slots.num, 701);
-        let compressed = Flate2::deflate(slots).unwrap();
-        assert_eq!(compressed.first_slot, 1);
-        assert_eq!(compressed.num, 701);
-        assert!(compressed.compressed.len() < 32);
-        let slots = compressed.inflate().unwrap();
-        assert_eq!(slots.first_slot, 1);
-        assert_eq!(slots.num, 701);
-        assert_eq!(slots.to_slots(1), vec![1, 2, 701]);
-    }
-    #[test]
-    fn test_epoch_slots_fill_range() {
-        let range: Vec<Slot> = (0..5000).into_iter().collect();
-        let mut slots = EpochSlots::default();
-        assert_eq!(slots.fill(&range, 1), 5000);
-        assert_eq!(slots.wallclock, 1);
-        assert_eq!(slots.to_slots(0), range);
-        assert_eq!(slots.to_slots(4999), vec![4999]);
-        assert_eq!(slots.to_slots(5000).is_empty(), true);
-    }
-    #[test]
-    fn test_epoch_slots_fill_sparce_range() {
-        let range: Vec<Slot> = (0..5000).into_iter().map(|x| x * 3).collect();
-        let mut slots = EpochSlots::default();
-        assert_eq!(slots.fill(&range, 2), 5000);
-        assert_eq!(slots.wallclock, 2);
-        assert_eq!(slots.slots.len(), 3);
-        assert_eq!(slots.slots[0].first_slot(), 0);
-        assert_ne!(slots.slots[0].num_slots(), 0);
-        let next = slots.slots[0].num_slots() as u64 + slots.slots[0].first_slot();
-        assert!(slots.slots[1].first_slot() >= next);
-        assert_ne!(slots.slots[1].num_slots(), 0);
-        assert_ne!(slots.slots[2].num_slots(), 0);
-        assert_eq!(slots.to_slots(0), range);
-        assert_eq!(slots.to_slots(4999 * 3), vec![4999 * 3]);
-    }
-
-    #[test]
-    fn test_epoch_slots_fill_large_sparce_range() {
-        let range: Vec<Slot> = (0..5000).into_iter().map(|x| x * 7).collect();
-        let mut slots = EpochSlots::default();
-        assert_eq!(slots.fill(&range, 2), 5000);
-        assert_eq!(slots.to_slots(0), range);
-    }
-
-    #[test]
-    fn test_epoch_slots_fill_uncompressed_random_range() {
-        use rand::Rng;
-        for _ in 0..10 {
-            let mut range: Vec<Slot> = vec![];
-            for _ in 0..5000 {
-                let last = *range.last().unwrap_or(&0);
-                range.push(last + rand::thread_rng().gen_range(1, 5));
-            }
-            let sz = EpochSlots::default().max_compressed_slot_size();
-            let mut slots = Uncompressed::new(sz as usize);
-            let sz = slots.add(&range);
-            let slots = slots.to_slots(0);
-            assert_eq!(slots.len(), sz);
-            assert_eq!(slots[..], range[..sz]);
-        }
-    }
-
-    #[test]
-    fn test_epoch_slots_fill_compressed_random_range() {
-        use rand::Rng;
-        for _ in 0..10 {
-            let mut range: Vec<Slot> = vec![];
-            for _ in 0..5000 {
-                let last = *range.last().unwrap_or(&0);
-                range.push(last + rand::thread_rng().gen_range(1, 5));
-            }
-            let sz = EpochSlots::default().max_compressed_slot_size();
-            let mut slots = Uncompressed::new(sz as usize);
-            let sz = slots.add(&range);
-            let mut slots = CompressedSlots::Uncompressed(slots);
-            slots.deflate().unwrap();
-            let slots = slots.to_slots(0).unwrap();
-            assert_eq!(slots.len(), sz);
-            assert_eq!(slots[..], range[..sz]);
-        }
-    }
-
-    #[test]
-    fn test_epoch_slots_fill_random_range() {
-        use rand::Rng;
-        for _ in 0..10 {
-            let mut range: Vec<Slot> = vec![];
-            for _ in 0..5000 {
-                let last = *range.last().unwrap_or(&0);
-                range.push(last + rand::thread_rng().gen_range(1, 5));
-            }
-            let mut slots = EpochSlots::default();
-            let sz = slots.fill(&range, 1);
-            let last = range[sz - 1];
-            assert_eq!(
-                last,
-                slots.slots.last().unwrap().first_slot()
-                    + slots.slots.last().unwrap().num_slots() as u64
-                    - 1
-            );
-            for s in &slots.slots {
-                assert!(s.to_slots(0).is_ok());
-            }
-            let slots = slots.to_slots(0);
-            assert_eq!(slots[..], range[..slots.len()]);
-            assert_eq!(sz, slots.len())
-        }
-    }
-}

core/src/fetch_stage.rs

@@ -1,14 +1,14 @@
 //! The `fetch_stage` batches input from a UDP socket and sends it to a channel.
 
 use crate::banking_stage::FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET;
+use crate::packet::PacketsRecycler;
 use crate::poh_recorder::PohRecorder;
 use crate::result::{Error, Result};
+use crate::streamer::{self, PacketReceiver, PacketSender};
 use solana_measure::thread_mem_usage;
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_info};
-use solana_perf::packet::PacketsRecycler;
 use solana_perf::recycler::Recycler;
 use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;
-use solana_streamer::streamer::{self, PacketReceiver, PacketSender};
 use std::net::UdpSocket;
 use std::sync::atomic::AtomicBool;
 use std::sync::mpsc::{channel, RecvTimeoutError};

core/src/genesis_utils.rs

@@ -0,0 +1 @@
+pub use solana_ledger::genesis_utils::*;

core/src/gossip_service.rs

@@ -2,13 +2,13 @@
 
 use crate::cluster_info::{ClusterInfo, VALIDATOR_PORT_RANGE};
 use crate::contact_info::ContactInfo;
+use crate::streamer;
 use rand::{thread_rng, Rng};
 use solana_client::thin_client::{create_client, ThinClient};
 use solana_ledger::bank_forks::BankForks;
 use solana_perf::recycler::Recycler;
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::signature::{Keypair, Signer};
-use solana_streamer::streamer;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, UdpSocket};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::mpsc::channel;

core/src/ledger_cleanup_service.rs

@@ -13,15 +13,18 @@ use std::thread;
 use std::thread::{Builder, JoinHandle};
 use std::time::Duration;
 
-// - To try and keep the RocksDB size under 512GB:
-//   Seeing about 1600b/shred, using 2000b/shred for margin, so 250m shreds can be stored in 512gb.
-//   at 5k shreds/slot at 50k tps, this is 500k slots (~5.5 hours).
+// - To try and keep the RocksDB size under 400GB:
+//   Seeing about 1600b/shred, using 2000b/shred for margin, so 200m shreds can be stored in 400gb.
+//   at 5k shreds/slot at 50k tps, this is 500k slots (~5 hours).
 //   At idle, 60 shreds/slot this is about 4m slots (18 days)
 // This is chosen to allow enough time for
 // - A validator to download a snapshot from a peer and boot from it
 // - To make sure that if a validator needs to reboot from its own snapshot, it has enough slots locally
 //   to catch back up to where it was when it stopped
-pub const DEFAULT_MAX_LEDGER_SHREDS: u64 = 250_000_000;
+pub const DEFAULT_MAX_LEDGER_SHREDS: u64 = 200_000_000;
+
+// Allow down to 50m, or 3.5 days at idle, 1hr at 50k load, around ~100GB
+pub const DEFAULT_MIN_MAX_LEDGER_SHREDS: u64 = 50_000_000;
 
 // Check for removing slots at this interval so we don't purge too often
 // and starve other blockstore users.

core/src/lib.rs

@@ -5,18 +5,17 @@
 //! command-line tools to spin up validators and a Rust library
 //!
 
-pub mod accounts_cleanup_service;
 pub mod accounts_hash_verifier;
 pub mod banking_stage;
 pub mod broadcast_stage;
 pub mod cluster_info_vote_listener;
 pub mod commitment;
-mod deprecated;
 pub mod shred_fetch_stage;
 #[macro_use]
 pub mod contact_info;
+pub mod blockstream;
+pub mod blockstream_service;
 pub mod cluster_info;
-pub mod cluster_slots;
 pub mod consensus;
 pub mod crds;
 pub mod crds_gossip;
@@ -24,15 +23,17 @@ pub mod crds_gossip_error;
 pub mod crds_gossip_pull;
 pub mod crds_gossip_push;
 pub mod crds_value;
-pub mod epoch_slots;
 pub mod fetch_stage;
 pub mod gen_keys;
+pub mod genesis_utils;
 pub mod gossip_service;
 pub mod ledger_cleanup_service;
 pub mod local_vote_signer_service;
+pub mod non_circulating_supply;
+pub mod packet;
 pub mod poh_recorder;
 pub mod poh_service;
-pub mod progress_map;
+pub mod recvmmsg;
 pub mod repair_service;
 pub mod replay_stage;
 mod result;
@@ -43,6 +44,7 @@ pub mod rpc_pubsub;
 pub mod rpc_pubsub_service;
 pub mod rpc_service;
 pub mod rpc_subscriptions;
+pub mod sendmmsg;
 pub mod serve_repair;
 pub mod serve_repair_service;
 pub mod sigverify;
@@ -50,11 +52,11 @@ pub mod sigverify_shreds;
 pub mod sigverify_stage;
 pub mod snapshot_packager_service;
 pub mod storage_stage;
+pub mod streamer;
 pub mod tpu;
 pub mod transaction_status_service;
 pub mod tvu;
 pub mod validator;
-pub mod verified_vote_packets;
 pub mod weighted_shuffle;
 pub mod window_service;
 

core/src/non_circulating_supply.rs

@@ -0,0 +1,193 @@
+use solana_runtime::bank::Bank;
+use solana_sdk::pubkey::Pubkey;
+use solana_stake_program::stake_state::StakeState;
+use std::{collections::HashSet, sync::Arc};
+
+pub struct NonCirculatingSupply {
+    pub lamports: u64,
+    pub accounts: Vec<Pubkey>,
+}
+
+pub fn calculate_non_circulating_supply(bank: Arc<Bank>) -> NonCirculatingSupply {
+    debug!("Updating Bank supply, epoch: {}", bank.epoch());
+    let mut non_circulating_accounts_set: HashSet<Pubkey> = HashSet::new();
+
+    for key in non_circulating_accounts() {
+        non_circulating_accounts_set.insert(key);
+    }
+
+    let clock = bank.clock();
+    let stake_accounts = bank.get_program_accounts(Some(&solana_stake_program::id()));
+    for (pubkey, account) in stake_accounts.iter() {
+        let stake_account = StakeState::from(&account).unwrap_or_default();
+        match stake_account {
+            StakeState::Initialized(meta) => {
+                if meta.lockup.is_in_force(&clock, &HashSet::default())
+                    || meta.authorized.withdrawer == withdraw_authority()
+                {
+                    non_circulating_accounts_set.insert(*pubkey);
+                }
+            }
+            StakeState::Stake(meta, _stake) => {
+                if meta.lockup.is_in_force(&clock, &HashSet::default())
+                    || meta.authorized.withdrawer == withdraw_authority()
+                {
+                    non_circulating_accounts_set.insert(*pubkey);
+                }
+            }
+            _ => {}
+        }
+    }
+
+    let lamports = non_circulating_accounts_set
+        .iter()
+        .fold(0, |acc, pubkey| acc + bank.get_balance(&pubkey));
+
+    NonCirculatingSupply {
+        lamports,
+        accounts: non_circulating_accounts_set.into_iter().collect(),
+    }
+}
+
+// Mainnet-beta accounts that should be considered non-circulating
+solana_sdk::pubkeys!(
+    non_circulating_accounts,
+    [
+        "9huDUZfxoJ7wGMTffUE7vh1xePqef7gyrLJu9NApncqA",
+        "GK2zqSsXLA2rwVZk347RYhh6jJpRsCA69FjLW93ZGi3B",
+        "HCV5dGFJXRrJ3jhDYA4DCeb9TEDTwGGYXtT3wHksu2Zr",
+        "25odAafVXnd63L6Hq5Cx6xGmhKqkhE2y6UrLVuqUfWZj",
+        "14FUT96s9swbmH7ZjpDvfEDywnAYy9zaNhv4xvezySGu",
+        "HbZ5FfmKWNHC7uwk6TF1hVi6TCs7dtYfdjEcuPGgzFAg",
+        "C7C8odR8oashR5Feyrq2tJKaXL18id1dSj2zbkDGL2C2",
+        "APnSR52EC1eH676m7qTBHUJ1nrGpHYpV7XKPxgRDD8gX",
+        "9ibqedFVnu5k4wo1mJRbH6KJ5HLBCyjpA9omPYkDeeT5",
+        "FopBKzQkG9pkyQqjdMFBLMQ995pSkjy83ziR4aism4c6",
+        "AiUHvJhTbMCcgFE2K26Ea9qCe74y3sFwqUt38iD5sfoR",
+        "3DndE3W53QdHSfBJiSJgzDKGvKJBoQLVmRHvy5LtqYfG",
+        "Eyr9P5XsjK2NUKNCnfu39eqpGoiLFgVAv1LSQgMZCwiQ",
+        "DE1bawNcRJB9rVm3buyMVfr8mBEoyyu73NBovf2oXJsJ",
+        "CakcnaRDHka2gXyfbEd2d3xsvkJkqsLw2akB3zsN1D2S",
+        "7Np41oeYqPefeNQEHSv1UDhYrehxin3NStELsSKCT4K2",
+        "GdnSyH3YtwcxFvQrVVJMm1JhTS4QVX7MFsX56uJLUfiZ",
+        "Mc5XB47H3DKJHym5RLa9mPzWv5snERsF3KNv5AauXK8",
+        "7cvkjYAkUYs4W8XcXsca7cBrEGFeSUjeZmKoNBvEwyri",
+        "AG3m2bAibcY8raMt4oXEGqRHwX4FWKPPJVjZxn1LySDX",
+        "5XdtyEDREHJXXW1CTtCsVjJRjBapAwK78ZquzvnNVRrV",
+        "6yKHERk8rsbmJxvMpPuwPs1ct3hRiP7xaJF2tvnGU6nK",
+        "CHmdL15akDcJgBkY6BP3hzs98Dqr6wbdDC5p8odvtSbq",
+        "FR84wZQy3Y3j2gWz6pgETUiUoJtreMEuWfbg6573UCj9",
+        "5q54XjQ7vDx4y6KphPeE97LUNiYGtP55spjvXAWPGBuf",
+    ]
+);
+
+// Withdraw authority for autostaked accounts on mainnet-beta
+solana_sdk::pubkeys!(
+    withdraw_authority,
+    "8CUUMKYNGxdgYio5CLHRHyzMEhhVRMcqefgE6dLqnVRK"
+);
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_sdk::{
+        account::Account, epoch_schedule::EpochSchedule, genesis_config::GenesisConfig,
+    };
+    use solana_stake_program::stake_state::{Authorized, Lockup, Meta, StakeState};
+    use std::{collections::BTreeMap, sync::Arc};
+
+    fn new_from_parent(parent: &Arc<Bank>) -> Bank {
+        Bank::new_from_parent(parent, &Pubkey::default(), parent.slot() + 1)
+    }
+
+    #[test]
+    fn test_calculate_non_circulating_supply() {
+        let mut accounts: BTreeMap<Pubkey, Account> = BTreeMap::new();
+        let balance = 10;
+        let num_genesis_accounts = 10;
+        for _ in 0..num_genesis_accounts {
+            accounts.insert(
+                Pubkey::new_rand(),
+                Account::new(balance, 0, &Pubkey::default()),
+            );
+        }
+        let non_circulating_accounts = non_circulating_accounts();
+        let num_non_circulating_accounts = non_circulating_accounts.len() as u64;
+        for key in non_circulating_accounts.clone() {
+            accounts.insert(key, Account::new(balance, 0, &Pubkey::default()));
+        }
+
+        let num_stake_accounts = 3;
+        for _ in 0..num_stake_accounts {
+            let pubkey = Pubkey::new_rand();
+            let meta = Meta {
+                authorized: Authorized::auto(&pubkey),
+                lockup: Lockup {
+                    epoch: 1,
+                    ..Lockup::default()
+                },
+                ..Meta::default()
+            };
+            let stake_account = Account::new_data_with_space(
+                balance,
+                &StakeState::Initialized(meta),
+                std::mem::size_of::<StakeState>(),
+                &solana_stake_program::id(),
+            )
+            .unwrap();
+            accounts.insert(pubkey, stake_account);
+        }
+
+        let slots_per_epoch = 32;
+        let genesis_config = GenesisConfig {
+            accounts,
+            epoch_schedule: EpochSchedule::new(slots_per_epoch),
+            ..GenesisConfig::default()
+        };
+        let mut bank = Arc::new(Bank::new(&genesis_config));
+        assert_eq!(
+            bank.capitalization(),
+            (num_genesis_accounts + num_non_circulating_accounts + num_stake_accounts) * balance
+        );
+
+        let non_circulating_supply = calculate_non_circulating_supply(bank.clone());
+        assert_eq!(
+            non_circulating_supply.lamports,
+            (num_non_circulating_accounts + num_stake_accounts) * balance
+        );
+        assert_eq!(
+            non_circulating_supply.accounts.len(),
+            num_non_circulating_accounts as usize + num_stake_accounts as usize
+        );
+
+        bank = Arc::new(new_from_parent(&bank));
+        let new_balance = 11;
+        for key in non_circulating_accounts {
+            bank.store_account(&key, &Account::new(new_balance, 0, &Pubkey::default()));
+        }
+        let non_circulating_supply = calculate_non_circulating_supply(bank.clone());
+        assert_eq!(
+            non_circulating_supply.lamports,
+            (num_non_circulating_accounts * new_balance) + (num_stake_accounts * balance)
+        );
+        assert_eq!(
+            non_circulating_supply.accounts.len(),
+            num_non_circulating_accounts as usize + num_stake_accounts as usize
+        );
+
+        // Advance bank an epoch, which should unlock stakes
+        for _ in 0..slots_per_epoch {
+            bank = Arc::new(new_from_parent(&bank));
+        }
+        assert_eq!(bank.epoch(), 1);
+        let non_circulating_supply = calculate_non_circulating_supply(bank.clone());
+        assert_eq!(
+            non_circulating_supply.lamports,
+            num_non_circulating_accounts * new_balance
+        );
+        assert_eq!(
+            non_circulating_supply.accounts.len(),
+            num_non_circulating_accounts as usize
+        );
+    }
+}

core/src/packet.rs

@@ -72,7 +72,7 @@ mod tests {
     #[test]
     fn test_packets_set_addr() {
         // test that the address is actually being updated
-        let send_addr: SocketAddr = "127.0.0.1:123".parse().unwrap();
+        let send_addr = socketaddr!([127, 0, 0, 1], 123);
         let packets = vec![Packet::default()];
         let mut msgs = Packets::new(packets);
         msgs.set_addr(&send_addr);

core/src/poh_recorder.rs

@@ -511,8 +511,8 @@ impl PohRecorder {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use bincode::serialize;
-    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_ledger::{blockstore::Blockstore, blockstore_meta::SlotMeta, get_tmp_ledger_path};
     use solana_perf::test_tx::test_tx;
     use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;

core/src/poh_service.rs

@@ -1,10 +1,8 @@
 //! The `poh_service` module implements a service that records the passing of
 //! "ticks", a measure of time in the PoH stream
 use crate::poh_recorder::PohRecorder;
-use core_affinity;
 use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;
 use solana_sdk::poh_config::PohConfig;
-use solana_sys_tuner;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread::{self, sleep, Builder, JoinHandle};
@@ -120,8 +118,8 @@ impl PohService {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use crate::poh_recorder::WorkingBank;
-    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
     use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
     use solana_perf::test_tx::test_tx;

core/src/progress_map.rs

@@ -1,571 +0,0 @@
-use crate::{
-    cluster_info_vote_listener::SlotVoteTracker, cluster_slots::SlotPubkeys,
-    consensus::StakeLockout, replay_stage::SUPERMINORITY_THRESHOLD,
-};
-use solana_ledger::{
-    bank_forks::BankForks,
-    blockstore_processor::{ConfirmationProgress, ConfirmationTiming},
-};
-use solana_runtime::bank::Bank;
-use solana_sdk::{account::Account, clock::Slot, hash::Hash, pubkey::Pubkey};
-use std::{
-    collections::{HashMap, HashSet},
-    rc::Rc,
-    sync::{Arc, RwLock},
-};
-
-#[derive(Default)]
-pub(crate) struct ReplaySlotStats(ConfirmationTiming);
-impl std::ops::Deref for ReplaySlotStats {
-    type Target = ConfirmationTiming;
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-impl std::ops::DerefMut for ReplaySlotStats {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
-}
-
-impl ReplaySlotStats {
-    pub fn report_stats(&self, slot: Slot, num_entries: usize, num_shreds: u64) {
-        datapoint_info!(
-            "replay-slot-stats",
-            ("slot", slot as i64, i64),
-            ("fetch_entries_time", self.fetch_elapsed as i64, i64),
-            (
-                "fetch_entries_fail_time",
-                self.fetch_fail_elapsed as i64,
-                i64
-            ),
-            ("entry_verification_time", self.verify_elapsed as i64, i64),
-            ("replay_time", self.replay_elapsed as i64, i64),
-            (
-                "replay_total_elapsed",
-                self.started.elapsed().as_micros() as i64,
-                i64
-            ),
-            ("total_entries", num_entries as i64, i64),
-            ("total_shreds", num_shreds as i64, i64),
-        );
-    }
-}
-
-#[derive(Debug)]
-pub(crate) struct ValidatorStakeInfo {
-    pub validator_vote_pubkey: Pubkey,
-    pub stake: u64,
-    pub total_epoch_stake: u64,
-}
-
-impl Default for ValidatorStakeInfo {
-    fn default() -> Self {
-        Self {
-            stake: 0,
-            validator_vote_pubkey: Pubkey::default(),
-            total_epoch_stake: 1,
-        }
-    }
-}
-
-impl ValidatorStakeInfo {
-    pub fn new(validator_vote_pubkey: Pubkey, stake: u64, total_epoch_stake: u64) -> Self {
-        Self {
-            validator_vote_pubkey,
-            stake,
-            total_epoch_stake,
-        }
-    }
-}
-
-pub(crate) struct ForkProgress {
-    pub(crate) is_dead: bool,
-    pub(crate) fork_stats: ForkStats,
-    pub(crate) propagated_stats: PropagatedStats,
-    pub(crate) replay_stats: ReplaySlotStats,
-    pub(crate) replay_progress: ConfirmationProgress,
-}
-
-impl ForkProgress {
-    pub fn new(
-        last_entry: Hash,
-        prev_leader_slot: Option<Slot>,
-        validator_stake_info: Option<ValidatorStakeInfo>,
-    ) -> Self {
-        let (
-            is_leader_slot,
-            propagated_validators_stake,
-            propagated_validators,
-            is_propagated,
-            total_epoch_stake,
-        ) = validator_stake_info
-            .map(|info| {
-                (
-                    true,
-                    info.stake,
-                    vec![Rc::new(info.validator_vote_pubkey)]
-                        .into_iter()
-                        .collect(),
-                    {
-                        if info.total_epoch_stake == 0 {
-                            true
-                        } else {
-                            info.stake as f64 / info.total_epoch_stake as f64
-                                > SUPERMINORITY_THRESHOLD
-                        }
-                    },
-                    info.total_epoch_stake,
-                )
-            })
-            .unwrap_or((false, 0, HashSet::new(), false, 0));
-        Self {
-            is_dead: false,
-            fork_stats: ForkStats::default(),
-            replay_stats: ReplaySlotStats::default(),
-            replay_progress: ConfirmationProgress::new(last_entry),
-            propagated_stats: PropagatedStats {
-                prev_leader_slot,
-                is_leader_slot,
-                propagated_validators_stake,
-                propagated_validators,
-                is_propagated,
-                total_epoch_stake,
-                ..PropagatedStats::default()
-            },
-        }
-    }
-
-    pub fn new_from_bank(
-        bank: &Bank,
-        my_pubkey: &Pubkey,
-        voting_pubkey: &Pubkey,
-        prev_leader_slot: Option<Slot>,
-    ) -> Self {
-        let validator_fork_info = {
-            if bank.collector_id() == my_pubkey {
-                let stake = bank.epoch_vote_account_stake(voting_pubkey);
-                Some(ValidatorStakeInfo::new(
-                    *voting_pubkey,
-                    stake,
-                    bank.total_epoch_stake(),
-                ))
-            } else {
-                None
-            }
-        };
-
-        Self::new(bank.last_blockhash(), prev_leader_slot, validator_fork_info)
-    }
-}
-
-#[derive(Debug, Clone, Default)]
-pub(crate) struct ForkStats {
-    pub(crate) weight: u128,
-    pub(crate) fork_weight: u128,
-    pub(crate) total_staked: u64,
-    pub(crate) slot: Slot,
-    pub(crate) block_height: u64,
-    pub(crate) has_voted: bool,
-    pub(crate) is_recent: bool,
-    pub(crate) is_empty: bool,
-    pub(crate) vote_threshold: bool,
-    pub(crate) is_locked_out: bool,
-    pub(crate) stake_lockouts: HashMap<u64, StakeLockout>,
-    pub(crate) confirmation_reported: bool,
-    pub(crate) computed: bool,
-}
-
-#[derive(Clone, Default)]
-pub(crate) struct PropagatedStats {
-    pub(crate) propagated_validators: HashSet<Rc<Pubkey>>,
-    pub(crate) propagated_node_ids: HashSet<Rc<Pubkey>>,
-    pub(crate) propagated_validators_stake: u64,
-    pub(crate) is_propagated: bool,
-    pub(crate) is_leader_slot: bool,
-    pub(crate) prev_leader_slot: Option<Slot>,
-    pub(crate) slot_vote_tracker: Option<Arc<RwLock<SlotVoteTracker>>>,
-    pub(crate) cluster_slot_pubkeys: Option<Arc<RwLock<SlotPubkeys>>>,
-    pub(crate) total_epoch_stake: u64,
-}
-
-impl PropagatedStats {
-    pub fn add_vote_pubkey(
-        &mut self,
-        vote_pubkey: &Pubkey,
-        all_pubkeys: &mut HashSet<Rc<Pubkey>>,
-        stake: u64,
-    ) {
-        if !self.propagated_validators.contains(vote_pubkey) {
-            let mut cached_pubkey: Option<Rc<Pubkey>> = all_pubkeys.get(vote_pubkey).cloned();
-            if cached_pubkey.is_none() {
-                let new_pubkey = Rc::new(*vote_pubkey);
-                all_pubkeys.insert(new_pubkey.clone());
-                cached_pubkey = Some(new_pubkey);
-            }
-            let vote_pubkey = cached_pubkey.unwrap();
-            self.propagated_validators.insert(vote_pubkey);
-            self.propagated_validators_stake += stake;
-        }
-    }
-
-    pub fn add_node_pubkey(
-        &mut self,
-        node_pubkey: &Pubkey,
-        all_pubkeys: &mut HashSet<Rc<Pubkey>>,
-        bank: &Bank,
-    ) {
-        if !self.propagated_node_ids.contains(node_pubkey) {
-            let node_vote_accounts = bank
-                .epoch_vote_accounts_for_node_id(&node_pubkey)
-                .map(|v| &v.vote_accounts);
-
-            if let Some(node_vote_accounts) = node_vote_accounts {
-                self.add_node_pubkey_internal(
-                    node_pubkey,
-                    all_pubkeys,
-                    node_vote_accounts,
-                    bank.epoch_vote_accounts(bank.epoch())
-                        .expect("Epoch stakes for bank's own epoch must exist"),
-                );
-            }
-        }
-    }
-
-    fn add_node_pubkey_internal(
-        &mut self,
-        node_pubkey: &Pubkey,
-        all_pubkeys: &mut HashSet<Rc<Pubkey>>,
-        vote_account_pubkeys: &[Pubkey],
-        epoch_vote_accounts: &HashMap<Pubkey, (u64, Account)>,
-    ) {
-        let mut cached_pubkey: Option<Rc<Pubkey>> = all_pubkeys.get(node_pubkey).cloned();
-        if cached_pubkey.is_none() {
-            let new_pubkey = Rc::new(*node_pubkey);
-            all_pubkeys.insert(new_pubkey.clone());
-            cached_pubkey = Some(new_pubkey);
-        }
-        let node_pubkey = cached_pubkey.unwrap();
-        self.propagated_node_ids.insert(node_pubkey);
-        for vote_account_pubkey in vote_account_pubkeys.iter() {
-            let stake = epoch_vote_accounts
-                .get(vote_account_pubkey)
-                .map(|(stake, _)| *stake)
-                .unwrap_or(0);
-            self.add_vote_pubkey(vote_account_pubkey, all_pubkeys, stake);
-        }
-    }
-}
-
-#[derive(Default)]
-pub(crate) struct ProgressMap {
-    progress_map: HashMap<Slot, ForkProgress>,
-}
-
-impl std::ops::Deref for ProgressMap {
-    type Target = HashMap<Slot, ForkProgress>;
-    fn deref(&self) -> &Self::Target {
-        &self.progress_map
-    }
-}
-
-impl std::ops::DerefMut for ProgressMap {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.progress_map
-    }
-}
-
-impl ProgressMap {
-    pub fn insert(&mut self, slot: Slot, fork_progress: ForkProgress) {
-        self.progress_map.insert(slot, fork_progress);
-    }
-
-    pub fn get_propagated_stats(&self, slot: Slot) -> Option<&PropagatedStats> {
-        self.progress_map
-            .get(&slot)
-            .map(|fork_progress| &fork_progress.propagated_stats)
-    }
-
-    pub fn get_propagated_stats_mut(&mut self, slot: Slot) -> Option<&mut PropagatedStats> {
-        self.progress_map
-            .get_mut(&slot)
-            .map(|fork_progress| &mut fork_progress.propagated_stats)
-    }
-
-    pub fn get_fork_stats(&self, slot: Slot) -> Option<&ForkStats> {
-        self.progress_map
-            .get(&slot)
-            .map(|fork_progress| &fork_progress.fork_stats)
-    }
-
-    pub fn get_fork_stats_mut(&mut self, slot: Slot) -> Option<&mut ForkStats> {
-        self.progress_map
-            .get_mut(&slot)
-            .map(|fork_progress| &mut fork_progress.fork_stats)
-    }
-
-    pub fn is_propagated(&self, slot: Slot) -> bool {
-        let leader_slot_to_check = self.get_latest_leader_slot(slot);
-
-        // prev_leader_slot doesn't exist because already rooted
-        // or this validator hasn't been scheduled as a leader
-        // yet. In both cases the latest leader is vacuously
-        // confirmed
-        leader_slot_to_check
-            .map(|leader_slot_to_check| {
-                // If the leader's stats are None (isn't in the
-                // progress map), this means that prev_leader slot is
-                // rooted, so return true
-                self.get_propagated_stats(leader_slot_to_check)
-                    .map(|stats| stats.is_propagated)
-                    .unwrap_or(true)
-            })
-            .unwrap_or(true)
-    }
-
-    pub fn get_latest_leader_slot(&self, slot: Slot) -> Option<Slot> {
-        let propagated_stats = self
-            .get_propagated_stats(slot)
-            .expect("All frozen banks must exist in the Progress map");
-
-        if propagated_stats.is_leader_slot {
-            Some(slot)
-        } else {
-            propagated_stats.prev_leader_slot
-        }
-    }
-
-    pub fn get_bank_prev_leader_slot(&self, bank: &Bank) -> Option<Slot> {
-        let parent_slot = bank.parent_slot();
-        self.get_propagated_stats(parent_slot)
-            .map(|stats| {
-                if stats.is_leader_slot {
-                    Some(parent_slot)
-                } else {
-                    stats.prev_leader_slot
-                }
-            })
-            .unwrap_or(None)
-    }
-
-    pub fn handle_new_root(&mut self, bank_forks: &BankForks) {
-        self.progress_map
-            .retain(|k, _| bank_forks.get(*k).is_some());
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn test_add_vote_pubkey() {
-        let mut stats = PropagatedStats::default();
-        let mut all_pubkeys = HashSet::new();
-        let mut vote_pubkey = Pubkey::new_rand();
-        all_pubkeys.insert(Rc::new(vote_pubkey.clone()));
-
-        // Add a vote pubkey, the number of references in all_pubkeys
-        // should be 2
-        stats.add_vote_pubkey(&vote_pubkey, &mut all_pubkeys, 1);
-        assert!(stats.propagated_validators.contains(&vote_pubkey));
-        assert_eq!(stats.propagated_validators_stake, 1);
-        assert_eq!(Rc::strong_count(all_pubkeys.get(&vote_pubkey).unwrap()), 2);
-
-        // Adding it again should change no state since the key already existed
-        stats.add_vote_pubkey(&vote_pubkey, &mut all_pubkeys, 1);
-        assert!(stats.propagated_validators.contains(&vote_pubkey));
-        assert_eq!(stats.propagated_validators_stake, 1);
-
-        // Addding another pubkey should succeed
-        vote_pubkey = Pubkey::new_rand();
-        stats.add_vote_pubkey(&vote_pubkey, &mut all_pubkeys, 2);
-        assert!(stats.propagated_validators.contains(&vote_pubkey));
-        assert_eq!(stats.propagated_validators_stake, 3);
-        assert_eq!(Rc::strong_count(all_pubkeys.get(&vote_pubkey).unwrap()), 2);
-    }
-
-    #[test]
-    fn test_add_node_pubkey_internal() {
-        let num_vote_accounts = 10;
-        let staked_vote_accounts = 5;
-        let vote_account_pubkeys: Vec<_> = std::iter::repeat_with(|| Pubkey::new_rand())
-            .take(num_vote_accounts)
-            .collect();
-        let epoch_vote_accounts: HashMap<_, _> = vote_account_pubkeys
-            .iter()
-            .skip(num_vote_accounts - staked_vote_accounts)
-            .map(|pubkey| (*pubkey, (1, Account::default())))
-            .collect();
-
-        let mut stats = PropagatedStats::default();
-        let mut all_pubkeys = HashSet::new();
-        let mut node_pubkey = Pubkey::new_rand();
-        all_pubkeys.insert(Rc::new(node_pubkey.clone()));
-
-        // Add a vote pubkey, the number of references in all_pubkeys
-        // should be 2
-        stats.add_node_pubkey_internal(
-            &node_pubkey,
-            &mut all_pubkeys,
-            &vote_account_pubkeys,
-            &epoch_vote_accounts,
-        );
-        assert!(stats.propagated_node_ids.contains(&node_pubkey));
-        assert_eq!(
-            stats.propagated_validators_stake,
-            staked_vote_accounts as u64
-        );
-        assert_eq!(Rc::strong_count(all_pubkeys.get(&node_pubkey).unwrap()), 2);
-
-        // Adding it again should not change any state
-        stats.add_node_pubkey_internal(
-            &node_pubkey,
-            &mut all_pubkeys,
-            &vote_account_pubkeys,
-            &epoch_vote_accounts,
-        );
-        assert!(stats.propagated_node_ids.contains(&node_pubkey));
-        assert_eq!(
-            stats.propagated_validators_stake,
-            staked_vote_accounts as u64
-        );
-
-        // Addding another pubkey with same vote accounts should succeed, but stake
-        // shouldn't increase
-        node_pubkey = Pubkey::new_rand();
-        stats.add_node_pubkey_internal(
-            &node_pubkey,
-            &mut all_pubkeys,
-            &vote_account_pubkeys,
-            &epoch_vote_accounts,
-        );
-        assert!(stats.propagated_node_ids.contains(&node_pubkey));
-        assert_eq!(
-            stats.propagated_validators_stake,
-            staked_vote_accounts as u64
-        );
-        assert_eq!(Rc::strong_count(all_pubkeys.get(&node_pubkey).unwrap()), 2);
-
-        // Addding another pubkey with different vote accounts should succeed
-        // and increase stake
-        node_pubkey = Pubkey::new_rand();
-        let vote_account_pubkeys: Vec<_> = std::iter::repeat_with(|| Pubkey::new_rand())
-            .take(num_vote_accounts)
-            .collect();
-        let epoch_vote_accounts: HashMap<_, _> = vote_account_pubkeys
-            .iter()
-            .skip(num_vote_accounts - staked_vote_accounts)
-            .map(|pubkey| (*pubkey, (1, Account::default())))
-            .collect();
-        stats.add_node_pubkey_internal(
-            &node_pubkey,
-            &mut all_pubkeys,
-            &vote_account_pubkeys,
-            &epoch_vote_accounts,
-        );
-        assert!(stats.propagated_node_ids.contains(&node_pubkey));
-        assert_eq!(
-            stats.propagated_validators_stake,
-            2 * staked_vote_accounts as u64
-        );
-        assert_eq!(Rc::strong_count(all_pubkeys.get(&node_pubkey).unwrap()), 2);
-    }
-
-    #[test]
-    fn test_is_propagated_status_on_construction() {
-        // If the given ValidatorStakeInfo == None, then this is not
-        // a leader slot and is_propagated == false
-        let progress = ForkProgress::new(Hash::default(), Some(9), None);
-        assert!(!progress.propagated_stats.is_propagated);
-
-        // If the stake is zero, then threshold is always achieved
-        let progress = ForkProgress::new(
-            Hash::default(),
-            Some(9),
-            Some(ValidatorStakeInfo {
-                total_epoch_stake: 0,
-                ..ValidatorStakeInfo::default()
-            }),
-        );
-        assert!(progress.propagated_stats.is_propagated);
-
-        // If the stake is non zero, then threshold is not achieved unless
-        // validator has enough stake by itself to pass threshold
-        let progress = ForkProgress::new(
-            Hash::default(),
-            Some(9),
-            Some(ValidatorStakeInfo {
-                total_epoch_stake: 2,
-                ..ValidatorStakeInfo::default()
-            }),
-        );
-        assert!(!progress.propagated_stats.is_propagated);
-
-        // Give the validator enough stake by itself to pass threshold
-        let progress = ForkProgress::new(
-            Hash::default(),
-            Some(9),
-            Some(ValidatorStakeInfo {
-                stake: 1,
-                total_epoch_stake: 2,
-                ..ValidatorStakeInfo::default()
-            }),
-        );
-        assert!(progress.propagated_stats.is_propagated);
-
-        // Check that the default ValidatorStakeInfo::default() constructs a ForkProgress
-        // with is_propagated == false, otherwise propagation tests will fail to run
-        // the proper checks (most will auto-pass without checking anything)
-        let progress = ForkProgress::new(
-            Hash::default(),
-            Some(9),
-            Some(ValidatorStakeInfo::default()),
-        );
-        assert!(!progress.propagated_stats.is_propagated);
-    }
-
-    #[test]
-    fn test_is_propagated() {
-        let mut progress_map = ProgressMap::default();
-
-        // Insert new ForkProgress for slot 10 (not a leader slot) and its
-        // previous leader slot 9 (leader slot)
-        progress_map.insert(10, ForkProgress::new(Hash::default(), Some(9), None));
-        progress_map.insert(
-            9,
-            ForkProgress::new(Hash::default(), None, Some(ValidatorStakeInfo::default())),
-        );
-
-        // None of these slot have parents which are confirmed
-        assert!(!progress_map.is_propagated(9));
-        assert!(!progress_map.is_propagated(10));
-
-        // Insert new ForkProgress for slot 8 with no previous leader.
-        // The previous leader before 8, slot 7, does not exist in
-        // progress map, so is_propagated(8) should return true as
-        // this implies the parent is rooted
-        progress_map.insert(8, ForkProgress::new(Hash::default(), Some(7), None));
-        assert!(progress_map.is_propagated(8));
-
-        // If we set the is_propagated = true, is_propagated should return true
-        progress_map
-            .get_propagated_stats_mut(9)
-            .unwrap()
-            .is_propagated = true;
-        assert!(progress_map.is_propagated(9));
-        assert!(progress_map.get(&9).unwrap().propagated_stats.is_propagated);
-
-        // Because slot 9 is now confirmed, then slot 10 is also confirmed b/c 9
-        // is the last leader slot before 10
-        assert!(progress_map.is_propagated(10));
-
-        // If we make slot 10 a leader slot though, even though its previous
-        // leader slot 9 has been confirmed, slot 10 itself is not confirmed
-        progress_map
-            .get_propagated_stats_mut(10)
-            .unwrap()
-            .is_leader_slot = true;
-        assert!(!progress_map.is_propagated(10));
-    }
-}

core/src/repair_service.rs

@@ -2,7 +2,6 @@
 //! regularly finds missing shreds in the ledger and sends repair requests for those shreds
 use crate::{
     cluster_info::ClusterInfo,
-    cluster_slots::ClusterSlots,
     result::Result,
     serve_repair::{RepairType, ServeRepair},
 };
@@ -10,45 +9,27 @@ use solana_ledger::{
     bank_forks::BankForks,
     blockstore::{Blockstore, CompletedSlotsReceiver, SlotMeta},
 };
+use solana_sdk::clock::DEFAULT_SLOTS_PER_EPOCH;
 use solana_sdk::{clock::Slot, epoch_schedule::EpochSchedule, pubkey::Pubkey};
 use std::{
-    collections::HashMap,
+    collections::{BTreeSet, HashSet},
     iter::Iterator,
-    net::SocketAddr,
     net::UdpSocket,
+    ops::Bound::{Included, Unbounded},
     sync::atomic::{AtomicBool, Ordering},
     sync::{Arc, RwLock},
     thread::sleep,
     thread::{self, Builder, JoinHandle},
-    time::{Duration, Instant},
+    time::Duration,
 };
 
-#[derive(Default)]
-pub struct RepairStatsGroup {
-    pub count: u64,
-    pub min: u64,
-    pub max: u64,
-}
-
-impl RepairStatsGroup {
-    pub fn update(&mut self, slot: u64) {
-        self.count += 1;
-        self.min = std::cmp::min(self.min, slot);
-        self.max = std::cmp::max(self.max, slot);
-    }
-}
-
-#[derive(Default)]
-pub struct RepairStats {
-    pub shred: RepairStatsGroup,
-    pub highest_shred: RepairStatsGroup,
-    pub orphan: RepairStatsGroup,
-}
-
 pub const MAX_REPAIR_LENGTH: usize = 512;
 pub const REPAIR_MS: u64 = 100;
 pub const MAX_ORPHANS: usize = 5;
 
+const MAX_COMPLETED_SLOT_CACHE_LEN: usize = 256;
+const COMPLETED_SLOT_CACHE_FLUSH_TRIGGER: usize = 512;
+
 pub enum RepairStrategy {
     RepairRange(RepairSlotRange),
     RepairAll {
@@ -83,7 +64,6 @@ impl RepairService {
         repair_socket: Arc<UdpSocket>,
         cluster_info: Arc<RwLock<ClusterInfo>>,
         repair_strategy: RepairStrategy,
-        cluster_slots: Arc<ClusterSlots>,
     ) -> Self {
         let t_repair = Builder::new()
             .name("solana-repair-service".to_string())
@@ -94,7 +74,6 @@ impl RepairService {
                     &repair_socket,
                     &cluster_info,
                     repair_strategy,
-                    &cluster_slots,
                 )
             })
             .unwrap();
@@ -103,20 +82,31 @@ impl RepairService {
     }
 
     fn run(
-        blockstore: &Blockstore,
-        exit: &AtomicBool,
-        repair_socket: &UdpSocket,
+        blockstore: &Arc<Blockstore>,
+        exit: &Arc<AtomicBool>,
+        repair_socket: &Arc<UdpSocket>,
         cluster_info: &Arc<RwLock<ClusterInfo>>,
         repair_strategy: RepairStrategy,
-        cluster_slots: &Arc<ClusterSlots>,
     ) {
         let serve_repair = ServeRepair::new(cluster_info.clone());
+        let mut epoch_slots: BTreeSet<Slot> = BTreeSet::new();
+        let mut old_incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
         let id = cluster_info.read().unwrap().id();
-        if let RepairStrategy::RepairAll { .. } = repair_strategy {
-            Self::initialize_lowest_slot(id, blockstore, cluster_info);
+        if let RepairStrategy::RepairAll {
+            ref epoch_schedule, ..
+        } = repair_strategy
+        {
+            let current_root = blockstore.last_root();
+            Self::initialize_epoch_slots(
+                id,
+                blockstore,
+                &mut epoch_slots,
+                &old_incomplete_slots,
+                current_root,
+                epoch_schedule,
+                cluster_info,
+            );
         }
-        let mut repair_stats = RepairStats::default();
-        let mut last_stats = Instant::now();
         loop {
             if exit.load(Ordering::Relaxed) {
                 break;
@@ -135,38 +125,30 @@ impl RepairService {
 
                     RepairStrategy::RepairAll {
                         ref completed_slots_receiver,
-                        ref bank_forks,
                         ..
                     } => {
                         let new_root = blockstore.last_root();
                         let lowest_slot = blockstore.lowest_slot();
-                        Self::update_lowest_slot(&id, lowest_slot, &cluster_info);
-                        Self::update_completed_slots(
-                            &id,
+                        Self::update_epoch_slots(
+                            id,
                             new_root,
-                            &cluster_slots,
-                            blockstore,
-                            completed_slots_receiver,
+                            lowest_slot,
+                            &mut epoch_slots,
+                            &mut old_incomplete_slots,
                             &cluster_info,
+                            completed_slots_receiver,
                         );
-                        cluster_slots.update(new_root, cluster_info, bank_forks);
                         Self::generate_repairs(blockstore, new_root, MAX_REPAIR_LENGTH)
                     }
                 }
             };
 
             if let Ok(repairs) = repairs {
-                let mut cache = HashMap::new();
-                let reqs: Vec<((SocketAddr, Vec<u8>), RepairType)> = repairs
+                let reqs: Vec<_> = repairs
                     .into_iter()
                     .filter_map(|repair_request| {
                         serve_repair
-                            .repair_request(
-                                &cluster_slots,
-                                &repair_request,
-                                &mut cache,
-                                &mut repair_stats,
-                            )
+                            .repair_request(&repair_request)
                             .map(|result| (result, repair_request))
                             .ok()
                     })
@@ -179,24 +161,6 @@ impl RepairService {
                     });
                 }
             }
-            if last_stats.elapsed().as_secs() > 1 {
-                let repair_total = repair_stats.shred.count
-                    + repair_stats.highest_shred.count
-                    + repair_stats.orphan.count;
-                if repair_total > 0 {
-                    datapoint_info!(
-                        "serve_repair-repair",
-                        ("repair-total", repair_total, i64),
-                        ("shred-count", repair_stats.shred.count, i64),
-                        ("highest-shred-count", repair_stats.highest_shred.count, i64),
-                        ("orphan-count", repair_stats.orphan.count, i64),
-                        ("repair-highest-slot", repair_stats.highest_shred.max, i64),
-                        ("repair-orphan", repair_stats.orphan.max, i64),
-                    );
-                }
-                repair_stats = RepairStats::default();
-                last_stats = Instant::now();
-            }
             sleep(Duration::from_millis(REPAIR_MS));
         }
     }
@@ -308,54 +272,150 @@ impl RepairService {
         }
     }
 
-    fn initialize_lowest_slot(
+    fn get_completed_slots_past_root(
+        blockstore: &Blockstore,
+        slots_in_gossip: &mut BTreeSet<Slot>,
+        root: Slot,
+        epoch_schedule: &EpochSchedule,
+    ) {
+        let last_confirmed_epoch = epoch_schedule.get_leader_schedule_epoch(root);
+        let last_epoch_slot = epoch_schedule.get_last_slot_in_epoch(last_confirmed_epoch);
+
+        let meta_iter = blockstore
+            .slot_meta_iterator(root + 1)
+            .expect("Couldn't get db iterator");
+
+        for (current_slot, meta) in meta_iter {
+            if current_slot > last_epoch_slot {
+                break;
+            }
+            if meta.is_full() {
+                slots_in_gossip.insert(current_slot);
+            }
+        }
+    }
+
+    fn initialize_epoch_slots(
         id: Pubkey,
         blockstore: &Blockstore,
+        slots_in_gossip: &mut BTreeSet<Slot>,
+        old_incomplete_slots: &BTreeSet<Slot>,
+        root: Slot,
+        epoch_schedule: &EpochSchedule,
         cluster_info: &RwLock<ClusterInfo>,
     ) {
+        Self::get_completed_slots_past_root(blockstore, slots_in_gossip, root, epoch_schedule);
+
         // Safe to set into gossip because by this time, the leader schedule cache should
         // also be updated with the latest root (done in blockstore_processor) and thus
         // will provide a schedule to window_service for any incoming shreds up to the
         // last_confirmed_epoch.
-        cluster_info
-            .write()
-            .unwrap()
-            .push_lowest_slot(id, blockstore.lowest_slot());
+        cluster_info.write().unwrap().push_epoch_slots(
+            id,
+            root,
+            blockstore.lowest_slot(),
+            slots_in_gossip.clone(),
+            old_incomplete_slots,
+        );
     }
 
-    fn update_completed_slots(
-        id: &Pubkey,
-        root: Slot,
-        cluster_slots: &ClusterSlots,
-        blockstore: &Blockstore,
-        completed_slots_receiver: &CompletedSlotsReceiver,
+    // Update the gossiped structure used for the "Repairmen" repair protocol. See docs
+    // for details.
+    fn update_epoch_slots(
+        id: Pubkey,
+        latest_known_root: Slot,
+        lowest_slot: Slot,
+        completed_slot_cache: &mut BTreeSet<Slot>,
+        incomplete_slot_stash: &mut BTreeSet<Slot>,
         cluster_info: &RwLock<ClusterInfo>,
+        completed_slots_receiver: &CompletedSlotsReceiver,
     ) {
-        let mine = cluster_slots.collect(id);
-        let mut slots: Vec<Slot> = vec![];
-        while let Ok(mut more) = completed_slots_receiver.try_recv() {
-            more.retain(|x| !mine.contains(x));
-            slots.append(&mut more);
+        let mut should_update = false;
+        while let Ok(completed_slots) = completed_slots_receiver.try_recv() {
+            for slot in completed_slots {
+                let last_slot_in_stash = *incomplete_slot_stash.iter().next_back().unwrap_or(&0);
+                let removed_from_stash = incomplete_slot_stash.remove(&slot);
+                // If the newly completed slot was not being tracked in stash, and is > last
+                // slot being tracked in stash, add it to cache. Also, update gossip
+                if !removed_from_stash && slot >= last_slot_in_stash {
+                    should_update |= completed_slot_cache.insert(slot);
+                }
+                // If the slot was removed from stash, update gossip
+                should_update |= removed_from_stash;
+            }
         }
+
+        if should_update {
+            if completed_slot_cache.len() >= COMPLETED_SLOT_CACHE_FLUSH_TRIGGER {
+                Self::stash_old_incomplete_slots(completed_slot_cache, incomplete_slot_stash);
+                let lowest_completed_slot_in_cache =
+                    *completed_slot_cache.iter().next().unwrap_or(&0);
+                Self::prune_incomplete_slot_stash(
+                    incomplete_slot_stash,
+                    lowest_completed_slot_in_cache,
+                );
+            }
+
+            cluster_info.write().unwrap().push_epoch_slots(
+                id,
+                latest_known_root,
+                lowest_slot,
+                completed_slot_cache.clone(),
+                incomplete_slot_stash,
+            );
+        }
+    }
+
+    fn stash_old_incomplete_slots(cache: &mut BTreeSet<Slot>, stash: &mut BTreeSet<Slot>) {
+        if cache.len() > MAX_COMPLETED_SLOT_CACHE_LEN {
+            let mut prev = *cache.iter().next().expect("Expected to find some slot");
+            cache.remove(&prev);
+            while cache.len() >= MAX_COMPLETED_SLOT_CACHE_LEN {
+                let next = *cache.iter().next().expect("Expected to find some slot");
+                cache.remove(&next);
+                // Prev slot and next slot are not included in incomplete slot list.
+                (prev + 1..next).for_each(|slot| {
+                    stash.insert(slot);
+                });
+                prev = next;
+            }
+        }
+    }
+
+    fn prune_incomplete_slot_stash(
+        stash: &mut BTreeSet<Slot>,
+        lowest_completed_slot_in_cache: Slot,
+    ) {
+        if let Some(oldest_incomplete_slot) = stash.iter().next() {
+            // Prune old slots
+            // Prune in batches to reduce overhead. Pruning starts when oldest slot is 1.5 epochs
+            // earlier than the new root. But, we prune all the slots that are older than 1 epoch.
+            // So slots in a batch of half epoch are getting pruned
+            if oldest_incomplete_slot + DEFAULT_SLOTS_PER_EPOCH + DEFAULT_SLOTS_PER_EPOCH / 2
+                < lowest_completed_slot_in_cache
+            {
+                let oldest_slot_to_retain =
+                    lowest_completed_slot_in_cache.saturating_sub(DEFAULT_SLOTS_PER_EPOCH);
+                *stash = stash
+                    .range((Included(&oldest_slot_to_retain), Unbounded))
+                    .cloned()
+                    .collect();
+            }
+        }
+    }
+
+    #[allow(dead_code)]
+    fn find_incomplete_slots(blockstore: &Blockstore, root: Slot) -> HashSet<Slot> {
         blockstore
             .live_slots_iterator(root)
-            .for_each(|(slot, slot_meta)| {
-                if slot_meta.is_full() && !mine.contains(&slot) {
-                    slots.push(slot)
+            .filter_map(|(slot, slot_meta)| {
+                if !slot_meta.is_full() {
+                    Some(slot)
+                } else {
+                    None
                 }
-            });
-        slots.sort();
-        slots.dedup();
-        if !slots.is_empty() {
-            cluster_info.write().unwrap().push_epoch_slots(&slots);
-        }
-    }
-
-    fn update_lowest_slot(id: &Pubkey, lowest_slot: Slot, cluster_info: &RwLock<ClusterInfo>) {
-        cluster_info
-            .write()
-            .unwrap()
-            .push_lowest_slot(*id, lowest_slot);
+            })
+            .collect()
     }
 
     pub fn join(self) -> thread::Result<()> {
@@ -367,11 +427,15 @@ impl RepairService {
 mod test {
     use super::*;
     use crate::cluster_info::Node;
+    use itertools::Itertools;
+    use rand::seq::SliceRandom;
+    use rand::{thread_rng, Rng};
     use solana_ledger::blockstore::{
         make_chaining_slot_entries, make_many_slot_entries, make_slot_entries,
     };
     use solana_ledger::shred::max_ticks_per_n_shreds;
     use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
+    use std::thread::Builder;
 
     #[test]
     pub fn test_repair_orphan() {
@@ -587,19 +651,342 @@ mod test {
     }
 
     #[test]
-    pub fn test_update_lowest_slot() {
-        let node_info = Node::new_localhost_with_pubkey(&Pubkey::default());
-        let cluster_info = RwLock::new(ClusterInfo::new_with_invalid_keypair(
-            node_info.info.clone(),
-        ));
-        RepairService::update_lowest_slot(&Pubkey::default(), 5, &cluster_info);
-        let lowest = cluster_info
-            .read()
-            .unwrap()
-            .get_lowest_slot_for_node(&Pubkey::default(), None)
-            .unwrap()
-            .0
-            .clone();
-        assert_eq!(lowest.lowest, 5);
+    pub fn test_get_completed_slots_past_root() {
+        let blockstore_path = get_tmp_ledger_path!();
+        {
+            let blockstore = Blockstore::open(&blockstore_path).unwrap();
+            let num_entries_per_slot = 10;
+            let root = 10;
+
+            let fork1 = vec![5, 7, root, 15, 20, 21];
+            let fork1_shreds: Vec<_> = make_chaining_slot_entries(&fork1, num_entries_per_slot)
+                .into_iter()
+                .flat_map(|(shreds, _)| shreds)
+                .collect();
+            let fork2 = vec![8, 12];
+            let fork2_shreds = make_chaining_slot_entries(&fork2, num_entries_per_slot);
+
+            // Remove the last shred from each slot to make an incomplete slot
+            let fork2_incomplete_shreds: Vec<_> = fork2_shreds
+                .into_iter()
+                .flat_map(|(mut shreds, _)| {
+                    shreds.pop();
+                    shreds
+                })
+                .collect();
+            let mut full_slots = BTreeSet::new();
+
+            blockstore.insert_shreds(fork1_shreds, None, false).unwrap();
+            blockstore
+                .insert_shreds(fork2_incomplete_shreds, None, false)
+                .unwrap();
+
+            // Test that only slots > root from fork1 were included
+            let epoch_schedule = EpochSchedule::custom(32, 32, false);
+
+            RepairService::get_completed_slots_past_root(
+                &blockstore,
+                &mut full_slots,
+                root,
+                &epoch_schedule,
+            );
+
+            let mut expected: BTreeSet<_> = fork1.into_iter().filter(|x| *x > root).collect();
+            assert_eq!(full_slots, expected);
+
+            // Test that slots past the last confirmed epoch boundary don't get included
+            let last_epoch = epoch_schedule.get_leader_schedule_epoch(root);
+            let last_slot = epoch_schedule.get_last_slot_in_epoch(last_epoch);
+            let fork3 = vec![last_slot, last_slot + 1];
+            let fork3_shreds: Vec<_> = make_chaining_slot_entries(&fork3, num_entries_per_slot)
+                .into_iter()
+                .flat_map(|(shreds, _)| shreds)
+                .collect();
+            blockstore.insert_shreds(fork3_shreds, None, false).unwrap();
+            RepairService::get_completed_slots_past_root(
+                &blockstore,
+                &mut full_slots,
+                root,
+                &epoch_schedule,
+            );
+            expected.insert(last_slot);
+            assert_eq!(full_slots, expected);
+        }
+        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
+    }
+
+    #[test]
+    pub fn test_update_epoch_slots() {
+        let blockstore_path = get_tmp_ledger_path!();
+        {
+            // Create blockstore
+            let (blockstore, _, completed_slots_receiver) =
+                Blockstore::open_with_signal(&blockstore_path).unwrap();
+
+            let blockstore = Arc::new(blockstore);
+
+            let mut root = 0;
+            let num_slots = 100;
+            let entries_per_slot = 5;
+            let blockstore_ = blockstore.clone();
+
+            // Spin up thread to write to blockstore
+            let writer = Builder::new()
+                .name("writer".to_string())
+                .spawn(move || {
+                    let slots: Vec<_> = (1..num_slots + 1).collect();
+                    let mut shreds: Vec<_> = make_chaining_slot_entries(&slots, entries_per_slot)
+                        .into_iter()
+                        .flat_map(|(shreds, _)| shreds)
+                        .collect();
+                    shreds.shuffle(&mut thread_rng());
+                    let mut i = 0;
+                    let max_step = entries_per_slot * 4;
+                    let repair_interval_ms = 10;
+                    let mut rng = rand::thread_rng();
+                    let num_shreds = shreds.len();
+                    while i < num_shreds {
+                        let step = rng.gen_range(1, max_step + 1) as usize;
+                        let step = std::cmp::min(step, num_shreds - i);
+                        let shreds_to_insert = shreds.drain(..step).collect_vec();
+                        blockstore_
+                            .insert_shreds(shreds_to_insert, None, false)
+                            .unwrap();
+                        sleep(Duration::from_millis(repair_interval_ms));
+                        i += step;
+                    }
+                })
+                .unwrap();
+
+            let mut completed_slots = BTreeSet::new();
+            let node_info = Node::new_localhost_with_pubkey(&Pubkey::default());
+            let cluster_info = RwLock::new(ClusterInfo::new_with_invalid_keypair(
+                node_info.info.clone(),
+            ));
+
+            let mut old_incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
+            while completed_slots.len() < num_slots as usize {
+                RepairService::update_epoch_slots(
+                    Pubkey::default(),
+                    root,
+                    blockstore.lowest_slot(),
+                    &mut completed_slots,
+                    &mut old_incomplete_slots,
+                    &cluster_info,
+                    &completed_slots_receiver,
+                );
+            }
+
+            let mut expected: BTreeSet<_> = (1..num_slots + 1).collect();
+            assert_eq!(completed_slots, expected);
+
+            // Update with new root, should filter out the slots <= root
+            root = num_slots / 2;
+            let (shreds, _) = make_slot_entries(num_slots + 2, num_slots + 1, entries_per_slot);
+            blockstore.insert_shreds(shreds, None, false).unwrap();
+            RepairService::update_epoch_slots(
+                Pubkey::default(),
+                root,
+                0,
+                &mut completed_slots,
+                &mut old_incomplete_slots,
+                &cluster_info,
+                &completed_slots_receiver,
+            );
+            expected.insert(num_slots + 2);
+            assert_eq!(completed_slots, expected);
+            writer.join().unwrap();
+        }
+        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
+    }
+
+    #[test]
+    fn test_stash_old_incomplete_slots() {
+        let mut cache: BTreeSet<Slot> = BTreeSet::new();
+        let mut stash: BTreeSet<Slot> = BTreeSet::new();
+
+        // When cache is empty.
+        RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
+        assert_eq!(stash.len(), 0);
+
+        // Insert some slots in cache ( < MAX_COMPLETED_SLOT_CACHE_LEN + 1)
+        cache.insert(101);
+        cache.insert(102);
+        cache.insert(104);
+        cache.insert(105);
+
+        // Not enough slots in cache. So stash should remain empty.
+        RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
+        assert_eq!(stash.len(), 0);
+        assert_eq!(cache.len(), 4);
+
+        // Insert slots in cache ( = MAX_COMPLETED_SLOT_CACHE_LEN)
+        let mut cache: BTreeSet<Slot> = BTreeSet::new();
+        (0..MAX_COMPLETED_SLOT_CACHE_LEN as u64)
+            .into_iter()
+            .for_each(|slot| {
+                cache.insert(slot);
+            });
+
+        // Not enough slots in cache. So stash should remain empty.
+        RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
+        assert_eq!(stash.len(), 0);
+        assert_eq!(cache.len(), MAX_COMPLETED_SLOT_CACHE_LEN);
+
+        // Insert 1 more to cross the threshold
+        cache.insert(MAX_COMPLETED_SLOT_CACHE_LEN as u64);
+        RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
+        // Stash is still empty, as no missing slots
+        assert_eq!(stash.len(), 0);
+        // It removed some entries from cache
+        assert_eq!(cache.len(), MAX_COMPLETED_SLOT_CACHE_LEN - 1);
+
+        // Insert more slots to create a missing slot
+        let mut cache: BTreeSet<Slot> = BTreeSet::new();
+        cache.insert(0);
+        (2..=MAX_COMPLETED_SLOT_CACHE_LEN as u64 + 2)
+            .into_iter()
+            .for_each(|slot| {
+                cache.insert(slot);
+            });
+        RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
+
+        // Stash is not empty
+        assert!(stash.contains(&1));
+        // It removed some entries from cache
+        assert_eq!(cache.len(), MAX_COMPLETED_SLOT_CACHE_LEN - 1);
+
+        // Test multiple missing slots at dispersed locations
+        let mut cache: BTreeSet<Slot> = BTreeSet::new();
+        (0..MAX_COMPLETED_SLOT_CACHE_LEN as u64 * 2)
+            .into_iter()
+            .for_each(|slot| {
+                cache.insert(slot);
+            });
+
+        cache.remove(&10);
+        cache.remove(&11);
+
+        cache.remove(&28);
+        cache.remove(&29);
+
+        cache.remove(&148);
+        cache.remove(&149);
+        cache.remove(&150);
+        cache.remove(&151);
+
+        RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
+
+        // Stash is not empty
+        assert!(stash.contains(&10));
+        assert!(stash.contains(&11));
+        assert!(stash.contains(&28));
+        assert!(stash.contains(&29));
+        assert!(stash.contains(&148));
+        assert!(stash.contains(&149));
+        assert!(stash.contains(&150));
+        assert!(stash.contains(&151));
+
+        assert!(!stash.contains(&147));
+        assert!(!stash.contains(&152));
+        // It removed some entries from cache
+        assert_eq!(cache.len(), MAX_COMPLETED_SLOT_CACHE_LEN - 1);
+        (MAX_COMPLETED_SLOT_CACHE_LEN + 1..MAX_COMPLETED_SLOT_CACHE_LEN * 2)
+            .into_iter()
+            .for_each(|slot| {
+                let slot: u64 = slot as u64;
+                assert!(cache.contains(&slot));
+            });
+    }
+
+    #[test]
+    fn test_prune_incomplete_slot_stash() {
+        // Prune empty stash
+        let mut stash: BTreeSet<Slot> = BTreeSet::new();
+        RepairService::prune_incomplete_slot_stash(&mut stash, 0);
+        assert!(stash.is_empty());
+
+        // Prune stash with slots < DEFAULT_SLOTS_PER_EPOCH
+        stash.insert(0);
+        stash.insert(10);
+        stash.insert(11);
+        stash.insert(50);
+        assert_eq!(stash.len(), 4);
+        RepairService::prune_incomplete_slot_stash(&mut stash, 100);
+        assert_eq!(stash.len(), 4);
+
+        // Prune stash with slots > DEFAULT_SLOTS_PER_EPOCH, but < 1.5 * DEFAULT_SLOTS_PER_EPOCH
+        stash.insert(DEFAULT_SLOTS_PER_EPOCH + 50);
+        assert_eq!(stash.len(), 5);
+        RepairService::prune_incomplete_slot_stash(&mut stash, DEFAULT_SLOTS_PER_EPOCH + 100);
+        assert_eq!(stash.len(), 5);
+
+        // Prune stash with slots > 1.5 * DEFAULT_SLOTS_PER_EPOCH
+        stash.insert(DEFAULT_SLOTS_PER_EPOCH + DEFAULT_SLOTS_PER_EPOCH / 2);
+        assert_eq!(stash.len(), 6);
+        RepairService::prune_incomplete_slot_stash(
+            &mut stash,
+            DEFAULT_SLOTS_PER_EPOCH + DEFAULT_SLOTS_PER_EPOCH / 2 + 1,
+        );
+        assert_eq!(stash.len(), 2);
+    }
+
+    #[test]
+    fn test_find_incomplete_slots() {
+        let blockstore_path = get_tmp_ledger_path!();
+        {
+            let blockstore = Blockstore::open(&blockstore_path).unwrap();
+            let num_entries_per_slot = 100;
+            let (mut shreds, _) = make_slot_entries(0, 0, num_entries_per_slot);
+            assert!(shreds.len() > 1);
+            let (shreds4, _) = make_slot_entries(4, 0, num_entries_per_slot);
+            shreds.extend(shreds4);
+            blockstore.insert_shreds(shreds, None, false).unwrap();
+
+            // Nothing is incomplete
+            assert!(RepairService::find_incomplete_slots(&blockstore, 0).is_empty());
+
+            // Insert a slot 5 that chains to an incomplete orphan slot 3
+            let (shreds5, _) = make_slot_entries(5, 3, num_entries_per_slot);
+            blockstore.insert_shreds(shreds5, None, false).unwrap();
+            assert_eq!(
+                RepairService::find_incomplete_slots(&blockstore, 0),
+                vec![3].into_iter().collect()
+            );
+
+            // Insert another incomplete orphan slot 2 that is the parent of slot 3.
+            // Both should be incomplete
+            let (shreds3, _) = make_slot_entries(3, 2, num_entries_per_slot);
+            blockstore
+                .insert_shreds(shreds3[1..].to_vec(), None, false)
+                .unwrap();
+            assert_eq!(
+                RepairService::find_incomplete_slots(&blockstore, 0),
+                vec![2, 3].into_iter().collect()
+            );
+
+            // Insert a incomplete slot 6 that chains to the root 0,
+            // should also be incomplete
+            let (shreds6, _) = make_slot_entries(6, 0, num_entries_per_slot);
+            blockstore
+                .insert_shreds(shreds6[1..].to_vec(), None, false)
+                .unwrap();
+            assert_eq!(
+                RepairService::find_incomplete_slots(&blockstore, 0),
+                vec![2, 3, 6].into_iter().collect()
+            );
+
+            // Complete slot 3, should no longer be marked incomplete
+            blockstore
+                .insert_shreds(shreds3[..].to_vec(), None, false)
+                .unwrap();
+
+            assert_eq!(
+                RepairService::find_incomplete_slots(&blockstore, 0),
+                vec![2, 6].into_iter().collect()
+            );
+        }
+
+        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
 }

core/src/retransmit_stage.rs

@@ -2,9 +2,10 @@
 
 use crate::{
     cluster_info::{compute_retransmit_peers, ClusterInfo, DATA_PLANE_FANOUT},
-    cluster_slots::ClusterSlots,
+    packet::Packets,
     repair_service::RepairStrategy,
     result::{Error, Result},
+    streamer::PacketReceiver,
     window_service::{should_retransmit_and_persist, WindowService},
 };
 use crossbeam_channel::Receiver as CrossbeamReceiver;
@@ -16,9 +17,7 @@ use solana_ledger::{
 };
 use solana_measure::measure::Measure;
 use solana_metrics::inc_new_counter_error;
-use solana_perf::packet::Packets;
 use solana_sdk::epoch_schedule::EpochSchedule;
-use solana_streamer::streamer::PacketReceiver;
 use std::{
     cmp,
     net::UdpSocket,
@@ -215,7 +214,6 @@ impl RetransmitStage {
         epoch_schedule: EpochSchedule,
         cfg: Option<Arc<AtomicBool>>,
         shred_version: u16,
-        cluster_slots: Arc<ClusterSlots>,
     ) -> Self {
         let (retransmit_sender, retransmit_receiver) = channel();
 
@@ -258,7 +256,6 @@ impl RetransmitStage {
                 );
                 rv && is_connected
             },
-            cluster_slots,
         );
 
         let thread_hdls = t_retransmit;
@@ -281,11 +278,11 @@ impl RetransmitStage {
 mod tests {
     use super::*;
     use crate::contact_info::ContactInfo;
+    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use crate::packet::{self, Meta, Packet, Packets};
     use solana_ledger::blockstore_processor::{process_blockstore, ProcessOptions};
     use solana_ledger::create_new_tmp_ledger;
-    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_net_utils::find_available_port_in_range;
-    use solana_perf::packet::{Meta, Packet, Packets};
     use solana_sdk::pubkey::Pubkey;
     use std::net::{IpAddr, Ipv4Addr};
 
@@ -336,7 +333,7 @@ mod tests {
         // it should send this over the sockets.
         retransmit_sender.send(packets).unwrap();
         let mut packets = Packets::new(vec![]);
-        solana_streamer::packet::recv_from(&mut packets, &me_retransmit, 1).unwrap();
+        packet::recv_from(&mut packets, &me_retransmit, 1).unwrap();
         assert_eq!(packets.packets.len(), 1);
         assert_eq!(packets.packets[0].meta.repair, false);
 
@@ -352,7 +349,7 @@ mod tests {
         let packets = Packets::new(vec![repair, Packet::default()]);
         retransmit_sender.send(packets).unwrap();
         let mut packets = Packets::new(vec![]);
-        solana_streamer::packet::recv_from(&mut packets, &me_retransmit, 1).unwrap();
+        packet::recv_from(&mut packets, &me_retransmit, 1).unwrap();
         assert_eq!(packets.packets.len(), 1);
         assert_eq!(packets.packets[0].meta.repair, false);
     }

core/src/rpc_pubsub.rs

@@ -4,9 +4,16 @@ use crate::rpc_subscriptions::{Confirmations, RpcSubscriptions, SlotInfo};
 use jsonrpc_core::{Error, ErrorCode, Result};
 use jsonrpc_derive::rpc;
 use jsonrpc_pubsub::{typed::Subscriber, Session, SubscriptionId};
-use solana_client::rpc_response::{Response as RpcResponse, RpcAccount, RpcKeyedAccount};
-use solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature, transaction};
-use std::sync::{atomic, Arc};
+use solana_client::rpc_response::{
+    Response as RpcResponse, RpcAccount, RpcKeyedAccount, RpcSignatureResult,
+};
+#[cfg(test)]
+use solana_ledger::blockstore::Blockstore;
+use solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature};
+use std::{
+    str::FromStr,
+    sync::{atomic, Arc},
+};
 
 // Suppress needless_return due to
 //   https://github.com/paritytech/jsonrpc/blob/2d38e6424d8461cdf72e78425ce67d51af9c6586/derive/src/lib.rs#L204
@@ -74,7 +81,7 @@ pub trait RpcSolPubSub {
     fn signature_subscribe(
         &self,
         meta: Self::Metadata,
-        subscriber: Subscriber<RpcResponse<transaction::Result<()>>>,
+        subscriber: Subscriber<RpcResponse<RpcSignatureResult>>,
         signature_str: String,
         confirmations: Option<Confirmations>,
     );
@@ -116,7 +123,6 @@ pub trait RpcSolPubSub {
     fn root_unsubscribe(&self, meta: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool>;
 }
 
-#[derive(Default)]
 pub struct RpcSolPubSubImpl {
     uid: Arc<atomic::AtomicUsize>,
     subscriptions: Arc<RpcSubscriptions>,
@@ -127,9 +133,14 @@ impl RpcSolPubSubImpl {
         let uid = Arc::new(atomic::AtomicUsize::default());
         Self { uid, subscriptions }
     }
-}
 
-use std::str::FromStr;
+    #[cfg(test)]
+    fn default_with_blockstore(blockstore: Arc<Blockstore>) -> Self {
+        let uid = Arc::new(atomic::AtomicUsize::default());
+        let subscriptions = Arc::new(RpcSubscriptions::default_with_blockstore(blockstore));
+        Self { uid, subscriptions }
+    }
+}
 
 fn param<T: FromStr>(param_str: &str, thing: &str) -> Result<T> {
     param_str.parse::<T>().map_err(|_e| Error {
@@ -225,7 +236,7 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
     fn signature_subscribe(
         &self,
         _meta: Self::Metadata,
-        subscriber: Subscriber<RpcResponse<transaction::Result<()>>>,
+        subscriber: Subscriber<RpcResponse<RpcSignatureResult>>,
         signature_str: String,
         confirmations: Option<Confirmations>,
     ) {
@@ -314,13 +325,14 @@ mod tests {
     use super::*;
     use crate::{
         commitment::{BlockCommitment, BlockCommitmentCache},
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
         rpc_subscriptions::tests::robust_poll_or_panic,
     };
     use jsonrpc_core::{futures::sync::mpsc, Response};
     use jsonrpc_pubsub::{PubSubHandler, Session};
+    use serial_test_derive::serial;
     use solana_budget_program::{self, budget_instruction};
-    use solana_ledger::bank_forks::BankForks;
-    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use solana_ledger::{bank_forks::BankForks, get_tmp_ledger_path};
     use solana_runtime::bank::Bank;
     use solana_sdk::{
         pubkey::Pubkey,
@@ -355,6 +367,7 @@ mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_signature_subscribe() {
         let GenesisConfigInfo {
             genesis_config,
@@ -366,12 +379,16 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let rpc = RpcSolPubSubImpl {
             subscriptions: Arc::new(RpcSubscriptions::new(
                 &Arc::new(AtomicBool::new(false)),
-                Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+                Arc::new(RwLock::new(
+                    BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+                )),
             )),
-            ..RpcSolPubSubImpl::default()
+            uid: Arc::new(atomic::AtomicUsize::default()),
         };
 
         // Test signature subscriptions
@@ -385,7 +402,7 @@ mod tests {
 
         // Test signature confirmation notification
         let (response, _) = robust_poll_or_panic(receiver);
-        let expected_res: Option<transaction::Result<()>> = Some(Ok(()));
+        let expected_res = RpcSignatureResult { err: None };
         let expected = json!({
            "jsonrpc": "2.0",
            "method": "signatureNotification",
@@ -401,6 +418,7 @@ mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_signature_unsubscribe() {
         let GenesisConfigInfo {
             genesis_config,
@@ -411,11 +429,13 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let arc_bank = Arc::new(bank);
         let blockhash = arc_bank.last_blockhash();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
         let session = create_session();
 
         let mut io = PubSubHandler::default();
-        let rpc = RpcSolPubSubImpl::default();
+        let rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore);
         io.extend_with(rpc.to_delegate());
 
         let tx = system_transaction::transfer(&alice, &bob_pubkey, 20, blockhash);
@@ -449,6 +469,7 @@ mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_account_subscribe() {
         let GenesisConfigInfo {
             mut genesis_config,
@@ -469,13 +490,17 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
         let rpc = RpcSolPubSubImpl {
             subscriptions: Arc::new(RpcSubscriptions::new(
                 &Arc::new(AtomicBool::new(false)),
-                Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+                Arc::new(RwLock::new(
+                    BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+                )),
             )),
-            ..RpcSolPubSubImpl::default()
+            uid: Arc::new(atomic::AtomicUsize::default()),
         };
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification");
@@ -559,12 +584,15 @@ mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_account_unsubscribe() {
         let bob_pubkey = Pubkey::new_rand();
         let session = create_session();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
         let mut io = PubSubHandler::default();
-        let rpc = RpcSolPubSubImpl::default();
+        let rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore);
 
         io.extend_with(rpc.to_delegate());
 
@@ -608,13 +636,17 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bob = Keypair::new();
 
-        let mut rpc = RpcSolPubSubImpl::default();
+        let mut rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore.clone());
         let exit = Arc::new(AtomicBool::new(false));
         let subscriptions = RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         );
         rpc.subscriptions = Arc::new(subscriptions);
         let session = create_session();
@@ -645,11 +677,15 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bob = Keypair::new();
 
-        let mut rpc = RpcSolPubSubImpl::default();
+        let mut rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore.clone());
         let exit = Arc::new(AtomicBool::new(false));
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests()));
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_blockstore(blockstore.clone()),
+        ));
 
         let subscriptions = RpcSubscriptions::new(&exit, block_commitment_cache.clone());
         rpc.subscriptions = Arc::new(subscriptions);
@@ -676,8 +712,14 @@ mod tests {
         cache0.increase_confirmation_stake(1, 10);
         let mut block_commitment = HashMap::new();
         block_commitment.entry(0).or_insert(cache0.clone());
-        let mut new_block_commitment =
-            BlockCommitmentCache::new(block_commitment, 10, bank1.clone(), 0);
+        let mut new_block_commitment = BlockCommitmentCache::new(
+            block_commitment,
+            0,
+            10,
+            bank1.clone(),
+            blockstore.clone(),
+            0,
+        );
         let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
         std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
         drop(w_block_commitment_cache);
@@ -691,7 +733,8 @@ mod tests {
         cache0.increase_confirmation_stake(2, 10);
         let mut block_commitment = HashMap::new();
         block_commitment.entry(0).or_insert(cache0.clone());
-        let mut new_block_commitment = BlockCommitmentCache::new(block_commitment, 10, bank2, 0);
+        let mut new_block_commitment =
+            BlockCommitmentCache::new(block_commitment, 0, 10, bank2, blockstore.clone(), 0);
         let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
         std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
         drop(w_block_commitment_cache);
@@ -719,8 +762,11 @@ mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_slot_subscribe() {
-        let rpc = RpcSolPubSubImpl::default();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification");
         rpc.slot_subscribe(session, subscriber);
@@ -743,8 +789,11 @@ mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_slot_unsubscribe() {
-        let rpc = RpcSolPubSubImpl::default();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification");
         rpc.slot_subscribe(session, subscriber);

core/src/rpc_pubsub_service.rs

@@ -73,6 +73,7 @@ impl PubSubService {
 mod tests {
     use super::*;
     use crate::commitment::BlockCommitmentCache;
+    use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
     use std::{
         net::{IpAddr, Ipv4Addr},
         sync::RwLock,
@@ -82,9 +83,13 @@ mod tests {
     fn test_pubsub_new() {
         let pubsub_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0);
         let exit = Arc::new(AtomicBool::new(false));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let subscriptions = Arc::new(RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         ));
         let pubsub_service = PubSubService::new(&subscriptions, pubsub_addr, &exit);
         let thread = pubsub_service.thread_hdl.thread();

core/src/rpc_service.rs

@@ -15,8 +15,9 @@ use solana_ledger::{
     blockstore::Blockstore,
     snapshot_utils,
 };
-use solana_sdk::hash::Hash;
+use solana_sdk::{hash::Hash, pubkey::Pubkey};
 use std::{
+    collections::HashSet,
     net::SocketAddr,
     path::{Path, PathBuf},
     sync::{mpsc::channel, Arc, RwLock},
@@ -24,6 +25,10 @@ use std::{
 };
 use tokio::prelude::Future;
 
+// If trusted validators are specified, consider this validator healthy if its latest account hash
+// is no further behind than this distance from the latest trusted validator account hash
+const HEALTH_CHECK_SLOT_DISTANCE: u64 = 150;
+
 pub struct JsonRpcService {
     thread_hdl: JoinHandle<()>,
 
@@ -37,15 +42,24 @@ struct RpcRequestMiddleware {
     ledger_path: PathBuf,
     snapshot_archive_path_regex: Regex,
     snapshot_config: Option<SnapshotConfig>,
+    cluster_info: Arc<RwLock<ClusterInfo>>,
+    trusted_validators: Option<HashSet<Pubkey>>,
 }
 
 impl RpcRequestMiddleware {
-    pub fn new(ledger_path: PathBuf, snapshot_config: Option<SnapshotConfig>) -> Self {
+    pub fn new(
+        ledger_path: PathBuf,
+        snapshot_config: Option<SnapshotConfig>,
+        cluster_info: Arc<RwLock<ClusterInfo>>,
+        trusted_validators: Option<HashSet<Pubkey>>,
+    ) -> Self {
         Self {
             ledger_path,
             snapshot_archive_path_regex: Regex::new(r"/snapshot-\d+-[[:alnum:]]+\.tar\.bz2$")
                 .unwrap(),
             snapshot_config,
+            cluster_info,
+            trusted_validators,
         }
     }
 
@@ -85,19 +99,9 @@ impl RpcRequestMiddleware {
     }
 
     fn get(&self, path: &str) -> RequestMiddlewareAction {
-        let stem = path.split_at(1).1; // Drop leading '/' from path
-        let filename = {
-            match path {
-                "/genesis.tar.bz2" => self.ledger_path.join(stem),
-                _ => self
-                    .snapshot_config
-                    .as_ref()
-                    .unwrap()
-                    .snapshot_package_output_path
-                    .join(stem),
-            }
-        };
-
+        let filename = self.ledger_path.join(
+            path.split_at(1).1, // Drop leading '/' from path
+        );
         info!("get {} -> {:?}", path, filename);
 
         RequestMiddlewareAction::Respond {
@@ -114,6 +118,58 @@ impl RpcRequestMiddleware {
             ),
         }
     }
+
+    fn health_check(&self) -> &'static str {
+        let response = if let Some(trusted_validators) = &self.trusted_validators {
+            let (latest_account_hash_slot, latest_trusted_validator_account_hash_slot) = {
+                let cluster_info = self.cluster_info.read().unwrap();
+                (
+                    cluster_info
+                        .get_accounts_hash_for_node(&cluster_info.id())
+                        .map(|hashes| hashes.iter().max_by(|a, b| a.0.cmp(&b.0)))
+                        .flatten()
+                        .map(|slot_hash| slot_hash.0)
+                        .unwrap_or(0),
+                    trusted_validators
+                        .iter()
+                        .map(|trusted_validator| {
+                            cluster_info
+                                .get_accounts_hash_for_node(&trusted_validator)
+                                .map(|hashes| hashes.iter().max_by(|a, b| a.0.cmp(&b.0)))
+                                .flatten()
+                                .map(|slot_hash| slot_hash.0)
+                                .unwrap_or(0)
+                        })
+                        .max()
+                        .unwrap_or(0),
+                )
+            };
+
+            // This validator is considered healthy if its latest account hash slot is within
+            // `HEALTH_CHECK_SLOT_DISTANCE` of the latest trusted validator's account hash slot
+            if latest_account_hash_slot > 0
+                && latest_trusted_validator_account_hash_slot > 0
+                && latest_account_hash_slot
+                    > latest_trusted_validator_account_hash_slot
+                        .saturating_sub(HEALTH_CHECK_SLOT_DISTANCE)
+            {
+                "ok"
+            } else {
+                warn!(
+                    "health check: me={}, latest trusted_validator={}",
+                    latest_account_hash_slot, latest_trusted_validator_account_hash_slot
+                );
+                "behind"
+            }
+        } else {
+            // No trusted validator point of reference available, so this validator is healthy
+            // because it's running
+            "ok"
+        };
+
+        info!("health check: {}", response);
+        response
+    }
 }
 
 impl RequestMiddleware for RpcRequestMiddleware {
@@ -148,6 +204,16 @@ impl RequestMiddleware for RpcRequestMiddleware {
         }
         if self.is_get_path(request.uri().path()) {
             self.get(request.uri().path())
+        } else if request.uri().path() == "/health" {
+            RequestMiddlewareAction::Respond {
+                should_validate_hosts: true,
+                response: Box::new(jsonrpc_core::futures::future::ok(
+                    hyper::Response::builder()
+                        .status(hyper::StatusCode::OK)
+                        .body(hyper::Body::from(self.health_check()))
+                        .unwrap(),
+                )),
+            }
         } else {
             RequestMiddlewareAction::Proceed {
                 should_continue_on_invalid_cors: false,
@@ -171,6 +237,7 @@ impl JsonRpcService {
         ledger_path: &Path,
         storage_state: StorageState,
         validator_exit: Arc<RwLock<Option<ValidatorExit>>>,
+        trusted_validators: Option<HashSet<Pubkey>>,
     ) -> Self {
         info!("rpc bound to {:?}", rpc_addr);
         info!("rpc configuration: {:?}", config);
@@ -196,20 +263,35 @@ impl JsonRpcService {
                 let rpc = RpcSolImpl;
                 io.extend_with(rpc.to_delegate());
 
-                let server =
-                    ServerBuilder::with_meta_extractor(io, move |_req: &hyper::Request<hyper::Body>| Meta {
+                let request_middleware = RpcRequestMiddleware::new(
+                    ledger_path,
+                    snapshot_config,
+                    cluster_info.clone(),
+                    trusted_validators,
+                );
+                let server = ServerBuilder::with_meta_extractor(
+                    io,
+                    move |_req: &hyper::Request<hyper::Body>| Meta {
                         request_processor: request_processor.clone(),
                         cluster_info: cluster_info.clone(),
-                        genesis_hash
-                    }).threads(4)
-                        .cors(DomainsValidation::AllowOnly(vec![
-                            AccessControlAllowOrigin::Any,
-                        ]))
-                        .cors_max_age(86400)
-                        .request_middleware(RpcRequestMiddleware::new(ledger_path, snapshot_config))
-                        .start_http(&rpc_addr);
+                        genesis_hash,
+                    },
+                )
+                .threads(4)
+                .cors(DomainsValidation::AllowOnly(vec![
+                    AccessControlAllowOrigin::Any,
+                ]))
+                .cors_max_age(86400)
+                .request_middleware(request_middleware)
+                .start_http(&rpc_addr);
+
                 if let Err(e) = server {
-                    warn!("JSON RPC service unavailable error: {:?}. \nAlso, check that port {} is not already in use by another application", e, rpc_addr.port());
+                    warn!(
+                        "JSON RPC service unavailable error: {:?}. \n\
+                           Also, check that port {} is not already in use by another application",
+                        e,
+                        rpc_addr.port()
+                    );
                     return;
                 }
 
@@ -248,11 +330,13 @@ impl JsonRpcService {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{contact_info::ContactInfo, rpc::tests::create_validator_exit};
-    use solana_ledger::{
+    use crate::{
+        contact_info::ContactInfo,
+        crds_value::{CrdsData, CrdsValue, SnapshotHash},
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
-        get_tmp_ledger_path,
+        rpc::tests::create_validator_exit,
     };
+    use solana_ledger::get_tmp_ledger_path;
     use solana_runtime::bank::Bank;
     use solana_sdk::signature::Signer;
     use std::net::{IpAddr, Ipv4Addr, SocketAddr};
@@ -277,21 +361,24 @@ mod tests {
             solana_net_utils::find_available_port_in_range(ip_addr, (10000, 65535)).unwrap(),
         );
         let bank_forks = Arc::new(RwLock::new(BankForks::new(bank.slot(), bank)));
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
         let ledger_path = get_tmp_ledger_path!();
-        let blockstore = Blockstore::open(&ledger_path).unwrap();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::default_with_blockstore(blockstore.clone()),
+        ));
         let mut rpc_service = JsonRpcService::new(
             rpc_addr,
             JsonRpcConfig::default(),
             None,
             bank_forks,
             block_commitment_cache,
-            Arc::new(blockstore),
+            blockstore,
             cluster_info,
             Hash::default(),
             &PathBuf::from("farf"),
             StorageState::default(),
             validator_exit,
+            None,
         );
         let thread = rpc_service.thread_hdl.thread();
         assert_eq!(thread.name().unwrap(), "solana-jsonrpc");
@@ -312,7 +399,11 @@ mod tests {
 
     #[test]
     fn test_is_get_path() {
-        let rrm = RpcRequestMiddleware::new(PathBuf::from("/"), None);
+        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
+            ContactInfo::default(),
+        )));
+
+        let rrm = RpcRequestMiddleware::new(PathBuf::from("/"), None, cluster_info.clone(), None);
         let rrm_with_snapshot_config = RpcRequestMiddleware::new(
             PathBuf::from("/"),
             Some(SnapshotConfig {
@@ -320,6 +411,8 @@ mod tests {
                 snapshot_package_output_path: PathBuf::from("/"),
                 snapshot_path: PathBuf::from("/"),
             }),
+            cluster_info,
+            None,
         );
 
         assert!(rrm.is_get_path("/genesis.tar.bz2"));
@@ -341,4 +434,95 @@ mod tests {
         assert!(!rrm.is_get_path(".."));
         assert!(!rrm.is_get_path("🎣"));
     }
+
+    #[test]
+    fn test_health_check_with_no_trusted_validators() {
+        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
+            ContactInfo::default(),
+        )));
+
+        let rm = RpcRequestMiddleware::new(PathBuf::from("/"), None, cluster_info.clone(), None);
+        assert_eq!(rm.health_check(), "ok");
+    }
+
+    #[test]
+    fn test_health_check_with_trusted_validators() {
+        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
+            ContactInfo::default(),
+        )));
+
+        let trusted_validators = vec![Pubkey::new_rand(), Pubkey::new_rand(), Pubkey::new_rand()];
+        let rm = RpcRequestMiddleware::new(
+            PathBuf::from("/"),
+            None,
+            cluster_info.clone(),
+            Some(trusted_validators.clone().into_iter().collect()),
+        );
+
+        // No account hashes for this node or any trusted validators == "behind"
+        assert_eq!(rm.health_check(), "behind");
+
+        // No account hashes for any trusted validators == "behind"
+        {
+            let mut cluster_info = cluster_info.write().unwrap();
+            cluster_info
+                .push_accounts_hashes(vec![(1000, Hash::default()), (900, Hash::default())]);
+        }
+        assert_eq!(rm.health_check(), "behind");
+
+        // This node is ahead of the trusted validators == "ok"
+        {
+            let mut cluster_info = cluster_info.write().unwrap();
+            cluster_info
+                .gossip
+                .crds
+                .insert(
+                    CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
+                        trusted_validators[0].clone(),
+                        vec![
+                            (1, Hash::default()),
+                            (1001, Hash::default()),
+                            (2, Hash::default()),
+                        ],
+                    ))),
+                    1,
+                )
+                .unwrap();
+        }
+        assert_eq!(rm.health_check(), "ok");
+
+        // Node is slightly behind the trusted validators == "ok"
+        {
+            let mut cluster_info = cluster_info.write().unwrap();
+            cluster_info
+                .gossip
+                .crds
+                .insert(
+                    CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
+                        trusted_validators[1].clone(),
+                        vec![(1000 + HEALTH_CHECK_SLOT_DISTANCE - 1, Hash::default())],
+                    ))),
+                    1,
+                )
+                .unwrap();
+        }
+        assert_eq!(rm.health_check(), "ok");
+
+        // Node is far behind the trusted validators == "behind"
+        {
+            let mut cluster_info = cluster_info.write().unwrap();
+            cluster_info
+                .gossip
+                .crds
+                .insert(
+                    CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
+                        trusted_validators[2].clone(),
+                        vec![(1000 + HEALTH_CHECK_SLOT_DISTANCE, Hash::default())],
+                    ))),
+                    1,
+                )
+                .unwrap();
+        }
+        assert_eq!(rm.health_check(), "behind");
+    }
 }

core/src/rpc_subscriptions.rs

@@ -8,8 +8,10 @@ use jsonrpc_pubsub::{
     SubscriptionId,
 };
 use serde::Serialize;
-use solana_client::rpc_response::{Response, RpcAccount, RpcKeyedAccount, RpcResponseContext};
-use solana_ledger::bank_forks::BankForks;
+use solana_client::rpc_response::{
+    Response, RpcAccount, RpcKeyedAccount, RpcResponseContext, RpcSignatureResult,
+};
+use solana_ledger::{bank_forks::BankForks, blockstore::Blockstore};
 use solana_runtime::bank::Bank;
 use solana_sdk::{
     account::Account, clock::Slot, pubkey::Pubkey, signature::Signature, transaction,
@@ -66,7 +68,7 @@ type RpcProgramSubscriptions = RwLock<
 type RpcSignatureSubscriptions = RwLock<
     HashMap<
         Signature,
-        HashMap<SubscriptionId, (Sink<Response<transaction::Result<()>>>, Confirmations)>,
+        HashMap<SubscriptionId, (Sink<Response<RpcSignatureResult>>, Confirmations)>,
     >,
 >;
 type RpcSlotSubscriptions = RwLock<HashMap<SubscriptionId, Sink<SlotInfo>>>;
@@ -207,11 +209,15 @@ fn filter_account_result(
     Box::new(iter::empty())
 }
 
-fn filter_signature_result<S>(result: Option<S>, _root: Slot) -> Box<dyn Iterator<Item = S>>
-where
-    S: 'static + Clone + Serialize,
-{
-    Box::new(result.into_iter())
+fn filter_signature_result(
+    result: Option<transaction::Result<()>>,
+    _root: Slot,
+) -> Box<dyn Iterator<Item = RpcSignatureResult>> {
+    Box::new(
+        result
+            .into_iter()
+            .map(|result| RpcSignatureResult { err: result.err() }),
+    )
 }
 
 fn filter_program_results(
@@ -240,15 +246,6 @@ pub struct RpcSubscriptions {
     exit: Arc<AtomicBool>,
 }
 
-impl Default for RpcSubscriptions {
-    fn default() -> Self {
-        Self::new(
-            &Arc::new(AtomicBool::new(false)),
-            Arc::new(RwLock::new(BlockCommitmentCache::default())),
-        )
-    }
-}
-
 impl Drop for RpcSubscriptions {
     fn drop(&mut self) {
         self.shutdown().unwrap_or_else(|err| {
@@ -318,6 +315,15 @@ impl RpcSubscriptions {
         }
     }
 
+    pub fn default_with_blockstore(blockstore: Arc<Blockstore>) -> Self {
+        Self::new(
+            &Arc::new(AtomicBool::new(false)),
+            Arc::new(RwLock::new(BlockCommitmentCache::default_with_blockstore(
+                blockstore,
+            ))),
+        )
+    }
+
     fn check_account(
         pubkey: &Pubkey,
         bank_forks: &Arc<RwLock<BankForks>>,
@@ -430,7 +436,7 @@ impl RpcSubscriptions {
         signature: Signature,
         confirmations: Option<Confirmations>,
         sub_id: SubscriptionId,
-        subscriber: Subscriber<Response<transaction::Result<()>>>,
+        subscriber: Subscriber<Response<RpcSignatureResult>>,
     ) {
         let mut subscriptions = self.signature_subscriptions.write().unwrap();
         add_subscription(
@@ -610,11 +616,15 @@ impl RpcSubscriptions {
 #[cfg(test)]
 pub(crate) mod tests {
     use super::*;
-    use crate::commitment::BlockCommitment;
+    use crate::{
+        commitment::BlockCommitment,
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+    };
     use jsonrpc_core::futures::{self, stream::Stream};
     use jsonrpc_pubsub::typed::Subscriber;
+    use serial_test_derive::serial;
     use solana_budget_program;
-    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use solana_ledger::get_tmp_ledger_path;
     use solana_sdk::{
         signature::{Keypair, Signer},
         system_transaction,
@@ -648,12 +658,15 @@ pub(crate) mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_check_account_subscribe() {
         let GenesisConfigInfo {
             genesis_config,
             mint_keypair,
             ..
         } = create_genesis_config(100);
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
@@ -680,7 +693,9 @@ pub(crate) mod tests {
         let exit = Arc::new(AtomicBool::new(false));
         let subscriptions = RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         );
         subscriptions.add_account_subscription(alice.pubkey(), None, sub_id.clone(), subscriber);
 
@@ -720,12 +735,15 @@ pub(crate) mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_check_program_subscribe() {
         let GenesisConfigInfo {
             genesis_config,
             mint_keypair,
             ..
         } = create_genesis_config(100);
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
@@ -752,7 +770,9 @@ pub(crate) mod tests {
         let exit = Arc::new(AtomicBool::new(false));
         let subscriptions = RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         );
         subscriptions.add_program_subscription(
             solana_budget_program::id(),
@@ -800,12 +820,15 @@ pub(crate) mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_check_signature_subscribe() {
         let GenesisConfigInfo {
             genesis_config,
             mint_keypair,
             ..
         } = create_genesis_config(100);
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let mut bank_forks = BankForks::new(0, bank);
@@ -844,7 +867,8 @@ pub(crate) mod tests {
         let mut block_commitment = HashMap::new();
         block_commitment.entry(0).or_insert(cache0.clone());
         block_commitment.entry(1).or_insert(cache1.clone());
-        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 10, bank1, 0);
+        let block_commitment_cache =
+            BlockCommitmentCache::new(block_commitment, 0, 10, bank1, blockstore, 0);
 
         let exit = Arc::new(AtomicBool::new(false));
         let subscriptions =
@@ -890,7 +914,7 @@ pub(crate) mod tests {
         }
 
         subscriptions.notify_subscribers(1, &bank_forks);
-        let expected_res: Option<transaction::Result<()>> = Some(Ok(()));
+        let expected_res = RpcSignatureResult { err: None };
 
         struct Notification {
             slot: Slot,
@@ -941,14 +965,19 @@ pub(crate) mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_check_slot_subscribe() {
         let (subscriber, _id_receiver, transport_receiver) =
             Subscriber::new_test("slotNotification");
         let sub_id = SubscriptionId::Number(0 as u64);
         let exit = Arc::new(AtomicBool::new(false));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let subscriptions = RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         );
         subscriptions.add_slot_subscription(sub_id.clone(), subscriber);
 
@@ -982,14 +1011,19 @@ pub(crate) mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_check_root_subscribe() {
         let (subscriber, _id_receiver, mut transport_receiver) =
             Subscriber::new_test("rootNotification");
         let sub_id = SubscriptionId::Number(0 as u64);
         let exit = Arc::new(AtomicBool::new(false));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let subscriptions = RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         );
         subscriptions.add_root_subscription(sub_id.clone(), subscriber);
 
@@ -1022,6 +1056,7 @@ pub(crate) mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_add_and_remove_subscription() {
         let mut subscriptions: HashMap<u64, HashMap<SubscriptionId, (Sink<()>, Confirmations)>> =
             HashMap::new();

core/src/sendmmsg.rs

@@ -164,8 +164,8 @@ mod tests {
             .map(|_| (vec![0u8; PACKET_DATA_SIZE], &addr))
             .collect();
 
-        let sent = send_mmsg(&sender, &mut packets).ok();
-        assert_eq!(sent, Some(32));
+        let sent = send_mmsg(&sender, &mut packets);
+        assert_matches!(sent, Ok(32));
 
         let mut packets = vec![Packet::default(); 32];
         let recv = recv_mmsg(&reader, &mut packets[..]).unwrap().1;
@@ -192,8 +192,8 @@ mod tests {
             })
             .collect();
 
-        let sent = send_mmsg(&sender, &mut packets).ok();
-        assert_eq!(sent, Some(32));
+        let sent = send_mmsg(&sender, &mut packets);
+        assert_matches!(sent, Ok(32));
 
         let mut packets = vec![Packet::default(); 32];
         let recv = recv_mmsg(&reader, &mut packets[..]).unwrap().1;
@@ -226,9 +226,8 @@ mod tests {
             &sender,
             &mut packet.data[..packet.meta.size],
             &[&addr, &addr2, &addr3, &addr4],
-        )
-        .ok();
-        assert_eq!(sent, Some(4));
+        );
+        assert_matches!(sent, Ok(4));
 
         let mut packets = vec![Packet::default(); 32];
         let recv = recv_mmsg(&reader, &mut packets[..]).unwrap().1;

core/src/serve_repair.rs

@@ -1,26 +1,24 @@
+use crate::packet::limited_deserialize;
+use crate::streamer::{PacketReceiver, PacketSender};
 use crate::{
     cluster_info::{ClusterInfo, ClusterInfoError},
-    cluster_slots::ClusterSlots,
     contact_info::ContactInfo,
-    repair_service::RepairStats,
+    packet::Packet,
     result::{Error, Result},
-    weighted_shuffle::weighted_best,
 };
 use bincode::serialize;
+use rand::{thread_rng, Rng};
 use solana_ledger::blockstore::Blockstore;
 use solana_measure::measure::Measure;
 use solana_measure::thread_mem_usage;
 use solana_metrics::{datapoint_debug, inc_new_counter_debug};
-use solana_perf::packet::{limited_deserialize, Packet, Packets, PacketsRecycler};
+use solana_perf::packet::{Packets, PacketsRecycler};
 use solana_sdk::{
     clock::Slot,
-    pubkey::Pubkey,
     signature::{Keypair, Signer},
     timing::duration_as_ms,
 };
-use solana_streamer::streamer::{PacketReceiver, PacketSender};
 use std::{
-    collections::HashMap,
     net::SocketAddr,
     sync::atomic::{AtomicBool, Ordering},
     sync::{Arc, RwLock},
@@ -48,17 +46,6 @@ impl RepairType {
     }
 }
 
-#[derive(Default)]
-pub struct ServeRepairStats {
-    pub total_packets: usize,
-    pub dropped_packets: usize,
-    pub processed: usize,
-    pub self_repair: usize,
-    pub window_index: usize,
-    pub highest_window_index: usize,
-    pub orphan: usize,
-}
-
 /// Window protocol messages
 #[derive(Serialize, Deserialize, Debug)]
 enum RepairProtocol {
@@ -75,8 +62,6 @@ pub struct ServeRepair {
     cluster_info: Arc<RwLock<ClusterInfo>>,
 }
 
-type RepairCache = HashMap<Slot, (Vec<ContactInfo>, Vec<(u64, usize)>)>;
-
 impl ServeRepair {
     /// Without a valid keypair gossip will not function. Only useful for tests.
     pub fn new_with_invalid_keypair(contact_info: ContactInfo) -> Self {
@@ -119,7 +104,6 @@ impl ServeRepair {
         from_addr: &SocketAddr,
         blockstore: Option<&Arc<Blockstore>>,
         request: RepairProtocol,
-        stats: &mut ServeRepairStats,
     ) -> Option<Packets> {
         let now = Instant::now();
 
@@ -127,14 +111,18 @@ impl ServeRepair {
         let my_id = me.read().unwrap().keypair.pubkey();
         let from = Self::get_repair_sender(&request);
         if from.id == my_id {
-            stats.self_repair += 1;
+            warn!(
+                "{}: Ignored received repair request from ME {}",
+                my_id, from.id,
+            );
+            inc_new_counter_debug!("serve_repair-handle-repair--eq", 1);
             return None;
         }
 
         let (res, label) = {
             match &request {
                 RepairProtocol::WindowIndex(from, slot, shred_index) => {
-                    stats.window_index += 1;
+                    inc_new_counter_debug!("serve_repair-request-window-index", 1);
                     (
                         Self::run_window_request(
                             recycler,
@@ -150,7 +138,7 @@ impl ServeRepair {
                 }
 
                 RepairProtocol::HighestWindowIndex(_, slot, highest_index) => {
-                    stats.highest_window_index += 1;
+                    inc_new_counter_debug!("serve_repair-request-highest-window-index", 1);
                     (
                         Self::run_highest_window_request(
                             recycler,
@@ -163,7 +151,7 @@ impl ServeRepair {
                     )
                 }
                 RepairProtocol::Orphan(_, slot) => {
-                    stats.orphan += 1;
+                    inc_new_counter_debug!("serve_repair-request-orphan", 1);
                     (
                         Self::run_orphan(
                             recycler,
@@ -197,7 +185,6 @@ impl ServeRepair {
         blockstore: Option<&Arc<Blockstore>>,
         requests_receiver: &PacketReceiver,
         response_sender: &PacketSender,
-        stats: &mut ServeRepairStats,
         max_packets: &mut usize,
     ) -> Result<()> {
         //TODO cache connections
@@ -205,23 +192,17 @@ impl ServeRepair {
         let mut reqs_v = vec![requests_receiver.recv_timeout(timeout)?];
         let mut total_packets = reqs_v[0].packets.len();
 
-        let mut dropped_packets = 0;
         while let Ok(more) = requests_receiver.try_recv() {
             total_packets += more.packets.len();
             if total_packets < *max_packets {
                 // Drop the rest in the channel in case of dos
                 reqs_v.push(more);
-            } else {
-                dropped_packets += more.packets.len();
             }
         }
 
-        stats.dropped_packets += dropped_packets;
-        stats.total_packets += total_packets;
-
         let mut time = Measure::start("repair::handle_packets");
         for reqs in reqs_v {
-            Self::handle_packets(obj, &recycler, blockstore, reqs, response_sender, stats);
+            Self::handle_packets(obj, &recycler, blockstore, reqs, response_sender);
         }
         time.stop();
         if total_packets >= *max_packets {
@@ -234,34 +215,6 @@ impl ServeRepair {
         Ok(())
     }
 
-    fn report_reset_stats(me: &Arc<RwLock<Self>>, stats: &mut ServeRepairStats) {
-        if stats.self_repair > 0 {
-            let my_id = me.read().unwrap().keypair.pubkey();
-            warn!(
-                "{}: Ignored received repair requests from ME: {}",
-                my_id, stats.self_repair,
-            );
-            inc_new_counter_debug!("serve_repair-handle-repair--eq", stats.self_repair);
-        }
-
-        inc_new_counter_info!("serve_repair-total_packets", stats.total_packets);
-        inc_new_counter_info!("serve_repair-dropped_packets", stats.dropped_packets);
-
-        debug!(
-            "repair_listener: total_packets: {} passed: {}",
-            stats.total_packets, stats.processed
-        );
-
-        inc_new_counter_debug!("serve_repair-request-window-index", stats.window_index);
-        inc_new_counter_debug!(
-            "serve_repair-request-highest-window-index",
-            stats.highest_window_index
-        );
-        inc_new_counter_debug!("serve_repair-request-orphan", stats.orphan);
-
-        *stats = ServeRepairStats::default();
-    }
-
     pub fn listen(
         me: Arc<RwLock<Self>>,
         blockstore: Option<Arc<Blockstore>>,
@@ -274,8 +227,6 @@ impl ServeRepair {
         Builder::new()
             .name("solana-repair-listen".to_string())
             .spawn(move || {
-                let mut last_print = Instant::now();
-                let mut stats = ServeRepairStats::default();
                 let mut max_packets = 1024;
                 loop {
                     let result = Self::run_listen(
@@ -284,7 +235,6 @@ impl ServeRepair {
                         blockstore.as_ref(),
                         &requests_receiver,
                         &response_sender,
-                        &mut stats,
                         &mut max_packets,
                     );
                     match result {
@@ -294,10 +244,6 @@ impl ServeRepair {
                     if exit.load(Ordering::Relaxed) {
                         return;
                     }
-                    if last_print.elapsed().as_secs() > 2 {
-                        Self::report_reset_stats(&me, &mut stats);
-                        last_print = Instant::now();
-                    }
                     thread_mem_usage::datapoint("solana-repair-listen");
                 }
             })
@@ -310,7 +256,6 @@ impl ServeRepair {
         blockstore: Option<&Arc<Blockstore>>,
         packets: Packets,
         response_sender: &PacketSender,
-        stats: &mut ServeRepairStats,
     ) {
         // iter over the packets, collect pulls separately and process everything else
         let allocated = thread_mem_usage::Allocatedp::default();
@@ -320,9 +265,7 @@ impl ServeRepair {
             limited_deserialize(&packet.data[..packet.meta.size])
                 .into_iter()
                 .for_each(|request| {
-                    stats.processed += 1;
-                    let rsp =
-                        Self::handle_repair(me, recycler, &from_addr, blockstore, request, stats);
+                    let rsp = Self::handle_repair(me, recycler, &from_addr, blockstore, request);
                     if let Some(rsp) = rsp {
                         let _ignore_disconnect = response_sender.send(rsp);
                     }
@@ -352,50 +295,44 @@ impl ServeRepair {
         Ok(out)
     }
 
-    pub fn repair_request(
-        &self,
-        cluster_slots: &ClusterSlots,
-        repair_request: &RepairType,
-        cache: &mut RepairCache,
-        repair_stats: &mut RepairStats,
-    ) -> Result<(SocketAddr, Vec<u8>)> {
+    pub fn repair_request(&self, repair_request: &RepairType) -> Result<(SocketAddr, Vec<u8>)> {
         // find a peer that appears to be accepting replication and has the desired slot, as indicated
         // by a valid tvu port location
-        if cache.get(&repair_request.slot()).is_none() {
-            let repair_peers: Vec<_> = self
-                .cluster_info
-                .read()
-                .unwrap()
-                .repair_peers(repair_request.slot());
-            if repair_peers.is_empty() {
-                return Err(ClusterInfoError::NoPeers.into());
-            }
-            let weights = cluster_slots.compute_weights(repair_request.slot(), &repair_peers);
-            cache.insert(repair_request.slot(), (repair_peers, weights));
+        let valid: Vec<_> = self
+            .cluster_info
+            .read()
+            .unwrap()
+            .repair_peers(repair_request.slot());
+        if valid.is_empty() {
+            return Err(ClusterInfoError::NoPeers.into());
         }
-        let (repair_peers, weights) = cache.get(&repair_request.slot()).unwrap();
-        let n = weighted_best(&weights, Pubkey::new_rand().to_bytes());
-        let addr = repair_peers[n].serve_repair; // send the request to the peer's serve_repair port
-        let out = self.map_repair_request(repair_request, repair_stats)?;
+        let n = thread_rng().gen::<usize>() % valid.len();
+        let addr = valid[n].serve_repair; // send the request to the peer's serve_repair port
+        let out = self.map_repair_request(repair_request)?;
+
         Ok((addr, out))
     }
 
-    pub fn map_repair_request(
-        &self,
-        repair_request: &RepairType,
-        repair_stats: &mut RepairStats,
-    ) -> Result<Vec<u8>> {
+    pub fn map_repair_request(&self, repair_request: &RepairType) -> Result<Vec<u8>> {
         match repair_request {
             RepairType::Shred(slot, shred_index) => {
-                repair_stats.shred.update(*slot);
+                datapoint_debug!(
+                    "serve_repair-repair",
+                    ("repair-slot", *slot, i64),
+                    ("repair-ix", *shred_index, i64)
+                );
                 Ok(self.window_index_request_bytes(*slot, *shred_index)?)
             }
             RepairType::HighestShred(slot, shred_index) => {
-                repair_stats.highest_shred.update(*slot);
+                datapoint_info!(
+                    "serve_repair-repair_highest",
+                    ("repair-highest-slot", *slot, i64),
+                    ("repair-highest-ix", *shred_index, i64)
+                );
                 Ok(self.window_highest_index_request_bytes(*slot, *shred_index)?)
             }
             RepairType::Orphan(slot) => {
-                repair_stats.orphan.update(*slot);
+                datapoint_info!("serve_repair-repair_orphan", ("repair-orphan", *slot, i64));
                 Ok(self.orphan_bytes(*slot)?)
             }
         }
@@ -652,16 +589,10 @@ mod tests {
 
     #[test]
     fn window_index_request() {
-        let cluster_slots = ClusterSlots::default();
         let me = ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp());
         let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(me)));
         let serve_repair = ServeRepair::new(cluster_info.clone());
-        let rv = serve_repair.repair_request(
-            &cluster_slots,
-            &RepairType::Shred(0, 0),
-            &mut HashMap::new(),
-            &mut RepairStats::default(),
-        );
+        let rv = serve_repair.repair_request(&RepairType::Shred(0, 0));
         assert_matches!(rv, Err(Error::ClusterInfoError(ClusterInfoError::NoPeers)));
 
         let serve_repair_addr = socketaddr!([127, 0, 0, 1], 1243);
@@ -682,12 +613,7 @@ mod tests {
         };
         cluster_info.write().unwrap().insert_info(nxt.clone());
         let rv = serve_repair
-            .repair_request(
-                &cluster_slots,
-                &RepairType::Shred(0, 0),
-                &mut HashMap::new(),
-                &mut RepairStats::default(),
-            )
+            .repair_request(&RepairType::Shred(0, 0))
             .unwrap();
         assert_eq!(nxt.serve_repair, serve_repair_addr);
         assert_eq!(rv.0, nxt.serve_repair);
@@ -714,12 +640,7 @@ mod tests {
         while !one || !two {
             //this randomly picks an option, so eventually it should pick both
             let rv = serve_repair
-                .repair_request(
-                    &cluster_slots,
-                    &RepairType::Shred(0, 0),
-                    &mut HashMap::new(),
-                    &mut RepairStats::default(),
-                )
+                .repair_request(&RepairType::Shred(0, 0))
                 .unwrap();
             if rv.0 == serve_repair_addr {
                 one = true;