cli: Add --follow option to catchup command to allow for easy ongoing monitoring between two nodes (bp #9260) (#9278)

automerge

.appveyor.yml

@@ -1,42 +0,0 @@
-version: '{build}'
-
-branches:
-  only:
-  - master
-  - /^v[0-9.]+\.[0-9.]+/
-
-cache:
-  - '%USERPROFILE%\.cargo'
-  - '%APPVEYOR_BUILD_FOLDER%\target'
-
-clone_folder: d:\projects\solana
-
-build_script:
-  - bash ci/publish-tarball.sh
-
-notifications:
-  - provider: Slack
-    incoming_webhook:
-      secure: GJsBey+F5apAtUm86MHVJ68Uqa6WN1SImcuIc4TsTZrDhA8K1QWUNw9FFQPybUWDyOcS5dly3kubnUqlGt9ux6Ad2efsfRIQYWv0tOVXKeY=
-    channel: ci-status
-    on_build_success: false
-    on_build_failure: true
-    on_build_status_changed: true
-
-deploy:
-  - provider: S3
-    access_key_id:
-      secure: fTbJl6JpFebR40J7cOWZ2mXBa3kIvEiXgzxAj6L3N7A=
-    secret_access_key:
-      secure: vItsBXb2rEFLvkWtVn/Rcxu5a5+2EwC+b7GsA0waJy9hXh6XuBAD0lnHd9re3g/4
-    bucket: release.solana.com
-    region: us-west-1
-    set_public: true
-
-  - provider: GitHub
-    auth_token:
-      secure: 81fEmPZ0cV1wLtNuUrcmtgxKF6ROQF1+/ft5m+fHX21z6PoeCbaNo8cTyLioWBj7
-    draft: false
-    prerelease: false
-    on:
-      appveyor_repo_tag: true

.gitignore

@@ -1,5 +1,6 @@
 /docs/html/
 /docs/src/tests.ok
+/docs/src/cli/usage.md
 /docs/src/.gitbook/assets/*.svg
 /farf/
 /solana-release/

.mergify.yml

@@ -19,20 +19,13 @@ pull_request_rules:
       label:
         add:
           - automerge
-  - name: v0.23 backport
-    conditions:
-      - base=master
-      - label=v0.23
-    actions:
-      backport:
-        branches:
-          - v0.23
   - name: v1.0 backport
     conditions:
       - base=master
       - label=v1.0
     actions:
       backport:
+        ignore_conflicts: true
         branches:
           - v1.0
   - name: v1.1 backport
@@ -41,6 +34,7 @@ pull_request_rules:
       - label=v1.1
     actions:
       backport:
+        ignore_conflicts: true
         branches:
           - v1.1
   - name: v1.2 backport
@@ -49,5 +43,6 @@ pull_request_rules:
       - label=v1.2
     actions:
       backport:
+        ignore_conflicts: true
         branches:
           - v1.2

CONTRIBUTING.md

@@ -45,7 +45,7 @@ $ git pull --rebase upstream master
 
 If there are no functional changes, PRs can be very large and that's no
 problem. If, however, your changes are making meaningful changes or additions,
-then about 1000 lines of changes is about the most you should ask a Solana
+then about 1,000 lines of changes is about the most you should ask a Solana
 maintainer to review.
 
 ### Should I send small PRs as I develop large, new components?

Cargo.toml

@@ -3,6 +3,7 @@ members = [
     "bench-exchange",
     "bench-streamer",
     "bench-tps",
+    "accounts-bench",
     "banking-bench",
     "chacha",
     "chacha-cuda",
@@ -10,6 +11,8 @@ members = [
     "cli-config",
     "client",
     "core",
+    "dos",
+    "download-utils",
     "faucet",
     "perf",
     "validator",
@@ -24,6 +27,7 @@ members = [
     "logger",
     "log-analyzer",
     "merkle-tree",
+    "streamer",
     "measure",
     "metrics",
     "net-shaper",
@@ -48,7 +52,10 @@ members = [
     "sdk",
     "sdk-c",
     "scripts",
+    "stake-accounts",
+    "stake-monitor",
     "sys-tuner",
+    "transaction-status",
     "upload-perf",
     "net-utils",
     "vote-signer",

README.md

@@ -9,46 +9,7 @@ Blockchain Rebuilt for Scale
 Solana™ is a new blockchain architecture built from the ground up for scale. The architecture supports
 up to 710 thousand transactions per second on a gigabit network.
 
-Documentation
-===
-
-Before you jump into the code, review the documentation [Solana: Blockchain Rebuilt for Scale](https://docs.solana.com).
-
-(The _latest_ development version of the docs is [available here](https://docs.solana.com/v/master).)
-
-Release Binaries
-===
-Official release binaries are available at [Github Releases](https://github.com/solana-labs/solana/releases).
-
-Additionally we provide pre-release binaries for the latest code on the edge and
-beta channels.  Note that these pre-release binaries may be less stable than an
-official release.
-
-### Edge channel
-#### Linux (x86_64-unknown-linux-gnu)
-* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-unknown-linux-gnu.tar.bz2)
-* [solana-install-init](http://release.solana.com/edge/solana-install-init-x86_64-unknown-linux-gnu) as a stand-alone executable
-#### mac OS (x86_64-apple-darwin)
-* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-apple-darwin.tar.bz2)
-* [solana-install-init](http://release.solana.com/edge/solana-install-init-x86_64-apple-darwin) as a stand-alone executable
-#### Windows (x86_64-pc-windows-msvc)
-* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-pc-windows-msvc.tar.bz2)
-* [solana-install-init.exe](http://release.solana.com/edge/solana-install-init-x86_64-pc-windows-msvc.exe) as a stand-alone executable
-#### All platforms
-* [solana-metrics.tar.bz2](http://release.solana.com.s3.amazonaws.com/edge/solana-metrics.tar.bz2)
-
-### Beta channel
-#### Linux (x86_64-unknown-linux-gnu)
-* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-unknown-linux-gnu.tar.bz2)
-* [solana-install-init](http://release.solana.com/beta/solana-install-init-x86_64-unknown-linux-gnu) as a stand-alone executable
-#### mac OS (x86_64-apple-darwin)
-* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-apple-darwin.tar.bz2)
-* [solana-install-init](http://release.solana.com/beta/solana-install-init-x86_64-apple-darwin) as a stand-alone executable
-#### Windows (x86_64-pc-windows-msvc)
-* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-pc-windows-msvc.tar.bz2)
-* [solana-install-init.exe](http://release.solana.com/beta/solana-install-init-x86_64-pc-windows-msvc.exe) as a stand-alone executable
-#### All platforms
-* [solana-metrics.tar.bz2](http://release.solana.com.s3.amazonaws.com/beta/solana-metrics.tar.bz2)
+Read all about it at [Solana: Blockchain Rebuilt for Scale](https://docs.solana.com/v/master).
 
 Developing
 ===

accounts-bench/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.com>"]
+edition = "2018"
+name = "solana-accounts-bench"
+version = "1.1.1"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+
+[dependencies]
+log = "0.4.6"
+rayon = "1.3.0"
+solana-logger = { path = "../logger", version = "1.1.1" }
+solana-runtime = { path = "../runtime", version = "1.1.1" }
+solana-measure = { path = "../measure", version = "1.1.1" }
+solana-sdk = { path = "../sdk", version = "1.1.1" }
+rand = "0.6.5"
+clap = "2.33.0"
+crossbeam-channel = "0.4"

accounts-bench/src/main.rs

@@ -0,0 +1,103 @@
+use clap::{value_t, App, Arg};
+use rayon::prelude::*;
+use solana_measure::measure::Measure;
+use solana_runtime::accounts::{create_test_accounts, update_accounts, Accounts};
+use solana_sdk::pubkey::Pubkey;
+use std::collections::HashMap;
+use std::fs;
+use std::path::PathBuf;
+
+fn main() {
+    solana_logger::setup();
+
+    let matches = App::new("crate")
+        .about("about")
+        .version("version")
+        .arg(
+            Arg::with_name("num_slots")
+                .long("num_slots")
+                .takes_value(true)
+                .value_name("SLOTS")
+                .help("Number of slots to store to."),
+        )
+        .arg(
+            Arg::with_name("num_accounts")
+                .long("num_accounts")
+                .takes_value(true)
+                .value_name("NUM_ACCOUNTS")
+                .help("Total number of accounts"),
+        )
+        .arg(
+            Arg::with_name("iterations")
+                .long("iterations")
+                .takes_value(true)
+                .value_name("ITERATIONS")
+                .help("Number of bench iterations"),
+        )
+        .arg(
+            Arg::with_name("clean")
+                .long("clean")
+                .takes_value(false)
+                .help("Run clean"),
+        )
+        .get_matches();
+
+    let num_slots = value_t!(matches, "num_slots", usize).unwrap_or(4);
+    let num_accounts = value_t!(matches, "num_accounts", usize).unwrap_or(10_000);
+    let iterations = value_t!(matches, "iterations", usize).unwrap_or(20);
+    let clean = matches.is_present("clean");
+    println!("clean: {:?}", clean);
+
+    let path = PathBuf::from("farf/accounts-bench");
+    if fs::remove_dir_all(path.clone()).is_err() {
+        println!("Warning: Couldn't remove {:?}", path);
+    }
+    let accounts = Accounts::new(vec![path]);
+    println!("Creating {} accounts", num_accounts);
+    let mut create_time = Measure::start("create accounts");
+    let pubkeys: Vec<_> = (0..num_slots)
+        .into_par_iter()
+        .map(|slot| {
+            let mut pubkeys: Vec<Pubkey> = vec![];
+            create_test_accounts(
+                &accounts,
+                &mut pubkeys,
+                num_accounts / num_slots,
+                slot as u64,
+            );
+            pubkeys
+        })
+        .collect();
+    let pubkeys: Vec<_> = pubkeys.into_iter().flatten().collect();
+    create_time.stop();
+    println!(
+        "created {} accounts in {} slots {}",
+        (num_accounts / num_slots) * num_slots,
+        num_slots,
+        create_time
+    );
+    let mut ancestors: HashMap<u64, usize> = vec![(0, 0)].into_iter().collect();
+    for i in 1..num_slots {
+        ancestors.insert(i as u64, i - 1);
+        accounts.add_root(i as u64);
+    }
+    for x in 0..iterations {
+        if clean {
+            let mut time = Measure::start("clean");
+            accounts.accounts_db.clean_accounts();
+            time.stop();
+            println!("{}", time);
+            for slot in 0..num_slots {
+                update_accounts(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64);
+                accounts.add_root((x * num_slots + slot) as u64);
+            }
+        } else {
+            let mut pubkeys: Vec<Pubkey> = vec![];
+            let mut time = Measure::start("hash");
+            let hash = accounts.accounts_db.update_accounts_hash(0, &ancestors);
+            time.stop();
+            println!("hash: {} {}", hash, time);
+            create_test_accounts(&accounts, &mut pubkeys, 1, 0);
+        }
+    }
+}

archiver-lib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-archiver-lib"
-version = "1.0.10"
+version = "1.1.1"
 description = "Solana Archiver Library"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,30 +10,31 @@ edition = "2018"
 
 [dependencies]
 bincode = "1.2.1"
-crossbeam-channel = "0.3"
+crossbeam-channel = "0.4"
 ed25519-dalek = "=1.0.0-pre.1"
 log = "0.4.8"
 rand = "0.6.5"
 rand_chacha = "0.1.1"
-solana-client = { path = "../client", version = "1.0.10" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.10" }
+solana-client = { path = "../client", version = "1.1.1" }
+solana-storage-program = { path = "../programs/storage", version = "1.1.1" }
 thiserror = "1.0"
-serde = "1.0.104"
-serde_json = "1.0.46"
+serde = "1.0.105"
+serde_json = "1.0.48"
 serde_derive = "1.0.103"
-solana-net-utils = { path = "../net-utils", version = "1.0.10" }
-solana-chacha = { path = "../chacha", version = "1.0.10" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.10" }
-solana-ledger = { path = "../ledger", version = "1.0.10" }
-solana-logger = { path = "../logger", version = "1.0.10" }
-solana-perf = { path = "../perf", version = "1.0.10" }
-solana-sdk = { path = "../sdk", version = "1.0.10" }
-solana-core = { path = "../core", version = "1.0.10" }
-solana-archiver-utils = { path = "../archiver-utils", version = "1.0.10" }
-solana-metrics = { path = "../metrics", version = "1.0.10" }
+solana-net-utils = { path = "../net-utils", version = "1.1.1" }
+solana-chacha = { path = "../chacha", version = "1.1.1" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.1.1" }
+solana-ledger = { path = "../ledger", version = "1.1.1" }
+solana-logger = { path = "../logger", version = "1.1.1" }
+solana-perf = { path = "../perf", version = "1.1.1" }
+solana-sdk = { path = "../sdk", version = "1.1.1" }
+solana-core = { path = "../core", version = "1.1.1" }
+solana-streamer = { path = "../streamer", version = "1.1.1" }
+solana-archiver-utils = { path = "../archiver-utils", version = "1.1.1" }
+solana-metrics = { path = "../metrics", version = "1.1.1" }
 
 [dev-dependencies]
-hex = "0.4.0"
+hex = "0.4.2"
 
 [lib]
 name = "solana_archiver_lib"

archiver-lib/src/archiver.rs

@@ -10,16 +10,15 @@ use solana_client::{
 };
 use solana_core::{
     cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE},
+    cluster_slots::ClusterSlots,
     contact_info::ContactInfo,
     gossip_service::GossipService,
-    packet::{limited_deserialize, PACKET_DATA_SIZE},
     repair_service,
-    repair_service::{RepairService, RepairSlotRange, RepairStrategy},
+    repair_service::{RepairService, RepairSlotRange, RepairStats, RepairStrategy},
     serve_repair::ServeRepair,
     shred_fetch_stage::ShredFetchStage,
     sigverify_stage::{DisabledSigVerifier, SigVerifyStage},
     storage_stage::NUM_STORAGE_SAMPLES,
-    streamer::{receiver, responder, PacketReceiver},
     window_service::WindowService,
 };
 use solana_ledger::{
@@ -27,6 +26,7 @@ use solana_ledger::{
 };
 use solana_net_utils::bind_in_range;
 use solana_perf::packet::Packets;
+use solana_perf::packet::{limited_deserialize, PACKET_DATA_SIZE};
 use solana_perf::recycler::Recycler;
 use solana_sdk::packet::Packet;
 use solana_sdk::{
@@ -45,6 +45,7 @@ use solana_storage_program::{
     storage_contract::StorageContract,
     storage_instruction::{self, StorageAccountType},
 };
+use solana_streamer::streamer::{receiver, responder, PacketReceiver};
 use std::{
     io::{self, ErrorKind},
     net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
@@ -187,7 +188,7 @@ impl Archiver {
         let mut cluster_info = ClusterInfo::new(node.info.clone(), keypair.clone());
         cluster_info.set_entrypoint(cluster_entrypoint.clone());
         let cluster_info = Arc::new(RwLock::new(cluster_info));
-
+        let cluster_slots = Arc::new(ClusterSlots::default());
         // Note for now, this ledger will not contain any of the existing entries
         // in the ledger located at ledger_path, and will only append on newly received
         // entries after being passed to window_service
@@ -262,6 +263,7 @@ impl Archiver {
                     repair_socket,
                     shred_fetch_receiver,
                     slot_sender,
+                    cluster_slots,
                 ) {
                     Ok(window_service) => window_service,
                     Err(e) => {
@@ -400,6 +402,7 @@ impl Archiver {
     }
 
     // Find a segment to replicate and download it.
+    #[allow(clippy::too_many_arguments)]
     fn setup(
         meta: &mut ArchiverMeta,
         cluster_info: Arc<RwLock<ClusterInfo>>,
@@ -410,6 +413,7 @@ impl Archiver {
         repair_socket: Arc<UdpSocket>,
         shred_fetch_receiver: PacketReceiver,
         slot_sender: Sender<u64>,
+        cluster_slots: Arc<ClusterSlots>,
     ) -> Result<WindowService> {
         let slots_per_segment =
             match Self::get_segment_config(&cluster_info, meta.client_commitment) {
@@ -467,6 +471,7 @@ impl Archiver {
             RepairStrategy::RepairRange(repair_slot_range),
             &Arc::new(LeaderScheduleCache::default()),
             |_, _, _, _| true,
+            cluster_slots,
         );
         info!("waiting for ledger download");
         Self::wait_for_segment_download(
@@ -839,13 +844,14 @@ impl Archiver {
                 repair_service::MAX_REPAIR_LENGTH,
                 &repair_slot_range,
             );
+            let mut repair_stats = RepairStats::default();
             //iter over the repairs and send them
             if let Ok(repairs) = repairs {
                 let reqs: Vec<_> = repairs
                     .into_iter()
                     .filter_map(|repair_request| {
                         serve_repair
-                            .map_repair_request(&repair_request)
+                            .map_repair_request(&repair_request, &mut repair_stats)
                             .map(|result| ((archiver_info.gossip, result), repair_request))
                             .ok()
                     })

archiver-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-archiver-utils"
-version = "1.0.10"
+version = "1.1.1"
 description = "Solana Archiver Utils"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,15 +11,15 @@ edition = "2018"
 [dependencies]
 log = "0.4.8"
 rand = "0.6.5"
-solana-chacha = { path = "../chacha", version = "1.0.10" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.10" }
-solana-ledger = { path = "../ledger", version = "1.0.10" }
-solana-logger = { path = "../logger", version = "1.0.10" }
-solana-perf = { path = "../perf", version = "1.0.10" }
-solana-sdk = { path = "../sdk", version = "1.0.10" }
+solana-chacha = { path = "../chacha", version = "1.1.1" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.1.1" }
+solana-ledger = { path = "../ledger", version = "1.1.1" }
+solana-logger = { path = "../logger", version = "1.1.1" }
+solana-perf = { path = "../perf", version = "1.1.1" }
+solana-sdk = { path = "../sdk", version = "1.1.1" }
 
 [dev-dependencies]
-hex = "0.4.0"
+hex = "0.4.2"
 
 [lib]
 name = "solana_archiver_utils"

archiver/Cargo.toml

@@ -2,19 +2,19 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-archiver"
-version = "1.0.10"
+version = "1.1.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-console = "0.9.2"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.10" }
-solana-core = { path = "../core", version = "1.0.10" }
-solana-logger = { path = "../logger", version = "1.0.10" }
-solana-metrics = { path = "../metrics", version = "1.0.10" }
-solana-archiver-lib = { path = "../archiver-lib", version = "1.0.10" }
-solana-net-utils = { path = "../net-utils", version = "1.0.10" }
-solana-sdk = { path = "../sdk", version = "1.0.10" }
+console = "0.10.0"
+solana-clap-utils = { path = "../clap-utils", version = "1.1.1" }
+solana-core = { path = "../core", version = "1.1.1" }
+solana-logger = { path = "../logger", version = "1.1.1" }
+solana-metrics = { path = "../metrics", version = "1.1.1" }
+solana-archiver-lib = { path = "../archiver-lib", version = "1.1.1" }
+solana-net-utils = { path = "../net-utils", version = "1.1.1" }
+solana-sdk = { path = "../sdk", version = "1.1.1" }
 

banking-bench/Cargo.toml

@@ -2,19 +2,21 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.0.10"
+version = "1.1.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 log = "0.4.6"
-rayon = "1.2.0"
-solana-core = { path = "../core", version = "1.0.10" }
-solana-ledger = { path = "../ledger", version = "1.0.10" }
-solana-logger = { path = "../logger", version = "1.0.10" }
-solana-runtime = { path = "../runtime", version = "1.0.10" }
-solana-measure = { path = "../measure", version = "1.0.10" }
-solana-sdk = { path = "../sdk", version = "1.0.10" }
+rayon = "1.3.0"
+solana-core = { path = "../core", version = "1.1.1" }
+solana-streamer = { path = "../streamer", version = "1.1.1" }
+solana-perf = { path = "../perf", version = "1.1.1" }
+solana-ledger = { path = "../ledger", version = "1.1.1" }
+solana-logger = { path = "../logger", version = "1.1.1" }
+solana-runtime = { path = "../runtime", version = "1.1.1" }
+solana-measure = { path = "../measure", version = "1.1.1" }
+solana-sdk = { path = "../sdk", version = "1.1.1" }
 rand = "0.6.5"
-crossbeam-channel = "0.3"
+crossbeam-channel = "0.4"

banking-bench/src/main.rs

@@ -2,29 +2,36 @@ use crossbeam_channel::unbounded;
 use log::*;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
-use solana_core::banking_stage::{create_test_recorder, BankingStage};
-use solana_core::cluster_info::ClusterInfo;
-use solana_core::cluster_info::Node;
-use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-use solana_core::packet::to_packets_chunked;
-use solana_core::poh_recorder::PohRecorder;
-use solana_core::poh_recorder::WorkingBankEntry;
-use solana_ledger::bank_forks::BankForks;
-use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
+use solana_core::{
+    banking_stage::{create_test_recorder, BankingStage},
+    cluster_info::ClusterInfo,
+    cluster_info::Node,
+    poh_recorder::PohRecorder,
+    poh_recorder::WorkingBankEntry,
+};
+use solana_ledger::{
+    bank_forks::BankForks,
+    blockstore::Blockstore,
+    genesis_utils::{create_genesis_config, GenesisConfigInfo},
+    get_tmp_ledger_path,
+};
 use solana_measure::measure::Measure;
+use solana_perf::packet::to_packets_chunked;
 use solana_runtime::bank::Bank;
-use solana_sdk::hash::Hash;
-use solana_sdk::pubkey::Pubkey;
-use solana_sdk::signature::Keypair;
-use solana_sdk::signature::Signature;
-use solana_sdk::system_transaction;
-use solana_sdk::timing::{duration_as_us, timestamp};
-use solana_sdk::transaction::Transaction;
-use std::sync::atomic::Ordering;
-use std::sync::mpsc::Receiver;
-use std::sync::{Arc, Mutex, RwLock};
-use std::thread::sleep;
-use std::time::{Duration, Instant};
+use solana_sdk::{
+    hash::Hash,
+    pubkey::Pubkey,
+    signature::Keypair,
+    signature::Signature,
+    system_transaction,
+    timing::{duration_as_us, timestamp},
+    transaction::Transaction,
+};
+use std::{
+    sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex, RwLock},
+    thread::sleep,
+    time::{Duration, Instant},
+};
 
 fn check_txs(
     receiver: &Arc<Receiver<WorkingBankEntry>>,

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.0.10"
+version = "1.1.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,25 +10,25 @@ publish = false
 
 [dependencies]
 clap = "2.32.0"
-itertools = "0.8.2"
+itertools = "0.9.0"
 log = "0.4.8"
 num-derive = "0.3"
 num-traits = "0.2"
 rand = "0.6.5"
-rayon = "1.2.0"
-serde_json = "1.0.46"
+rayon = "1.3.0"
+serde_json = "1.0.48"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.10" }
-solana-core = { path = "../core", version = "1.0.10" }
-solana-genesis = { path = "../genesis", version = "1.0.10" }
-solana-client = { path = "../client", version = "1.0.10" }
-solana-faucet = { path = "../faucet", version = "1.0.10" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.0.10" }
-solana-logger = { path = "../logger", version = "1.0.10" }
-solana-metrics = { path = "../metrics", version = "1.0.10" }
-solana-net-utils = { path = "../net-utils", version = "1.0.10" }
-solana-runtime = { path = "../runtime", version = "1.0.10" }
-solana-sdk = { path = "../sdk", version = "1.0.10" }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.1" }
+solana-core = { path = "../core", version = "1.1.1" }
+solana-genesis = { path = "../genesis", version = "1.1.1" }
+solana-client = { path = "../client", version = "1.1.1" }
+solana-faucet = { path = "../faucet", version = "1.1.1" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.1.1" }
+solana-logger = { path = "../logger", version = "1.1.1" }
+solana-metrics = { path = "../metrics", version = "1.1.1" }
+solana-net-utils = { path = "../net-utils", version = "1.1.1" }
+solana-runtime = { path = "../runtime", version = "1.1.1" }
+solana-sdk = { path = "../sdk", version = "1.1.1" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.0.10" }
+solana-local-cluster = { path = "../local-cluster", version = "1.1.1" }

bench-streamer/Cargo.toml

@@ -2,14 +2,14 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.0.10"
+version = "1.1.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.10" }
-solana-core = { path = "../core", version = "1.0.10" }
-solana-logger = { path = "../logger", version = "1.0.10" }
-solana-net-utils = { path = "../net-utils", version = "1.0.10" }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.1" }
+solana-streamer = { path = "../streamer", version = "1.1.1" }
+solana-logger = { path = "../logger", version = "1.1.1" }
+solana-net-utils = { path = "../net-utils", version = "1.1.1" }

bench-streamer/src/main.rs

@@ -1,6 +1,6 @@
 use clap::{crate_description, crate_name, App, Arg};
-use solana_core::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
-use solana_core::streamer::{receiver, PacketReceiver};
+use solana_streamer::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
+use solana_streamer::streamer::{receiver, PacketReceiver};
 use std::cmp::max;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.0.10"
+version = "1.1.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,27 +11,27 @@ homepage = "https://solana.com/"
 bincode = "1.2.1"
 clap = "2.33.0"
 log = "0.4.8"
-rayon = "1.2.0"
-serde_json = "1.0.46"
+rayon = "1.3.0"
+serde_json = "1.0.48"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.10" }
-solana-core = { path = "../core", version = "1.0.10" }
-solana-genesis = { path = "../genesis", version = "1.0.10" }
-solana-client = { path = "../client", version = "1.0.10" }
-solana-faucet = { path = "../faucet", version = "1.0.10" }
-solana-librapay = { path = "../programs/librapay", version = "1.0.10", optional = true }
-solana-logger = { path = "../logger", version = "1.0.10" }
-solana-metrics = { path = "../metrics", version = "1.0.10" }
-solana-measure = { path = "../measure", version = "1.0.10" }
-solana-net-utils = { path = "../net-utils", version = "1.0.10" }
-solana-runtime = { path = "../runtime", version = "1.0.10" }
-solana-sdk = { path = "../sdk", version = "1.0.10" }
-solana-move-loader-program = { path = "../programs/move_loader", version = "1.0.10", optional = true }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.1" }
+solana-core = { path = "../core", version = "1.1.1" }
+solana-genesis = { path = "../genesis", version = "1.1.1" }
+solana-client = { path = "../client", version = "1.1.1" }
+solana-faucet = { path = "../faucet", version = "1.1.1" }
+solana-librapay = { path = "../programs/librapay", version = "1.1.1", optional = true }
+solana-logger = { path = "../logger", version = "1.1.1" }
+solana-metrics = { path = "../metrics", version = "1.1.1" }
+solana-measure = { path = "../measure", version = "1.1.1" }
+solana-net-utils = { path = "../net-utils", version = "1.1.1" }
+solana-runtime = { path = "../runtime", version = "1.1.1" }
+solana-sdk = { path = "../sdk", version = "1.1.1" }
+solana-move-loader-program = { path = "../programs/move_loader", version = "1.1.1", optional = true }
 
 [dev-dependencies]
-serial_test = "0.3.2"
+serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.0.10" }
+solana-local-cluster = { path = "../local-cluster", version = "1.1.1" }
 
 [features]
 move = ["solana-librapay", "solana-move-loader-program"]

chacha-cuda/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha-cuda"
-version = "1.0.10"
+version = "1.1.1"
 description = "Solana Chacha Cuda APIs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,12 +10,12 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-solana-archiver-utils = { path = "../archiver-utils", version = "1.0.10" }
-solana-chacha = { path = "../chacha", version = "1.0.10" }
-solana-ledger = { path = "../ledger", version = "1.0.10" }
-solana-logger = { path = "../logger", version = "1.0.10" }
-solana-perf = { path = "../perf", version = "1.0.10" }
-solana-sdk = { path = "../sdk", version = "1.0.10" }
+solana-archiver-utils = { path = "../archiver-utils", version = "1.1.1" }
+solana-chacha = { path = "../chacha", version = "1.1.1" }
+solana-ledger = { path = "../ledger", version = "1.1.1" }
+solana-logger = { path = "../logger", version = "1.1.1" }
+solana-perf = { path = "../perf", version = "1.1.1" }
+solana-sdk = { path = "../sdk", version = "1.1.1" }
 
 [dev-dependencies]
 hex-literal = "0.2.1"

chacha-sys/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha-sys"
-version = "1.0.10"
+version = "1.1.1"
 description = "Solana chacha-sys"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

chacha/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha"
-version = "1.0.10"
+version = "1.1.1"
 description = "Solana Chacha APIs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,11 +12,11 @@ edition = "2018"
 log = "0.4.8"
 rand = "0.6.5"
 rand_chacha = "0.1.1"
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.10" }
-solana-ledger = { path = "../ledger", version = "1.0.10" }
-solana-logger = { path = "../logger", version = "1.0.10" }
-solana-perf = { path = "../perf", version = "1.0.10" }
-solana-sdk = { path = "../sdk", version = "1.0.10" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.1.1" }
+solana-ledger = { path = "../ledger", version = "1.1.1" }
+solana-logger = { path = "../logger", version = "1.1.1" }
+solana-perf = { path = "../perf", version = "1.1.1" }
+solana-sdk = { path = "../sdk", version = "1.1.1" }
 
 [dev-dependencies]
 hex-literal = "0.2.1"

ci/buildkite-tests.yml

@@ -0,0 +1,37 @@
+# These steps are conditionally triggered by ci/buildkite.yml when files
+# other than those in docs/ are modified
+
+steps:
+  - command: "ci/shellcheck.sh"
+    name: "shellcheck"
+    timeout_in_minutes: 5
+
+  - wait
+
+  - command: "ci/test-stable-perf.sh"
+    name: "stable-perf"
+    timeout_in_minutes: 40
+    artifact_paths: "log-*.txt"
+    agents:
+      - "queue=cuda"
+  - command: "ci/test-bench.sh"
+    name: "bench"
+    timeout_in_minutes: 30
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh"
+    name: "stable"
+    timeout_in_minutes: 60
+    artifact_paths: "log-*.txt"
+    agents:
+      - "queue=rpc-test-capable"
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
+    name: "move"
+    timeout_in_minutes: 20
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-local-cluster.sh"
+    name: "local-cluster"
+    timeout_in_minutes: 45
+    artifact_paths: "log-*.txt"
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh"
+    name: "coverage"
+    timeout_in_minutes: 30
+    agents:
+      - "queue=rpc-test-capable"

ci/buildkite.yml

@@ -1,42 +1,22 @@
 # Build steps that run on pushes and pull requests.
+# If files other than those in docs/ were modified, this will be followed up by
+# ci/buildkite-tests.yml
 #
 # Release tags use buildkite-release.yml instead
+
 steps:
-  - command: "ci/shellcheck.sh"
-    name: "shellcheck"
-    timeout_in_minutes: 5
   - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-checks.sh"
     name: "checks"
     timeout_in_minutes: 20
+
   - wait
-  - command: "ci/test-stable-perf.sh"
-    name: "stable-perf"
-    timeout_in_minutes: 40
-    artifact_paths: "log-*.txt"
-    agents:
-      - "queue=cuda"
-  - command: "ci/test-bench.sh"
-    name: "bench"
-    timeout_in_minutes: 30
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh"
-    name: "stable"
-    timeout_in_minutes: 60
-    artifact_paths: "log-*.txt"
-    agents:
-      - "queue=rpc-test-capable"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
-    name: "move"
-    timeout_in_minutes: 20
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-local-cluster.sh"
-    name: "local-cluster"
-    timeout_in_minutes: 45
-    artifact_paths: "log-*.txt"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh"
-    name: "coverage"
-    timeout_in_minutes: 30
-    agents:
-      - "queue=rpc-test-capable"
+
+  - command: "ci/maybe-trigger-tests.sh"
+    name: "maybe-trigger-tests"
+    timeout_in_minutes: 2
+
   - wait
+
   - trigger: "solana-secondary"
     branches: "!pull/*"
     async: true

ci/docker-rust/Dockerfile

@@ -32,6 +32,7 @@ RUN set -x \
  && cargo install cargo-audit \
  && cargo install svgbob_cli \
  && cargo install mdbook \
+ && cargo install mdbook-linkcheck \
  && rustc --version \
  && cargo --version \
  && curl -OL https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP \

ci/maybe-trigger-tests.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -e
+cd "$(dirname "$0")/.."
+
+annotate() {
+  ${BUILDKITE:-false} && {
+    buildkite-agent annotate "$@"
+  }
+}
+
+# Skip if only the docs have been modified
+ci/affects-files.sh \
+  \!^docs/ \
+|| {
+  annotate --style info \
+    "Skipping all further tests as only docs/ files were modified"
+  exit 0
+}
+
+annotate --style info "Triggering tests"
+buildkite-agent pipeline upload ci/buildkite-tests.yml

ci/test-bench.sh

@@ -67,8 +67,9 @@ _ cargo +$rust_nightly bench --manifest-path core/Cargo.toml ${V:+--verbose} \
 _ cargo +$rust_nightly bench --manifest-path programs/bpf/Cargo.toml ${V:+--verbose} --features=bpf_c \
   -- -Z unstable-options --format=json --nocapture | tee -a "$BENCH_FILE"
 
-# Run banking bench. Doesn't require nightly, but use since it is already built.
+# Run banking/accounts bench. Doesn't require nightly, but use since it is already built.
 _ cargo +$rust_nightly run --release --manifest-path banking-bench/Cargo.toml ${V:+--verbose} | tee -a "$BENCH_FILE"
+_ cargo +$rust_nightly run --release --manifest-path accounts-bench/Cargo.toml ${V:+--verbose} -- --num_accounts 10000 --num_slots 4 | tee -a "$BENCH_FILE"
 
 # `solana-upload-perf` disabled as it can take over 30 minutes to complete for some
 # reason

ci/test-checks.sh

@@ -22,7 +22,7 @@ _ cargo +"$rust_stable" clippy --all --exclude solana-sdk-c -- --deny=warnings
 _ cargo +"$rust_stable" clippy --manifest-path sdk-c/Cargo.toml -- --deny=warnings
 
 _ cargo +"$rust_stable" audit --version
-_ cargo +"$rust_stable" audit --ignore RUSTSEC-2020-0002
+_ cargo +"$rust_stable" audit --ignore RUSTSEC-2020-0002 --ignore RUSTSEC-2020-0008
 _ ci/nits.sh
 _ ci/order-crates-for-publishing.py
 _ docs/build.sh

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.0.10"
+version = "1.1.1"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.0.10" }
-solana-sdk = { path = "../sdk", version = "1.0.10" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.1.1" }
+solana-sdk = { path = "../sdk", version = "1.1.1" }
 thiserror = "1.0.11"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"

clap-utils/src/input_parsers.rs

@@ -62,6 +62,21 @@ pub fn keypair_of(matches: &ArgMatches<'_>, name: &str) -> Option<Keypair> {
     }
 }
 
+pub fn keypairs_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<Keypair>> {
+    matches.values_of(name).map(|values| {
+        values
+            .filter_map(|value| {
+                if value == ASK_KEYWORD {
+                    let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
+                    keypair_from_seed_phrase(name, skip_validation, true).ok()
+                } else {
+                    read_keypair_file(value).ok()
+                }
+            })
+            .collect()
+    })
+}
+
 // Return a pubkey for an argument that can itself be parsed into a pubkey,
 // or is a filename that can be read as a keypair
 pub fn pubkey_of(matches: &ArgMatches<'_>, name: &str) -> Option<Pubkey> {

clap-utils/src/keypair.rs

@@ -47,6 +47,13 @@ pub fn parse_keypair_path(path: &str) -> KeypairUrl {
     }
 }
 
+pub fn check_for_usb<S>(mut items: impl Iterator<Item = S>) -> bool
+where
+    S: Into<String>,
+{
+    items.any(|arg| matches!(parse_keypair_path(&arg.into()), KeypairUrl::Usb(_)))
+}
+
 pub fn presigner_from_pubkey_sigs(
     pubkey: &Pubkey,
     signers: &[(Pubkey, Signature)],
@@ -256,4 +263,20 @@ mod tests {
             sanitize_seed_phrase(seed_phrase)
         );
     }
+
+    #[test]
+    fn test_check_for_usb() {
+        let args: Vec<&str> = vec![];
+        assert_eq!(check_for_usb(args.into_iter()), false);
+        let args = vec!["usb://"];
+        assert_eq!(check_for_usb(args.into_iter()), true);
+        let args = vec!["other"];
+        assert_eq!(check_for_usb(args.into_iter()), false);
+        let args = vec!["other", "usb://", "another"];
+        assert_eq!(check_for_usb(args.into_iter()), true);
+        let args = vec!["other", "another"];
+        assert_eq!(check_for_usb(args.into_iter()), false);
+        let args = vec!["usb://", "usb://"];
+        assert_eq!(check_for_usb(args.into_iter()), true);
+    }
 }

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.10"
+version = "1.1.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,7 +11,7 @@ homepage = "https://solana.com/"
 [dependencies]
 dirs = "2.0.2"
 lazy_static = "1.4.0"
-serde = "1.0.104"
+serde = "1.0.105"
 serde_derive = "1.0.103"
 serde_yaml = "0.8.11"
 url = "2.1.1"

cli-config/src/config.rs

@@ -1,10 +1,6 @@
 // Wallet settings that can be configured for long-term use
 use serde_derive::{Deserialize, Serialize};
-use std::{
-    fs::{create_dir_all, File},
-    io::{self, Write},
-    path::Path,
-};
+use std::io;
 use url::Url;
 
 lazy_static! {
@@ -46,23 +42,11 @@ impl Default for Config {
 
 impl Config {
     pub fn load(config_file: &str) -> Result<Self, io::Error> {
-        let file = File::open(config_file.to_string())?;
-        let config = serde_yaml::from_reader(file)
-            .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
-        Ok(config)
+        crate::load_config_file(config_file)
     }
 
     pub fn save(&self, config_file: &str) -> Result<(), io::Error> {
-        let serialized = serde_yaml::to_string(self)
-            .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
-
-        if let Some(outdir) = Path::new(&config_file).parent() {
-            create_dir_all(outdir)?;
-        }
-        let mut file = File::create(config_file)?;
-        file.write_all(&serialized.into_bytes())?;
-
-        Ok(())
+        crate::save_config_file(self, config_file)
     }
 
     pub fn compute_websocket_url(json_rpc_url: &str) -> String {

cli-config/src/lib.rs

@@ -3,3 +3,37 @@ extern crate lazy_static;
 
 mod config;
 pub use config::{Config, CONFIG_FILE};
+
+use std::{
+    fs::{create_dir_all, File},
+    io::{self, Write},
+    path::Path,
+};
+
+pub fn load_config_file<T, P>(config_file: P) -> Result<T, io::Error>
+where
+    T: serde::de::DeserializeOwned,
+    P: AsRef<Path>,
+{
+    let file = File::open(config_file)?;
+    let config = serde_yaml::from_reader(file)
+        .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
+    Ok(config)
+}
+
+pub fn save_config_file<T, P>(config: &T, config_file: P) -> Result<(), io::Error>
+where
+    T: serde::ser::Serialize,
+    P: AsRef<Path>,
+{
+    let serialized = serde_yaml::to_string(config)
+        .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
+
+    if let Some(outdir) = config_file.as_ref().parent() {
+        create_dir_all(outdir)?;
+    }
+    let mut file = File::create(config_file)?;
+    file.write_all(&serialized.into_bytes())?;
+
+    Ok(())
+}

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.10"
+version = "1.1.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,43 +11,43 @@ homepage = "https://solana.com/"
 [dependencies]
 bincode = "1.2.1"
 bs58 = "0.3.0"
-chrono = { version = "0.4.10", features = ["serde"] }
+chrono = { version = "0.4.11", features = ["serde"] }
 clap = "2.33.0"
 criterion-stats = "0.3.0"
 ctrlc = { version = "3.1.4", features = ["termination"] }
-console = "0.9.2"
+console = "0.10.0"
 dirs = "2.0.2"
 log = "0.4.8"
 indicatif = "0.14.0"
 humantime = "2.0.0"
 num-traits = "0.2"
 pretty-hex = "0.1.1"
-reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
-serde = "1.0.104"
+reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+serde = "1.0.105"
 serde_derive = "1.0.103"
-serde_json = "1.0.46"
-solana-budget-program = { path = "../programs/budget", version = "1.0.10" }
-solana-clap-utils = { path = "../clap-utils", version = "1.0.10" }
-solana-cli-config = { path = "../cli-config", version = "1.0.10" }
-solana-client = { path = "../client", version = "1.0.10" }
-solana-config-program = { path = "../programs/config", version = "1.0.10" }
-solana-faucet = { path = "../faucet", version = "1.0.10" }
-solana-logger = { path = "../logger", version = "1.0.10" }
-solana-net-utils = { path = "../net-utils", version = "1.0.10" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.0.10" }
-solana-runtime = { path = "../runtime", version = "1.0.10" }
-solana-sdk = { path = "../sdk", version = "1.0.10" }
-solana-stake-program = { path = "../programs/stake", version = "1.0.10" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.10" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.10" }
-solana-vote-signer = { path = "../vote-signer", version = "1.0.10" }
+serde_json = "1.0.48"
+solana-budget-program = { path = "../programs/budget", version = "1.1.1" }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.1" }
+solana-cli-config = { path = "../cli-config", version = "1.1.1" }
+solana-client = { path = "../client", version = "1.1.1" }
+solana-config-program = { path = "../programs/config", version = "1.1.1" }
+solana-faucet = { path = "../faucet", version = "1.1.1" }
+solana-logger = { path = "../logger", version = "1.1.1" }
+solana-net-utils = { path = "../net-utils", version = "1.1.1" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.1.1" }
+solana-runtime = { path = "../runtime", version = "1.1.1" }
+solana-sdk = { path = "../sdk", version = "1.1.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.1.1" }
+solana-storage-program = { path = "../programs/storage", version = "1.1.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.1.1" }
+solana-vote-signer = { path = "../vote-signer", version = "1.1.1" }
 titlecase = "1.1.0"
-thiserror = "1.0.11"
+thiserror = "1.0.13"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.0.10" }
-solana-budget-program = { path = "../programs/budget", version = "1.0.10" }
+solana-core = { path = "../core", version = "1.1.1" }
+solana-budget-program = { path = "../programs/budget", version = "1.1.1" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cli.rs

@@ -38,9 +38,9 @@ use solana_sdk::{
     message::Message,
     native_token::lamports_to_sol,
     program_utils::DecodeError,
-    pubkey::Pubkey,
+    pubkey::{Pubkey, MAX_SEED_LEN},
     signature::{Keypair, Signature, Signer, SignerError},
-    system_instruction::{self, create_address_with_seed, SystemError, MAX_ADDRESS_SEED_LEN},
+    system_instruction::{self, SystemError},
     system_program,
     transaction::{Transaction, TransactionError},
 };
@@ -173,6 +173,8 @@ pub enum CliCommand {
     Catchup {
         node_pubkey: Pubkey,
         node_json_rpc_url: Option<String>,
+        commitment_config: CommitmentConfig,
+        follow: bool,
     },
     ClusterVersion,
     CreateAddressWithSeed {
@@ -188,9 +190,15 @@ pub enum CliCommand {
         commitment_config: CommitmentConfig,
     },
     GetGenesisHash,
+    GetEpoch {
+        commitment_config: CommitmentConfig,
+    },
     GetSlot {
         commitment_config: CommitmentConfig,
     },
+    TotalSupply {
+        commitment_config: CommitmentConfig,
+    },
     GetTransactionCount {
         commitment_config: CommitmentConfig,
     },
@@ -583,7 +591,9 @@ pub fn parse_command(
             command: CliCommand::GetGenesisHash,
             signers: vec![],
         }),
+        ("epoch", Some(matches)) => parse_get_epoch(matches),
         ("slot", Some(matches)) => parse_get_slot(matches),
+        ("total-supply", Some(matches)) => parse_total_supply(matches),
         ("transaction-count", Some(matches)) => parse_get_transaction_count(matches),
         ("leader-schedule", Some(_matches)) => Ok(CliCommandInfo {
             command: CliCommand::LeaderSchedule,
@@ -1061,7 +1071,7 @@ pub fn parse_create_address_with_seed(
 
     let seed = matches.value_of("seed").unwrap().to_string();
 
-    if seed.len() > MAX_ADDRESS_SEED_LEN {
+    if seed.len() > MAX_SEED_LEN {
         return Err(CliError::BadParameter(
             "Address seed must not be longer than 32 bytes".to_string(),
         ));
@@ -1088,7 +1098,7 @@ fn process_create_address_with_seed(
     } else {
         config.pubkey()?
     };
-    let address = create_address_with_seed(&from_pubkey, seed, program_id)?;
+    let address = Pubkey::create_with_seed(&from_pubkey, seed, program_id)?;
     Ok(address.to_string())
 }
 
@@ -1149,7 +1159,7 @@ fn process_balance(
 }
 
 fn process_confirm(rpc_client: &RpcClient, signature: &Signature) -> ProcessResult {
-    match rpc_client.get_signature_status(&signature.to_string()) {
+    match rpc_client.get_signature_status(&signature) {
         Ok(status) => {
             if let Some(result) = status {
                 match result {
@@ -1181,6 +1191,7 @@ fn process_show_account(
     );
     println_name_value("Owner:", &account.owner.to_string());
     println_name_value("Executable:", &account.executable.to_string());
+    println_name_value("Rent Epoch:", &account.rent_epoch.to_string());
 
     if let Some(output_file) = output_file {
         let mut f = File::create(output_file)?;
@@ -1584,7 +1595,15 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         CliCommand::Catchup {
             node_pubkey,
             node_json_rpc_url,
-        } => process_catchup(&rpc_client, node_pubkey, node_json_rpc_url),
+            commitment_config,
+            follow,
+        } => process_catchup(
+            &rpc_client,
+            node_pubkey,
+            node_json_rpc_url,
+            *commitment_config,
+            *follow,
+        ),
         CliCommand::ClusterVersion => process_cluster_version(&rpc_client),
         CliCommand::CreateAddressWithSeed {
             from_pubkey,
@@ -1597,9 +1616,15 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         CliCommand::GetEpochInfo { commitment_config } => {
             process_get_epoch_info(&rpc_client, *commitment_config)
         }
+        CliCommand::GetEpoch { commitment_config } => {
+            process_get_epoch(&rpc_client, *commitment_config)
+        }
         CliCommand::GetSlot { commitment_config } => {
             process_get_slot(&rpc_client, *commitment_config)
         }
+        CliCommand::TotalSupply { commitment_config } => {
+            process_total_supply(&rpc_client, *commitment_config)
+        }
         CliCommand::GetTransactionCount { commitment_config } => {
             process_get_transaction_count(&rpc_client, *commitment_config)
         }
@@ -2157,7 +2182,7 @@ pub fn request_and_confirm_airdrop(
     log_instruction_custom_error::<SystemError>(result)
 }
 
-pub fn log_instruction_custom_error<E>(result: ClientResult<String>) -> ProcessResult
+pub fn log_instruction_custom_error<E>(result: ClientResult<Signature>) -> ProcessResult
 where
     E: 'static + std::error::Error + DecodeError<E> + FromPrimitive,
 {
@@ -2174,7 +2199,7 @@ where
             }
             Err(err.into())
         }
-        Ok(sig) => Ok(sig),
+        Ok(sig) => Ok(sig.to_string()),
     }
 }
 
@@ -3368,7 +3393,7 @@ mod tests {
         };
         let address = process_command(&config);
         let expected_address =
-            create_address_with_seed(&from_pubkey, "seed", &solana_stake_program::id()).unwrap();
+            Pubkey::create_with_seed(&from_pubkey, "seed", &solana_stake_program::id()).unwrap();
         assert_eq!(address.unwrap(), expected_address.to_string());
 
         // Need airdrop cases

cli/src/cluster_query.rs

@@ -5,6 +5,7 @@ use crate::{
     },
     display::println_name_value,
 };
+use chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc};
 use clap::{value_t, value_t_or_exit, App, Arg, ArgMatches, SubCommand};
 use console::{style, Emoji};
 use indicatif::{ProgressBar, ProgressStyle};
@@ -22,6 +23,7 @@ use solana_sdk::{
     epoch_schedule::Epoch,
     hash::Hash,
     message::Message,
+    native_token::lamports_to_sol,
     pubkey::Pubkey,
     signature::{Keypair, Signer},
     system_instruction,
@@ -67,6 +69,20 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .takes_value(true)
                         .validator(is_url)
                         .help("JSON RPC URL for validator, which is useful for validators with a private RPC service")
+                )
+                .arg(
+                    Arg::with_name("confirmed")
+                        .long("confirmed")
+                        .takes_value(false)
+                        .help(
+                            "Return information at maximum-lockout commitment level",
+                        ),
+                )
+                .arg(
+                    Arg::with_name("follow")
+                        .long("follow")
+                        .takes_value(false)
+                        .help("Continue reporting progress even after the validator has caught up"),
                 ),
         )
         .subcommand(
@@ -117,6 +133,28 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                     ),
             ),
         )
+        .subcommand(
+            SubCommand::with_name("epoch").about("Get current epoch")
+            .arg(
+                Arg::with_name("confirmed")
+                    .long("confirmed")
+                    .takes_value(false)
+                    .help(
+                        "Return epoch at maximum-lockout commitment level",
+                    ),
+            ),
+        )
+        .subcommand(
+            SubCommand::with_name("total-supply").about("Get total number of SOL")
+            .arg(
+                Arg::with_name("confirmed")
+                    .long("confirmed")
+                    .takes_value(false)
+                    .help(
+                        "Return count at maximum-lockout commitment level",
+                    ),
+            ),
+        )
         .subcommand(
             SubCommand::with_name("transaction-count").about("Get current transaction count")
             .alias("get-transaction-count")
@@ -249,10 +287,18 @@ pub fn parse_catchup(
 ) -> Result<CliCommandInfo, CliError> {
     let node_pubkey = pubkey_of_signer(matches, "node_pubkey", wallet_manager)?.unwrap();
     let node_json_rpc_url = value_t!(matches, "node_json_rpc_url", String).ok();
+    let commitment_config = if matches.is_present("confirmed") {
+        CommitmentConfig::default()
+    } else {
+        CommitmentConfig::recent()
+    };
+    let follow = matches.is_present("follow");
     Ok(CliCommandInfo {
         command: CliCommand::Catchup {
             node_pubkey,
             node_json_rpc_url,
+            commitment_config,
+            follow,
         },
         signers: vec![],
     })
@@ -325,6 +371,30 @@ pub fn parse_get_slot(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliErr
     })
 }
 
+pub fn parse_get_epoch(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+    let commitment_config = if matches.is_present("confirmed") {
+        CommitmentConfig::default()
+    } else {
+        CommitmentConfig::recent()
+    };
+    Ok(CliCommandInfo {
+        command: CliCommand::GetEpoch { commitment_config },
+        signers: vec![],
+    })
+}
+
+pub fn parse_total_supply(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+    let commitment_config = if matches.is_present("confirmed") {
+        CommitmentConfig::default()
+    } else {
+        CommitmentConfig::recent()
+    };
+    Ok(CliCommandInfo {
+        command: CliCommand::TotalSupply { commitment_config },
+        signers: vec![],
+    })
+}
+
 pub fn parse_get_transaction_count(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let commitment_config = if matches.is_present("confirmed") {
         CommitmentConfig::default()
@@ -384,20 +454,37 @@ pub fn process_catchup(
     rpc_client: &RpcClient,
     node_pubkey: &Pubkey,
     node_json_rpc_url: &Option<String>,
+    commitment_config: CommitmentConfig,
+    follow: bool,
 ) -> ProcessResult {
-    let cluster_nodes = rpc_client.get_cluster_nodes()?;
+    let sleep_interval = 5;
+
+    let progress_bar = new_spinner_progress_bar();
+    progress_bar.set_message("Connecting...");
 
     let node_client = if let Some(node_json_rpc_url) = node_json_rpc_url {
         RpcClient::new(node_json_rpc_url.to_string())
     } else {
-        RpcClient::new_socket(
-            cluster_nodes
+        let rpc_addr = loop {
+            let cluster_nodes = rpc_client.get_cluster_nodes()?;
+            if let Some(contact_info) = cluster_nodes
                 .iter()
                 .find(|contact_info| contact_info.pubkey == node_pubkey.to_string())
-                .ok_or_else(|| format!("Contact information not found for {}", node_pubkey))?
-                .rpc
-                .ok_or_else(|| format!("RPC service not found for {}", node_pubkey))?,
-        )
+            {
+                if let Some(rpc_addr) = contact_info.rpc {
+                    break rpc_addr;
+                }
+                progress_bar.set_message(&format!("RPC service not found for {}", node_pubkey));
+            } else {
+                progress_bar.set_message(&format!(
+                    "Contact information not found for {}",
+                    node_pubkey
+                ));
+            }
+            sleep(Duration::from_secs(sleep_interval as u64));
+        };
+
+        RpcClient::new_socket(rpc_addr)
     };
 
     let reported_node_pubkey = node_client.get_identity()?;
@@ -413,16 +500,12 @@ pub fn process_catchup(
         return Err("Both RPC URLs reference the same node, unable to monitor for catchup.  Try a different --url".into());
     }
 
-    let progress_bar = new_spinner_progress_bar();
-    progress_bar.set_message("Connecting...");
-
     let mut previous_rpc_slot = std::u64::MAX;
     let mut previous_slot_distance = 0;
-    let sleep_interval = 5;
     loop {
-        let rpc_slot = rpc_client.get_slot_with_commitment(CommitmentConfig::recent())?;
-        let node_slot = node_client.get_slot_with_commitment(CommitmentConfig::recent())?;
-        if node_slot > std::cmp::min(previous_rpc_slot, rpc_slot) {
+        let rpc_slot = rpc_client.get_slot_with_commitment(commitment_config)?;
+        let node_slot = node_client.get_slot_with_commitment(commitment_config)?;
+        if !follow && node_slot > std::cmp::min(previous_rpc_slot, rpc_slot) {
             progress_bar.finish_and_clear();
             return Ok(format!(
                 "{} has caught up (us:{} them:{})",
@@ -436,7 +519,7 @@ pub fn process_catchup(
             slot_distance,
             node_slot,
             rpc_slot,
-            if previous_rpc_slot == std::u64::MAX {
+            if slot_distance == 0 || previous_rpc_slot == std::u64::MAX {
                 "".to_string()
             } else {
                 let slots_per_second =
@@ -511,7 +594,13 @@ pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
 
 pub fn process_get_block_time(rpc_client: &RpcClient, slot: Slot) -> ProcessResult {
     let timestamp = rpc_client.get_block_time(slot)?;
-    Ok(timestamp.to_string())
+    let result = format!(
+        "{} (UnixTimestamp: {})",
+        DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(timestamp, 0), Utc)
+            .to_rfc3339_opts(SecondsFormat::Secs, true),
+        timestamp
+    );
+    Ok(result)
 }
 
 fn slot_to_human_time(slot: Slot) -> String {
@@ -575,6 +664,14 @@ pub fn process_get_slot(
     Ok(slot.to_string())
 }
 
+pub fn process_get_epoch(
+    rpc_client: &RpcClient,
+    commitment_config: CommitmentConfig,
+) -> ProcessResult {
+    let epoch_info = rpc_client.get_epoch_info_with_commitment(commitment_config.clone())?;
+    Ok(epoch_info.epoch.to_string())
+}
+
 pub fn parse_show_block_production(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let epoch = value_t!(matches, "epoch", Epoch).ok();
     let slot_limit = value_t!(matches, "slot_limit", u64).ok();
@@ -756,6 +853,14 @@ pub fn process_show_block_production(
     Ok("".to_string())
 }
 
+pub fn process_total_supply(
+    rpc_client: &RpcClient,
+    commitment_config: CommitmentConfig,
+) -> ProcessResult {
+    let total_supply = rpc_client.total_supply_with_commitment(commitment_config.clone())?;
+    Ok(format!("{} SOL", lamports_to_sol(total_supply)))
+}
+
 pub fn process_get_transaction_count(
     rpc_client: &RpcClient,
     commitment_config: CommitmentConfig,
@@ -1280,6 +1385,32 @@ mod tests {
             }
         );
 
+        let test_get_epoch = test_commands
+            .clone()
+            .get_matches_from(vec!["test", "epoch"]);
+        assert_eq!(
+            parse_command(&test_get_epoch, &default_keypair_file, None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::GetEpoch {
+                    commitment_config: CommitmentConfig::recent(),
+                },
+                signers: vec![],
+            }
+        );
+
+        let test_total_supply = test_commands
+            .clone()
+            .get_matches_from(vec!["test", "total-supply"]);
+        assert_eq!(
+            parse_command(&test_total_supply, &default_keypair_file, None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::TotalSupply {
+                    commitment_config: CommitmentConfig::recent(),
+                },
+                signers: vec![],
+            }
+        );
+
         let test_transaction_count = test_commands
             .clone()
             .get_matches_from(vec!["test", "transaction-count"]);

cli/src/main.rs

@@ -2,7 +2,9 @@ use clap::{crate_description, crate_name, AppSettings, Arg, ArgGroup, ArgMatches
 use console::style;
 
 use solana_clap_utils::{
-    input_validators::is_url, keypair::SKIP_SEED_PHRASE_VALIDATION_ARG, offline::SIGN_ONLY_ARG,
+    input_validators::is_url,
+    keypair::{check_for_usb, SKIP_SEED_PHRASE_VALIDATION_ARG},
+    offline::SIGN_ONLY_ARG,
     DisplayError,
 };
 use solana_cli::{
@@ -233,12 +235,20 @@ fn main() -> Result<(), Box<dyn error::Error>> {
     )
     .get_matches();
 
-    do_main(&matches).map_err(|err| DisplayError::new_as_boxed(err).into())
+    do_main(&matches, check_for_usb(std::env::args()))
+        .map_err(|err| DisplayError::new_as_boxed(err).into())
 }
 
-fn do_main(matches: &ArgMatches<'_>) -> Result<(), Box<dyn error::Error>> {
+fn do_main(
+    matches: &ArgMatches<'_>,
+    need_wallet_manager: bool,
+) -> Result<(), Box<dyn error::Error>> {
     if parse_settings(&matches)? {
-        let wallet_manager = maybe_wallet_manager()?;
+        let wallet_manager = if need_wallet_manager {
+            maybe_wallet_manager()?
+        } else {
+            None
+        };
 
         let (mut config, signers) = parse_args(&matches, wallet_manager)?;
         config.signers = signers.iter().map(|s| s.as_ref()).collect();

cli/src/nonce.rs

@@ -21,9 +21,8 @@ use solana_sdk::{
     },
     pubkey::Pubkey,
     system_instruction::{
-        advance_nonce_account, authorize_nonce_account, create_address_with_seed,
-        create_nonce_account, create_nonce_account_with_seed, withdraw_nonce_account, NonceError,
-        SystemError,
+        advance_nonce_account, authorize_nonce_account, create_nonce_account,
+        create_nonce_account_with_seed, withdraw_nonce_account, NonceError, SystemError,
     },
     system_program,
     transaction::Transaction,
@@ -474,7 +473,7 @@ pub fn process_create_nonce_account(
 ) -> ProcessResult {
     let nonce_account_pubkey = config.signers[nonce_account].pubkey();
     let nonce_account_address = if let Some(seed) = seed.clone() {
-        create_address_with_seed(&nonce_account_pubkey, &seed, &system_program::id())?
+        Pubkey::create_with_seed(&nonce_account_pubkey, &seed, &system_program::id())?
     } else {
         nonce_account_pubkey
     };

cli/src/stake.rs

@@ -17,7 +17,7 @@ use solana_sdk::{
     account_utils::StateMut,
     message::Message,
     pubkey::Pubkey,
-    system_instruction::{create_address_with_seed, SystemError},
+    system_instruction::SystemError,
     sysvar::{
         stake_history::{self, StakeHistory},
         Sysvar,
@@ -86,7 +86,7 @@ impl StakeSubCommands for App<'_, '_> {
                         .takes_value(true)
                         .validator(is_amount)
                         .required(true)
-                        .help("The amount of send to the vote account, in SOL")
+                        .help("The amount to send to the stake account, in SOL")
                 )
                 .arg(
                     Arg::with_name("custodian")
@@ -790,7 +790,7 @@ pub fn process_create_stake_account(
 ) -> ProcessResult {
     let stake_account = config.signers[stake_account];
     let stake_account_address = if let Some(seed) = seed {
-        create_address_with_seed(&stake_account.pubkey(), &seed, &solana_stake_program::id())?
+        Pubkey::create_with_seed(&stake_account.pubkey(), &seed, &solana_stake_program::id())?
     } else {
         stake_account.pubkey()
     };
@@ -1108,7 +1108,7 @@ pub fn process_split_stake(
     let stake_authority = config.signers[stake_authority];
 
     let split_stake_account_address = if let Some(seed) = split_stake_account_seed {
-        create_address_with_seed(
+        Pubkey::create_with_seed(
             &split_stake_account.pubkey(),
             &seed,
             &solana_stake_program::id(),

cli/src/storage.rs

@@ -266,8 +266,8 @@ pub fn process_claim_storage_reward(
         &fee_calculator,
         &tx.message,
     )?;
-    let signature_str = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &signers)?;
-    Ok(signature_str)
+    let signature = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &signers)?;
+    Ok(signature.to_string())
 }
 
 pub fn process_show_storage_account(

cli/src/vote.rs

@@ -8,12 +8,8 @@ use solana_clap_utils::{input_parsers::*, input_validators::*};
 use solana_client::rpc_client::RpcClient;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
-    account::Account,
-    commitment_config::CommitmentConfig,
-    message::Message,
-    pubkey::Pubkey,
-    system_instruction::{create_address_with_seed, SystemError},
-    transaction::Transaction,
+    account::Account, commitment_config::CommitmentConfig, message::Message, pubkey::Pubkey,
+    system_instruction::SystemError, transaction::Transaction,
 };
 use solana_vote_program::{
     vote_instruction::{self, withdraw, VoteError},
@@ -381,7 +377,7 @@ pub fn process_create_vote_account(
     let vote_account = config.signers[1];
     let vote_account_pubkey = vote_account.pubkey();
     let vote_account_address = if let Some(seed) = seed {
-        create_address_with_seed(&vote_account_pubkey, &seed, &solana_vote_program::id())?
+        Pubkey::create_with_seed(&vote_account_pubkey, &seed, &solana_vote_program::id())?
     } else {
         vote_account_pubkey
     };

cli/tests/nonce.rs

@@ -13,7 +13,6 @@ use solana_sdk::{
     hash::Hash,
     pubkey::Pubkey,
     signature::{keypair_from_seed, Keypair, Signer},
-    system_instruction::create_address_with_seed,
     system_program,
 };
 use std::{fs::remove_dir_all, sync::mpsc::channel, thread::sleep, time::Duration};
@@ -130,7 +129,7 @@ fn full_battery_tests(
     config_nonce.signers = vec![&nonce_keypair];
 
     let nonce_account = if let Some(seed) = seed.as_ref() {
-        create_address_with_seed(
+        Pubkey::create_with_seed(
             &config_nonce.signers[0].pubkey(),
             seed,
             &system_program::id(),
@@ -301,7 +300,7 @@ fn test_create_account_with_seed() {
     let authority_pubkey = offline_nonce_authority_signer.pubkey();
     let seed = authority_pubkey.to_string()[0..32].to_string();
     let nonce_address =
-        create_address_with_seed(&creator_pubkey, &seed, &system_program::id()).unwrap();
+        Pubkey::create_with_seed(&creator_pubkey, &seed, &system_program::id()).unwrap();
     check_balance(0, &rpc_client, &nonce_address);
 
     let mut creator_config = CliConfig::default();

cli/tests/stake.rs

@@ -14,7 +14,6 @@ use solana_sdk::{
     nonce::State as NonceState,
     pubkey::Pubkey,
     signature::{keypair_from_seed, Keypair, Signer},
-    system_instruction::create_address_with_seed,
 };
 use solana_stake_program::{
     stake_instruction::LockupArgs,
@@ -160,7 +159,7 @@ fn test_seed_stake_delegation_and_deactivation() {
     .unwrap();
     check_balance(100_000, &rpc_client, &config_validator.signers[0].pubkey());
 
-    let stake_address = create_address_with_seed(
+    let stake_address = Pubkey::create_with_seed(
         &config_validator.signers[0].pubkey(),
         "hi there",
         &solana_stake_program::id(),
@@ -1523,7 +1522,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     };
     process_command(&config).unwrap();
     let seed_address =
-        create_address_with_seed(&stake_pubkey, seed, &solana_stake_program::id()).unwrap();
+        Pubkey::create_with_seed(&stake_pubkey, seed, &solana_stake_program::id()).unwrap();
     check_balance(50_000, &rpc_client, &seed_address);
 
     server.close().unwrap();

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.0.10"
+version = "1.1.1"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,14 +14,15 @@ bs58 = "0.3.0"
 indicatif = "0.14.0"
 jsonrpc-core = "14.0.5"
 log = "0.4.8"
-rayon = "1.2.0"
-reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
-serde = "1.0.104"
+rayon = "1.3.0"
+reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+serde = "1.0.105"
 serde_derive = "1.0.103"
-serde_json = "1.0.46"
-solana-net-utils = { path = "../net-utils", version = "1.0.10" }
-solana-sdk = { path = "../sdk", version = "1.0.10" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.10" }
+serde_json = "1.0.48"
+solana-transaction-status = { path = "../transaction-status", version = "1.1.1" }
+solana-net-utils = { path = "../net-utils", version = "1.1.1" }
+solana-sdk = { path = "../sdk", version = "1.1.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.1.1" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -30,4 +31,4 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "14.0.5"
 jsonrpc-http-server = "14.0.6"
-solana-logger = { path = "../logger", version = "1.0.10" }
+solana-logger = { path = "../logger", version = "1.1.1" }

client/src/mock_rpc_client_request.rs

@@ -2,7 +2,7 @@ use crate::{
     client_error::Result,
     generic_rpc_client_request::GenericRpcClientRequest,
     rpc_request::RpcRequest,
-    rpc_response::{Response, RpcResponseContext, RpcTransactionStatus},
+    rpc_response::{Response, RpcResponseContext},
 };
 use serde_json::{Number, Value};
 use solana_sdk::{
@@ -10,6 +10,7 @@ use solana_sdk::{
     instruction::InstructionError,
     transaction::{self, TransactionError},
 };
+use solana_transaction_status::TransactionStatus;
 use std::{collections::HashMap, sync::RwLock};
 
 pub const PUBKEY: &str = "7RoSF9fUmdphVCpabEoefH81WwrW7orsWonXWqTXkKV8";
@@ -87,6 +88,21 @@ impl GenericRpcClientRequest for MockRpcClientRequest {
                 value: serde_json::to_value(FeeRateGovernor::default()).unwrap(),
             })?,
             RpcRequest::GetSignatureStatus => {
+                let response: Option<transaction::Result<()>> = if self.url == "account_in_use" {
+                    Some(Err(TransactionError::AccountInUse))
+                } else if self.url == "instruction_error" {
+                    Some(Err(TransactionError::InstructionError(
+                        0,
+                        InstructionError::UninitializedAccount,
+                    )))
+                } else if self.url == "sig_not_found" {
+                    None
+                } else {
+                    Some(Ok(()))
+                };
+                serde_json::to_value(response).unwrap()
+            }
+            RpcRequest::GetSignatureStatuses => {
                 let status: transaction::Result<()> = if self.url == "account_in_use" {
                     Err(TransactionError::AccountInUse)
                 } else if self.url == "instruction_error" {
@@ -100,9 +116,16 @@ impl GenericRpcClientRequest for MockRpcClientRequest {
                 let status = if self.url == "sig_not_found" {
                     None
                 } else {
-                    Some(RpcTransactionStatus { status, slot: 1 })
+                    Some(TransactionStatus {
+                        status,
+                        slot: 1,
+                        confirmations: Some(0),
+                    })
                 };
-                serde_json::to_value(vec![status])?
+                serde_json::to_value(Response {
+                    context: RpcResponseContext { slot: 1 },
+                    value: vec![status],
+                })?
             }
             RpcRequest::GetTransactionCount => Value::Number(Number::from(1234)),
             RpcRequest::GetSlot => Value::Number(Number::from(0)),

client/src/rpc_client.rs

@@ -4,11 +4,7 @@ use crate::{
     mock_rpc_client_request::{MockRpcClientRequest, Mocks},
     rpc_client_request::RpcClientRequest,
     rpc_request::{RpcError, RpcRequest},
-    rpc_response::{
-        Response, RpcAccount, RpcBlockhashFeeCalculator, RpcConfirmedBlock, RpcContactInfo,
-        RpcEpochInfo, RpcFeeCalculator, RpcFeeRateGovernor, RpcIdentity, RpcKeyedAccount,
-        RpcLeaderSchedule, RpcResult, RpcTransactionStatus, RpcVersionInfo, RpcVoteAccountStatus,
-    },
+    rpc_response::*,
 };
 use bincode::serialize;
 use indicatif::{ProgressBar, ProgressStyle};
@@ -27,11 +23,11 @@ use solana_sdk::{
     signers::Signers,
     transaction::{self, Transaction, TransactionError},
 };
+use solana_transaction_status::{ConfirmedBlock, TransactionEncoding, TransactionStatus};
 use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
 use std::{
     error,
     net::SocketAddr,
-    str::FromStr,
     thread::sleep,
     time::{Duration, Instant},
 };
@@ -70,7 +66,7 @@ impl RpcClient {
         }
     }
 
-    pub fn confirm_transaction(&self, signature: &str) -> ClientResult<bool> {
+    pub fn confirm_transaction(&self, signature: &Signature) -> ClientResult<bool> {
         Ok(self
             .confirm_transaction_with_commitment(signature, CommitmentConfig::default())?
             .value)
@@ -78,14 +74,14 @@ impl RpcClient {
 
     pub fn confirm_transaction_with_commitment(
         &self,
-        signature: &str,
+        signature: &Signature,
         commitment_config: CommitmentConfig,
     ) -> RpcResult<bool> {
         let response = self
             .client
             .send(
                 &RpcRequest::ConfirmTransaction,
-                json!([signature, commitment_config]),
+                json!([signature.to_string(), commitment_config]),
                 0,
             )
             .map_err(|err| err.into_with_command("ConfirmTransaction"))?;
@@ -94,38 +90,45 @@ impl RpcClient {
             .map_err(|err| ClientError::new_with_command(err.into(), "ConfirmTransaction"))
     }
 
-    pub fn send_transaction(&self, transaction: &Transaction) -> ClientResult<String> {
+    pub fn send_transaction(&self, transaction: &Transaction) -> ClientResult<Signature> {
         let serialized_encoded = bs58::encode(serialize(transaction).unwrap()).into_string();
-        let signature =
+        let response =
             self.client
                 .send(&RpcRequest::SendTransaction, json!([serialized_encoded]), 5)?;
-        if signature.as_str().is_none() {
-            Err(RpcError::ForUser("Received result of an unexpected type".to_string()).into())
-        } else {
-            Ok(signature.as_str().unwrap().to_string())
+
+        match response.as_str() {
+            None => {
+                Err(RpcError::ForUser("Received result of an unexpected type".to_string()).into())
+            }
+            Some(signature_base58_str) => signature_base58_str
+                .parse::<Signature>()
+                .map_err(|err| RpcError::ParseError(err.to_string()).into()),
         }
     }
 
     pub fn get_signature_status(
         &self,
-        signature: &str,
+        signature: &Signature,
     ) -> ClientResult<Option<transaction::Result<()>>> {
         self.get_signature_status_with_commitment(signature, CommitmentConfig::default())
     }
 
     pub fn get_signature_status_with_commitment(
         &self,
-        signature: &str,
+        signature: &Signature,
         commitment_config: CommitmentConfig,
     ) -> ClientResult<Option<transaction::Result<()>>> {
         let signature_status = self.client.send(
-            &RpcRequest::GetSignatureStatus,
+            &RpcRequest::GetSignatureStatuses,
             json!([[signature.to_string()], commitment_config]),
             5,
         )?;
-        let result: Vec<Option<RpcTransactionStatus>> =
-            serde_json::from_value(signature_status).unwrap();
-        Ok(result[0].clone().map(|status_meta| status_meta.status))
+        let result: Response<Vec<Option<TransactionStatus>>> =
+            serde_json::from_value(signature_status)
+                .map_err(|err| ClientError::new_with_command(err.into(), "GetSignatureStatuses"))?;
+        Ok(result.value[0]
+            .clone()
+            .map(|status_meta| status_meta.status))
     }
 
     pub fn get_slot(&self) -> ClientResult<Slot> {
@@ -145,6 +148,23 @@ impl RpcClient {
             .map_err(|err| ClientError::new_with_command(err.into(), "GetSlot"))
     }
 
+    pub fn total_supply(&self) -> ClientResult<u64> {
+        self.total_supply_with_commitment(CommitmentConfig::default())
+    }
+
+    pub fn total_supply_with_commitment(
+        &self,
+        commitment_config: CommitmentConfig,
+    ) -> ClientResult<u64> {
+        let response = self
+            .client
+            .send(&RpcRequest::GetTotalSupply, json!([commitment_config]), 0)
+            .map_err(|err| err.into_with_command("GetTotalSupply"))?;
+
+        serde_json::from_value(response)
+            .map_err(|err| ClientError::new_with_command(err.into(), "GetTotalSupply"))
+    }
+
     pub fn get_vote_accounts(&self) -> ClientResult<RpcVoteAccountStatus> {
         self.get_vote_accounts_with_commitment(CommitmentConfig::default())
     }
@@ -172,10 +192,18 @@ impl RpcClient {
             .map_err(|err| ClientError::new_with_command(err.into(), "GetClusterNodes"))
     }
 
-    pub fn get_confirmed_block(&self, slot: Slot) -> ClientResult<RpcConfirmedBlock> {
+    pub fn get_confirmed_block(&self, slot: Slot) -> ClientResult<ConfirmedBlock> {
+        self.get_confirmed_block_with_encoding(slot, TransactionEncoding::Json)
+    }
+
+    pub fn get_confirmed_block_with_encoding(
+        &self,
+        slot: Slot,
+        encoding: TransactionEncoding,
+    ) -> ClientResult<ConfirmedBlock> {
         let response = self
             .client
-            .send(&RpcRequest::GetConfirmedBlock, json!([slot]), 0)
+            .send(&RpcRequest::GetConfirmedBlock, json!([slot, encoding]), 0)
             .map_err(|err| err.into_with_command("GetConfirmedBlock"))?;
 
         serde_json::from_value(response)
@@ -322,13 +350,13 @@ impl RpcClient {
         &self,
         transaction: &mut Transaction,
         signer_keys: &T,
-    ) -> ClientResult<String> {
+    ) -> ClientResult<Signature> {
         let mut send_retries = 20;
         loop {
             let mut status_retries = 15;
-            let signature_str = self.send_transaction(transaction)?;
+            let signature = self.send_transaction(transaction)?;
             let status = loop {
-                let status = self.get_signature_status(&signature_str)?;
+                let status = self.get_signature_status(&signature)?;
                 if status.is_none() {
                     status_retries -= 1;
                     if status_retries == 0 {
@@ -344,7 +372,7 @@ impl RpcClient {
             };
             send_retries = if let Some(result) = status.clone() {
                 match result {
-                    Ok(_) => return Ok(signature_str),
+                    Ok(_) => return Ok(signature),
                     Err(TransactionError::AccountInUse) => {
                         // Fetch a new blockhash and re-sign the transaction before sending it again
                         self.resign_transaction(transaction, signer_keys)?;
@@ -806,10 +834,9 @@ impl RpcClient {
     ) -> ClientResult<()> {
         let now = Instant::now();
         loop {
-            if let Ok(Some(_)) = self.get_signature_status_with_commitment(
-                &signature.to_string(),
-                commitment_config.clone(),
-            ) {
+            if let Ok(Some(_)) =
+                self.get_signature_status_with_commitment(&signature, commitment_config.clone())
+            {
                 break;
             }
             if now.elapsed().as_secs() > 15 {
@@ -922,21 +949,32 @@ impl RpcClient {
         let response = self
             .client
             .send(
-                &RpcRequest::GetNumBlocksSinceSignatureConfirmation,
-                json!([signature.to_string(), CommitmentConfig::recent().ok()]),
+                &RpcRequest::GetSignatureStatuses,
+                json!([[signature.to_string()], CommitmentConfig::recent().ok()]),
                 1,
             )
-            .map_err(|err| err.into_with_command("GetNumBlocksSinceSignatureConfirmation"))?;
-        serde_json::from_value(response).map_err(|err| {
-            ClientError::new_with_command(err.into(), "GetNumBlocksSinceSignatureConfirmation")
-        })
+            .map_err(|err| err.into_with_command("GetSignatureStatuses"))?;
+        let result: Response<Vec<Option<TransactionStatus>>> = serde_json::from_value(response)
+            .map_err(|err| ClientError::new_with_command(err.into(), "GetSignatureStatuses"))?;
+
+        let confirmations = result.value[0]
+            .clone()
+            .ok_or_else(|| {
+                ClientError::new_with_command(
+                    ClientErrorKind::Custom("signature not found".to_string()),
+                    "GetSignatureStatuses",
+                )
+            })?
+            .confirmations
+            .unwrap_or(MAX_LOCKOUT_HISTORY + 1);
+        Ok(confirmations)
     }
 
     pub fn send_and_confirm_transaction_with_spinner<T: Signers>(
         &self,
         transaction: &mut Transaction,
         signer_keys: &T,
-    ) -> ClientResult<String> {
+    ) -> ClientResult<Signature> {
         let mut confirmations = 0;
 
         let progress_bar = new_spinner_progress_bar();
@@ -947,23 +985,21 @@ impl RpcClient {
         ));
 
         let mut send_retries = 20;
-        let signature_str = loop {
+        let signature = loop {
             let mut status_retries = 15;
-            let (signature_str, status) = loop {
-                let signature_str = self.send_transaction(transaction)?;
+            let (signature, status) = loop {
+                let signature = self.send_transaction(transaction)?;
 
                 // Get recent commitment in order to count confirmations for successful transactions
-                let status = self.get_signature_status_with_commitment(
-                    &signature_str,
-                    CommitmentConfig::recent(),
-                )?;
+                let status = self
+                    .get_signature_status_with_commitment(&signature, CommitmentConfig::recent())?;
                 if status.is_none() {
                     status_retries -= 1;
                     if status_retries == 0 {
-                        break (signature_str, status);
+                        break (signature, status);
                     }
                 } else {
-                    break (signature_str, status);
+                    break (signature, status);
                 }
 
                 if cfg!(not(test)) {
@@ -988,7 +1024,7 @@ impl RpcClient {
                 if let Some(result) = status {
                     match result {
                         Ok(_) => {
-                            break signature_str;
+                            break signature;
                         }
                         Err(err) => {
                             return Err(err.into());
@@ -1001,19 +1037,13 @@ impl RpcClient {
                 }
             }
         };
-        let signature = Signature::from_str(&signature_str).map_err(|_| {
-            ClientError::from(ClientErrorKind::Custom(format!(
-                "Returned string {} cannot be parsed as a signature",
-                signature_str
-            )))
-        })?;
         loop {
             // Return when default (max) commitment is reached
             // Failed transactions have already been eliminated, `is_some` check is sufficient
-            if self.get_signature_status(&signature_str)?.is_some() {
+            if self.get_signature_status(&signature)?.is_some() {
                 progress_bar.set_message("Transaction confirmed");
                 progress_bar.finish_and_clear();
-                return Ok(signature_str);
+                return Ok(signature);
             }
             progress_bar.set_message(&format!(
                 "[{}/{}] Waiting for confirmations",
@@ -1175,7 +1205,7 @@ mod tests {
         let tx = system_transaction::transfer(&key, &to, 50, blockhash);
 
         let signature = rpc_client.send_transaction(&tx);
-        assert_eq!(signature.unwrap(), SIGNATURE.to_string());
+        assert_eq!(signature.unwrap(), SIGNATURE.parse().unwrap());
 
         let rpc_client = RpcClient::new_mock("fails".to_string());
 
@@ -1198,18 +1228,17 @@ mod tests {
 
     #[test]
     fn test_get_signature_status() {
+        let signature = Signature::default();
+
         let rpc_client = RpcClient::new_mock("succeeds".to_string());
-        let signature = "good_signature";
         let status = rpc_client.get_signature_status(&signature).unwrap();
         assert_eq!(status, Some(Ok(())));
 
         let rpc_client = RpcClient::new_mock("sig_not_found".to_string());
-        let signature = "sig_not_found";
         let status = rpc_client.get_signature_status(&signature).unwrap();
         assert_eq!(status, None);
 
         let rpc_client = RpcClient::new_mock("account_in_use".to_string());
-        let signature = "account_in_use";
         let status = rpc_client.get_signature_status(&signature).unwrap();
         assert_eq!(status, Some(Err(TransactionError::AccountInUse)));
     }

client/src/rpc_request.rs

@@ -18,18 +18,19 @@ pub enum RpcRequest {
     GetIdentity,
     GetInflation,
     GetLeaderSchedule,
-    GetNumBlocksSinceSignatureConfirmation,
     GetProgramAccounts,
     GetRecentBlockhash,
     GetFeeCalculatorForBlockhash,
     GetFeeRateGovernor,
     GetSignatureStatus,
+    GetSignatureStatuses,
     GetSlot,
     GetSlotLeader,
     GetStorageTurn,
     GetStorageTurnRate,
     GetSlotsPerSegment,
     GetStoragePubkeysForSlot,
+    GetTotalSupply,
     GetTransactionCount,
     GetVersion,
     GetVoteAccounts,
@@ -60,20 +61,19 @@ impl RpcRequest {
             RpcRequest::GetIdentity => "getIdentity",
             RpcRequest::GetInflation => "getInflation",
             RpcRequest::GetLeaderSchedule => "getLeaderSchedule",
-            RpcRequest::GetNumBlocksSinceSignatureConfirmation => {
-                "getNumBlocksSinceSignatureConfirmation"
-            }
             RpcRequest::GetProgramAccounts => "getProgramAccounts",
             RpcRequest::GetRecentBlockhash => "getRecentBlockhash",
             RpcRequest::GetFeeCalculatorForBlockhash => "getFeeCalculatorForBlockhash",
             RpcRequest::GetFeeRateGovernor => "getFeeRateGovernor",
             RpcRequest::GetSignatureStatus => "getSignatureStatus",
+            RpcRequest::GetSignatureStatuses => "getSignatureStatuses",
             RpcRequest::GetSlot => "getSlot",
             RpcRequest::GetSlotLeader => "getSlotLeader",
             RpcRequest::GetStorageTurn => "getStorageTurn",
             RpcRequest::GetStorageTurnRate => "getStorageTurnRate",
             RpcRequest::GetSlotsPerSegment => "getSlotsPerSegment",
             RpcRequest::GetStoragePubkeysForSlot => "getStoragePubkeysForSlot",
+            RpcRequest::GetTotalSupply => "getTotalSupply",
             RpcRequest::GetTransactionCount => "getTransactionCount",
             RpcRequest::GetVersion => "getVersion",
             RpcRequest::GetVoteAccounts => "getVoteAccounts",

client/src/rpc_response.rs

@@ -1,12 +1,10 @@
 use crate::{client_error, rpc_request::RpcError};
-use bincode::serialize;
 use solana_sdk::{
     account::Account,
     clock::{Epoch, Slot},
     fee_calculator::{FeeCalculator, FeeRateGovernor},
-    message::MessageHeader,
     pubkey::Pubkey,
-    transaction::{Result, Transaction},
+    transaction::Result,
 };
 use std::{collections::HashMap, net::SocketAddr, str::FromStr};
 
@@ -30,126 +28,6 @@ pub struct RpcBlockCommitment<T> {
     pub total_stake: u64,
 }
 
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-pub struct RpcReward {
-    pub pubkey: String,
-    pub lamports: i64,
-}
-
-pub type RpcRewards = Vec<RpcReward>;
-
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcConfirmedBlock {
-    pub previous_blockhash: String,
-    pub blockhash: String,
-    pub parent_slot: Slot,
-    pub transactions: Vec<RpcTransactionWithStatusMeta>,
-    pub rewards: RpcRewards,
-}
-
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcTransactionWithStatusMeta {
-    pub transaction: RpcEncodedTransaction,
-    pub meta: Option<RpcTransactionStatusMeta>,
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-#[serde(rename_all = "camelCase")]
-pub enum RpcTransactionEncoding {
-    Binary,
-    Json,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase", untagged)]
-pub enum RpcEncodedTransaction {
-    Binary(String),
-    Json(RpcTransaction),
-}
-
-impl RpcEncodedTransaction {
-    pub fn encode(transaction: Transaction, encoding: RpcTransactionEncoding) -> Self {
-        if encoding == RpcTransactionEncoding::Json {
-            RpcEncodedTransaction::Json(RpcTransaction {
-                signatures: transaction
-                    .signatures
-                    .iter()
-                    .map(|sig| sig.to_string())
-                    .collect(),
-                message: RpcMessage {
-                    header: transaction.message.header,
-                    account_keys: transaction
-                        .message
-                        .account_keys
-                        .iter()
-                        .map(|pubkey| pubkey.to_string())
-                        .collect(),
-                    recent_blockhash: transaction.message.recent_blockhash.to_string(),
-                    instructions: transaction
-                        .message
-                        .instructions
-                        .iter()
-                        .map(|instruction| RpcCompiledInstruction {
-                            program_id_index: instruction.program_id_index,
-                            accounts: instruction.accounts.clone(),
-                            data: bs58::encode(instruction.data.clone()).into_string(),
-                        })
-                        .collect(),
-                },
-            })
-        } else {
-            RpcEncodedTransaction::Binary(
-                bs58::encode(serialize(&transaction).unwrap()).into_string(),
-            )
-        }
-    }
-}
-
-/// A duplicate representation of a Transaction for pretty JSON serialization
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcTransaction {
-    pub signatures: Vec<String>,
-    pub message: RpcMessage,
-}
-
-/// A duplicate representation of a Message for pretty JSON serialization
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcMessage {
-    pub header: MessageHeader,
-    pub account_keys: Vec<String>,
-    pub recent_blockhash: String,
-    pub instructions: Vec<RpcCompiledInstruction>,
-}
-
-/// A duplicate representation of a Message for pretty JSON serialization
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcCompiledInstruction {
-    pub program_id_index: u8,
-    pub accounts: Vec<u8>,
-    pub data: String,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcTransactionStatusMeta {
-    pub status: Result<()>,
-    pub fee: u64,
-    pub pre_balances: Vec<u64>,
-    pub post_balances: Vec<u64>,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcTransactionStatus {
-    pub slot: Slot,
-    pub status: Result<()>,
-}
-
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcBlockhashFeeCalculator {

client/src/thin_client.rs

@@ -471,7 +471,7 @@ impl SyncClient for ThinClient {
     ) -> TransportResult<Option<transaction::Result<()>>> {
         let status = self
             .rpc_client()
-            .get_signature_status(&signature.to_string())
+            .get_signature_status(&signature)
             .map_err(|err| {
                 io::Error::new(
                     io::ErrorKind::Other,
@@ -488,7 +488,7 @@ impl SyncClient for ThinClient {
     ) -> TransportResult<Option<transaction::Result<()>>> {
         let status = self
             .rpc_client()
-            .get_signature_status_with_commitment(&signature.to_string(), commitment_config)
+            .get_signature_status_with_commitment(&signature, commitment_config)
             .map_err(|err| {
                 io::Error::new(
                     io::ErrorKind::Other,

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.10"
+version = "1.1.1"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -17,65 +17,65 @@ codecov = { repository = "solana-labs/solana", branch = "master", service = "git
 bincode = "1.2.1"
 bv = { version = "0.11.1", features = ["serde"] }
 bs58 = "0.3.0"
-byteorder = "1.3.2"
-chrono = { version = "0.4.10", features = ["serde"] }
-compression = "0.1.5"
+byteorder = "1.3.4"
+chrono = { version = "0.4.11", features = ["serde"] }
 core_affinity = "0.5.10"
-crossbeam-channel = "0.3"
+crossbeam-channel = "0.4"
 fs_extra = "1.1.0"
+flate2 = "1.0"
 indexmap = "1.3"
-itertools = "0.8.2"
+itertools = "0.9.0"
 jsonrpc-core = "14.0.5"
 jsonrpc-core-client = { version = "14.0.5", features = ["ws"] }
 jsonrpc-derive = "14.0.5"
 jsonrpc-http-server = "14.0.6"
 jsonrpc-pubsub = "14.0.6"
 jsonrpc-ws-server = "14.0.6"
-libc = "0.2.66"
 log = "0.4.8"
-nix = "0.17.0"
 num_cpus = "1.0.0"
 num-traits = "0.2"
 rand = "0.6.5"
 rand_chacha = "0.1.1"
-rayon = "1.2.0"
-regex = "1.3.4"
-serde = "1.0.104"
+rayon = "1.3.0"
+regex = "1.3.6"
+serde = "1.0.105"
 serde_derive = "1.0.103"
-serde_json = "1.0.46"
-solana-budget-program = { path = "../programs/budget", version = "1.0.10" }
-solana-clap-utils = { path = "../clap-utils", version = "1.0.10" }
-solana-client = { path = "../client", version = "1.0.10" }
-solana-faucet = { path = "../faucet", version = "1.0.10" }
+serde_json = "1.0.48"
+solana-budget-program = { path = "../programs/budget", version = "1.1.1" }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.1" }
+solana-client = { path = "../client", version = "1.1.1" }
+solana-transaction-status = { path = "../transaction-status", version = "1.1.1" }
+solana-faucet = { path = "../faucet", version = "1.1.1" }
 ed25519-dalek = "=1.0.0-pre.1"
-solana-ledger = { path = "../ledger", version = "1.0.10" }
-solana-logger = { path = "../logger", version = "1.0.10" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.0.10" }
-solana-metrics = { path = "../metrics", version = "1.0.10" }
-solana-measure = { path = "../measure", version = "1.0.10" }
-solana-net-utils = { path = "../net-utils", version = "1.0.10" }
-solana-chacha-cuda = { path = "../chacha-cuda", version = "1.0.10" }
-solana-perf = { path = "../perf", version = "1.0.10" }
-solana-runtime = { path = "../runtime", version = "1.0.10" }
-solana-sdk = { path = "../sdk", version = "1.0.10" }
-solana-stake-program = { path = "../programs/stake", version = "1.0.10" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.10" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.10" }
-solana-vote-signer = { path = "../vote-signer", version = "1.0.10" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.0.10" }
+solana-ledger = { path = "../ledger", version = "1.1.1" }
+solana-logger = { path = "../logger", version = "1.1.1" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.1.1" }
+solana-metrics = { path = "../metrics", version = "1.1.1" }
+solana-measure = { path = "../measure", version = "1.1.1" }
+solana-net-utils = { path = "../net-utils", version = "1.1.1" }
+solana-chacha-cuda = { path = "../chacha-cuda", version = "1.1.1" }
+solana-perf = { path = "../perf", version = "1.1.1" }
+solana-runtime = { path = "../runtime", version = "1.1.1" }
+solana-sdk = { path = "../sdk", version = "1.1.1" }
+solana-stake-program = { path = "../programs/stake", version = "1.1.1" }
+solana-storage-program = { path = "../programs/storage", version = "1.1.1" }
+solana-streamer = { path = "../streamer", version = "1.1.1" }
+solana-vote-program = { path = "../programs/vote", version = "1.1.1" }
+solana-vote-signer = { path = "../vote-signer", version = "1.1.1" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.1.1" }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = "0.1"
 tokio-codec = "0.1"
 tokio-fs = "0.1"
 tokio-io = "0.1"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.0.10" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.1.1" }
 trees = "0.2.1"
 
 [dev-dependencies]
 matches = "0.1.6"
-reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
-serial_test = "0.3.2"
+reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
 systemstat = "0.1.5"
 

core/benches/banking_stage.rs

@@ -9,12 +9,12 @@ use rayon::prelude::*;
 use solana_core::banking_stage::{create_test_recorder, BankingStage};
 use solana_core::cluster_info::ClusterInfo;
 use solana_core::cluster_info::Node;
-use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-use solana_core::packet::to_packets_chunked;
 use solana_core::poh_recorder::WorkingBankEntry;
 use solana_ledger::blockstore_processor::process_entries;
 use solana_ledger::entry::{next_hash, Entry};
+use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
 use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
+use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_runtime::bank::Bank;
 use solana_sdk::genesis_config::GenesisConfig;

core/benches/retransmit_stage.rs

@@ -6,12 +6,12 @@ extern crate test;
 use log::*;
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::contact_info::ContactInfo;
-use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-use solana_core::packet::to_packets_chunked;
 use solana_core::retransmit_stage::retransmitter;
 use solana_ledger::bank_forks::BankForks;
+use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
 use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
 use solana_measure::measure::Measure;
+use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_runtime::bank::Bank;
 use solana_sdk::pubkey::Pubkey;

core/benches/sigverify_stage.rs

@@ -6,9 +6,9 @@ extern crate test;
 use crossbeam_channel::unbounded;
 use log::*;
 use rand::{thread_rng, Rng};
-use solana_core::packet::to_packets_chunked;
 use solana_core::sigverify::TransactionSigVerifier;
 use solana_core::sigverify_stage::SigVerifyStage;
+use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_sdk::hash::Hash;
 use solana_sdk::signature::{Keypair, Signer};

core/src/accounts_cleanup_service.rs

@@ -0,0 +1,38 @@
+// Service to clean up dead slots in accounts_db
+//
+// This can be expensive since we have to walk the append vecs being cleaned up.
+
+use solana_ledger::bank_forks::BankForks;
+use std::sync::{
+    atomic::{AtomicBool, Ordering},
+    Arc, RwLock,
+};
+use std::thread::{self, sleep, Builder, JoinHandle};
+use std::time::Duration;
+
+pub struct AccountsCleanupService {
+    t_cleanup: JoinHandle<()>,
+}
+
+impl AccountsCleanupService {
+    pub fn new(bank_forks: Arc<RwLock<BankForks>>, exit: &Arc<AtomicBool>) -> Self {
+        info!("AccountsCleanupService active");
+        let exit = exit.clone();
+        let t_cleanup = Builder::new()
+            .name("solana-accounts-cleanup".to_string())
+            .spawn(move || loop {
+                if exit.load(Ordering::Relaxed) {
+                    break;
+                }
+                let bank = bank_forks.read().unwrap().working_bank();
+                bank.clean_dead_slots();
+                sleep(Duration::from_millis(100));
+            })
+            .unwrap();
+        Self { t_cleanup }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.t_cleanup.join()
+    }
+}

core/src/accounts_hash_verifier.rs

@@ -4,7 +4,7 @@
 // hash on gossip. Monitor gossip for messages from validators in the --trusted-validators
 // set and halt the node if a mismatch is detected.
 
-use crate::cluster_info::ClusterInfo;
+use crate::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES};
 use solana_ledger::{
     snapshot_package::SnapshotPackage, snapshot_package::SnapshotPackageReceiver,
     snapshot_package::SnapshotPackageSender,
@@ -94,6 +94,10 @@ impl AccountsHashVerifier {
             hashes.push((snapshot_package.root, snapshot_package.hash));
         }
 
+        while hashes.len() > MAX_SNAPSHOT_HASHES {
+            hashes.remove(0);
+        }
+
         if halt_on_trusted_validator_accounts_hash_mismatch {
             let mut slot_to_hash = HashMap::new();
             for (slot, hash) in hashes.iter() {
@@ -119,6 +123,7 @@ impl AccountsHashVerifier {
         slot_to_hash: &mut HashMap<Slot, Hash>,
     ) -> bool {
         let mut verified_count = 0;
+        let mut highest_slot = 0;
         if let Some(trusted_validators) = trusted_validators.as_ref() {
             for trusted_validator in trusted_validators {
                 let cluster_info_r = cluster_info.read().unwrap();
@@ -140,6 +145,7 @@ impl AccountsHashVerifier {
                                 verified_count += 1;
                             }
                         } else {
+                            highest_slot = std::cmp::max(*slot, highest_slot);
                             slot_to_hash.insert(*slot, *hash);
                         }
                     }
@@ -147,6 +153,10 @@ impl AccountsHashVerifier {
             }
         }
         inc_new_counter_info!("accounts_hash_verifier-hashes_verified", verified_count);
+        datapoint_info!(
+            "accounts_hash_verifier",
+            ("highest_slot_verified", highest_slot, i64),
+        );
         false
     }
 
@@ -197,4 +207,57 @@ mod tests {
             &mut slot_to_hash,
         ));
     }
+
+    #[test]
+    fn test_max_hashes() {
+        solana_logger::setup();
+        use std::path::PathBuf;
+        use tempfile::TempDir;
+        let keypair = Keypair::new();
+
+        let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0);
+        let cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
+        let cluster_info = Arc::new(RwLock::new(cluster_info));
+
+        let trusted_validators = HashSet::new();
+        let exit = Arc::new(AtomicBool::new(false));
+        let mut hashes = vec![];
+        for i in 0..MAX_SNAPSHOT_HASHES + 1 {
+            let snapshot_links = TempDir::new().unwrap();
+            let snapshot_package = SnapshotPackage {
+                hash: hash(&[i as u8]),
+                root: 100 + i as u64,
+                slot_deltas: vec![],
+                snapshot_links,
+                tar_output_file: PathBuf::from("."),
+                storages: vec![],
+            };
+
+            AccountsHashVerifier::process_snapshot(
+                snapshot_package,
+                &cluster_info,
+                &Some(trusted_validators.clone()),
+                false,
+                &None,
+                &mut hashes,
+                &exit,
+                0,
+            );
+        }
+        let cluster_info_r = cluster_info.read().unwrap();
+        let cluster_hashes = cluster_info_r
+            .get_accounts_hash_for_node(&keypair.pubkey())
+            .unwrap();
+        info!("{:?}", cluster_hashes);
+        assert_eq!(hashes.len(), MAX_SNAPSHOT_HASHES);
+        assert_eq!(cluster_hashes.len(), MAX_SNAPSHOT_HASHES);
+        assert_eq!(cluster_hashes[0], (101, hash(&[1])));
+        assert_eq!(
+            cluster_hashes[MAX_SNAPSHOT_HASHES - 1],
+            (
+                100 + MAX_SNAPSHOT_HASHES as u64,
+                hash(&[MAX_SNAPSHOT_HASHES as u8])
+            )
+        );
+    }
 }

core/src/banking_stage.rs

@@ -3,7 +3,6 @@
 //! can do its processing in parallel with signature verification on the GPU.
 use crate::{
     cluster_info::ClusterInfo,
-    packet::{limited_deserialize, Packet, Packets, PACKETS_PER_BATCH},
     poh_recorder::{PohRecorder, PohRecorderError, WorkingBankEntry},
     poh_service::PohService,
 };
@@ -17,7 +16,11 @@ use solana_ledger::{
 };
 use solana_measure::{measure::Measure, thread_mem_usage};
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_info, inc_new_counter_warn};
-use solana_perf::{cuda_runtime::PinnedVec, perf_libs};
+use solana_perf::{
+    cuda_runtime::PinnedVec,
+    packet::{limited_deserialize, Packet, Packets, PACKETS_PER_BATCH},
+    perf_libs,
+};
 use solana_runtime::{
     accounts_db::ErrorCounters,
     bank::{Bank, TransactionBalancesSet, TransactionProcessResult},
@@ -1009,20 +1012,18 @@ pub fn create_test_recorder(
 mod tests {
     use super::*;
     use crate::{
-        cluster_info::Node,
-        genesis_utils::{create_genesis_config, GenesisConfigInfo},
-        packet::to_packets,
-        poh_recorder::WorkingBank,
+        cluster_info::Node, poh_recorder::WorkingBank,
         transaction_status_service::TransactionStatusService,
     };
     use crossbeam_channel::unbounded;
     use itertools::Itertools;
-    use solana_client::rpc_response::{RpcEncodedTransaction, RpcTransactionWithStatusMeta};
     use solana_ledger::{
         blockstore::entries_to_test_shreds,
         entry::{next_entry, Entry, EntrySlice},
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
         get_tmp_ledger_path,
     };
+    use solana_perf::packet::to_packets;
     use solana_runtime::bank::HashAgeKind;
     use solana_sdk::{
         instruction::InstructionError,
@@ -1030,6 +1031,7 @@ mod tests {
         system_transaction,
         transaction::TransactionError,
     };
+    use solana_transaction_status::{EncodedTransaction, TransactionWithStatusMeta};
     use std::{sync::atomic::Ordering, thread::sleep};
 
     #[test]
@@ -1975,10 +1977,10 @@ mod tests {
             let confirmed_block = blockstore.get_confirmed_block(bank.slot(), None).unwrap();
             assert_eq!(confirmed_block.transactions.len(), 3);
 
-            for RpcTransactionWithStatusMeta { transaction, meta } in
+            for TransactionWithStatusMeta { transaction, meta } in
                 confirmed_block.transactions.into_iter()
             {
-                if let RpcEncodedTransaction::Json(transaction) = transaction {
+                if let EncodedTransaction::Json(transaction) = transaction {
                     if transaction.signatures[0] == success_signature.to_string() {
                         assert_eq!(meta.unwrap().status, Ok(()));
                     } else if transaction.signatures[0] == ix_error_signature.to_string() {

core/src/blockstream.rs

@@ -1,283 +0,0 @@
-//! The `blockstream` module provides a method for streaming entries out via a
-//! local unix socket, to provide client services such as a block explorer with
-//! real-time access to entries.
-
-use bincode::serialize;
-use chrono::{SecondsFormat, Utc};
-use serde_json::json;
-use solana_ledger::entry::Entry;
-use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
-use std::cell::RefCell;
-use std::io::Result;
-use std::path::{Path, PathBuf};
-
-pub trait EntryWriter: std::fmt::Debug {
-    fn write(&self, payload: String) -> Result<()>;
-}
-
-#[derive(Debug, Default)]
-pub struct EntryVec {
-    values: RefCell<Vec<String>>,
-}
-
-impl EntryWriter for EntryVec {
-    fn write(&self, payload: String) -> Result<()> {
-        self.values.borrow_mut().push(payload);
-        Ok(())
-    }
-}
-
-impl EntryVec {
-    pub fn new() -> Self {
-        EntryVec {
-            values: RefCell::new(Vec::new()),
-        }
-    }
-
-    pub fn entries(&self) -> Vec<String> {
-        self.values.borrow().clone()
-    }
-}
-
-#[derive(Debug)]
-pub struct EntrySocket {
-    unix_socket: PathBuf,
-}
-
-impl EntryWriter for EntrySocket {
-    #[cfg(not(windows))]
-    fn write(&self, payload: String) -> Result<()> {
-        use std::io::prelude::*;
-        use std::net::Shutdown;
-        use std::os::unix::net::UnixStream;
-
-        const MESSAGE_TERMINATOR: &str = "\n";
-
-        let mut socket = UnixStream::connect(&self.unix_socket)?;
-        socket.write_all(payload.as_bytes())?;
-        socket.write_all(MESSAGE_TERMINATOR.as_bytes())?;
-        socket.shutdown(Shutdown::Write)?;
-        Ok(())
-    }
-    #[cfg(windows)]
-    fn write(&self, _payload: String) -> Result<()> {
-        Err(std::io::Error::new(
-            std::io::ErrorKind::Other,
-            "EntryWriter::write() not implemented for windows",
-        ))
-    }
-}
-
-pub trait BlockstreamEvents {
-    fn emit_entry_event(
-        &self,
-        slot: Slot,
-        tick_height: u64,
-        leader_pubkey: &Pubkey,
-        entries: &Entry,
-    ) -> Result<()>;
-    fn emit_block_event(
-        &self,
-        slot: Slot,
-        tick_height: u64,
-        leader_pubkey: &Pubkey,
-        blockhash: Hash,
-    ) -> Result<()>;
-}
-
-#[derive(Debug)]
-pub struct Blockstream<T: EntryWriter> {
-    pub output: T,
-}
-
-impl<T> BlockstreamEvents for Blockstream<T>
-where
-    T: EntryWriter,
-{
-    fn emit_entry_event(
-        &self,
-        slot: Slot,
-        tick_height: u64,
-        leader_pubkey: &Pubkey,
-        entry: &Entry,
-    ) -> Result<()> {
-        let transactions: Vec<Vec<u8>> = serialize_transactions(entry);
-        let stream_entry = json!({
-            "num_hashes": entry.num_hashes,
-            "hash": entry.hash,
-            "transactions": transactions
-        });
-        let json_entry = serde_json::to_string(&stream_entry)?;
-        let payload = format!(
-            r#"{{"dt":"{}","t":"entry","s":{},"h":{},"l":"{:?}","entry":{}}}"#,
-            Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
-            slot,
-            tick_height,
-            leader_pubkey,
-            json_entry,
-        );
-        self.output.write(payload)?;
-        Ok(())
-    }
-
-    fn emit_block_event(
-        &self,
-        slot: Slot,
-        tick_height: u64,
-        leader_pubkey: &Pubkey,
-        blockhash: Hash,
-    ) -> Result<()> {
-        let payload = format!(
-            r#"{{"dt":"{}","t":"block","s":{},"h":{},"l":"{:?}","hash":"{:?}"}}"#,
-            Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
-            slot,
-            tick_height,
-            leader_pubkey,
-            blockhash,
-        );
-        self.output.write(payload)?;
-        Ok(())
-    }
-}
-
-pub type SocketBlockstream = Blockstream<EntrySocket>;
-
-impl SocketBlockstream {
-    pub fn new(unix_socket: &Path) -> Self {
-        Blockstream {
-            output: EntrySocket {
-                unix_socket: unix_socket.to_path_buf(),
-            },
-        }
-    }
-}
-
-pub type MockBlockstream = Blockstream<EntryVec>;
-
-impl MockBlockstream {
-    pub fn new(_: &Path) -> Self {
-        Blockstream {
-            output: EntryVec::new(),
-        }
-    }
-
-    pub fn entries(&self) -> Vec<String> {
-        self.output.entries()
-    }
-}
-
-fn serialize_transactions(entry: &Entry) -> Vec<Vec<u8>> {
-    entry
-        .transactions
-        .iter()
-        .map(|tx| serialize(&tx).unwrap())
-        .collect()
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use chrono::{DateTime, FixedOffset};
-    use serde_json::Value;
-    use solana_sdk::hash::Hash;
-    use solana_sdk::signature::{Keypair, Signer};
-    use solana_sdk::system_transaction;
-    use std::collections::HashSet;
-    use std::path::PathBuf;
-
-    #[test]
-    fn test_serialize_transactions() {
-        let entry = Entry::new(&Hash::default(), 1, vec![]);
-        let empty_vec: Vec<Vec<u8>> = vec![];
-        assert_eq!(serialize_transactions(&entry), empty_vec);
-
-        let keypair0 = Keypair::new();
-        let keypair1 = Keypair::new();
-        let tx0 = system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default());
-        let tx1 = system_transaction::transfer(&keypair1, &keypair0.pubkey(), 2, Hash::default());
-        let serialized_tx0 = serialize(&tx0).unwrap();
-        let serialized_tx1 = serialize(&tx1).unwrap();
-        let entry = Entry::new(&Hash::default(), 1, vec![tx0, tx1]);
-        assert_eq!(
-            serialize_transactions(&entry),
-            vec![serialized_tx0, serialized_tx1]
-        );
-    }
-
-    #[test]
-    fn test_blockstream() -> () {
-        let blockstream = MockBlockstream::new(&PathBuf::from("test_stream"));
-        let ticks_per_slot = 5;
-
-        let mut blockhash = Hash::default();
-        let mut entries = Vec::new();
-        let mut expected_entries = Vec::new();
-
-        let tick_height_initial = 1;
-        let tick_height_final = tick_height_initial + ticks_per_slot + 2;
-        let mut curr_slot = 0;
-        let leader_pubkey = Pubkey::new_rand();
-
-        for tick_height in tick_height_initial..=tick_height_final {
-            if tick_height == 5 {
-                blockstream
-                    .emit_block_event(curr_slot, tick_height, &leader_pubkey, blockhash)
-                    .unwrap();
-                curr_slot += 1;
-            }
-            let entry = Entry::new(&mut blockhash, 1, vec![]); // just ticks
-            blockhash = entry.hash;
-            blockstream
-                .emit_entry_event(curr_slot, tick_height, &leader_pubkey, &entry)
-                .unwrap();
-            expected_entries.push(entry.clone());
-            entries.push(entry);
-        }
-
-        assert_eq!(
-            blockstream.entries().len() as u64,
-            // one entry per tick (1..=N+2) is +3, plus one block
-            ticks_per_slot + 3 + 1
-        );
-
-        let mut j = 0;
-        let mut matched_entries = 0;
-        let mut matched_slots = HashSet::new();
-        let mut matched_blocks = HashSet::new();
-
-        for item in blockstream.entries() {
-            let json: Value = serde_json::from_str(&item).unwrap();
-            let dt_str = json["dt"].as_str().unwrap();
-
-            // Ensure `ts` field parses as valid DateTime
-            let _dt: DateTime<FixedOffset> = DateTime::parse_from_rfc3339(dt_str).unwrap();
-
-            let item_type = json["t"].as_str().unwrap();
-            match item_type {
-                "block" => {
-                    let hash = json["hash"].to_string();
-                    matched_blocks.insert(hash);
-                }
-
-                "entry" => {
-                    let slot = json["s"].as_u64().unwrap();
-                    matched_slots.insert(slot);
-                    let entry_obj = json["entry"].clone();
-                    let entry: Entry = serde_json::from_value(entry_obj).unwrap();
-
-                    assert_eq!(entry, expected_entries[j]);
-                    matched_entries += 1;
-                    j += 1;
-                }
-
-                _ => {
-                    assert!(false, "unknown item type {}", item);
-                }
-            }
-        }
-
-        assert_eq!(matched_entries, expected_entries.len());
-        assert_eq!(matched_slots.len(), 2);
-        assert_eq!(matched_blocks.len(), 1);
-    }
-}

core/src/blockstream_service.rs

@@ -1,228 +0,0 @@
-//! The `blockstream_service` implements optional streaming of entries and block metadata
-//! using the `blockstream` module, providing client services such as a block explorer with
-//! real-time access to entries.
-
-use crate::blockstream::BlockstreamEvents;
-#[cfg(test)]
-use crate::blockstream::MockBlockstream as Blockstream;
-#[cfg(not(test))]
-use crate::blockstream::SocketBlockstream as Blockstream;
-use crate::result::{Error, Result};
-use solana_ledger::blockstore::Blockstore;
-use solana_sdk::pubkey::Pubkey;
-use std::path::Path;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::mpsc::{Receiver, RecvTimeoutError};
-use std::sync::Arc;
-use std::thread::{self, Builder, JoinHandle};
-use std::time::Duration;
-
-pub struct BlockstreamService {
-    t_blockstream: JoinHandle<()>,
-}
-
-impl BlockstreamService {
-    #[allow(clippy::new_ret_no_self)]
-    pub fn new(
-        slot_full_receiver: Receiver<(u64, Pubkey)>,
-        blockstore: Arc<Blockstore>,
-        unix_socket: &Path,
-        exit: &Arc<AtomicBool>,
-    ) -> Self {
-        let mut blockstream = Blockstream::new(unix_socket);
-        let exit = exit.clone();
-        let t_blockstream = Builder::new()
-            .name("solana-blockstream".to_string())
-            .spawn(move || loop {
-                if exit.load(Ordering::Relaxed) {
-                    break;
-                }
-                if let Err(e) =
-                    Self::process_entries(&slot_full_receiver, &blockstore, &mut blockstream)
-                {
-                    match e {
-                        Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
-                        Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
-                        _ => info!("Error from process_entries: {:?}", e),
-                    }
-                }
-            })
-            .unwrap();
-        Self { t_blockstream }
-    }
-    fn process_entries(
-        slot_full_receiver: &Receiver<(u64, Pubkey)>,
-        blockstore: &Arc<Blockstore>,
-        blockstream: &mut Blockstream,
-    ) -> Result<()> {
-        let timeout = Duration::new(1, 0);
-        let (slot, slot_leader) = slot_full_receiver.recv_timeout(timeout)?;
-
-        // Slot might not exist due to LedgerCleanupService, check first
-        let blockstore_meta = blockstore.meta(slot).unwrap();
-        if let Some(blockstore_meta) = blockstore_meta {
-            // Return error to main loop. Thread won't exit, will just log the error
-            let entries = blockstore.get_slot_entries(slot, 0, None)?;
-            let _parent_slot = if slot == 0 {
-                None
-            } else {
-                Some(blockstore_meta.parent_slot)
-            };
-            let ticks_per_slot = entries.iter().filter(|entry| entry.is_tick()).count() as u64;
-            let mut tick_height = ticks_per_slot * slot;
-
-            for (i, entry) in entries.iter().enumerate() {
-                if entry.is_tick() {
-                    tick_height += 1;
-                }
-                blockstream
-                    .emit_entry_event(slot, tick_height, &slot_leader, &entry)
-                    .unwrap_or_else(|e| {
-                        debug!("Blockstream error: {:?}, {:?}", e, blockstream.output);
-                    });
-                if i == entries.len() - 1 {
-                    blockstream
-                        .emit_block_event(slot, tick_height, &slot_leader, entry.hash)
-                        .unwrap_or_else(|e| {
-                            debug!("Blockstream error: {:?}, {:?}", e, blockstream.output);
-                        });
-                }
-            }
-        }
-        Ok(())
-    }
-
-    pub fn join(self) -> thread::Result<()> {
-        self.t_blockstream.join()
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-    use bincode::{deserialize, serialize};
-    use chrono::{DateTime, FixedOffset};
-    use serde_json::Value;
-    use solana_ledger::create_new_tmp_ledger;
-    use solana_ledger::entry::{create_ticks, Entry};
-    use solana_sdk::hash::Hash;
-    use solana_sdk::signature::{Keypair, Signer};
-    use solana_sdk::system_transaction;
-    use std::path::PathBuf;
-    use std::sync::mpsc::channel;
-
-    #[test]
-    fn test_blockstream_service_process_entries() {
-        let ticks_per_slot = 5;
-        let leader_pubkey = Pubkey::new_rand();
-
-        // Set up genesis config and blockstore
-        let GenesisConfigInfo {
-            mut genesis_config, ..
-        } = create_genesis_config(1000);
-        genesis_config.ticks_per_slot = ticks_per_slot;
-
-        let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
-        let blockstore = Blockstore::open(&ledger_path).unwrap();
-
-        // Set up blockstream
-        let mut blockstream = Blockstream::new(&PathBuf::from("test_stream"));
-
-        // Set up dummy channel to receive a full-slot notification
-        let (slot_full_sender, slot_full_receiver) = channel();
-
-        // Create entries - 4 ticks + 1 populated entry + 1 tick
-        let mut entries = create_ticks(4, 0, Hash::default());
-
-        let keypair = Keypair::new();
-        let mut blockhash = entries[3].hash;
-        let tx = system_transaction::transfer(&keypair, &keypair.pubkey(), 1, Hash::default());
-        let entry = Entry::new(&mut blockhash, 1, vec![tx]);
-        blockhash = entry.hash;
-        entries.push(entry);
-        let final_tick = create_ticks(1, 0, blockhash);
-        entries.extend_from_slice(&final_tick);
-
-        let expected_entries = entries.clone();
-        let expected_tick_heights = [6, 7, 8, 9, 9, 10];
-
-        blockstore
-            .write_entries(
-                1,
-                0,
-                0,
-                ticks_per_slot,
-                None,
-                true,
-                &Arc::new(Keypair::new()),
-                entries,
-                0,
-            )
-            .unwrap();
-
-        slot_full_sender.send((1, leader_pubkey)).unwrap();
-        BlockstreamService::process_entries(
-            &slot_full_receiver,
-            &Arc::new(blockstore),
-            &mut blockstream,
-        )
-        .unwrap();
-        assert_eq!(blockstream.entries().len(), 7);
-
-        let (entry_events, block_events): (Vec<Value>, Vec<Value>) = blockstream
-            .entries()
-            .iter()
-            .map(|item| {
-                let json: Value = serde_json::from_str(&item).unwrap();
-                let dt_str = json["dt"].as_str().unwrap();
-                // Ensure `ts` field parses as valid DateTime
-                let _dt: DateTime<FixedOffset> = DateTime::parse_from_rfc3339(dt_str).unwrap();
-                json
-            })
-            .partition(|json| {
-                let item_type = json["t"].as_str().unwrap();
-                item_type == "entry"
-            });
-        for (i, json) in entry_events.iter().enumerate() {
-            let height = json["h"].as_u64().unwrap();
-            assert_eq!(height, expected_tick_heights[i]);
-            let entry_obj = json["entry"].clone();
-            let tx = entry_obj["transactions"].as_array().unwrap();
-            let entry: Entry;
-            if tx.len() == 0 {
-                entry = serde_json::from_value(entry_obj).unwrap();
-            } else {
-                let entry_json = entry_obj.as_object().unwrap();
-                entry = Entry {
-                    num_hashes: entry_json.get("num_hashes").unwrap().as_u64().unwrap(),
-                    hash: serde_json::from_value(entry_json.get("hash").unwrap().clone()).unwrap(),
-                    transactions: entry_json
-                        .get("transactions")
-                        .unwrap()
-                        .as_array()
-                        .unwrap()
-                        .into_iter()
-                        .enumerate()
-                        .map(|(j, tx)| {
-                            let tx_vec: Vec<u8> = serde_json::from_value(tx.clone()).unwrap();
-                            // Check explicitly that transaction matches bincode-serialized format
-                            assert_eq!(
-                                tx_vec,
-                                serialize(&expected_entries[i].transactions[j]).unwrap()
-                            );
-                            deserialize(&tx_vec).unwrap()
-                        })
-                        .collect(),
-                };
-            }
-            assert_eq!(entry, expected_entries[i]);
-        }
-        for json in block_events {
-            let slot = json["s"].as_u64().unwrap();
-            assert_eq!(1, slot);
-            let height = json["h"].as_u64().unwrap();
-            assert_eq!(2 * ticks_per_slot, height);
-        }
-    }
-}

core/src/broadcast_stage.rs

@@ -1,30 +1,41 @@
 //! A stage to broadcast data from a leader node to validators
-use self::broadcast_fake_shreds_run::BroadcastFakeShredsRun;
-use self::fail_entry_verification_broadcast_run::FailEntryVerificationBroadcastRun;
-use self::standard_broadcast_run::StandardBroadcastRun;
-use crate::cluster_info::{ClusterInfo, ClusterInfoError};
-use crate::poh_recorder::WorkingBankEntry;
-use crate::result::{Error, Result};
-use solana_ledger::blockstore::Blockstore;
-use solana_ledger::shred::Shred;
-use solana_ledger::staking_utils;
+use self::{
+    broadcast_fake_shreds_run::BroadcastFakeShredsRun,
+    fail_entry_verification_broadcast_run::FailEntryVerificationBroadcastRun,
+    standard_broadcast_run::StandardBroadcastRun,
+};
+use crate::{
+    cluster_info::{ClusterInfo, ClusterInfoError},
+    poh_recorder::WorkingBankEntry,
+    result::{Error, Result},
+};
+use crossbeam_channel::{
+    Receiver as CrossbeamReceiver, RecvTimeoutError as CrossbeamRecvTimeoutError,
+    Sender as CrossbeamSender,
+};
+use solana_ledger::{blockstore::Blockstore, shred::Shred, staking_utils};
 use solana_metrics::{inc_new_counter_error, inc_new_counter_info};
-use solana_sdk::pubkey::Pubkey;
-use std::collections::HashMap;
-use std::net::UdpSocket;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender};
-use std::sync::{Arc, Mutex, RwLock};
-use std::thread::{self, Builder, JoinHandle};
-use std::time::Instant;
-
-pub const NUM_INSERT_THREADS: usize = 2;
+use solana_runtime::bank::Bank;
+use solana_sdk::{clock::Slot, pubkey::Pubkey};
+use std::{
+    collections::HashMap,
+    net::UdpSocket,
+    sync::atomic::{AtomicBool, Ordering},
+    sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender},
+    sync::{Arc, Mutex, RwLock},
+    thread::{self, Builder, JoinHandle},
+    time::{Duration, Instant},
+};
 
 mod broadcast_fake_shreds_run;
 pub(crate) mod broadcast_utils;
 mod fail_entry_verification_broadcast_run;
 mod standard_broadcast_run;
 
+pub const NUM_INSERT_THREADS: usize = 2;
+pub type RetransmitSlotsSender = CrossbeamSender<HashMap<Slot, Arc<Bank>>>;
+pub type RetransmitSlotsReceiver = CrossbeamReceiver<HashMap<Slot, Arc<Bank>>>;
+
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub enum BroadcastStageReturnType {
     ChannelDisconnected,
@@ -43,6 +54,7 @@ impl BroadcastStageType {
         sock: Vec<UdpSocket>,
         cluster_info: Arc<RwLock<ClusterInfo>>,
         receiver: Receiver<WorkingBankEntry>,
+        retransmit_slots_receiver: RetransmitSlotsReceiver,
         exit_sender: &Arc<AtomicBool>,
         blockstore: &Arc<Blockstore>,
         shred_version: u16,
@@ -53,6 +65,7 @@ impl BroadcastStageType {
                 sock,
                 cluster_info,
                 receiver,
+                retransmit_slots_receiver,
                 exit_sender,
                 blockstore,
                 StandardBroadcastRun::new(keypair, shred_version),
@@ -62,6 +75,7 @@ impl BroadcastStageType {
                 sock,
                 cluster_info,
                 receiver,
+                retransmit_slots_receiver,
                 exit_sender,
                 blockstore,
                 FailEntryVerificationBroadcastRun::new(keypair, shred_version),
@@ -71,6 +85,7 @@ impl BroadcastStageType {
                 sock,
                 cluster_info,
                 receiver,
+                retransmit_slots_receiver,
                 exit_sender,
                 blockstore,
                 BroadcastFakeShredsRun::new(keypair, 0, shred_version),
@@ -79,7 +94,7 @@ impl BroadcastStageType {
     }
 }
 
-type TransmitShreds = (Option<Arc<HashMap<Pubkey, u64>>>, Arc<Vec<Shred>>);
+pub type TransmitShreds = (Option<Arc<HashMap<Pubkey, u64>>>, Arc<Vec<Shred>>);
 trait BroadcastRun {
     fn run(
         &mut self,
@@ -135,25 +150,27 @@ impl BroadcastStage {
         loop {
             let res =
                 broadcast_stage_run.run(blockstore, receiver, socket_sender, blockstore_sender);
-            let res = Self::handle_error(res);
+            let res = Self::handle_error(res, "run");
             if let Some(res) = res {
                 return res;
             }
         }
     }
-    fn handle_error(r: Result<()>) -> Option<BroadcastStageReturnType> {
+    fn handle_error(r: Result<()>, name: &str) -> Option<BroadcastStageReturnType> {
         if let Err(e) = r {
             match e {
                 Error::RecvTimeoutError(RecvTimeoutError::Disconnected)
                 | Error::SendError
-                | Error::RecvError(RecvError) => {
+                | Error::RecvError(RecvError)
+                | Error::CrossbeamRecvTimeoutError(CrossbeamRecvTimeoutError::Disconnected) => {
                     return Some(BroadcastStageReturnType::ChannelDisconnected);
                 }
-                Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
+                Error::RecvTimeoutError(RecvTimeoutError::Timeout)
+                | Error::CrossbeamRecvTimeoutError(CrossbeamRecvTimeoutError::Timeout) => (),
                 Error::ClusterInfoError(ClusterInfoError::NoPeers) => (), // TODO: Why are the unit-tests throwing hundreds of these?
                 _ => {
                     inc_new_counter_error!("streamer-broadcaster-error", 1, 1);
-                    error!("broadcaster error: {:?}", e);
+                    error!("{} broadcaster error: {:?}", name, e);
                 }
             }
         }
@@ -180,6 +197,7 @@ impl BroadcastStage {
         socks: Vec<UdpSocket>,
         cluster_info: Arc<RwLock<ClusterInfo>>,
         receiver: Receiver<WorkingBankEntry>,
+        retransmit_slots_receiver: RetransmitSlotsReceiver,
         exit_sender: &Arc<AtomicBool>,
         blockstore: &Arc<Blockstore>,
         broadcast_stage_run: impl BroadcastRun + Send + 'static + Clone,
@@ -189,6 +207,8 @@ impl BroadcastStage {
         let (socket_sender, socket_receiver) = channel();
         let (blockstore_sender, blockstore_receiver) = channel();
         let bs_run = broadcast_stage_run.clone();
+
+        let socket_sender_ = socket_sender.clone();
         let thread_hdl = Builder::new()
             .name("solana-broadcaster".to_string())
             .spawn(move || {
@@ -196,7 +216,7 @@ impl BroadcastStage {
                 Self::run(
                     &btree,
                     &receiver,
-                    &socket_sender,
+                    &socket_sender_,
                     &blockstore_sender,
                     bs_run,
                 )
@@ -212,7 +232,7 @@ impl BroadcastStage {
                 .name("solana-broadcaster-transmit".to_string())
                 .spawn(move || loop {
                     let res = bs_transmit.transmit(&socket_receiver, &cluster_info, &sock);
-                    let res = Self::handle_error(res);
+                    let res = Self::handle_error(res, "solana-broadcaster-transmit");
                     if let Some(res) = res {
                         return res;
                     }
@@ -229,7 +249,7 @@ impl BroadcastStage {
                 .name("solana-broadcaster-record".to_string())
                 .spawn(move || loop {
                     let res = bs_record.record(&blockstore_receiver, &btree);
-                    let res = Self::handle_error(res);
+                    let res = Self::handle_error(res, "solana-broadcaster-record");
                     if let Some(res) = res {
                         return res;
                     }
@@ -238,9 +258,68 @@ impl BroadcastStage {
             thread_hdls.push(t);
         }
 
+        let blockstore = blockstore.clone();
+        let retransmit_thread = Builder::new()
+            .name("solana-broadcaster-retransmit".to_string())
+            .spawn(move || loop {
+                if let Some(res) = Self::handle_error(
+                    Self::check_retransmit_signals(
+                        &blockstore,
+                        &retransmit_slots_receiver,
+                        &socket_sender,
+                    ),
+                    "solana-broadcaster-retransmit-check_retransmit_signals",
+                ) {
+                    return res;
+                }
+            })
+            .unwrap();
+
+        thread_hdls.push(retransmit_thread);
         Self { thread_hdls }
     }
 
+    fn check_retransmit_signals(
+        blockstore: &Blockstore,
+        retransmit_slots_receiver: &RetransmitSlotsReceiver,
+        socket_sender: &Sender<TransmitShreds>,
+    ) -> Result<()> {
+        let timer = Duration::from_millis(100);
+
+        // Check for a retransmit signal
+        let mut retransmit_slots = retransmit_slots_receiver.recv_timeout(timer)?;
+        while let Ok(new_retransmit_slots) = retransmit_slots_receiver.try_recv() {
+            retransmit_slots.extend(new_retransmit_slots);
+        }
+
+        for (_, bank) in retransmit_slots.iter() {
+            let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
+            let stakes = staking_utils::staked_nodes_at_epoch(&bank, bank_epoch);
+            let stakes = stakes.map(Arc::new);
+            let data_shreds = Arc::new(
+                blockstore
+                    .get_data_shreds_for_slot(bank.slot(), 0)
+                    .expect("My own shreds must be reconstructable"),
+            );
+
+            if !data_shreds.is_empty() {
+                socket_sender.send((stakes.clone(), data_shreds))?;
+            }
+
+            let coding_shreds = Arc::new(
+                blockstore
+                    .get_coding_shreds_for_slot(bank.slot(), 0)
+                    .expect("My own shreds must be reconstructable"),
+            );
+
+            if !coding_shreds.is_empty() {
+                socket_sender.send((stakes.clone(), coding_shreds))?;
+            }
+        }
+
+        Ok(())
+    }
+
     pub fn join(self) -> thread::Result<BroadcastStageReturnType> {
         for thread_hdl in self.thread_hdls.into_iter() {
             let _ = thread_hdl.join();
@@ -250,22 +329,137 @@ impl BroadcastStage {
 }
 
 #[cfg(test)]
-mod test {
+pub mod test {
     use super::*;
     use crate::cluster_info::{ClusterInfo, Node};
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-    use solana_ledger::entry::create_ticks;
-    use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
+    use crossbeam_channel::unbounded;
+    use solana_ledger::{
+        blockstore::{make_slot_entries, Blockstore},
+        entry::create_ticks,
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        get_tmp_ledger_path,
+        shred::{max_ticks_per_n_shreds, Shredder, RECOMMENDED_FEC_RATE},
+    };
     use solana_runtime::bank::Bank;
-    use solana_sdk::hash::Hash;
-    use solana_sdk::pubkey::Pubkey;
-    use solana_sdk::signature::{Keypair, Signer};
-    use std::path::Path;
-    use std::sync::atomic::AtomicBool;
-    use std::sync::mpsc::channel;
-    use std::sync::{Arc, RwLock};
-    use std::thread::sleep;
-    use std::time::Duration;
+    use solana_sdk::{
+        hash::Hash,
+        pubkey::Pubkey,
+        signature::{Keypair, Signer},
+    };
+    use std::{
+        path::Path,
+        sync::atomic::AtomicBool,
+        sync::mpsc::channel,
+        sync::{Arc, RwLock},
+        thread::sleep,
+    };
+
+    pub fn make_transmit_shreds(
+        slot: Slot,
+        num: u64,
+        stakes: Option<Arc<HashMap<Pubkey, u64>>>,
+    ) -> (
+        Vec<Shred>,
+        Vec<Shred>,
+        Vec<TransmitShreds>,
+        Vec<TransmitShreds>,
+    ) {
+        let num_entries = max_ticks_per_n_shreds(num);
+        let (data_shreds, _) = make_slot_entries(slot, 0, num_entries);
+        let keypair = Arc::new(Keypair::new());
+        let shredder = Shredder::new(slot, 0, RECOMMENDED_FEC_RATE, keypair, 0, 0)
+            .expect("Expected to create a new shredder");
+
+        let coding_shreds = shredder.data_shreds_to_coding_shreds(&data_shreds[0..]);
+        (
+            data_shreds.clone(),
+            coding_shreds.clone(),
+            data_shreds
+                .into_iter()
+                .map(|s| (stakes.clone(), Arc::new(vec![s])))
+                .collect(),
+            coding_shreds
+                .into_iter()
+                .map(|s| (stakes.clone(), Arc::new(vec![s])))
+                .collect(),
+        )
+    }
+
+    fn check_all_shreds_received(
+        transmit_receiver: &Receiver<TransmitShreds>,
+        mut data_index: u64,
+        mut coding_index: u64,
+        num_expected_data_shreds: u64,
+        num_expected_coding_shreds: u64,
+    ) {
+        while let Ok(new_retransmit_slots) = transmit_receiver.try_recv() {
+            if new_retransmit_slots.1[0].is_data() {
+                for data_shred in new_retransmit_slots.1.iter() {
+                    assert_eq!(data_shred.index() as u64, data_index);
+                    data_index += 1;
+                }
+            } else {
+                assert_eq!(new_retransmit_slots.1[0].index() as u64, coding_index);
+                for coding_shred in new_retransmit_slots.1.iter() {
+                    assert_eq!(coding_shred.index() as u64, coding_index);
+                    coding_index += 1;
+                }
+            }
+        }
+
+        assert_eq!(num_expected_data_shreds, data_index);
+        assert_eq!(num_expected_coding_shreds, coding_index);
+    }
+
+    #[test]
+    fn test_duplicate_retransmit_signal() {
+        // Setup
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let (transmit_sender, transmit_receiver) = channel();
+        let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
+        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(100_000);
+        let bank0 = Arc::new(Bank::new(&genesis_config));
+
+        // Make some shreds
+        let updated_slot = 0;
+        let (all_data_shreds, all_coding_shreds, _, _all_coding_transmit_shreds) =
+            make_transmit_shreds(updated_slot, 10, None);
+        let num_data_shreds = all_data_shreds.len();
+        let num_coding_shreds = all_coding_shreds.len();
+        assert!(num_data_shreds >= 10);
+
+        // Insert all the shreds
+        blockstore
+            .insert_shreds(all_data_shreds, None, true)
+            .unwrap();
+        blockstore
+            .insert_shreds(all_coding_shreds, None, true)
+            .unwrap();
+
+        // Insert duplicate retransmit signal, blocks should
+        // only be retransmitted once
+        retransmit_slots_sender
+            .send(vec![(updated_slot, bank0.clone())].into_iter().collect())
+            .unwrap();
+        retransmit_slots_sender
+            .send(vec![(updated_slot, bank0.clone())].into_iter().collect())
+            .unwrap();
+        BroadcastStage::check_retransmit_signals(
+            &blockstore,
+            &retransmit_slots_receiver,
+            &transmit_sender,
+        )
+        .unwrap();
+        // Check all the data shreds were received only once
+        check_all_shreds_received(
+            &transmit_receiver,
+            0,
+            0,
+            num_data_shreds as u64,
+            num_coding_shreds as u64,
+        );
+    }
 
     struct MockBroadcastStage {
         blockstore: Arc<Blockstore>,
@@ -277,6 +471,7 @@ mod test {
         leader_pubkey: &Pubkey,
         ledger_path: &Path,
         entry_receiver: Receiver<WorkingBankEntry>,
+        retransmit_slots_receiver: RetransmitSlotsReceiver,
     ) -> MockBroadcastStage {
         // Make the database ledger
         let blockstore = Arc::new(Blockstore::open(ledger_path).unwrap());
@@ -304,6 +499,7 @@ mod test {
             leader_info.sockets.broadcast,
             cluster_info,
             entry_receiver,
+            retransmit_slots_receiver,
             &exit_sender,
             &blockstore,
             StandardBroadcastRun::new(leader_keypair, 0),
@@ -326,10 +522,12 @@ mod test {
             let leader_keypair = Keypair::new();
 
             let (entry_sender, entry_receiver) = channel();
+            let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
             let broadcast_service = setup_dummy_broadcast_service(
                 &leader_keypair.pubkey(),
                 &ledger_path,
                 entry_receiver,
+                retransmit_slots_receiver,
             );
             let start_tick_height;
             let max_tick_height;
@@ -348,6 +546,7 @@ mod test {
                         .expect("Expect successful send to broadcast service");
                 }
             }
+
             sleep(Duration::from_millis(2000));
 
             trace!(
@@ -364,6 +563,7 @@ mod test {
             assert_eq!(entries.len(), max_tick_height as usize);
 
             drop(entry_sender);
+            drop(retransmit_slots_sender);
             broadcast_service
                 .broadcast_service
                 .join()

core/src/broadcast_stage/broadcast_utils.rs

@@ -76,7 +76,7 @@ pub(super) fn recv_slot_entries(receiver: &Receiver<WorkingBankEntry>) -> Result
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_sdk::genesis_config::GenesisConfig;
     use solana_sdk::pubkey::Pubkey;
     use solana_sdk::system_transaction;

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -1,11 +1,11 @@
 use super::broadcast_utils::{self, ReceiveResults};
 use super::*;
 use crate::broadcast_stage::broadcast_utils::UnfinishedSlotInfo;
-use solana_ledger::entry::Entry;
-use solana_ledger::shred::{Shred, Shredder, RECOMMENDED_FEC_RATE, SHRED_TICK_REFERENCE_MASK};
-use solana_sdk::pubkey::Pubkey;
-use solana_sdk::signature::Keypair;
-use solana_sdk::timing::duration_as_us;
+use solana_ledger::{
+    entry::Entry,
+    shred::{Shred, Shredder, RECOMMENDED_FEC_RATE, SHRED_TICK_REFERENCE_MASK},
+};
+use solana_sdk::{pubkey::Pubkey, signature::Keypair, timing::duration_as_us};
 use std::collections::HashMap;
 use std::time::Duration;
 
@@ -212,7 +212,8 @@ impl StandardBroadcastRun {
         blockstore_sender.send(data_shreds.clone())?;
         let coding_shreds = shredder.data_shreds_to_coding_shreds(&data_shreds[0..last_data_shred]);
         let coding_shreds = Arc::new(coding_shreds);
-        socket_sender.send((stakes, coding_shreds))?;
+        socket_sender.send((stakes, coding_shreds.clone()))?;
+        blockstore_sender.send(coding_shreds)?;
         self.update_broadcast_stats(BroadcastStats {
             shredding_elapsed: duration_as_us(&to_shreds_elapsed),
             receive_elapsed: duration_as_us(&receive_elapsed),
@@ -353,14 +354,13 @@ impl BroadcastRun for StandardBroadcastRun {
 mod test {
     use super::*;
     use crate::cluster_info::{ClusterInfo, Node};
-    use crate::genesis_utils::create_genesis_config;
+    use solana_ledger::genesis_utils::create_genesis_config;
     use solana_ledger::{
         blockstore::Blockstore, entry::create_ticks, get_tmp_ledger_path,
         shred::max_ticks_per_n_shreds,
     };
     use solana_runtime::bank::Bank;
     use solana_sdk::{
-        clock::Slot,
         genesis_config::GenesisConfig,
         signature::{Keypair, Signer},
     };

core/src/cluster_info.rs

@@ -12,23 +12,19 @@
 //! * layer 2 - Everyone else, if layer 1 is `2^10`, layer 2 should be able to fit `2^20` number of nodes.
 //!
 //! Bank needs to provide an interface for us to query the stake weight
-use crate::crds_value::CompressionType::*;
-use crate::crds_value::EpochIncompleteSlots;
-use crate::packet::limited_deserialize;
-use crate::streamer::{PacketReceiver, PacketSender};
 use crate::{
     contact_info::ContactInfo,
     crds_gossip::CrdsGossip,
     crds_gossip_error::CrdsGossipError,
     crds_gossip_pull::{CrdsFilter, CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS},
-    crds_value::{self, CrdsData, CrdsValue, CrdsValueLabel, EpochSlots, SnapshotHash, Vote},
-    packet::{Packet, PACKET_DATA_SIZE},
+    crds_value::{
+        self, CrdsData, CrdsValue, CrdsValueLabel, EpochSlotsIndex, LowestSlot, SnapshotHash, Vote,
+    },
+    epoch_slots::EpochSlots,
     result::{Error, Result},
-    sendmmsg::{multicast, send_mmsg},
     weighted_shuffle::{weighted_best, weighted_shuffle},
 };
 use bincode::{serialize, serialized_size};
-use compression::prelude::*;
 use core::cmp;
 use itertools::Itertools;
 use rayon::iter::IntoParallelIterator;
@@ -41,21 +37,26 @@ use solana_net_utils::{
     bind_common, bind_common_in_range, bind_in_range, find_available_port_in_range,
     multi_bind_in_range, PortRange,
 };
-use solana_perf::packet::{to_packets_with_destination, Packets, PacketsRecycler};
+use solana_perf::packet::{
+    limited_deserialize, to_packets_with_destination, Packet, Packets, PacketsRecycler,
+    PACKET_DATA_SIZE,
+};
 use solana_rayon_threadlimit::get_thread_count;
 use solana_sdk::hash::Hash;
 use solana_sdk::timing::duration_as_s;
 use solana_sdk::{
-    clock::{Slot, DEFAULT_MS_PER_SLOT},
+    clock::{Slot, DEFAULT_MS_PER_SLOT, DEFAULT_SLOTS_PER_EPOCH},
     pubkey::Pubkey,
     signature::{Keypair, Signable, Signature, Signer},
     timing::{duration_as_ms, timestamp},
     transaction::Transaction,
 };
+use solana_streamer::sendmmsg::{multicast, send_mmsg};
+use solana_streamer::streamer::{PacketReceiver, PacketSender};
 use std::{
     borrow::Cow,
     cmp::min,
-    collections::{BTreeSet, HashMap, HashSet},
+    collections::{HashMap, HashSet},
     fmt,
     net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, UdpSocket},
     sync::atomic::{AtomicBool, Ordering},
@@ -71,7 +72,8 @@ pub const DATA_PLANE_FANOUT: usize = 200;
 /// milliseconds we sleep for between gossip requests
 pub const GOSSIP_SLEEP_MILLIS: u64 = 100;
 /// The maximum size of a bloom filter
-pub const MAX_BLOOM_SIZE: usize = 1018;
+pub const MAX_BLOOM_SIZE: usize = MAX_CRDS_OBJECT_SIZE;
+pub const MAX_CRDS_OBJECT_SIZE: usize = 928;
 /// The maximum size of a protocol payload
 const MAX_PROTOCOL_PAYLOAD_SIZE: u64 = PACKET_DATA_SIZE as u64 - MAX_PROTOCOL_HEADER_SIZE;
 /// The largest protocol header size
@@ -81,9 +83,6 @@ const MAX_PROTOCOL_HEADER_SIZE: u64 = 214;
 /// 128MB/PACKET_DATA_SIZE
 const MAX_GOSSIP_TRAFFIC: usize = 128_000_000 / PACKET_DATA_SIZE;
 
-const NUM_BITS_PER_BYTE: u64 = 8;
-const MIN_SIZE_TO_COMPRESS_GZIP: u64 = 64;
-
 /// Keep the number of snapshot hashes a node publishes under MAX_PROTOCOL_PAYLOAD_SIZE
 pub const MAX_SNAPSHOT_HASHES: usize = 16;
 
@@ -262,6 +261,16 @@ impl ClusterInfo {
         self.lookup(&self.id()).cloned().unwrap()
     }
 
+    pub fn lookup_epoch_slots(&self, ix: EpochSlotsIndex) -> EpochSlots {
+        let entry = CrdsValueLabel::EpochSlots(ix, self.id());
+        self.gossip
+            .crds
+            .lookup(&entry)
+            .and_then(CrdsValue::epoch_slots)
+            .cloned()
+            .unwrap_or_else(|| EpochSlots::new(self.id(), timestamp()))
+    }
+
     pub fn contact_info_trace(&self) -> String {
         let now = timestamp();
         let mut spy_nodes = 0;
@@ -336,119 +345,80 @@ impl ClusterInfo {
         )
     }
 
-    pub fn compress_incomplete_slots(incomplete_slots: &BTreeSet<Slot>) -> EpochIncompleteSlots {
-        if !incomplete_slots.is_empty() {
-            let first_slot = incomplete_slots
-                .iter()
-                .next()
-                .expect("expected to find at least one slot");
-            let last_slot = incomplete_slots
-                .iter()
-                .next_back()
-                .expect("expected to find last slot");
-            let num_uncompressed_bits = last_slot.saturating_sub(*first_slot) + 1;
-            let num_uncompressed_bytes = if num_uncompressed_bits % NUM_BITS_PER_BYTE > 0 {
-                1
-            } else {
-                0
-            } + num_uncompressed_bits / NUM_BITS_PER_BYTE;
-            let mut uncompressed = vec![0u8; num_uncompressed_bytes as usize];
-            incomplete_slots.iter().for_each(|slot| {
-                let offset_from_first_slot = slot.saturating_sub(*first_slot);
-                let index = offset_from_first_slot / NUM_BITS_PER_BYTE;
-                let bit_index = offset_from_first_slot % NUM_BITS_PER_BYTE;
-                uncompressed[index as usize] |= 1 << bit_index;
-            });
-            if num_uncompressed_bytes >= MIN_SIZE_TO_COMPRESS_GZIP {
-                if let Ok(compressed) = uncompressed
-                    .iter()
-                    .cloned()
-                    .encode(&mut GZipEncoder::new(), Action::Finish)
-                    .collect::<std::result::Result<Vec<u8>, _>>()
-                {
-                    return EpochIncompleteSlots {
-                        first: *first_slot,
-                        compression: GZip,
-                        compressed_list: compressed,
-                    };
-                }
-            } else {
-                return EpochIncompleteSlots {
-                    first: *first_slot,
-                    compression: Uncompressed,
-                    compressed_list: uncompressed,
-                };
-            }
-        }
-        EpochIncompleteSlots::default()
-    }
-
-    fn bitmap_to_slot_list(first: Slot, bitmap: &[u8]) -> BTreeSet<Slot> {
-        let mut old_incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
-        bitmap.iter().enumerate().for_each(|(i, val)| {
-            if *val != 0 {
-                (0..8).for_each(|bit_index| {
-                    if (1 << bit_index & *val) != 0 {
-                        let slot = first + i as u64 * NUM_BITS_PER_BYTE + bit_index as u64;
-                        old_incomplete_slots.insert(slot);
-                    }
-                })
-            }
-        });
-        old_incomplete_slots
-    }
-
-    pub fn decompress_incomplete_slots(slots: &EpochIncompleteSlots) -> BTreeSet<Slot> {
-        match slots.compression {
-            Uncompressed => Self::bitmap_to_slot_list(slots.first, &slots.compressed_list),
-            GZip => {
-                if let Ok(decompressed) = slots
-                    .compressed_list
-                    .iter()
-                    .cloned()
-                    .decode(&mut GZipDecoder::new())
-                    .collect::<std::result::Result<Vec<u8>, _>>()
-                {
-                    Self::bitmap_to_slot_list(slots.first, &decompressed)
-                } else {
-                    BTreeSet::new()
-                }
-            }
-            BZip2 => {
-                if let Ok(decompressed) = slots
-                    .compressed_list
-                    .iter()
-                    .cloned()
-                    .decode(&mut BZip2Decoder::new())
-                    .collect::<std::result::Result<Vec<u8>, _>>()
-                {
-                    Self::bitmap_to_slot_list(slots.first, &decompressed)
-                } else {
-                    BTreeSet::new()
-                }
-            }
-        }
-    }
-
-    pub fn push_epoch_slots(
-        &mut self,
-        id: Pubkey,
-        root: Slot,
-        min: Slot,
-        slots: BTreeSet<Slot>,
-        incomplete_slots: &BTreeSet<Slot>,
-    ) {
-        let compressed = Self::compress_incomplete_slots(incomplete_slots);
+    pub fn push_lowest_slot(&mut self, id: Pubkey, min: Slot) {
         let now = timestamp();
-        let entry = CrdsValue::new_signed(
-            CrdsData::EpochSlots(
-                0,
-                EpochSlots::new(id, root, min, slots, vec![compressed], now),
-            ),
-            &self.keypair,
-        );
-        self.gossip
-            .process_push_message(&self.id(), vec![entry], now);
+        let last = self
+            .gossip
+            .crds
+            .lookup(&CrdsValueLabel::LowestSlot(self.id()))
+            .and_then(|x| x.lowest_slot())
+            .map(|x| x.lowest)
+            .unwrap_or(0);
+        if min > last {
+            let entry = CrdsValue::new_signed(
+                CrdsData::LowestSlot(0, LowestSlot::new(id, min, now)),
+                &self.keypair,
+            );
+            self.gossip
+                .process_push_message(&self.id(), vec![entry], now);
+        }
+    }
+
+    pub fn push_epoch_slots(&mut self, update: &[Slot]) {
+        let mut num = 0;
+        let mut current_slots: Vec<_> = (0..crds_value::MAX_EPOCH_SLOTS)
+            .filter_map(|ix| {
+                Some((
+                    self.gossip
+                        .crds
+                        .lookup(&CrdsValueLabel::EpochSlots(ix, self.id()))
+                        .and_then(CrdsValue::epoch_slots)
+                        .and_then(|x| Some((x.wallclock, x.first_slot()?)))?,
+                    ix,
+                ))
+            })
+            .collect();
+        current_slots.sort();
+        let min_slot: Slot = current_slots
+            .iter()
+            .map(|((_, s), _)| *s)
+            .min()
+            .unwrap_or(0);
+        let max_slot: Slot = update.iter().max().cloned().unwrap_or(0);
+        let total_slots = max_slot as isize - min_slot as isize;
+        // WARN if CRDS is not storing at least a full epoch worth of slots
+        if DEFAULT_SLOTS_PER_EPOCH as isize > total_slots
+            && crds_value::MAX_EPOCH_SLOTS as usize <= current_slots.len()
+        {
+            inc_new_counter_warn!("cluster_info-epoch_slots-filled", 1);
+            warn!(
+                "EPOCH_SLOTS are filling up FAST {}/{}",
+                total_slots,
+                current_slots.len()
+            );
+        }
+        let mut reset = false;
+        let mut epoch_slot_index = current_slots.last().map(|(_, x)| *x).unwrap_or(0);
+        while num < update.len() {
+            let ix = (epoch_slot_index % crds_value::MAX_EPOCH_SLOTS) as u8;
+            let now = timestamp();
+            let mut slots = if !reset {
+                self.lookup_epoch_slots(ix)
+            } else {
+                EpochSlots::new(self.id(), now)
+            };
+            let n = slots.fill(&update[num..], now);
+            if n > 0 {
+                let entry = CrdsValue::new_signed(CrdsData::EpochSlots(ix, slots), &self.keypair);
+                self.gossip
+                    .process_push_message(&self.id(), vec![entry], now);
+            }
+            num += n;
+            if num < update.len() {
+                epoch_slot_index += 1;
+                reset = true;
+            }
+        }
     }
 
     pub fn push_message(&mut self, message: CrdsValue) {
@@ -504,23 +474,23 @@ impl ClusterInfo {
     /// since. This allows the bank to query for new votes only.
     ///
     /// * return - The votes, and the max timestamp from the new set.
-    pub fn get_votes(&self, since: u64) -> (Vec<Transaction>, u64) {
-        let votes: Vec<_> = self
+    pub fn get_votes(&self, since: u64) -> (Vec<CrdsValueLabel>, Vec<Transaction>, u64) {
+        let mut max_ts = since;
+        let (labels, txs): (Vec<CrdsValueLabel>, Vec<Transaction>) = self
             .gossip
             .crds
             .table
-            .values()
-            .filter(|x| x.insert_timestamp > since)
-            .filter_map(|x| {
+            .iter()
+            .filter(|(_, x)| x.insert_timestamp > since)
+            .filter_map(|(label, x)| {
+                max_ts = std::cmp::max(x.insert_timestamp, max_ts);
                 x.value
                     .vote()
-                    .map(|v| (x.insert_timestamp, v.transaction.clone()))
+                    .map(|v| (label.clone(), v.transaction.clone()))
             })
-            .collect();
-        let max_ts = votes.iter().map(|x| x.0).max().unwrap_or(since);
-        let txs: Vec<Transaction> = votes.into_iter().map(|x| x.1).collect();
+            .unzip();
         inc_new_counter_info!("cluster_info-get_votes-count", txs.len());
-        (txs, max_ts)
+        (labels, txs, max_ts)
     }
 
     pub fn get_snapshot_hash(&self, slot: Slot) -> Vec<(Pubkey, Hash)> {
@@ -556,21 +526,39 @@ impl ClusterInfo {
             .map(|x| &x.value.snapshot_hash().unwrap().hashes)
     }
 
-    pub fn get_epoch_state_for_node(
+    pub fn get_lowest_slot_for_node(
         &self,
         pubkey: &Pubkey,
         since: Option<u64>,
-    ) -> Option<(&EpochSlots, u64)> {
+    ) -> Option<(&LowestSlot, u64)> {
         self.gossip
             .crds
             .table
-            .get(&CrdsValueLabel::EpochSlots(*pubkey))
+            .get(&CrdsValueLabel::LowestSlot(*pubkey))
             .filter(|x| {
                 since
                     .map(|since| x.insert_timestamp > since)
                     .unwrap_or(true)
             })
-            .map(|x| (x.value.epoch_slots().unwrap(), x.insert_timestamp))
+            .map(|x| (x.value.lowest_slot().unwrap(), x.insert_timestamp))
+    }
+
+    pub fn get_epoch_slots_since(&self, since: Option<u64>) -> (Vec<EpochSlots>, Option<u64>) {
+        let vals: Vec<_> = self
+            .gossip
+            .crds
+            .table
+            .values()
+            .filter(|x| {
+                since
+                    .map(|since| x.insert_timestamp > since)
+                    .unwrap_or(true)
+            })
+            .filter_map(|x| Some((x.value.epoch_slots()?, x.insert_timestamp)))
+            .collect();
+        let max = vals.iter().map(|x| x.1).max().or(since);
+        let vec = vals.into_iter().map(|x| x.0).cloned().collect();
+        (vec, max)
     }
 
     pub fn get_contact_info_for_node(&self, pubkey: &Pubkey) -> Option<&ContactInfo> {
@@ -714,8 +702,8 @@ impl ClusterInfo {
                     && x.shred_version == me.shred_version
                     && ContactInfo::is_valid_address(&x.serve_repair)
                     && {
-                        self.get_epoch_state_for_node(&x.id, None)
-                            .map(|(epoch_slots, _)| epoch_slots.lowest <= slot)
+                        self.get_lowest_slot_for_node(&x.id, None)
+                            .map(|(lowest_slot, _)| lowest_slot.lowest <= slot)
                             .unwrap_or_else(|| /* fallback to legacy behavior */ true)
                     }
             })
@@ -2286,30 +2274,62 @@ mod tests {
     #[test]
     fn test_push_vote() {
         let keys = Keypair::new();
-        let now = timestamp();
         let contact_info = ContactInfo::new_localhost(&keys.pubkey(), 0);
         let mut cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
 
         // make sure empty crds is handled correctly
-        let (votes, max_ts) = cluster_info.get_votes(now);
+        let now = timestamp();
+        let (_, votes, max_ts) = cluster_info.get_votes(now);
         assert_eq!(votes, vec![]);
         assert_eq!(max_ts, now);
 
         // add a vote
         let tx = test_tx();
-        cluster_info.push_vote(0, tx.clone());
+        let index = 1;
+        cluster_info.push_vote(index, tx.clone());
 
         // -1 to make sure that the clock is strictly lower then when insert occurred
-        let (votes, max_ts) = cluster_info.get_votes(now - 1);
+        let (labels, votes, max_ts) = cluster_info.get_votes(now - 1);
         assert_eq!(votes, vec![tx]);
+        assert_eq!(labels.len(), 1);
+        match labels[0] {
+            CrdsValueLabel::Vote(_, pubkey) => {
+                assert_eq!(pubkey, keys.pubkey());
+            }
+
+            _ => panic!("Bad match"),
+        }
         assert!(max_ts >= now - 1);
 
         // make sure timestamp filter works
-        let (votes, new_max_ts) = cluster_info.get_votes(max_ts);
+        let (_, votes, new_max_ts) = cluster_info.get_votes(max_ts);
         assert_eq!(votes, vec![]);
         assert_eq!(max_ts, new_max_ts);
     }
 
+    #[test]
+    fn test_push_epoch_slots() {
+        let keys = Keypair::new();
+        let contact_info = ContactInfo::new_localhost(&keys.pubkey(), 0);
+        let mut cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
+        let (slots, since) = cluster_info.get_epoch_slots_since(None);
+        assert!(slots.is_empty());
+        assert!(since.is_none());
+        cluster_info.push_epoch_slots(&[0]);
+
+        let (slots, since) = cluster_info.get_epoch_slots_since(Some(std::u64::MAX));
+        assert!(slots.is_empty());
+        assert_eq!(since, Some(std::u64::MAX));
+
+        let (slots, since) = cluster_info.get_epoch_slots_since(None);
+        assert_eq!(slots.len(), 1);
+        assert!(since.is_some());
+
+        let (slots, since2) = cluster_info.get_epoch_slots_since(since.clone());
+        assert!(slots.is_empty());
+        assert_eq!(since2, since);
+    }
+
     #[test]
     fn test_add_entrypoint() {
         let node_keypair = Arc::new(Keypair::new());
@@ -2363,20 +2383,9 @@ mod tests {
 
     #[test]
     fn test_split_messages_large() {
-        let mut btree_slots = BTreeSet::new();
-        for i in 0..128 {
-            btree_slots.insert(i);
-        }
-        let value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
+        let value = CrdsValue::new_unsigned(CrdsData::LowestSlot(
             0,
-            EpochSlots {
-                from: Pubkey::default(),
-                root: 0,
-                lowest: 0,
-                slots: btree_slots,
-                stash: vec![],
-                wallclock: 0,
-            },
+            LowestSlot::new(Pubkey::default(), 0, 0),
         ));
         test_split_messages(value);
     }
@@ -2388,39 +2397,19 @@ mod tests {
         let payload: Vec<CrdsValue> = vec![];
         let vec_size = serialized_size(&payload).unwrap();
         let desired_size = MAX_PROTOCOL_PAYLOAD_SIZE - vec_size;
-        let mut value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
-            0,
-            EpochSlots {
-                from: Pubkey::default(),
-                root: 0,
-                lowest: 0,
-                slots: BTreeSet::new(),
-                stash: vec![],
-                wallclock: 0,
-            },
-        ));
+        let mut value = CrdsValue::new_unsigned(CrdsData::SnapshotHashes(SnapshotHash {
+            from: Pubkey::default(),
+            hashes: vec![],
+            wallclock: 0,
+        }));
 
         let mut i = 0;
         while value.size() <= desired_size {
-            let slots = (0..i).collect::<BTreeSet<_>>();
-            if slots.len() > 200 {
-                panic!(
-                    "impossible to match size: last {:?} vs desired {:?}",
-                    serialized_size(&value).unwrap(),
-                    desired_size
-                );
-            }
-            value.data = CrdsData::EpochSlots(
-                0,
-                EpochSlots {
-                    from: Pubkey::default(),
-                    root: 0,
-                    lowest: 0,
-                    slots,
-                    stash: vec![],
-                    wallclock: 0,
-                },
-            );
+            value.data = CrdsData::SnapshotHashes(SnapshotHash {
+                from: Pubkey::default(),
+                hashes: vec![(0, Hash::default()); i],
+                wallclock: 0,
+            });
             i += 1;
         }
         let split = ClusterInfo::split_gossip_messages(vec![value.clone()]);
@@ -2550,26 +2539,17 @@ mod tests {
             node_keypair,
         );
         for i in 0..10 {
-            let mut peer_root = 5;
             let mut peer_lowest = 0;
             if i >= 5 {
                 // make these invalid for the upcoming repair request
-                peer_root = 15;
                 peer_lowest = 10;
             }
             let other_node_pubkey = Pubkey::new_rand();
             let other_node = ContactInfo::new_localhost(&other_node_pubkey, timestamp());
             cluster_info.insert_info(other_node.clone());
-            let value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
+            let value = CrdsValue::new_unsigned(CrdsData::LowestSlot(
                 0,
-                EpochSlots::new(
-                    other_node_pubkey,
-                    peer_root,
-                    peer_lowest,
-                    BTreeSet::new(),
-                    vec![],
-                    timestamp(),
-                ),
+                LowestSlot::new(other_node_pubkey, peer_lowest, timestamp()),
             ));
             let _ = cluster_info.gossip.crds.insert(value, timestamp());
         }
@@ -2579,7 +2559,8 @@ mod tests {
 
     #[test]
     fn test_max_bloom_size() {
-        assert_eq!(MAX_BLOOM_SIZE, max_bloom_size());
+        // check that the constant fits into the dynamic size
+        assert!(MAX_BLOOM_SIZE <= max_bloom_size());
     }
 
     #[test]
@@ -2632,36 +2613,24 @@ mod tests {
     }
 
     #[test]
-    fn test_compress_incomplete_slots() {
-        let mut incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
-
-        assert_eq!(
-            EpochIncompleteSlots::default(),
-            ClusterInfo::compress_incomplete_slots(&incomplete_slots)
+    fn test_push_epoch_slots_large() {
+        use rand::Rng;
+        let node_keypair = Arc::new(Keypair::new());
+        let mut cluster_info = ClusterInfo::new(
+            ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()),
+            node_keypair,
         );
-
-        incomplete_slots.insert(100);
-        let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
-        assert_eq!(100, compressed.first);
-        let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
-        assert_eq!(incomplete_slots, decompressed);
-
-        incomplete_slots.insert(104);
-        let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
-        assert_eq!(100, compressed.first);
-        let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
-        assert_eq!(incomplete_slots, decompressed);
-
-        incomplete_slots.insert(80);
-        let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
-        assert_eq!(80, compressed.first);
-        let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
-        assert_eq!(incomplete_slots, decompressed);
-
-        incomplete_slots.insert(10000);
-        let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
-        assert_eq!(80, compressed.first);
-        let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
-        assert_eq!(incomplete_slots, decompressed);
+        let mut range: Vec<Slot> = vec![];
+        //random should be hard to compress
+        for _ in 0..32000 {
+            let last = *range.last().unwrap_or(&0);
+            range.push(last + rand::thread_rng().gen_range(1, 32));
+        }
+        cluster_info.push_epoch_slots(&range[..16000]);
+        cluster_info.push_epoch_slots(&range[16000..]);
+        let (slots, since) = cluster_info.get_epoch_slots_since(None);
+        let slots: Vec<_> = slots.iter().flat_map(|x| x.to_slots(0)).collect();
+        assert_eq!(slots, range);
+        assert!(since.is_some());
     }
 }

core/src/cluster_slots.rs

@@ -0,0 +1,348 @@
+use crate::{
+    cluster_info::ClusterInfo, contact_info::ContactInfo, epoch_slots::EpochSlots,
+    serve_repair::RepairType,
+};
+use solana_ledger::bank_forks::BankForks;
+use solana_runtime::epoch_stakes::NodeIdToVoteAccounts;
+use solana_sdk::{clock::Slot, pubkey::Pubkey};
+use std::{
+    collections::{HashMap, HashSet},
+    sync::{Arc, RwLock},
+};
+
+pub type SlotPubkeys = HashMap<Arc<Pubkey>, u64>;
+pub type ClusterSlotsMap = RwLock<HashMap<Slot, Arc<RwLock<SlotPubkeys>>>>;
+
+#[derive(Default)]
+pub struct ClusterSlots {
+    cluster_slots: ClusterSlotsMap,
+    keys: RwLock<HashSet<Arc<Pubkey>>>,
+    since: RwLock<Option<u64>>,
+    validator_stakes: RwLock<Arc<NodeIdToVoteAccounts>>,
+    epoch: RwLock<Option<u64>>,
+    self_id: RwLock<Pubkey>,
+}
+
+impl ClusterSlots {
+    pub fn lookup(&self, slot: Slot) -> Option<Arc<RwLock<SlotPubkeys>>> {
+        self.cluster_slots.read().unwrap().get(&slot).cloned()
+    }
+    pub fn update(
+        &self,
+        root: Slot,
+        cluster_info: &RwLock<ClusterInfo>,
+        bank_forks: &RwLock<BankForks>,
+    ) {
+        self.update_peers(cluster_info, bank_forks);
+        let since = *self.since.read().unwrap();
+        let epoch_slots = cluster_info.read().unwrap().get_epoch_slots_since(since);
+        self.update_internal(root, epoch_slots);
+    }
+    fn update_internal(&self, root: Slot, epoch_slots: (Vec<EpochSlots>, Option<u64>)) {
+        let (epoch_slots_list, since) = epoch_slots;
+        for epoch_slots in epoch_slots_list {
+            let slots = epoch_slots.to_slots(root);
+            for slot in &slots {
+                if *slot <= root {
+                    continue;
+                }
+                let pubkey = Arc::new(epoch_slots.from);
+                let exists = self.keys.read().unwrap().get(&pubkey).is_some();
+                if !exists {
+                    self.keys.write().unwrap().insert(pubkey.clone());
+                }
+                let from = self.keys.read().unwrap().get(&pubkey).unwrap().clone();
+                let balance = self
+                    .validator_stakes
+                    .read()
+                    .unwrap()
+                    .get(&from)
+                    .map(|v| v.total_stake)
+                    .unwrap_or(0);
+
+                let mut slot_pubkeys = self.cluster_slots.read().unwrap().get(slot).cloned();
+                if slot_pubkeys.is_none() {
+                    let new_slot_pubkeys = Arc::new(RwLock::new(HashMap::default()));
+                    self.cluster_slots
+                        .write()
+                        .unwrap()
+                        .insert(*slot, new_slot_pubkeys.clone());
+                    slot_pubkeys = Some(new_slot_pubkeys);
+                }
+
+                slot_pubkeys
+                    .unwrap()
+                    .write()
+                    .unwrap()
+                    .insert(from.clone(), balance);
+            }
+        }
+        self.cluster_slots.write().unwrap().retain(|x, _| *x > root);
+        self.keys
+            .write()
+            .unwrap()
+            .retain(|x| Arc::strong_count(x) > 1);
+        *self.since.write().unwrap() = since;
+    }
+    pub fn collect(&self, id: &Pubkey) -> HashSet<Slot> {
+        self.cluster_slots
+            .read()
+            .unwrap()
+            .iter()
+            .filter(|(_, keys)| keys.read().unwrap().get(id).is_some())
+            .map(|(slot, _)| slot)
+            .cloned()
+            .collect()
+    }
+
+    fn update_peers(&self, cluster_info: &RwLock<ClusterInfo>, bank_forks: &RwLock<BankForks>) {
+        let root_bank = bank_forks.read().unwrap().root_bank().clone();
+        let root_epoch = root_bank.epoch();
+        let my_epoch = *self.epoch.read().unwrap();
+
+        if Some(root_epoch) != my_epoch {
+            let validator_stakes = root_bank
+                .epoch_stakes(root_epoch)
+                .expect(
+                    "Bank must have epoch stakes
+                for its own epoch",
+                )
+                .node_id_to_vote_accounts()
+                .clone();
+
+            *self.validator_stakes.write().unwrap() = validator_stakes;
+            let id = cluster_info.read().unwrap().id();
+            *self.self_id.write().unwrap() = id;
+            *self.epoch.write().unwrap() = Some(root_epoch);
+        }
+    }
+
+    pub fn compute_weights(&self, slot: Slot, repair_peers: &[ContactInfo]) -> Vec<(u64, usize)> {
+        let slot_peers = self.lookup(slot);
+        repair_peers
+            .iter()
+            .enumerate()
+            .map(|(i, x)| {
+                let peer_stake = slot_peers
+                    .as_ref()
+                    .and_then(|v| v.read().unwrap().get(&x.id).cloned())
+                    .unwrap_or(0);
+                (
+                    1 + peer_stake
+                        + self
+                            .validator_stakes
+                            .read()
+                            .unwrap()
+                            .get(&x.id)
+                            .map(|v| v.total_stake)
+                            .unwrap_or(0),
+                    i,
+                )
+            })
+            .collect()
+    }
+
+    pub fn generate_repairs_for_missing_slots(
+        &self,
+        self_id: &Pubkey,
+        root: Slot,
+    ) -> Vec<RepairType> {
+        let my_slots = self.collect(self_id);
+        self.cluster_slots
+            .read()
+            .unwrap()
+            .keys()
+            .filter(|x| **x > root)
+            .filter(|x| !my_slots.contains(*x))
+            .map(|x| RepairType::HighestShred(*x, 0))
+            .collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_runtime::epoch_stakes::NodeVoteAccounts;
+
+    #[test]
+    fn test_default() {
+        let cs = ClusterSlots::default();
+        assert!(cs.cluster_slots.read().unwrap().is_empty());
+        assert!(cs.since.read().unwrap().is_none());
+    }
+
+    #[test]
+    fn test_update_noop() {
+        let cs = ClusterSlots::default();
+        cs.update_internal(0, (vec![], None));
+        assert!(cs.cluster_slots.read().unwrap().is_empty());
+        assert!(cs.since.read().unwrap().is_none());
+    }
+
+    #[test]
+    fn test_update_empty() {
+        let cs = ClusterSlots::default();
+        let epoch_slot = EpochSlots::default();
+        cs.update_internal(0, (vec![epoch_slot], Some(0)));
+        assert_eq!(*cs.since.read().unwrap(), Some(0));
+        assert!(cs.lookup(0).is_none());
+    }
+
+    #[test]
+    fn test_update_rooted() {
+        //root is 0, so it should clear out the slot
+        let cs = ClusterSlots::default();
+        let mut epoch_slot = EpochSlots::default();
+        epoch_slot.fill(&[0], 0);
+        cs.update_internal(0, (vec![epoch_slot], Some(0)));
+        assert_eq!(*cs.since.read().unwrap(), Some(0));
+        assert!(cs.lookup(0).is_none());
+    }
+
+    #[test]
+    fn test_update_new_slot() {
+        let cs = ClusterSlots::default();
+        let mut epoch_slot = EpochSlots::default();
+        epoch_slot.fill(&[1], 0);
+        cs.update_internal(0, (vec![epoch_slot], Some(0)));
+        assert_eq!(*cs.since.read().unwrap(), Some(0));
+        assert!(cs.lookup(0).is_none());
+        assert!(cs.lookup(1).is_some());
+        assert_eq!(
+            cs.lookup(1)
+                .unwrap()
+                .read()
+                .unwrap()
+                .get(&Pubkey::default()),
+            Some(&0)
+        );
+    }
+
+    #[test]
+    fn test_compute_weights() {
+        let cs = ClusterSlots::default();
+        let ci = ContactInfo::default();
+        assert_eq!(cs.compute_weights(0, &[ci]), vec![(1, 0)]);
+    }
+
+    #[test]
+    fn test_best_peer_2() {
+        let cs = ClusterSlots::default();
+        let mut c1 = ContactInfo::default();
+        let mut c2 = ContactInfo::default();
+        let mut map = HashMap::new();
+        let k1 = Pubkey::new_rand();
+        let k2 = Pubkey::new_rand();
+        map.insert(Arc::new(k1.clone()), std::u64::MAX / 2);
+        map.insert(Arc::new(k2.clone()), 0);
+        cs.cluster_slots
+            .write()
+            .unwrap()
+            .insert(0, Arc::new(RwLock::new(map)));
+        c1.id = k1;
+        c2.id = k2;
+        assert_eq!(
+            cs.compute_weights(0, &[c1, c2]),
+            vec![(std::u64::MAX / 2 + 1, 0), (1, 1)]
+        );
+    }
+
+    #[test]
+    fn test_best_peer_3() {
+        let cs = ClusterSlots::default();
+        let mut c1 = ContactInfo::default();
+        let mut c2 = ContactInfo::default();
+        let mut map = HashMap::new();
+        let k1 = Pubkey::new_rand();
+        let k2 = Pubkey::new_rand();
+        map.insert(Arc::new(k2.clone()), 0);
+        cs.cluster_slots
+            .write()
+            .unwrap()
+            .insert(0, Arc::new(RwLock::new(map)));
+        //make sure default weights are used as well
+        let validator_stakes: HashMap<_, _> = vec![(
+            *Arc::new(k1.clone()),
+            NodeVoteAccounts {
+                total_stake: std::u64::MAX / 2,
+                vote_accounts: vec![Pubkey::default()],
+            },
+        )]
+        .into_iter()
+        .collect();
+        *cs.validator_stakes.write().unwrap() = Arc::new(validator_stakes);
+        c1.id = k1;
+        c2.id = k2;
+        assert_eq!(
+            cs.compute_weights(0, &[c1, c2]),
+            vec![(std::u64::MAX / 2 + 1, 0), (1, 1)]
+        );
+    }
+
+    #[test]
+    fn test_update_new_staked_slot() {
+        let cs = ClusterSlots::default();
+        let mut epoch_slot = EpochSlots::default();
+        epoch_slot.fill(&[1], 0);
+
+        let map = Arc::new(
+            vec![(
+                Pubkey::default(),
+                NodeVoteAccounts {
+                    total_stake: 1,
+                    vote_accounts: vec![Pubkey::default()],
+                },
+            )]
+            .into_iter()
+            .collect(),
+        );
+
+        *cs.validator_stakes.write().unwrap() = map;
+        cs.update_internal(0, (vec![epoch_slot], None));
+        assert!(cs.lookup(1).is_some());
+        assert_eq!(
+            cs.lookup(1)
+                .unwrap()
+                .read()
+                .unwrap()
+                .get(&Pubkey::default()),
+            Some(&1)
+        );
+    }
+
+    #[test]
+    fn test_generate_repairs() {
+        let cs = ClusterSlots::default();
+        let mut epoch_slot = EpochSlots::default();
+        epoch_slot.fill(&[1], 0);
+        cs.update_internal(0, (vec![epoch_slot], None));
+        let self_id = Pubkey::new_rand();
+        assert_eq!(
+            cs.generate_repairs_for_missing_slots(&self_id, 0),
+            vec![RepairType::HighestShred(1, 0)]
+        )
+    }
+
+    #[test]
+    fn test_collect_my_slots() {
+        let cs = ClusterSlots::default();
+        let mut epoch_slot = EpochSlots::default();
+        epoch_slot.fill(&[1], 0);
+        let self_id = epoch_slot.from;
+        cs.update_internal(0, (vec![epoch_slot], None));
+        let slots: Vec<Slot> = cs.collect(&self_id).into_iter().collect();
+        assert_eq!(slots, vec![1]);
+    }
+
+    #[test]
+    fn test_generate_repairs_existing() {
+        let cs = ClusterSlots::default();
+        let mut epoch_slot = EpochSlots::default();
+        epoch_slot.fill(&[1], 0);
+        let self_id = epoch_slot.from;
+        cs.update_internal(0, (vec![epoch_slot], None));
+        assert!(cs
+            .generate_repairs_for_missing_slots(&self_id, 0)
+            .is_empty());
+    }
+}

core/src/commitment.rs

@@ -1,3 +1,6 @@
+use crate::consensus::VOTE_THRESHOLD_SIZE;
+use solana_measure::measure::Measure;
+use solana_metrics::inc_new_counter_info;
 use solana_runtime::bank::Bank;
 use solana_sdk::clock::Slot;
 use solana_vote_program::{vote_state::VoteState, vote_state::MAX_LOCKOUT_HISTORY};
@@ -31,17 +34,40 @@ impl BlockCommitment {
     }
 }
 
-#[derive(Debug, Default)]
+#[derive(Default)]
 pub struct BlockCommitmentCache {
     block_commitment: HashMap<Slot, BlockCommitment>,
     total_stake: u64,
+    bank: Arc<Bank>,
+    root: Slot,
+}
+
+impl std::fmt::Debug for BlockCommitmentCache {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("BlockCommitmentCache")
+            .field("block_commitment", &self.block_commitment)
+            .field("total_stake", &self.total_stake)
+            .field(
+                "bank",
+                &format_args!("Bank({{current_slot: {:?}}})", self.bank.slot()),
+            )
+            .field("root", &self.root)
+            .finish()
+    }
 }
 
 impl BlockCommitmentCache {
-    pub fn new(block_commitment: HashMap<Slot, BlockCommitment>, total_stake: u64) -> Self {
+    pub fn new(
+        block_commitment: HashMap<Slot, BlockCommitment>,
+        total_stake: u64,
+        bank: Arc<Bank>,
+        root: Slot,
+    ) -> Self {
         Self {
             block_commitment,
             total_stake,
+            bank,
+            root,
         }
     }
 
@@ -53,38 +79,62 @@ impl BlockCommitmentCache {
         self.total_stake
     }
 
-    pub fn get_block_with_depth_commitment(
-        &self,
-        minimum_depth: usize,
-        minimum_stake_percentage: f64,
-    ) -> Option<Slot> {
-        self.block_commitment
-            .iter()
-            .filter(|&(_, block_commitment)| {
-                let fork_stake_minimum_depth: u64 = block_commitment.commitment[minimum_depth..]
-                    .iter()
-                    .cloned()
-                    .sum();
-                fork_stake_minimum_depth as f64 / self.total_stake as f64
-                    >= minimum_stake_percentage
-            })
-            .map(|(slot, _)| *slot)
-            .max()
+    pub fn bank(&self) -> Arc<Bank> {
+        self.bank.clone()
     }
 
-    pub fn get_rooted_block_with_commitment(&self, minimum_stake_percentage: f64) -> Option<u64> {
-        self.get_block_with_depth_commitment(MAX_LOCKOUT_HISTORY - 1, minimum_stake_percentage)
+    pub fn slot(&self) -> Slot {
+        self.bank.slot()
+    }
+
+    pub fn root(&self) -> Slot {
+        self.root
+    }
+
+    pub fn get_confirmation_count(&self, slot: Slot) -> Option<usize> {
+        self.get_lockout_count(slot, VOTE_THRESHOLD_SIZE)
+    }
+
+    // Returns the lowest level at which at least `minimum_stake_percentage` of the total epoch
+    // stake is locked out
+    fn get_lockout_count(&self, slot: Slot, minimum_stake_percentage: f64) -> Option<usize> {
+        self.get_block_commitment(slot).map(|block_commitment| {
+            let iterator = block_commitment.commitment.iter().enumerate().rev();
+            let mut sum = 0;
+            for (i, stake) in iterator {
+                sum += stake;
+                if (sum as f64 / self.total_stake as f64) > minimum_stake_percentage {
+                    return i + 1;
+                }
+            }
+            0
+        })
+    }
+    #[cfg(test)]
+    pub fn new_for_tests() -> Self {
+        let mut block_commitment: HashMap<Slot, BlockCommitment> = HashMap::new();
+        block_commitment.insert(0, BlockCommitment::default());
+        Self {
+            block_commitment,
+            total_stake: 42,
+            ..Self::default()
+        }
     }
 }
 
 pub struct CommitmentAggregationData {
     bank: Arc<Bank>,
+    root: Slot,
     total_staked: u64,
 }
 
 impl CommitmentAggregationData {
-    pub fn new(bank: Arc<Bank>, total_staked: u64) -> Self {
-        Self { bank, total_staked }
+    pub fn new(bank: Arc<Bank>, root: Slot, total_staked: u64) -> Self {
+        Self {
+            bank,
+            root,
+            total_staked,
+        }
     }
 }
 
@@ -144,14 +194,24 @@ impl AggregateCommitmentService {
                 continue;
             }
 
+            let mut aggregate_commitment_time = Measure::start("aggregate-commitment-ms");
             let block_commitment = Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
 
-            let mut new_block_commitment =
-                BlockCommitmentCache::new(block_commitment, aggregation_data.total_staked);
+            let mut new_block_commitment = BlockCommitmentCache::new(
+                block_commitment,
+                aggregation_data.total_staked,
+                aggregation_data.bank,
+                aggregation_data.root,
+            );
 
             let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
 
             std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
+            aggregate_commitment_time.stop();
+            inc_new_counter_info!(
+                "aggregate-commitment-ms",
+                aggregate_commitment_time.as_ms() as usize
+            );
         }
     }
 
@@ -230,7 +290,7 @@ impl AggregateCommitmentService {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_sdk::pubkey::Pubkey;
     use solana_stake_program::stake_state;
     use solana_vote_program::vote_state::{self, VoteStateVersions};
@@ -246,84 +306,31 @@ mod tests {
     }
 
     #[test]
-    fn test_get_block_with_depth_commitment() {
+    fn test_get_confirmations() {
+        let bank = Arc::new(Bank::default());
         // Build BlockCommitmentCache with votes at depths 0 and 1 for 2 slots
         let mut cache0 = BlockCommitment::default();
-        cache0.increase_confirmation_stake(1, 15);
-        cache0.increase_confirmation_stake(2, 25);
+        cache0.increase_confirmation_stake(1, 5);
+        cache0.increase_confirmation_stake(2, 40);
 
         let mut cache1 = BlockCommitment::default();
-        cache1.increase_confirmation_stake(1, 10);
-        cache1.increase_confirmation_stake(2, 20);
+        cache1.increase_confirmation_stake(1, 40);
+        cache1.increase_confirmation_stake(2, 5);
+
+        let mut cache2 = BlockCommitment::default();
+        cache2.increase_confirmation_stake(1, 20);
+        cache2.increase_confirmation_stake(2, 5);
 
         let mut block_commitment = HashMap::new();
         block_commitment.entry(0).or_insert(cache0.clone());
         block_commitment.entry(1).or_insert(cache1.clone());
-        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 50);
+        block_commitment.entry(2).or_insert(cache2.clone());
+        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 50, bank, 0);
 
-        // Neither slot has rooted votes
-        assert_eq!(
-            block_commitment_cache.get_rooted_block_with_commitment(0.1),
-            None
-        );
-        // Neither slot meets the minimum level of commitment 0.6 at depth 1
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(1, 0.6),
-            None
-        );
-        // Only slot 0 meets the minimum level of commitment 0.5 at depth 1
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(1, 0.5),
-            Some(0)
-        );
-        // If multiple slots meet the minimum level of commitment, method should return the most recent
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(1, 0.4),
-            Some(1)
-        );
-        // If multiple slots meet the minimum level of commitment, method should return the most recent
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(0, 0.6),
-            Some(1)
-        );
-        // Neither slot meets the minimum level of commitment 0.9 at depth 0
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(0, 0.9),
-            None
-        );
-    }
-
-    #[test]
-    fn test_get_rooted_block_with_commitment() {
-        // Build BlockCommitmentCache with rooted votes
-        let mut cache0 = BlockCommitment::new([0; MAX_LOCKOUT_HISTORY]);
-        cache0.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, 40);
-        cache0.increase_confirmation_stake(MAX_LOCKOUT_HISTORY - 1, 10);
-        let mut cache1 = BlockCommitment::new([0; MAX_LOCKOUT_HISTORY]);
-        cache1.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, 30);
-        cache1.increase_confirmation_stake(MAX_LOCKOUT_HISTORY - 1, 10);
-        cache1.increase_confirmation_stake(MAX_LOCKOUT_HISTORY - 2, 10);
-
-        let mut block_commitment = HashMap::new();
-        block_commitment.entry(0).or_insert(cache0.clone());
-        block_commitment.entry(1).or_insert(cache1.clone());
-        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 50);
-
-        // Only slot 0 meets the minimum level of commitment 0.66 at root
-        assert_eq!(
-            block_commitment_cache.get_rooted_block_with_commitment(0.66),
-            Some(0)
-        );
-        // If multiple slots meet the minimum level of commitment, method should return the most recent
-        assert_eq!(
-            block_commitment_cache.get_rooted_block_with_commitment(0.6),
-            Some(1)
-        );
-        // Neither slot meets the minimum level of commitment 0.9 at root
-        assert_eq!(
-            block_commitment_cache.get_rooted_block_with_commitment(0.9),
-            None
-        );
+        assert_eq!(block_commitment_cache.get_confirmation_count(0), Some(2));
+        assert_eq!(block_commitment_cache.get_confirmation_count(1), Some(1));
+        assert_eq!(block_commitment_cache.get_confirmation_count(2), Some(0),);
+        assert_eq!(block_commitment_cache.get_confirmation_count(3), None,);
     }
 
     #[test]

core/src/consensus.rs

@@ -1,3 +1,4 @@
+use crate::progress_map::ProgressMap;
 use chrono::prelude::*;
 use solana_ledger::bank_forks::BankForks;
 use solana_runtime::bank::Bank;
@@ -355,6 +356,18 @@ impl Tower {
         }
     }
 
+    pub(crate) fn check_switch_threshold(
+        &self,
+        _slot: Slot,
+        _ancestors: &HashMap<Slot, HashSet<u64>>,
+        _descendants: &HashMap<Slot, HashSet<u64>>,
+        _progress: &ProgressMap,
+        _total_epoch_stake: u64,
+        _epoch_vote_accounts: &HashMap<Pubkey, (u64, Account)>,
+    ) -> bool {
+        true
+    }
+
     /// Update lockouts for all the ancestors
     fn update_ancestor_lockouts(
         stake_lockouts: &mut HashMap<Slot, StakeLockout>,
@@ -468,7 +481,12 @@ impl Tower {
 #[cfg(test)]
 pub mod test {
     use super::*;
-    use crate::replay_stage::{ForkProgress, ReplayStage};
+    use crate::{
+        cluster_info_vote_listener::VoteTracker,
+        cluster_slots::ClusterSlots,
+        progress_map::ForkProgress,
+        replay_stage::{HeaviestForkFailures, ReplayStage},
+    };
     use solana_ledger::bank_forks::BankForks;
     use solana_runtime::{
         bank::Bank,
@@ -511,9 +529,9 @@ pub mod test {
             cluster_votes: &mut HashMap<Pubkey, Vec<u64>>,
             validator_keypairs: &HashMap<Pubkey, ValidatorVoteKeypairs>,
             my_keypairs: &ValidatorVoteKeypairs,
-            progress: &mut HashMap<u64, ForkProgress>,
+            progress: &mut ProgressMap,
             tower: &mut Tower,
-        ) -> Vec<VoteFailures> {
+        ) -> Vec<HeaviestForkFailures> {
             let node = self
                 .find_node_and_update_simulation(vote_slot)
                 .expect("Vote to simulate must be for a slot in the tree");
@@ -550,7 +568,7 @@ pub mod test {
                 info!("parent of {} is {}", missing_slot, parent_bank.slot(),);
                 progress
                     .entry(missing_slot)
-                    .or_insert_with(|| ForkProgress::new(parent_bank.last_blockhash()));
+                    .or_insert_with(|| ForkProgress::new(parent_bank.last_blockhash(), None, None));
 
                 // Create the missing bank
                 let new_bank =
@@ -595,6 +613,10 @@ pub mod test {
                 &mut frozen_banks,
                 tower,
                 progress,
+                &VoteTracker::default(),
+                &ClusterSlots::default(),
+                bank_forks,
+                &mut HashSet::new(),
             );
 
             let bank = bank_forks
@@ -611,17 +633,23 @@ pub mod test {
             info!("lockouts: {:?}", fork_progress.fork_stats.stake_lockouts);
             let mut failures = vec![];
             if fork_progress.fork_stats.is_locked_out {
-                failures.push(VoteFailures::LockedOut(vote_slot));
+                failures.push(HeaviestForkFailures::LockedOut(vote_slot));
             }
             if !fork_progress.fork_stats.vote_threshold {
-                failures.push(VoteFailures::FailedThreshold(vote_slot));
+                failures.push(HeaviestForkFailures::FailedThreshold(vote_slot));
             }
             if !failures.is_empty() {
                 return failures;
             }
             let vote = tower.new_vote_from_bank(&bank, &my_vote_pubkey).0;
             if let Some(new_root) = tower.record_bank_vote(vote) {
-                ReplayStage::handle_new_root(new_root, bank_forks, progress, &None);
+                ReplayStage::handle_new_root(
+                    new_root,
+                    bank_forks,
+                    progress,
+                    &None,
+                    &mut HashSet::new(),
+                );
             }
 
             // Mark the vote for this bank under this node's pubkey so it will be
@@ -671,22 +699,17 @@ pub mod test {
         }
     }
 
-    #[derive(PartialEq, Debug)]
-    pub(crate) enum VoteFailures {
-        LockedOut(u64),
-        FailedThreshold(u64),
-    }
-
     // Setup BankForks with bank 0 and all the validator accounts
     pub(crate) fn initialize_state(
         validator_keypairs_map: &HashMap<Pubkey, ValidatorVoteKeypairs>,
-    ) -> (BankForks, HashMap<u64, ForkProgress>) {
+        stake: u64,
+    ) -> (BankForks, ProgressMap) {
         let validator_keypairs: Vec<_> = validator_keypairs_map.values().collect();
         let GenesisConfigInfo {
             genesis_config,
             mint_keypair,
             voting_keypair: _,
-        } = create_genesis_config_with_vote_accounts(1_000_000_000, &validator_keypairs);
+        } = create_genesis_config_with_vote_accounts(1_000_000_000, &validator_keypairs, stake);
 
         let bank0 = Bank::new(&genesis_config);
 
@@ -695,8 +718,8 @@ pub mod test {
         }
 
         bank0.freeze();
-        let mut progress = HashMap::new();
-        progress.insert(0, ForkProgress::new(bank0.last_blockhash()));
+        let mut progress = ProgressMap::default();
+        progress.insert(0, ForkProgress::new(bank0.last_blockhash(), None, None));
         (BankForks::new(0, bank0), progress)
     }
 
@@ -728,7 +751,7 @@ pub mod test {
         bank_forks: &RwLock<BankForks>,
         cluster_votes: &mut HashMap<Pubkey, Vec<u64>>,
         keypairs: &HashMap<Pubkey, ValidatorVoteKeypairs>,
-        progress: &mut HashMap<u64, ForkProgress>,
+        progress: &mut ProgressMap,
     ) -> bool {
         // Check that within some reasonable time, validator can make a new
         // root on this fork
@@ -773,7 +796,7 @@ pub mod test {
         );
 
         // Initialize BankForks
-        let (bank_forks, mut progress) = initialize_state(&keypairs);
+        let (bank_forks, mut progress) = initialize_state(&keypairs, 10_000);
         let bank_forks = RwLock::new(bank_forks);
 
         // Create the tree of banks
@@ -853,7 +876,7 @@ pub mod test {
         votes.extend((45..=50).into_iter());
 
         let mut cluster_votes: HashMap<Pubkey, Vec<Slot>> = HashMap::new();
-        let (bank_forks, mut progress) = initialize_state(&keypairs);
+        let (bank_forks, mut progress) = initialize_state(&keypairs, 10_000);
         let bank_forks = RwLock::new(bank_forks);
 
         // Simulate the votes. Should fail on trying to come back to the main fork

core/src/crds_value.rs

@@ -1,4 +1,6 @@
 use crate::contact_info::ContactInfo;
+use crate::deprecated;
+use crate::epoch_slots::EpochSlots;
 use bincode::{serialize, serialized_size};
 use solana_sdk::timing::timestamp;
 use solana_sdk::{
@@ -17,7 +19,8 @@ use std::{
 pub type VoteIndex = u8;
 pub const MAX_VOTES: VoteIndex = 32;
 
-pub type EpochSlotIndex = u8;
+pub type EpochSlotsIndex = u8;
+pub const MAX_EPOCH_SLOTS: EpochSlotsIndex = 255;
 
 /// CrdsValue that is replicated across the cluster
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -49,6 +52,7 @@ impl Signable for CrdsValue {
             .verify(&self.pubkey().as_ref(), self.signable_data().borrow());
         let data_check = match &self.data {
             CrdsData::Vote(ix, _) => *ix < MAX_VOTES,
+            CrdsData::EpochSlots(ix, _) => *ix < MAX_EPOCH_SLOTS,
             _ => true,
         };
         sig_check && data_check
@@ -57,34 +61,16 @@ impl Signable for CrdsValue {
 
 /// CrdsData that defines the different types of items CrdsValues can hold
 /// * Merge Strategy - Latest wallclock is picked
+/// * LowestSlot index is deprecated
 #[allow(clippy::large_enum_variant)]
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub enum CrdsData {
     ContactInfo(ContactInfo),
     Vote(VoteIndex, Vote),
-    EpochSlots(EpochSlotIndex, EpochSlots),
+    LowestSlot(u8, LowestSlot),
     SnapshotHashes(SnapshotHash),
     AccountsHashes(SnapshotHash),
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-pub enum CompressionType {
-    Uncompressed,
-    GZip,
-    BZip2,
-}
-
-impl Default for CompressionType {
-    fn default() -> Self {
-        Self::Uncompressed
-    }
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
-pub struct EpochIncompleteSlots {
-    pub first: Slot,
-    pub compression: CompressionType,
-    pub compressed_list: Vec<u8>,
+    EpochSlots(EpochSlotsIndex, EpochSlots),
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -103,32 +89,24 @@ impl SnapshotHash {
         }
     }
 }
-
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-pub struct EpochSlots {
+pub struct LowestSlot {
     pub from: Pubkey,
-    pub root: Slot,
+    root: Slot, //deprecated
     pub lowest: Slot,
-    pub slots: BTreeSet<Slot>,
-    pub stash: Vec<EpochIncompleteSlots>,
+    slots: BTreeSet<Slot>,                        //deprecated
+    stash: Vec<deprecated::EpochIncompleteSlots>, //deprecated
     pub wallclock: u64,
 }
 
-impl EpochSlots {
-    pub fn new(
-        from: Pubkey,
-        root: Slot,
-        lowest: Slot,
-        slots: BTreeSet<Slot>,
-        stash: Vec<EpochIncompleteSlots>,
-        wallclock: u64,
-    ) -> Self {
+impl LowestSlot {
+    pub fn new(from: Pubkey, lowest: Slot, wallclock: u64) -> Self {
         Self {
             from,
-            root,
+            root: 0,
             lowest,
-            slots,
-            stash,
+            slots: BTreeSet::new(),
+            stash: vec![],
             wallclock,
         }
     }
@@ -157,8 +135,9 @@ impl Vote {
 pub enum CrdsValueLabel {
     ContactInfo(Pubkey),
     Vote(VoteIndex, Pubkey),
-    EpochSlots(Pubkey),
+    LowestSlot(Pubkey),
     SnapshotHashes(Pubkey),
+    EpochSlots(EpochSlotsIndex, Pubkey),
     AccountsHashes(Pubkey),
 }
 
@@ -167,8 +146,9 @@ impl fmt::Display for CrdsValueLabel {
         match self {
             CrdsValueLabel::ContactInfo(_) => write!(f, "ContactInfo({})", self.pubkey()),
             CrdsValueLabel::Vote(ix, _) => write!(f, "Vote({}, {})", ix, self.pubkey()),
-            CrdsValueLabel::EpochSlots(_) => write!(f, "EpochSlots({})", self.pubkey()),
-            CrdsValueLabel::SnapshotHashes(_) => write!(f, "SnapshotHashes({})", self.pubkey()),
+            CrdsValueLabel::LowestSlot(_) => write!(f, "LowestSlot({})", self.pubkey()),
+            CrdsValueLabel::SnapshotHashes(_) => write!(f, "SnapshotHash({})", self.pubkey()),
+            CrdsValueLabel::EpochSlots(ix, _) => write!(f, "EpochSlots({}, {})", ix, self.pubkey()),
             CrdsValueLabel::AccountsHashes(_) => write!(f, "AccountsHashes({})", self.pubkey()),
         }
     }
@@ -179,8 +159,9 @@ impl CrdsValueLabel {
         match self {
             CrdsValueLabel::ContactInfo(p) => *p,
             CrdsValueLabel::Vote(_, p) => *p,
-            CrdsValueLabel::EpochSlots(p) => *p,
+            CrdsValueLabel::LowestSlot(p) => *p,
             CrdsValueLabel::SnapshotHashes(p) => *p,
+            CrdsValueLabel::EpochSlots(_, p) => *p,
             CrdsValueLabel::AccountsHashes(p) => *p,
         }
     }
@@ -206,27 +187,30 @@ impl CrdsValue {
         match &self.data {
             CrdsData::ContactInfo(contact_info) => contact_info.wallclock,
             CrdsData::Vote(_, vote) => vote.wallclock,
-            CrdsData::EpochSlots(_, vote) => vote.wallclock,
+            CrdsData::LowestSlot(_, obj) => obj.wallclock,
             CrdsData::SnapshotHashes(hash) => hash.wallclock,
             CrdsData::AccountsHashes(hash) => hash.wallclock,
+            CrdsData::EpochSlots(_, p) => p.wallclock,
         }
     }
     pub fn pubkey(&self) -> Pubkey {
         match &self.data {
             CrdsData::ContactInfo(contact_info) => contact_info.id,
             CrdsData::Vote(_, vote) => vote.from,
-            CrdsData::EpochSlots(_, slots) => slots.from,
+            CrdsData::LowestSlot(_, slots) => slots.from,
             CrdsData::SnapshotHashes(hash) => hash.from,
             CrdsData::AccountsHashes(hash) => hash.from,
+            CrdsData::EpochSlots(_, p) => p.from,
         }
     }
     pub fn label(&self) -> CrdsValueLabel {
         match &self.data {
             CrdsData::ContactInfo(_) => CrdsValueLabel::ContactInfo(self.pubkey()),
             CrdsData::Vote(ix, _) => CrdsValueLabel::Vote(*ix, self.pubkey()),
-            CrdsData::EpochSlots(_, _) => CrdsValueLabel::EpochSlots(self.pubkey()),
+            CrdsData::LowestSlot(_, _) => CrdsValueLabel::LowestSlot(self.pubkey()),
             CrdsData::SnapshotHashes(_) => CrdsValueLabel::SnapshotHashes(self.pubkey()),
             CrdsData::AccountsHashes(_) => CrdsValueLabel::AccountsHashes(self.pubkey()),
+            CrdsData::EpochSlots(ix, _) => CrdsValueLabel::EpochSlots(*ix, self.pubkey()),
         }
     }
     pub fn contact_info(&self) -> Option<&ContactInfo> {
@@ -249,9 +233,9 @@ impl CrdsValue {
         }
     }
 
-    pub fn epoch_slots(&self) -> Option<&EpochSlots> {
+    pub fn lowest_slot(&self) -> Option<&LowestSlot> {
         match &self.data {
-            CrdsData::EpochSlots(_, slots) => Some(slots),
+            CrdsData::LowestSlot(_, slots) => Some(slots),
             _ => None,
         }
     }
@@ -270,15 +254,23 @@ impl CrdsValue {
         }
     }
 
+    pub fn epoch_slots(&self) -> Option<&EpochSlots> {
+        match &self.data {
+            CrdsData::EpochSlots(_, slots) => Some(slots),
+            _ => None,
+        }
+    }
+
     /// Return all the possible labels for a record identified by Pubkey.
     pub fn record_labels(key: &Pubkey) -> Vec<CrdsValueLabel> {
         let mut labels = vec![
             CrdsValueLabel::ContactInfo(*key),
-            CrdsValueLabel::EpochSlots(*key),
+            CrdsValueLabel::LowestSlot(*key),
             CrdsValueLabel::SnapshotHashes(*key),
             CrdsValueLabel::AccountsHashes(*key),
         ];
         labels.extend((0..MAX_VOTES).map(|ix| CrdsValueLabel::Vote(ix, *key)));
+        labels.extend((0..MAX_EPOCH_SLOTS).map(|ix| CrdsValueLabel::EpochSlots(ix, *key)));
         labels
     }
 
@@ -326,15 +318,18 @@ mod test {
 
     #[test]
     fn test_labels() {
-        let mut hits = [false; 4 + MAX_VOTES as usize];
+        let mut hits = [false; 4 + MAX_VOTES as usize + MAX_EPOCH_SLOTS as usize];
         // this method should cover all the possible labels
         for v in &CrdsValue::record_labels(&Pubkey::default()) {
             match v {
                 CrdsValueLabel::ContactInfo(_) => hits[0] = true,
-                CrdsValueLabel::EpochSlots(_) => hits[1] = true,
+                CrdsValueLabel::LowestSlot(_) => hits[1] = true,
                 CrdsValueLabel::SnapshotHashes(_) => hits[2] = true,
                 CrdsValueLabel::AccountsHashes(_) => hits[3] = true,
                 CrdsValueLabel::Vote(ix, _) => hits[*ix as usize + 4] = true,
+                CrdsValueLabel::EpochSlots(ix, _) => {
+                    hits[*ix as usize + MAX_VOTES as usize + 4] = true
+                }
             }
         }
         assert!(hits.iter().all(|x| *x));
@@ -354,13 +349,13 @@ mod test {
         let key = v.clone().vote().unwrap().from;
         assert_eq!(v.label(), CrdsValueLabel::Vote(0, key));
 
-        let v = CrdsValue::new_unsigned(CrdsData::EpochSlots(
+        let v = CrdsValue::new_unsigned(CrdsData::LowestSlot(
             0,
-            EpochSlots::new(Pubkey::default(), 0, 0, BTreeSet::new(), vec![], 0),
+            LowestSlot::new(Pubkey::default(), 0, 0),
         ));
         assert_eq!(v.wallclock(), 0);
-        let key = v.clone().epoch_slots().unwrap().from;
-        assert_eq!(v.label(), CrdsValueLabel::EpochSlots(key));
+        let key = v.clone().lowest_slot().unwrap().from;
+        assert_eq!(v.label(), CrdsValueLabel::LowestSlot(key));
     }
 
     #[test]
@@ -377,10 +372,9 @@ mod test {
             Vote::new(&keypair.pubkey(), test_tx(), timestamp()),
         ));
         verify_signatures(&mut v, &keypair, &wrong_keypair);
-        let btreeset: BTreeSet<Slot> = vec![1, 2, 3, 6, 8].into_iter().collect();
-        v = CrdsValue::new_unsigned(CrdsData::EpochSlots(
+        v = CrdsValue::new_unsigned(CrdsData::LowestSlot(
             0,
-            EpochSlots::new(keypair.pubkey(), 0, 0, btreeset, vec![], timestamp()),
+            LowestSlot::new(keypair.pubkey(), 0, timestamp()),
         ));
         verify_signatures(&mut v, &keypair, &wrong_keypair);
     }
@@ -398,6 +392,18 @@ mod test {
         assert!(!vote.verify());
     }
 
+    #[test]
+    fn test_max_epoch_slots_index() {
+        let keypair = Keypair::new();
+        let item = CrdsValue::new_signed(
+            CrdsData::EpochSlots(
+                MAX_EPOCH_SLOTS,
+                EpochSlots::new(keypair.pubkey(), timestamp()),
+            ),
+            &keypair,
+        );
+        assert!(!item.verify());
+    }
     #[test]
     fn test_compute_vote_index_empty() {
         for i in 0..MAX_VOTES {

core/src/deprecated.rs

@@ -0,0 +1,21 @@
+use solana_sdk::clock::Slot;
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+enum CompressionType {
+    Uncompressed,
+    GZip,
+    BZip2,
+}
+
+impl Default for CompressionType {
+    fn default() -> Self {
+        Self::Uncompressed
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
+pub(crate) struct EpochIncompleteSlots {
+    first: Slot,
+    compression: CompressionType,
+    compressed_list: Vec<u8>,
+}

core/src/epoch_slots.rs

@@ -0,0 +1,401 @@
+use crate::cluster_info::MAX_CRDS_OBJECT_SIZE;
+use bincode::serialized_size;
+use bv::BitVec;
+use flate2::{Compress, Compression, Decompress, FlushCompress, FlushDecompress};
+use solana_sdk::clock::Slot;
+use solana_sdk::pubkey::Pubkey;
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+pub struct Uncompressed {
+    pub first_slot: Slot,
+    pub num: usize,
+    pub slots: BitVec<u8>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, PartialEq)]
+pub struct Flate2 {
+    pub first_slot: Slot,
+    pub num: usize,
+    pub compressed: Vec<u8>,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    CompressError,
+    DecompressError,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl std::convert::From<flate2::CompressError> for Error {
+    fn from(_e: flate2::CompressError) -> Error {
+        Error::CompressError
+    }
+}
+impl std::convert::From<flate2::DecompressError> for Error {
+    fn from(_e: flate2::DecompressError) -> Error {
+        Error::DecompressError
+    }
+}
+
+impl Flate2 {
+    fn deflate(mut unc: Uncompressed) -> Result<Self> {
+        let mut compressed = Vec::with_capacity(unc.slots.block_capacity());
+        let mut compressor = Compress::new(Compression::best(), false);
+        let first_slot = unc.first_slot;
+        let num = unc.num;
+        unc.slots.shrink_to_fit();
+        let bits = unc.slots.into_boxed_slice();
+        compressor.compress_vec(&bits, &mut compressed, FlushCompress::Finish)?;
+        let rv = Self {
+            first_slot,
+            num,
+            compressed,
+        };
+        let _ = rv.inflate()?;
+        Ok(rv)
+    }
+    pub fn inflate(&self) -> Result<Uncompressed> {
+        //add some head room for the decompressor which might spill more bits
+        let mut uncompressed = Vec::with_capacity(32 + (self.num + 4) / 8);
+        let mut decompress = Decompress::new(false);
+        decompress.decompress_vec(&self.compressed, &mut uncompressed, FlushDecompress::Finish)?;
+        Ok(Uncompressed {
+            first_slot: self.first_slot,
+            num: self.num,
+            slots: BitVec::from_bits(&uncompressed),
+        })
+    }
+}
+
+impl Uncompressed {
+    pub fn new(max_size: usize) -> Self {
+        Self {
+            num: 0,
+            first_slot: 0,
+            slots: BitVec::new_fill(false, 8 * max_size as u64),
+        }
+    }
+    pub fn to_slots(&self, min_slot: Slot) -> Vec<Slot> {
+        let mut rv = vec![];
+        let start = if min_slot < self.first_slot {
+            0 as usize
+        } else {
+            (min_slot - self.first_slot) as usize
+        };
+        for i in start..self.num {
+            if self.slots.get(i as u64) {
+                rv.push(self.first_slot + i as Slot);
+            }
+        }
+        rv
+    }
+    pub fn add(&mut self, slots: &[Slot]) -> usize {
+        for (i, s) in slots.iter().enumerate() {
+            if self.num == 0 {
+                self.first_slot = *s;
+            }
+            if *s < self.first_slot {
+                return i;
+            }
+            if *s - self.first_slot >= self.slots.capacity() {
+                return i;
+            }
+            self.slots.set(*s - self.first_slot, true);
+            self.num = std::cmp::max(self.num, 1 + (*s - self.first_slot) as usize);
+        }
+        slots.len()
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+pub enum CompressedSlots {
+    Flate2(Flate2),
+    Uncompressed(Uncompressed),
+}
+
+impl Default for CompressedSlots {
+    fn default() -> Self {
+        CompressedSlots::new(0)
+    }
+}
+
+impl CompressedSlots {
+    fn new(max_size: usize) -> Self {
+        CompressedSlots::Uncompressed(Uncompressed::new(max_size))
+    }
+
+    pub fn first_slot(&self) -> Slot {
+        match self {
+            CompressedSlots::Uncompressed(a) => a.first_slot,
+            CompressedSlots::Flate2(b) => b.first_slot,
+        }
+    }
+
+    pub fn num_slots(&self) -> usize {
+        match self {
+            CompressedSlots::Uncompressed(a) => a.num,
+            CompressedSlots::Flate2(b) => b.num,
+        }
+    }
+
+    pub fn add(&mut self, slots: &[Slot]) -> usize {
+        match self {
+            CompressedSlots::Uncompressed(vals) => vals.add(slots),
+            CompressedSlots::Flate2(_) => 0,
+        }
+    }
+    pub fn to_slots(&self, min_slot: Slot) -> Result<Vec<Slot>> {
+        match self {
+            CompressedSlots::Uncompressed(vals) => Ok(vals.to_slots(min_slot)),
+            CompressedSlots::Flate2(vals) => {
+                let unc = vals.inflate()?;
+                Ok(unc.to_slots(min_slot))
+            }
+        }
+    }
+    pub fn deflate(&mut self) -> Result<()> {
+        match self {
+            CompressedSlots::Uncompressed(vals) => {
+                let unc = vals.clone();
+                let compressed = Flate2::deflate(unc)?;
+                let mut new = CompressedSlots::Flate2(compressed);
+                std::mem::swap(self, &mut new);
+                Ok(())
+            }
+            CompressedSlots::Flate2(_) => Ok(()),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
+pub struct EpochSlots {
+    pub from: Pubkey,
+    pub slots: Vec<CompressedSlots>,
+    pub wallclock: u64,
+}
+
+impl EpochSlots {
+    pub fn new(from: Pubkey, now: u64) -> Self {
+        Self {
+            from,
+            wallclock: now,
+            slots: vec![],
+        }
+    }
+    pub fn fill(&mut self, slots: &[Slot], now: u64) -> usize {
+        let mut num = 0;
+        self.wallclock = std::cmp::max(now, self.wallclock + 1);
+        while num < slots.len() {
+            num += self.add(&slots[num..]);
+            if num < slots.len() {
+                if self.deflate().is_err() {
+                    return num;
+                }
+                let space = self.max_compressed_slot_size();
+                if space > 0 {
+                    let cslot = CompressedSlots::new(space as usize);
+                    self.slots.push(cslot);
+                } else {
+                    return num;
+                }
+            }
+        }
+        num
+    }
+    pub fn add(&mut self, slots: &[Slot]) -> usize {
+        let mut num = 0;
+        for s in &mut self.slots {
+            num += s.add(&slots[num..]);
+            if num >= slots.len() {
+                break;
+            }
+        }
+        num
+    }
+    pub fn deflate(&mut self) -> Result<()> {
+        for s in self.slots.iter_mut() {
+            s.deflate()?;
+        }
+        Ok(())
+    }
+    pub fn max_compressed_slot_size(&self) -> isize {
+        let len_header = serialized_size(self).unwrap();
+        let len_slot = serialized_size(&CompressedSlots::default()).unwrap();
+        MAX_CRDS_OBJECT_SIZE as isize - (len_header + len_slot) as isize
+    }
+
+    pub fn first_slot(&self) -> Option<Slot> {
+        self.slots.iter().map(|s| s.first_slot()).min()
+    }
+
+    pub fn to_slots(&self, min_slot: Slot) -> Vec<Slot> {
+        self.slots
+            .iter()
+            .filter(|s| min_slot < s.first_slot() + s.num_slots() as u64)
+            .filter_map(|s| s.to_slots(min_slot).ok())
+            .flatten()
+            .collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn test_epoch_slots_max_size() {
+        let epoch_slots = EpochSlots::default();
+        assert!(epoch_slots.max_compressed_slot_size() > 0);
+    }
+
+    #[test]
+    fn test_epoch_slots_uncompressed_add_1() {
+        let mut slots = Uncompressed::new(1);
+        assert_eq!(slots.slots.capacity(), 8);
+        assert_eq!(slots.add(&[1]), 1);
+        assert_eq!(slots.to_slots(1), vec![1]);
+        assert!(slots.to_slots(2).is_empty());
+    }
+    #[test]
+    fn test_epoch_slots_uncompressed_add_2() {
+        let mut slots = Uncompressed::new(1);
+        assert_eq!(slots.add(&[1, 2]), 2);
+        assert_eq!(slots.to_slots(1), vec![1, 2]);
+    }
+    #[test]
+    fn test_epoch_slots_uncompressed_add_3a() {
+        let mut slots = Uncompressed::new(1);
+        assert_eq!(slots.add(&[1, 3, 2]), 3);
+        assert_eq!(slots.to_slots(1), vec![1, 2, 3]);
+    }
+
+    #[test]
+    fn test_epoch_slots_uncompressed_add_3b() {
+        let mut slots = Uncompressed::new(1);
+        assert_eq!(slots.add(&[1, 10, 2]), 1);
+        assert_eq!(slots.to_slots(1), vec![1]);
+    }
+
+    #[test]
+    fn test_epoch_slots_uncompressed_add_3c() {
+        let mut slots = Uncompressed::new(2);
+        assert_eq!(slots.add(&[1, 10, 2]), 3);
+        assert_eq!(slots.to_slots(1), vec![1, 2, 10]);
+        assert_eq!(slots.to_slots(2), vec![2, 10]);
+        assert_eq!(slots.to_slots(3), vec![10]);
+        assert_eq!(slots.to_slots(11).is_empty(), true);
+    }
+    #[test]
+    fn test_epoch_slots_compressed() {
+        let mut slots = Uncompressed::new(100);
+        slots.add(&[1, 701, 2]);
+        assert_eq!(slots.num, 701);
+        let compressed = Flate2::deflate(slots).unwrap();
+        assert_eq!(compressed.first_slot, 1);
+        assert_eq!(compressed.num, 701);
+        assert!(compressed.compressed.len() < 32);
+        let slots = compressed.inflate().unwrap();
+        assert_eq!(slots.first_slot, 1);
+        assert_eq!(slots.num, 701);
+        assert_eq!(slots.to_slots(1), vec![1, 2, 701]);
+    }
+    #[test]
+    fn test_epoch_slots_fill_range() {
+        let range: Vec<Slot> = (0..5000).into_iter().collect();
+        let mut slots = EpochSlots::default();
+        assert_eq!(slots.fill(&range, 1), 5000);
+        assert_eq!(slots.wallclock, 1);
+        assert_eq!(slots.to_slots(0), range);
+        assert_eq!(slots.to_slots(4999), vec![4999]);
+        assert_eq!(slots.to_slots(5000).is_empty(), true);
+    }
+    #[test]
+    fn test_epoch_slots_fill_sparce_range() {
+        let range: Vec<Slot> = (0..5000).into_iter().map(|x| x * 3).collect();
+        let mut slots = EpochSlots::default();
+        assert_eq!(slots.fill(&range, 2), 5000);
+        assert_eq!(slots.wallclock, 2);
+        assert_eq!(slots.slots.len(), 3);
+        assert_eq!(slots.slots[0].first_slot(), 0);
+        assert_ne!(slots.slots[0].num_slots(), 0);
+        let next = slots.slots[0].num_slots() as u64 + slots.slots[0].first_slot();
+        assert!(slots.slots[1].first_slot() >= next);
+        assert_ne!(slots.slots[1].num_slots(), 0);
+        assert_ne!(slots.slots[2].num_slots(), 0);
+        assert_eq!(slots.to_slots(0), range);
+        assert_eq!(slots.to_slots(4999 * 3), vec![4999 * 3]);
+    }
+
+    #[test]
+    fn test_epoch_slots_fill_large_sparce_range() {
+        let range: Vec<Slot> = (0..5000).into_iter().map(|x| x * 7).collect();
+        let mut slots = EpochSlots::default();
+        assert_eq!(slots.fill(&range, 2), 5000);
+        assert_eq!(slots.to_slots(0), range);
+    }
+
+    #[test]
+    fn test_epoch_slots_fill_uncompressed_random_range() {
+        use rand::Rng;
+        for _ in 0..10 {
+            let mut range: Vec<Slot> = vec![];
+            for _ in 0..5000 {
+                let last = *range.last().unwrap_or(&0);
+                range.push(last + rand::thread_rng().gen_range(1, 5));
+            }
+            let sz = EpochSlots::default().max_compressed_slot_size();
+            let mut slots = Uncompressed::new(sz as usize);
+            let sz = slots.add(&range);
+            let slots = slots.to_slots(0);
+            assert_eq!(slots.len(), sz);
+            assert_eq!(slots[..], range[..sz]);
+        }
+    }
+
+    #[test]
+    fn test_epoch_slots_fill_compressed_random_range() {
+        use rand::Rng;
+        for _ in 0..10 {
+            let mut range: Vec<Slot> = vec![];
+            for _ in 0..5000 {
+                let last = *range.last().unwrap_or(&0);
+                range.push(last + rand::thread_rng().gen_range(1, 5));
+            }
+            let sz = EpochSlots::default().max_compressed_slot_size();
+            let mut slots = Uncompressed::new(sz as usize);
+            let sz = slots.add(&range);
+            let mut slots = CompressedSlots::Uncompressed(slots);
+            slots.deflate().unwrap();
+            let slots = slots.to_slots(0).unwrap();
+            assert_eq!(slots.len(), sz);
+            assert_eq!(slots[..], range[..sz]);
+        }
+    }
+
+    #[test]
+    fn test_epoch_slots_fill_random_range() {
+        use rand::Rng;
+        for _ in 0..10 {
+            let mut range: Vec<Slot> = vec![];
+            for _ in 0..5000 {
+                let last = *range.last().unwrap_or(&0);
+                range.push(last + rand::thread_rng().gen_range(1, 5));
+            }
+            let mut slots = EpochSlots::default();
+            let sz = slots.fill(&range, 1);
+            let last = range[sz - 1];
+            assert_eq!(
+                last,
+                slots.slots.last().unwrap().first_slot()
+                    + slots.slots.last().unwrap().num_slots() as u64
+                    - 1
+            );
+            for s in &slots.slots {
+                assert!(s.to_slots(0).is_ok());
+            }
+            let slots = slots.to_slots(0);
+            assert_eq!(slots[..], range[..slots.len()]);
+            assert_eq!(sz, slots.len())
+        }
+    }
+}

core/src/fetch_stage.rs

@@ -1,14 +1,14 @@
 //! The `fetch_stage` batches input from a UDP socket and sends it to a channel.
 
 use crate::banking_stage::FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET;
-use crate::packet::PacketsRecycler;
 use crate::poh_recorder::PohRecorder;
 use crate::result::{Error, Result};
-use crate::streamer::{self, PacketReceiver, PacketSender};
 use solana_measure::thread_mem_usage;
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_info};
+use solana_perf::packet::PacketsRecycler;
 use solana_perf::recycler::Recycler;
 use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;
+use solana_streamer::streamer::{self, PacketReceiver, PacketSender};
 use std::net::UdpSocket;
 use std::sync::atomic::AtomicBool;
 use std::sync::mpsc::{channel, RecvTimeoutError};

core/src/genesis_utils.rs

@@ -1 +0,0 @@
-pub use solana_ledger::genesis_utils::*;

core/src/gossip_service.rs

@@ -2,13 +2,13 @@
 
 use crate::cluster_info::{ClusterInfo, VALIDATOR_PORT_RANGE};
 use crate::contact_info::ContactInfo;
-use crate::streamer;
 use rand::{thread_rng, Rng};
 use solana_client::thin_client::{create_client, ThinClient};
 use solana_ledger::bank_forks::BankForks;
 use solana_perf::recycler::Recycler;
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::signature::{Keypair, Signer};
+use solana_streamer::streamer;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, UdpSocket};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::mpsc::channel;

core/src/ledger_cleanup_service.rs

@@ -1,6 +1,8 @@
 //! The `ledger_cleanup_service` drops older ledger data to limit disk space usage
 
 use solana_ledger::blockstore::Blockstore;
+use solana_ledger::blockstore_db::Result as BlockstoreResult;
+use solana_measure::measure::Measure;
 use solana_metrics::datapoint_debug;
 use solana_sdk::clock::Slot;
 use std::string::ToString;
@@ -11,13 +13,22 @@ use std::thread;
 use std::thread::{Builder, JoinHandle};
 use std::time::Duration;
 
+// - To try and keep the RocksDB size under 512GB:
+//   Seeing about 1600b/shred, using 2000b/shred for margin, so 250m shreds can be stored in 512gb.
+//   at 5k shreds/slot at 50k tps, this is 500k slots (~5.5 hours).
+//   At idle, 60 shreds/slot this is about 4m slots (18 days)
 // This is chosen to allow enough time for
-// - To try and keep the RocksDB size under 512GB at 50k tps (100 slots take ~2GB).
 // - A validator to download a snapshot from a peer and boot from it
 // - To make sure that if a validator needs to reboot from its own snapshot, it has enough slots locally
 //   to catch back up to where it was when it stopped
-pub const DEFAULT_MAX_LEDGER_SLOTS: u64 = 270_000;
-// Remove a fixed number of slots at a time, it's more efficient than doing it one-by-one
+pub const DEFAULT_MAX_LEDGER_SHREDS: u64 = 250_000_000;
+
+// Check for removing slots at this interval so we don't purge too often
+// and starve other blockstore users.
+pub const DEFAULT_PURGE_SLOT_INTERVAL: u64 = 512;
+
+// Remove a limited number of slots at a time, so the operation
+// does not take too long and block other blockstore users.
 pub const DEFAULT_PURGE_BATCH_SIZE: u64 = 256;
 
 pub struct LedgerCleanupService {
@@ -36,7 +47,7 @@ impl LedgerCleanupService {
             max_ledger_slots
         );
         let exit = exit.clone();
-        let mut next_purge_batch = max_ledger_slots;
+        let mut last_purge_slot = 0;
         let t_cleanup = Builder::new()
             .name("solana-ledger-cleanup".to_string())
             .spawn(move || loop {
@@ -47,7 +58,8 @@ impl LedgerCleanupService {
                     &new_root_receiver,
                     &blockstore,
                     max_ledger_slots,
-                    &mut next_purge_batch,
+                    &mut last_purge_slot,
+                    DEFAULT_PURGE_SLOT_INTERVAL,
                 ) {
                     match e {
                         RecvTimeoutError::Disconnected => break,
@@ -59,45 +71,123 @@ impl LedgerCleanupService {
         Self { t_cleanup }
     }
 
+    fn find_slots_to_clean(
+        blockstore: &Arc<Blockstore>,
+        root: Slot,
+        max_ledger_shreds: u64,
+    ) -> (u64, Slot, Slot) {
+        let mut shreds = Vec::new();
+        let mut iterate_time = Measure::start("iterate_time");
+        let mut total_shreds = 0;
+        let mut first_slot = 0;
+        for (i, (slot, meta)) in blockstore.slot_meta_iterator(0).unwrap().enumerate() {
+            if i == 0 {
+                first_slot = slot;
+                debug!("purge: searching from slot: {}", slot);
+            }
+            // Not exact since non-full slots will have holes
+            total_shreds += meta.received;
+            shreds.push((slot, meta.received));
+            if slot > root {
+                break;
+            }
+        }
+        iterate_time.stop();
+        info!(
+            "checking for ledger purge: max_shreds: {} slots: {} total_shreds: {} {}",
+            max_ledger_shreds,
+            shreds.len(),
+            total_shreds,
+            iterate_time
+        );
+        if (total_shreds as u64) < max_ledger_shreds {
+            return (0, 0, 0);
+        }
+        let mut cur_shreds = 0;
+        let mut lowest_slot_to_clean = shreds[0].0;
+        for (slot, num_shreds) in shreds.iter().rev() {
+            cur_shreds += *num_shreds as u64;
+            if cur_shreds > max_ledger_shreds {
+                lowest_slot_to_clean = *slot;
+                break;
+            }
+        }
+
+        (cur_shreds, lowest_slot_to_clean, first_slot)
+    }
+
     fn cleanup_ledger(
         new_root_receiver: &Receiver<Slot>,
         blockstore: &Arc<Blockstore>,
-        max_ledger_slots: u64,
-        next_purge_batch: &mut u64,
+        max_ledger_shreds: u64,
+        last_purge_slot: &mut u64,
+        purge_interval: u64,
     ) -> Result<(), RecvTimeoutError> {
-        let disk_utilization_pre = blockstore.storage_size();
-
-        let root = new_root_receiver.recv_timeout(Duration::from_secs(1))?;
-
-        // Notify blockstore of impending purge
-        if root > *next_purge_batch {
-            //cleanup
-            let lowest_slot = root - max_ledger_slots;
-            *blockstore.lowest_cleanup_slot.write().unwrap() = lowest_slot;
-            blockstore.purge_slots(0, Some(lowest_slot));
-            *next_purge_batch += DEFAULT_PURGE_BATCH_SIZE;
+        let mut root = new_root_receiver.recv_timeout(Duration::from_secs(1))?;
+        // Get the newest root
+        while let Ok(new_root) = new_root_receiver.try_recv() {
+            root = new_root;
         }
 
-        let disk_utilization_post = blockstore.storage_size();
-
-        if let (Ok(disk_utilization_pre), Ok(disk_utilization_post)) =
-            (disk_utilization_pre, disk_utilization_post)
-        {
-            datapoint_debug!(
-                "ledger_disk_utilization",
-                ("disk_utilization_pre", disk_utilization_pre as i64, i64),
-                ("disk_utilization_post", disk_utilization_post as i64, i64),
-                (
-                    "disk_utilization_delta",
-                    (disk_utilization_pre as i64 - disk_utilization_post as i64),
-                    i64
-                )
+        if root - *last_purge_slot > purge_interval {
+            let disk_utilization_pre = blockstore.storage_size();
+            info!(
+                "purge: new root: {} last_purge: {} purge_interval: {} disk: {:?}",
+                root, last_purge_slot, purge_interval, disk_utilization_pre
             );
+            *last_purge_slot = root;
+
+            let (num_shreds_to_clean, lowest_slot_to_clean, mut first_slot) =
+                Self::find_slots_to_clean(blockstore, root, max_ledger_shreds);
+
+            if num_shreds_to_clean > 0 {
+                debug!(
+                    "cleaning up to: {} shreds: {} first: {}",
+                    lowest_slot_to_clean, num_shreds_to_clean, first_slot
+                );
+                loop {
+                    let current_lowest =
+                        std::cmp::min(lowest_slot_to_clean, first_slot + DEFAULT_PURGE_BATCH_SIZE);
+
+                    let mut slot_update_time = Measure::start("slot_update");
+                    *blockstore.lowest_cleanup_slot.write().unwrap() = current_lowest;
+                    slot_update_time.stop();
+
+                    let mut clean_time = Measure::start("ledger_clean");
+                    blockstore.purge_slots(first_slot, Some(current_lowest));
+                    clean_time.stop();
+
+                    debug!(
+                        "ledger purge {} -> {}: {} {}",
+                        first_slot, current_lowest, slot_update_time, clean_time
+                    );
+                    first_slot += DEFAULT_PURGE_BATCH_SIZE;
+                    if current_lowest == lowest_slot_to_clean {
+                        break;
+                    }
+                    thread::sleep(Duration::from_millis(500));
+                }
+            }
+
+            let disk_utilization_post = blockstore.storage_size();
+
+            Self::report_disk_metrics(disk_utilization_pre, disk_utilization_post);
         }
 
         Ok(())
     }
 
+    fn report_disk_metrics(pre: BlockstoreResult<u64>, post: BlockstoreResult<u64>) {
+        if let (Ok(pre), Ok(post)) = (pre, post) {
+            datapoint_debug!(
+                "ledger_disk_utilization",
+                ("disk_utilization_pre", pre as i64, i64),
+                ("disk_utilization_post", post as i64, i64),
+                ("disk_utilization_delta", (pre as i64 - post as i64), i64)
+            );
+        }
+    }
+
     pub fn join(self) -> thread::Result<()> {
         self.t_cleanup.join()
     }
@@ -111,6 +201,7 @@ mod tests {
 
     #[test]
     fn test_cleanup() {
+        solana_logger::setup();
         let blockstore_path = get_tmp_ledger_path!();
         let blockstore = Blockstore::open(&blockstore_path).unwrap();
         let (shreds, _) = make_many_slot_entries(0, 50, 5);
@@ -118,10 +209,10 @@ mod tests {
         let blockstore = Arc::new(blockstore);
         let (sender, receiver) = channel();
 
-        //send a signal to kill slots 0-40
-        let mut next_purge_slot = 0;
+        //send a signal to kill all but 5 shreds, which will be in the newest slots
+        let mut last_purge_slot = 0;
         sender.send(50).unwrap();
-        LedgerCleanupService::cleanup_ledger(&receiver, &blockstore, 10, &mut next_purge_slot)
+        LedgerCleanupService::cleanup_ledger(&receiver, &blockstore, 5, &mut last_purge_slot, 10)
             .unwrap();
 
         //check that 0-40 don't exist
@@ -134,6 +225,62 @@ mod tests {
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
 
+    #[test]
+    fn test_cleanup_speed() {
+        solana_logger::setup();
+        let blockstore_path = get_tmp_ledger_path!();
+        let mut blockstore = Blockstore::open(&blockstore_path).unwrap();
+        blockstore.set_no_compaction(true);
+        let blockstore = Arc::new(blockstore);
+        let (sender, receiver) = channel();
+
+        let mut first_insert = Measure::start("first_insert");
+        let initial_slots = 50;
+        let initial_entries = 5;
+        let (shreds, _) = make_many_slot_entries(0, initial_slots, initial_entries);
+        blockstore.insert_shreds(shreds, None, false).unwrap();
+        first_insert.stop();
+        info!("{}", first_insert);
+
+        let mut last_purge_slot = 0;
+        let mut slot = initial_slots;
+        let mut num_slots = 6;
+        for _ in 0..5 {
+            let mut insert_time = Measure::start("insert time");
+            let batch_size = 2;
+            let batches = num_slots / batch_size;
+            for i in 0..batches {
+                let (shreds, _) = make_many_slot_entries(slot + i * batch_size, batch_size, 5);
+                blockstore.insert_shreds(shreds, None, false).unwrap();
+                if i % 100 == 0 {
+                    info!("inserting..{} of {}", i, batches);
+                }
+            }
+            insert_time.stop();
+
+            let mut time = Measure::start("purge time");
+            sender.send(slot + num_slots).unwrap();
+            LedgerCleanupService::cleanup_ledger(
+                &receiver,
+                &blockstore,
+                initial_slots,
+                &mut last_purge_slot,
+                10,
+            )
+            .unwrap();
+            time.stop();
+            info!(
+                "slot: {} size: {} {} {}",
+                slot, num_slots, insert_time, time
+            );
+            slot += num_slots;
+            num_slots *= 2;
+        }
+
+        drop(blockstore);
+        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
+    }
+
     #[test]
     fn test_compaction() {
         let blockstore_path = get_tmp_ledger_path!();
@@ -142,7 +289,7 @@ mod tests {
         let n = 10_000;
         let batch_size = 100;
         let batches = n / batch_size;
-        let max_ledger_slots = 100;
+        let max_ledger_shreds = 100;
 
         for i in 0..batches {
             let (shreds, _) = make_many_slot_entries(i * batch_size, batch_size, 1);
@@ -158,8 +305,9 @@ mod tests {
         LedgerCleanupService::cleanup_ledger(
             &receiver,
             &blockstore,
-            max_ledger_slots,
+            max_ledger_shreds,
             &mut next_purge_batch,
+            10,
         )
         .unwrap();
 
@@ -170,7 +318,7 @@ mod tests {
         assert!(u2 < u1, "insufficient compaction! pre={},post={}", u1, u2,);
 
         // check that early slots don't exist
-        let max_slot = n - max_ledger_slots;
+        let max_slot = n - max_ledger_shreds - 1;
         blockstore
             .slot_meta_iterator(0)
             .unwrap()

core/src/lib.rs

@@ -5,17 +5,18 @@
 //! command-line tools to spin up validators and a Rust library
 //!
 
+pub mod accounts_cleanup_service;
 pub mod accounts_hash_verifier;
 pub mod banking_stage;
 pub mod broadcast_stage;
 pub mod cluster_info_vote_listener;
 pub mod commitment;
+mod deprecated;
 pub mod shred_fetch_stage;
 #[macro_use]
 pub mod contact_info;
-pub mod blockstream;
-pub mod blockstream_service;
 pub mod cluster_info;
+pub mod cluster_slots;
 pub mod consensus;
 pub mod crds;
 pub mod crds_gossip;
@@ -23,16 +24,15 @@ pub mod crds_gossip_error;
 pub mod crds_gossip_pull;
 pub mod crds_gossip_push;
 pub mod crds_value;
+pub mod epoch_slots;
 pub mod fetch_stage;
 pub mod gen_keys;
-pub mod genesis_utils;
 pub mod gossip_service;
 pub mod ledger_cleanup_service;
 pub mod local_vote_signer_service;
-pub mod packet;
 pub mod poh_recorder;
 pub mod poh_service;
-pub mod recvmmsg;
+pub mod progress_map;
 pub mod repair_service;
 pub mod replay_stage;
 mod result;
@@ -43,7 +43,6 @@ pub mod rpc_pubsub;
 pub mod rpc_pubsub_service;
 pub mod rpc_service;
 pub mod rpc_subscriptions;
-pub mod sendmmsg;
 pub mod serve_repair;
 pub mod serve_repair_service;
 pub mod sigverify;
@@ -51,11 +50,11 @@ pub mod sigverify_shreds;
 pub mod sigverify_stage;
 pub mod snapshot_packager_service;
 pub mod storage_stage;
-pub mod streamer;
 pub mod tpu;
 pub mod transaction_status_service;
 pub mod tvu;
 pub mod validator;
+pub mod verified_vote_packets;
 pub mod weighted_shuffle;
 pub mod window_service;
 

core/src/poh_recorder.rs

@@ -511,8 +511,8 @@ impl PohRecorder {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use bincode::serialize;
+    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_ledger::{blockstore::Blockstore, blockstore_meta::SlotMeta, get_tmp_ledger_path};
     use solana_perf::test_tx::test_tx;
     use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;

core/src/poh_service.rs

@@ -120,8 +120,8 @@ impl PohService {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use crate::poh_recorder::WorkingBank;
+    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
     use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
     use solana_perf::test_tx::test_tx;

core/src/progress_map.rs

@@ -0,0 +1,571 @@
+use crate::{
+    cluster_info_vote_listener::SlotVoteTracker, cluster_slots::SlotPubkeys,
+    consensus::StakeLockout, replay_stage::SUPERMINORITY_THRESHOLD,
+};
+use solana_ledger::{
+    bank_forks::BankForks,
+    blockstore_processor::{ConfirmationProgress, ConfirmationTiming},
+};
+use solana_runtime::bank::Bank;
+use solana_sdk::{account::Account, clock::Slot, hash::Hash, pubkey::Pubkey};
+use std::{
+    collections::{HashMap, HashSet},
+    rc::Rc,
+    sync::{Arc, RwLock},
+};
+
+#[derive(Default)]
+pub(crate) struct ReplaySlotStats(ConfirmationTiming);
+impl std::ops::Deref for ReplaySlotStats {
+    type Target = ConfirmationTiming;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+impl std::ops::DerefMut for ReplaySlotStats {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+impl ReplaySlotStats {
+    pub fn report_stats(&self, slot: Slot, num_entries: usize, num_shreds: u64) {
+        datapoint_info!(
+            "replay-slot-stats",
+            ("slot", slot as i64, i64),
+            ("fetch_entries_time", self.fetch_elapsed as i64, i64),
+            (
+                "fetch_entries_fail_time",
+                self.fetch_fail_elapsed as i64,
+                i64
+            ),
+            ("entry_verification_time", self.verify_elapsed as i64, i64),
+            ("replay_time", self.replay_elapsed as i64, i64),
+            (
+                "replay_total_elapsed",
+                self.started.elapsed().as_micros() as i64,
+                i64
+            ),
+            ("total_entries", num_entries as i64, i64),
+            ("total_shreds", num_shreds as i64, i64),
+        );
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct ValidatorStakeInfo {
+    pub validator_vote_pubkey: Pubkey,
+    pub stake: u64,
+    pub total_epoch_stake: u64,
+}
+
+impl Default for ValidatorStakeInfo {
+    fn default() -> Self {
+        Self {
+            stake: 0,
+            validator_vote_pubkey: Pubkey::default(),
+            total_epoch_stake: 1,
+        }
+    }
+}
+
+impl ValidatorStakeInfo {
+    pub fn new(validator_vote_pubkey: Pubkey, stake: u64, total_epoch_stake: u64) -> Self {
+        Self {
+            validator_vote_pubkey,
+            stake,
+            total_epoch_stake,
+        }
+    }
+}
+
+pub(crate) struct ForkProgress {
+    pub(crate) is_dead: bool,
+    pub(crate) fork_stats: ForkStats,
+    pub(crate) propagated_stats: PropagatedStats,
+    pub(crate) replay_stats: ReplaySlotStats,
+    pub(crate) replay_progress: ConfirmationProgress,
+}
+
+impl ForkProgress {
+    pub fn new(
+        last_entry: Hash,
+        prev_leader_slot: Option<Slot>,
+        validator_stake_info: Option<ValidatorStakeInfo>,
+    ) -> Self {
+        let (
+            is_leader_slot,
+            propagated_validators_stake,
+            propagated_validators,
+            is_propagated,
+            total_epoch_stake,
+        ) = validator_stake_info
+            .map(|info| {
+                (
+                    true,
+                    info.stake,
+                    vec![Rc::new(info.validator_vote_pubkey)]
+                        .into_iter()
+                        .collect(),
+                    {
+                        if info.total_epoch_stake == 0 {
+                            true
+                        } else {
+                            info.stake as f64 / info.total_epoch_stake as f64
+                                > SUPERMINORITY_THRESHOLD
+                        }
+                    },
+                    info.total_epoch_stake,
+                )
+            })
+            .unwrap_or((false, 0, HashSet::new(), false, 0));
+        Self {
+            is_dead: false,
+            fork_stats: ForkStats::default(),
+            replay_stats: ReplaySlotStats::default(),
+            replay_progress: ConfirmationProgress::new(last_entry),
+            propagated_stats: PropagatedStats {
+                prev_leader_slot,
+                is_leader_slot,
+                propagated_validators_stake,
+                propagated_validators,
+                is_propagated,
+                total_epoch_stake,
+                ..PropagatedStats::default()
+            },
+        }
+    }
+
+    pub fn new_from_bank(
+        bank: &Bank,
+        my_pubkey: &Pubkey,
+        voting_pubkey: &Pubkey,
+        prev_leader_slot: Option<Slot>,
+    ) -> Self {
+        let validator_fork_info = {
+            if bank.collector_id() == my_pubkey {
+                let stake = bank.epoch_vote_account_stake(voting_pubkey);
+                Some(ValidatorStakeInfo::new(
+                    *voting_pubkey,
+                    stake,
+                    bank.total_epoch_stake(),
+                ))
+            } else {
+                None
+            }
+        };
+
+        Self::new(bank.last_blockhash(), prev_leader_slot, validator_fork_info)
+    }
+}
+
+#[derive(Debug, Clone, Default)]
+pub(crate) struct ForkStats {
+    pub(crate) weight: u128,
+    pub(crate) fork_weight: u128,
+    pub(crate) total_staked: u64,
+    pub(crate) slot: Slot,
+    pub(crate) block_height: u64,
+    pub(crate) has_voted: bool,
+    pub(crate) is_recent: bool,
+    pub(crate) is_empty: bool,
+    pub(crate) vote_threshold: bool,
+    pub(crate) is_locked_out: bool,
+    pub(crate) stake_lockouts: HashMap<u64, StakeLockout>,
+    pub(crate) confirmation_reported: bool,
+    pub(crate) computed: bool,
+}
+
+#[derive(Clone, Default)]
+pub(crate) struct PropagatedStats {
+    pub(crate) propagated_validators: HashSet<Rc<Pubkey>>,
+    pub(crate) propagated_node_ids: HashSet<Rc<Pubkey>>,
+    pub(crate) propagated_validators_stake: u64,
+    pub(crate) is_propagated: bool,
+    pub(crate) is_leader_slot: bool,
+    pub(crate) prev_leader_slot: Option<Slot>,
+    pub(crate) slot_vote_tracker: Option<Arc<RwLock<SlotVoteTracker>>>,
+    pub(crate) cluster_slot_pubkeys: Option<Arc<RwLock<SlotPubkeys>>>,
+    pub(crate) total_epoch_stake: u64,
+}
+
+impl PropagatedStats {
+    pub fn add_vote_pubkey(
+        &mut self,
+        vote_pubkey: &Pubkey,
+        all_pubkeys: &mut HashSet<Rc<Pubkey>>,
+        stake: u64,
+    ) {
+        if !self.propagated_validators.contains(vote_pubkey) {
+            let mut cached_pubkey: Option<Rc<Pubkey>> = all_pubkeys.get(vote_pubkey).cloned();
+            if cached_pubkey.is_none() {
+                let new_pubkey = Rc::new(*vote_pubkey);
+                all_pubkeys.insert(new_pubkey.clone());
+                cached_pubkey = Some(new_pubkey);
+            }
+            let vote_pubkey = cached_pubkey.unwrap();
+            self.propagated_validators.insert(vote_pubkey);
+            self.propagated_validators_stake += stake;
+        }
+    }
+
+    pub fn add_node_pubkey(
+        &mut self,
+        node_pubkey: &Pubkey,
+        all_pubkeys: &mut HashSet<Rc<Pubkey>>,
+        bank: &Bank,
+    ) {
+        if !self.propagated_node_ids.contains(node_pubkey) {
+            let node_vote_accounts = bank
+                .epoch_vote_accounts_for_node_id(&node_pubkey)
+                .map(|v| &v.vote_accounts);
+
+            if let Some(node_vote_accounts) = node_vote_accounts {
+                self.add_node_pubkey_internal(
+                    node_pubkey,
+                    all_pubkeys,
+                    node_vote_accounts,
+                    bank.epoch_vote_accounts(bank.epoch())
+                        .expect("Epoch stakes for bank's own epoch must exist"),
+                );
+            }
+        }
+    }
+
+    fn add_node_pubkey_internal(
+        &mut self,
+        node_pubkey: &Pubkey,
+        all_pubkeys: &mut HashSet<Rc<Pubkey>>,
+        vote_account_pubkeys: &[Pubkey],
+        epoch_vote_accounts: &HashMap<Pubkey, (u64, Account)>,
+    ) {
+        let mut cached_pubkey: Option<Rc<Pubkey>> = all_pubkeys.get(node_pubkey).cloned();
+        if cached_pubkey.is_none() {
+            let new_pubkey = Rc::new(*node_pubkey);
+            all_pubkeys.insert(new_pubkey.clone());
+            cached_pubkey = Some(new_pubkey);
+        }
+        let node_pubkey = cached_pubkey.unwrap();
+        self.propagated_node_ids.insert(node_pubkey);
+        for vote_account_pubkey in vote_account_pubkeys.iter() {
+            let stake = epoch_vote_accounts
+                .get(vote_account_pubkey)
+                .map(|(stake, _)| *stake)
+                .unwrap_or(0);
+            self.add_vote_pubkey(vote_account_pubkey, all_pubkeys, stake);
+        }
+    }
+}
+
+#[derive(Default)]
+pub(crate) struct ProgressMap {
+    progress_map: HashMap<Slot, ForkProgress>,
+}
+
+impl std::ops::Deref for ProgressMap {
+    type Target = HashMap<Slot, ForkProgress>;
+    fn deref(&self) -> &Self::Target {
+        &self.progress_map
+    }
+}
+
+impl std::ops::DerefMut for ProgressMap {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.progress_map
+    }
+}
+
+impl ProgressMap {
+    pub fn insert(&mut self, slot: Slot, fork_progress: ForkProgress) {
+        self.progress_map.insert(slot, fork_progress);
+    }
+
+    pub fn get_propagated_stats(&self, slot: Slot) -> Option<&PropagatedStats> {
+        self.progress_map
+            .get(&slot)
+            .map(|fork_progress| &fork_progress.propagated_stats)
+    }
+
+    pub fn get_propagated_stats_mut(&mut self, slot: Slot) -> Option<&mut PropagatedStats> {
+        self.progress_map
+            .get_mut(&slot)
+            .map(|fork_progress| &mut fork_progress.propagated_stats)
+    }
+
+    pub fn get_fork_stats(&self, slot: Slot) -> Option<&ForkStats> {
+        self.progress_map
+            .get(&slot)
+            .map(|fork_progress| &fork_progress.fork_stats)
+    }
+
+    pub fn get_fork_stats_mut(&mut self, slot: Slot) -> Option<&mut ForkStats> {
+        self.progress_map
+            .get_mut(&slot)
+            .map(|fork_progress| &mut fork_progress.fork_stats)
+    }
+
+    pub fn is_propagated(&self, slot: Slot) -> bool {
+        let leader_slot_to_check = self.get_latest_leader_slot(slot);
+
+        // prev_leader_slot doesn't exist because already rooted
+        // or this validator hasn't been scheduled as a leader
+        // yet. In both cases the latest leader is vacuously
+        // confirmed
+        leader_slot_to_check
+            .map(|leader_slot_to_check| {
+                // If the leader's stats are None (isn't in the
+                // progress map), this means that prev_leader slot is
+                // rooted, so return true
+                self.get_propagated_stats(leader_slot_to_check)
+                    .map(|stats| stats.is_propagated)
+                    .unwrap_or(true)
+            })
+            .unwrap_or(true)
+    }
+
+    pub fn get_latest_leader_slot(&self, slot: Slot) -> Option<Slot> {
+        let propagated_stats = self
+            .get_propagated_stats(slot)
+            .expect("All frozen banks must exist in the Progress map");
+
+        if propagated_stats.is_leader_slot {
+            Some(slot)
+        } else {
+            propagated_stats.prev_leader_slot
+        }
+    }
+
+    pub fn get_bank_prev_leader_slot(&self, bank: &Bank) -> Option<Slot> {
+        let parent_slot = bank.parent_slot();
+        self.get_propagated_stats(parent_slot)
+            .map(|stats| {
+                if stats.is_leader_slot {
+                    Some(parent_slot)
+                } else {
+                    stats.prev_leader_slot
+                }
+            })
+            .unwrap_or(None)
+    }
+
+    pub fn handle_new_root(&mut self, bank_forks: &BankForks) {
+        self.progress_map
+            .retain(|k, _| bank_forks.get(*k).is_some());
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_add_vote_pubkey() {
+        let mut stats = PropagatedStats::default();
+        let mut all_pubkeys = HashSet::new();
+        let mut vote_pubkey = Pubkey::new_rand();
+        all_pubkeys.insert(Rc::new(vote_pubkey.clone()));
+
+        // Add a vote pubkey, the number of references in all_pubkeys
+        // should be 2
+        stats.add_vote_pubkey(&vote_pubkey, &mut all_pubkeys, 1);
+        assert!(stats.propagated_validators.contains(&vote_pubkey));
+        assert_eq!(stats.propagated_validators_stake, 1);
+        assert_eq!(Rc::strong_count(all_pubkeys.get(&vote_pubkey).unwrap()), 2);
+
+        // Adding it again should change no state since the key already existed
+        stats.add_vote_pubkey(&vote_pubkey, &mut all_pubkeys, 1);
+        assert!(stats.propagated_validators.contains(&vote_pubkey));
+        assert_eq!(stats.propagated_validators_stake, 1);
+
+        // Addding another pubkey should succeed
+        vote_pubkey = Pubkey::new_rand();
+        stats.add_vote_pubkey(&vote_pubkey, &mut all_pubkeys, 2);
+        assert!(stats.propagated_validators.contains(&vote_pubkey));
+        assert_eq!(stats.propagated_validators_stake, 3);
+        assert_eq!(Rc::strong_count(all_pubkeys.get(&vote_pubkey).unwrap()), 2);
+    }
+
+    #[test]
+    fn test_add_node_pubkey_internal() {
+        let num_vote_accounts = 10;
+        let staked_vote_accounts = 5;
+        let vote_account_pubkeys: Vec<_> = std::iter::repeat_with(|| Pubkey::new_rand())
+            .take(num_vote_accounts)
+            .collect();
+        let epoch_vote_accounts: HashMap<_, _> = vote_account_pubkeys
+            .iter()
+            .skip(num_vote_accounts - staked_vote_accounts)
+            .map(|pubkey| (*pubkey, (1, Account::default())))
+            .collect();
+
+        let mut stats = PropagatedStats::default();
+        let mut all_pubkeys = HashSet::new();
+        let mut node_pubkey = Pubkey::new_rand();
+        all_pubkeys.insert(Rc::new(node_pubkey.clone()));
+
+        // Add a vote pubkey, the number of references in all_pubkeys
+        // should be 2
+        stats.add_node_pubkey_internal(
+            &node_pubkey,
+            &mut all_pubkeys,
+            &vote_account_pubkeys,
+            &epoch_vote_accounts,
+        );
+        assert!(stats.propagated_node_ids.contains(&node_pubkey));
+        assert_eq!(
+            stats.propagated_validators_stake,
+            staked_vote_accounts as u64
+        );
+        assert_eq!(Rc::strong_count(all_pubkeys.get(&node_pubkey).unwrap()), 2);
+
+        // Adding it again should not change any state
+        stats.add_node_pubkey_internal(
+            &node_pubkey,
+            &mut all_pubkeys,
+            &vote_account_pubkeys,
+            &epoch_vote_accounts,
+        );
+        assert!(stats.propagated_node_ids.contains(&node_pubkey));
+        assert_eq!(
+            stats.propagated_validators_stake,
+            staked_vote_accounts as u64
+        );
+
+        // Addding another pubkey with same vote accounts should succeed, but stake
+        // shouldn't increase
+        node_pubkey = Pubkey::new_rand();
+        stats.add_node_pubkey_internal(
+            &node_pubkey,
+            &mut all_pubkeys,
+            &vote_account_pubkeys,
+            &epoch_vote_accounts,
+        );
+        assert!(stats.propagated_node_ids.contains(&node_pubkey));
+        assert_eq!(
+            stats.propagated_validators_stake,
+            staked_vote_accounts as u64
+        );
+        assert_eq!(Rc::strong_count(all_pubkeys.get(&node_pubkey).unwrap()), 2);
+
+        // Addding another pubkey with different vote accounts should succeed
+        // and increase stake
+        node_pubkey = Pubkey::new_rand();
+        let vote_account_pubkeys: Vec<_> = std::iter::repeat_with(|| Pubkey::new_rand())
+            .take(num_vote_accounts)
+            .collect();
+        let epoch_vote_accounts: HashMap<_, _> = vote_account_pubkeys
+            .iter()
+            .skip(num_vote_accounts - staked_vote_accounts)
+            .map(|pubkey| (*pubkey, (1, Account::default())))
+            .collect();
+        stats.add_node_pubkey_internal(
+            &node_pubkey,
+            &mut all_pubkeys,
+            &vote_account_pubkeys,
+            &epoch_vote_accounts,
+        );
+        assert!(stats.propagated_node_ids.contains(&node_pubkey));
+        assert_eq!(
+            stats.propagated_validators_stake,
+            2 * staked_vote_accounts as u64
+        );
+        assert_eq!(Rc::strong_count(all_pubkeys.get(&node_pubkey).unwrap()), 2);
+    }
+
+    #[test]
+    fn test_is_propagated_status_on_construction() {
+        // If the given ValidatorStakeInfo == None, then this is not
+        // a leader slot and is_propagated == false
+        let progress = ForkProgress::new(Hash::default(), Some(9), None);
+        assert!(!progress.propagated_stats.is_propagated);
+
+        // If the stake is zero, then threshold is always achieved
+        let progress = ForkProgress::new(
+            Hash::default(),
+            Some(9),
+            Some(ValidatorStakeInfo {
+                total_epoch_stake: 0,
+                ..ValidatorStakeInfo::default()
+            }),
+        );
+        assert!(progress.propagated_stats.is_propagated);
+
+        // If the stake is non zero, then threshold is not achieved unless
+        // validator has enough stake by itself to pass threshold
+        let progress = ForkProgress::new(
+            Hash::default(),
+            Some(9),
+            Some(ValidatorStakeInfo {
+                total_epoch_stake: 2,
+                ..ValidatorStakeInfo::default()
+            }),
+        );
+        assert!(!progress.propagated_stats.is_propagated);
+
+        // Give the validator enough stake by itself to pass threshold
+        let progress = ForkProgress::new(
+            Hash::default(),
+            Some(9),
+            Some(ValidatorStakeInfo {
+                stake: 1,
+                total_epoch_stake: 2,
+                ..ValidatorStakeInfo::default()
+            }),
+        );
+        assert!(progress.propagated_stats.is_propagated);
+
+        // Check that the default ValidatorStakeInfo::default() constructs a ForkProgress
+        // with is_propagated == false, otherwise propagation tests will fail to run
+        // the proper checks (most will auto-pass without checking anything)
+        let progress = ForkProgress::new(
+            Hash::default(),
+            Some(9),
+            Some(ValidatorStakeInfo::default()),
+        );
+        assert!(!progress.propagated_stats.is_propagated);
+    }
+
+    #[test]
+    fn test_is_propagated() {
+        let mut progress_map = ProgressMap::default();
+
+        // Insert new ForkProgress for slot 10 (not a leader slot) and its
+        // previous leader slot 9 (leader slot)
+        progress_map.insert(10, ForkProgress::new(Hash::default(), Some(9), None));
+        progress_map.insert(
+            9,
+            ForkProgress::new(Hash::default(), None, Some(ValidatorStakeInfo::default())),
+        );
+
+        // None of these slot have parents which are confirmed
+        assert!(!progress_map.is_propagated(9));
+        assert!(!progress_map.is_propagated(10));
+
+        // Insert new ForkProgress for slot 8 with no previous leader.
+        // The previous leader before 8, slot 7, does not exist in
+        // progress map, so is_propagated(8) should return true as
+        // this implies the parent is rooted
+        progress_map.insert(8, ForkProgress::new(Hash::default(), Some(7), None));
+        assert!(progress_map.is_propagated(8));
+
+        // If we set the is_propagated = true, is_propagated should return true
+        progress_map
+            .get_propagated_stats_mut(9)
+            .unwrap()
+            .is_propagated = true;
+        assert!(progress_map.is_propagated(9));
+        assert!(progress_map.get(&9).unwrap().propagated_stats.is_propagated);
+
+        // Because slot 9 is now confirmed, then slot 10 is also confirmed b/c 9
+        // is the last leader slot before 10
+        assert!(progress_map.is_propagated(10));
+
+        // If we make slot 10 a leader slot though, even though its previous
+        // leader slot 9 has been confirmed, slot 10 itself is not confirmed
+        progress_map
+            .get_propagated_stats_mut(10)
+            .unwrap()
+            .is_leader_slot = true;
+        assert!(!progress_map.is_propagated(10));
+    }
+}

core/src/repair_service.rs

@@ -2,6 +2,7 @@
 //! regularly finds missing shreds in the ledger and sends repair requests for those shreds
 use crate::{
     cluster_info::ClusterInfo,
+    cluster_slots::ClusterSlots,
     result::Result,
     serve_repair::{RepairType, ServeRepair},
 };
@@ -9,27 +10,45 @@ use solana_ledger::{
     bank_forks::BankForks,
     blockstore::{Blockstore, CompletedSlotsReceiver, SlotMeta},
 };
-use solana_sdk::clock::DEFAULT_SLOTS_PER_EPOCH;
 use solana_sdk::{clock::Slot, epoch_schedule::EpochSchedule, pubkey::Pubkey};
 use std::{
-    collections::{BTreeSet, HashSet},
+    collections::HashMap,
     iter::Iterator,
+    net::SocketAddr,
     net::UdpSocket,
-    ops::Bound::{Included, Unbounded},
     sync::atomic::{AtomicBool, Ordering},
     sync::{Arc, RwLock},
     thread::sleep,
     thread::{self, Builder, JoinHandle},
-    time::Duration,
+    time::{Duration, Instant},
 };
 
+#[derive(Default)]
+pub struct RepairStatsGroup {
+    pub count: u64,
+    pub min: u64,
+    pub max: u64,
+}
+
+impl RepairStatsGroup {
+    pub fn update(&mut self, slot: u64) {
+        self.count += 1;
+        self.min = std::cmp::min(self.min, slot);
+        self.max = std::cmp::max(self.max, slot);
+    }
+}
+
+#[derive(Default)]
+pub struct RepairStats {
+    pub shred: RepairStatsGroup,
+    pub highest_shred: RepairStatsGroup,
+    pub orphan: RepairStatsGroup,
+}
+
 pub const MAX_REPAIR_LENGTH: usize = 512;
 pub const REPAIR_MS: u64 = 100;
 pub const MAX_ORPHANS: usize = 5;
 
-const MAX_COMPLETED_SLOT_CACHE_LEN: usize = 256;
-const COMPLETED_SLOT_CACHE_FLUSH_TRIGGER: usize = 512;
-
 pub enum RepairStrategy {
     RepairRange(RepairSlotRange),
     RepairAll {
@@ -64,6 +83,7 @@ impl RepairService {
         repair_socket: Arc<UdpSocket>,
         cluster_info: Arc<RwLock<ClusterInfo>>,
         repair_strategy: RepairStrategy,
+        cluster_slots: Arc<ClusterSlots>,
     ) -> Self {
         let t_repair = Builder::new()
             .name("solana-repair-service".to_string())
@@ -74,6 +94,7 @@ impl RepairService {
                     &repair_socket,
                     &cluster_info,
                     repair_strategy,
+                    &cluster_slots,
                 )
             })
             .unwrap();
@@ -82,31 +103,20 @@ impl RepairService {
     }
 
     fn run(
-        blockstore: &Arc<Blockstore>,
-        exit: &Arc<AtomicBool>,
-        repair_socket: &Arc<UdpSocket>,
+        blockstore: &Blockstore,
+        exit: &AtomicBool,
+        repair_socket: &UdpSocket,
         cluster_info: &Arc<RwLock<ClusterInfo>>,
         repair_strategy: RepairStrategy,
+        cluster_slots: &Arc<ClusterSlots>,
     ) {
         let serve_repair = ServeRepair::new(cluster_info.clone());
-        let mut epoch_slots: BTreeSet<Slot> = BTreeSet::new();
-        let mut old_incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
         let id = cluster_info.read().unwrap().id();
-        if let RepairStrategy::RepairAll {
-            ref epoch_schedule, ..
-        } = repair_strategy
-        {
-            let current_root = blockstore.last_root();
-            Self::initialize_epoch_slots(
-                id,
-                blockstore,
-                &mut epoch_slots,
-                &old_incomplete_slots,
-                current_root,
-                epoch_schedule,
-                cluster_info,
-            );
+        if let RepairStrategy::RepairAll { .. } = repair_strategy {
+            Self::initialize_lowest_slot(id, blockstore, cluster_info);
         }
+        let mut repair_stats = RepairStats::default();
+        let mut last_stats = Instant::now();
         loop {
             if exit.load(Ordering::Relaxed) {
                 break;
@@ -125,30 +135,38 @@ impl RepairService {
 
                     RepairStrategy::RepairAll {
                         ref completed_slots_receiver,
+                        ref bank_forks,
                         ..
                     } => {
                         let new_root = blockstore.last_root();
                         let lowest_slot = blockstore.lowest_slot();
-                        Self::update_epoch_slots(
-                            id,
+                        Self::update_lowest_slot(&id, lowest_slot, &cluster_info);
+                        Self::update_completed_slots(
+                            &id,
                             new_root,
-                            lowest_slot,
-                            &mut epoch_slots,
-                            &mut old_incomplete_slots,
-                            &cluster_info,
+                            &cluster_slots,
+                            blockstore,
                             completed_slots_receiver,
+                            &cluster_info,
                         );
+                        cluster_slots.update(new_root, cluster_info, bank_forks);
                         Self::generate_repairs(blockstore, new_root, MAX_REPAIR_LENGTH)
                     }
                 }
             };
 
             if let Ok(repairs) = repairs {
-                let reqs: Vec<_> = repairs
+                let mut cache = HashMap::new();
+                let reqs: Vec<((SocketAddr, Vec<u8>), RepairType)> = repairs
                     .into_iter()
                     .filter_map(|repair_request| {
                         serve_repair
-                            .repair_request(&repair_request)
+                            .repair_request(
+                                &cluster_slots,
+                                &repair_request,
+                                &mut cache,
+                                &mut repair_stats,
+                            )
                             .map(|result| (result, repair_request))
                             .ok()
                     })
@@ -161,6 +179,24 @@ impl RepairService {
                     });
                 }
             }
+            if last_stats.elapsed().as_secs() > 1 {
+                let repair_total = repair_stats.shred.count
+                    + repair_stats.highest_shred.count
+                    + repair_stats.orphan.count;
+                if repair_total > 0 {
+                    datapoint_info!(
+                        "serve_repair-repair",
+                        ("repair-total", repair_total, i64),
+                        ("shred-count", repair_stats.shred.count, i64),
+                        ("highest-shred-count", repair_stats.highest_shred.count, i64),
+                        ("orphan-count", repair_stats.orphan.count, i64),
+                        ("repair-highest-slot", repair_stats.highest_shred.max, i64),
+                        ("repair-orphan", repair_stats.orphan.max, i64),
+                    );
+                }
+                repair_stats = RepairStats::default();
+                last_stats = Instant::now();
+            }
             sleep(Duration::from_millis(REPAIR_MS));
         }
     }
@@ -272,150 +308,54 @@ impl RepairService {
         }
     }
 
-    fn get_completed_slots_past_root(
-        blockstore: &Blockstore,
-        slots_in_gossip: &mut BTreeSet<Slot>,
-        root: Slot,
-        epoch_schedule: &EpochSchedule,
-    ) {
-        let last_confirmed_epoch = epoch_schedule.get_leader_schedule_epoch(root);
-        let last_epoch_slot = epoch_schedule.get_last_slot_in_epoch(last_confirmed_epoch);
-
-        let meta_iter = blockstore
-            .slot_meta_iterator(root + 1)
-            .expect("Couldn't get db iterator");
-
-        for (current_slot, meta) in meta_iter {
-            if current_slot > last_epoch_slot {
-                break;
-            }
-            if meta.is_full() {
-                slots_in_gossip.insert(current_slot);
-            }
-        }
-    }
-
-    fn initialize_epoch_slots(
+    fn initialize_lowest_slot(
         id: Pubkey,
         blockstore: &Blockstore,
-        slots_in_gossip: &mut BTreeSet<Slot>,
-        old_incomplete_slots: &BTreeSet<Slot>,
-        root: Slot,
-        epoch_schedule: &EpochSchedule,
         cluster_info: &RwLock<ClusterInfo>,
     ) {
-        Self::get_completed_slots_past_root(blockstore, slots_in_gossip, root, epoch_schedule);
-
         // Safe to set into gossip because by this time, the leader schedule cache should
         // also be updated with the latest root (done in blockstore_processor) and thus
         // will provide a schedule to window_service for any incoming shreds up to the
         // last_confirmed_epoch.
-        cluster_info.write().unwrap().push_epoch_slots(
-            id,
-            root,
-            blockstore.lowest_slot(),
-            slots_in_gossip.clone(),
-            old_incomplete_slots,
-        );
+        cluster_info
+            .write()
+            .unwrap()
+            .push_lowest_slot(id, blockstore.lowest_slot());
     }
 
-    // Update the gossiped structure used for the "Repairmen" repair protocol. See docs
-    // for details.
-    fn update_epoch_slots(
-        id: Pubkey,
-        latest_known_root: Slot,
-        lowest_slot: Slot,
-        completed_slot_cache: &mut BTreeSet<Slot>,
-        incomplete_slot_stash: &mut BTreeSet<Slot>,
-        cluster_info: &RwLock<ClusterInfo>,
+    fn update_completed_slots(
+        id: &Pubkey,
+        root: Slot,
+        cluster_slots: &ClusterSlots,
+        blockstore: &Blockstore,
         completed_slots_receiver: &CompletedSlotsReceiver,
+        cluster_info: &RwLock<ClusterInfo>,
     ) {
-        let mut should_update = false;
-        while let Ok(completed_slots) = completed_slots_receiver.try_recv() {
-            for slot in completed_slots {
-                let last_slot_in_stash = *incomplete_slot_stash.iter().next_back().unwrap_or(&0);
-                let removed_from_stash = incomplete_slot_stash.remove(&slot);
-                // If the newly completed slot was not being tracked in stash, and is > last
-                // slot being tracked in stash, add it to cache. Also, update gossip
-                if !removed_from_stash && slot >= last_slot_in_stash {
-                    should_update |= completed_slot_cache.insert(slot);
-                }
-                // If the slot was removed from stash, update gossip
-                should_update |= removed_from_stash;
-            }
+        let mine = cluster_slots.collect(id);
+        let mut slots: Vec<Slot> = vec![];
+        while let Ok(mut more) = completed_slots_receiver.try_recv() {
+            more.retain(|x| !mine.contains(x));
+            slots.append(&mut more);
         }
-
-        if should_update {
-            if completed_slot_cache.len() >= COMPLETED_SLOT_CACHE_FLUSH_TRIGGER {
-                Self::stash_old_incomplete_slots(completed_slot_cache, incomplete_slot_stash);
-                let lowest_completed_slot_in_cache =
-                    *completed_slot_cache.iter().next().unwrap_or(&0);
-                Self::prune_incomplete_slot_stash(
-                    incomplete_slot_stash,
-                    lowest_completed_slot_in_cache,
-                );
-            }
-
-            cluster_info.write().unwrap().push_epoch_slots(
-                id,
-                latest_known_root,
-                lowest_slot,
-                completed_slot_cache.clone(),
-                incomplete_slot_stash,
-            );
-        }
-    }
-
-    fn stash_old_incomplete_slots(cache: &mut BTreeSet<Slot>, stash: &mut BTreeSet<Slot>) {
-        if cache.len() > MAX_COMPLETED_SLOT_CACHE_LEN {
-            let mut prev = *cache.iter().next().expect("Expected to find some slot");
-            cache.remove(&prev);
-            while cache.len() >= MAX_COMPLETED_SLOT_CACHE_LEN {
-                let next = *cache.iter().next().expect("Expected to find some slot");
-                cache.remove(&next);
-                // Prev slot and next slot are not included in incomplete slot list.
-                (prev + 1..next).for_each(|slot| {
-                    stash.insert(slot);
-                });
-                prev = next;
-            }
-        }
-    }
-
-    fn prune_incomplete_slot_stash(
-        stash: &mut BTreeSet<Slot>,
-        lowest_completed_slot_in_cache: Slot,
-    ) {
-        if let Some(oldest_incomplete_slot) = stash.iter().next() {
-            // Prune old slots
-            // Prune in batches to reduce overhead. Pruning starts when oldest slot is 1.5 epochs
-            // earlier than the new root. But, we prune all the slots that are older than 1 epoch.
-            // So slots in a batch of half epoch are getting pruned
-            if oldest_incomplete_slot + DEFAULT_SLOTS_PER_EPOCH + DEFAULT_SLOTS_PER_EPOCH / 2
-                < lowest_completed_slot_in_cache
-            {
-                let oldest_slot_to_retain =
-                    lowest_completed_slot_in_cache.saturating_sub(DEFAULT_SLOTS_PER_EPOCH);
-                *stash = stash
-                    .range((Included(&oldest_slot_to_retain), Unbounded))
-                    .cloned()
-                    .collect();
-            }
-        }
-    }
-
-    #[allow(dead_code)]
-    fn find_incomplete_slots(blockstore: &Blockstore, root: Slot) -> HashSet<Slot> {
         blockstore
             .live_slots_iterator(root)
-            .filter_map(|(slot, slot_meta)| {
-                if !slot_meta.is_full() {
-                    Some(slot)
-                } else {
-                    None
+            .for_each(|(slot, slot_meta)| {
+                if slot_meta.is_full() && !mine.contains(&slot) {
+                    slots.push(slot)
                 }
-            })
-            .collect()
+            });
+        slots.sort();
+        slots.dedup();
+        if !slots.is_empty() {
+            cluster_info.write().unwrap().push_epoch_slots(&slots);
+        }
+    }
+
+    fn update_lowest_slot(id: &Pubkey, lowest_slot: Slot, cluster_info: &RwLock<ClusterInfo>) {
+        cluster_info
+            .write()
+            .unwrap()
+            .push_lowest_slot(*id, lowest_slot);
     }
 
     pub fn join(self) -> thread::Result<()> {
@@ -427,15 +367,11 @@ impl RepairService {
 mod test {
     use super::*;
     use crate::cluster_info::Node;
-    use itertools::Itertools;
-    use rand::seq::SliceRandom;
-    use rand::{thread_rng, Rng};
     use solana_ledger::blockstore::{
         make_chaining_slot_entries, make_many_slot_entries, make_slot_entries,
     };
     use solana_ledger::shred::max_ticks_per_n_shreds;
     use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
-    use std::thread::Builder;
 
     #[test]
     pub fn test_repair_orphan() {
@@ -651,342 +587,19 @@ mod test {
     }
 
     #[test]
-    pub fn test_get_completed_slots_past_root() {
-        let blockstore_path = get_tmp_ledger_path!();
-        {
-            let blockstore = Blockstore::open(&blockstore_path).unwrap();
-            let num_entries_per_slot = 10;
-            let root = 10;
-
-            let fork1 = vec![5, 7, root, 15, 20, 21];
-            let fork1_shreds: Vec<_> = make_chaining_slot_entries(&fork1, num_entries_per_slot)
-                .into_iter()
-                .flat_map(|(shreds, _)| shreds)
-                .collect();
-            let fork2 = vec![8, 12];
-            let fork2_shreds = make_chaining_slot_entries(&fork2, num_entries_per_slot);
-
-            // Remove the last shred from each slot to make an incomplete slot
-            let fork2_incomplete_shreds: Vec<_> = fork2_shreds
-                .into_iter()
-                .flat_map(|(mut shreds, _)| {
-                    shreds.pop();
-                    shreds
-                })
-                .collect();
-            let mut full_slots = BTreeSet::new();
-
-            blockstore.insert_shreds(fork1_shreds, None, false).unwrap();
-            blockstore
-                .insert_shreds(fork2_incomplete_shreds, None, false)
-                .unwrap();
-
-            // Test that only slots > root from fork1 were included
-            let epoch_schedule = EpochSchedule::custom(32, 32, false);
-
-            RepairService::get_completed_slots_past_root(
-                &blockstore,
-                &mut full_slots,
-                root,
-                &epoch_schedule,
-            );
-
-            let mut expected: BTreeSet<_> = fork1.into_iter().filter(|x| *x > root).collect();
-            assert_eq!(full_slots, expected);
-
-            // Test that slots past the last confirmed epoch boundary don't get included
-            let last_epoch = epoch_schedule.get_leader_schedule_epoch(root);
-            let last_slot = epoch_schedule.get_last_slot_in_epoch(last_epoch);
-            let fork3 = vec![last_slot, last_slot + 1];
-            let fork3_shreds: Vec<_> = make_chaining_slot_entries(&fork3, num_entries_per_slot)
-                .into_iter()
-                .flat_map(|(shreds, _)| shreds)
-                .collect();
-            blockstore.insert_shreds(fork3_shreds, None, false).unwrap();
-            RepairService::get_completed_slots_past_root(
-                &blockstore,
-                &mut full_slots,
-                root,
-                &epoch_schedule,
-            );
-            expected.insert(last_slot);
-            assert_eq!(full_slots, expected);
-        }
-        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
-    }
-
-    #[test]
-    pub fn test_update_epoch_slots() {
-        let blockstore_path = get_tmp_ledger_path!();
-        {
-            // Create blockstore
-            let (blockstore, _, completed_slots_receiver) =
-                Blockstore::open_with_signal(&blockstore_path).unwrap();
-
-            let blockstore = Arc::new(blockstore);
-
-            let mut root = 0;
-            let num_slots = 100;
-            let entries_per_slot = 5;
-            let blockstore_ = blockstore.clone();
-
-            // Spin up thread to write to blockstore
-            let writer = Builder::new()
-                .name("writer".to_string())
-                .spawn(move || {
-                    let slots: Vec<_> = (1..num_slots + 1).collect();
-                    let mut shreds: Vec<_> = make_chaining_slot_entries(&slots, entries_per_slot)
-                        .into_iter()
-                        .flat_map(|(shreds, _)| shreds)
-                        .collect();
-                    shreds.shuffle(&mut thread_rng());
-                    let mut i = 0;
-                    let max_step = entries_per_slot * 4;
-                    let repair_interval_ms = 10;
-                    let mut rng = rand::thread_rng();
-                    let num_shreds = shreds.len();
-                    while i < num_shreds {
-                        let step = rng.gen_range(1, max_step + 1) as usize;
-                        let step = std::cmp::min(step, num_shreds - i);
-                        let shreds_to_insert = shreds.drain(..step).collect_vec();
-                        blockstore_
-                            .insert_shreds(shreds_to_insert, None, false)
-                            .unwrap();
-                        sleep(Duration::from_millis(repair_interval_ms));
-                        i += step;
-                    }
-                })
-                .unwrap();
-
-            let mut completed_slots = BTreeSet::new();
-            let node_info = Node::new_localhost_with_pubkey(&Pubkey::default());
-            let cluster_info = RwLock::new(ClusterInfo::new_with_invalid_keypair(
-                node_info.info.clone(),
-            ));
-
-            let mut old_incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
-            while completed_slots.len() < num_slots as usize {
-                RepairService::update_epoch_slots(
-                    Pubkey::default(),
-                    root,
-                    blockstore.lowest_slot(),
-                    &mut completed_slots,
-                    &mut old_incomplete_slots,
-                    &cluster_info,
-                    &completed_slots_receiver,
-                );
-            }
-
-            let mut expected: BTreeSet<_> = (1..num_slots + 1).collect();
-            assert_eq!(completed_slots, expected);
-
-            // Update with new root, should filter out the slots <= root
-            root = num_slots / 2;
-            let (shreds, _) = make_slot_entries(num_slots + 2, num_slots + 1, entries_per_slot);
-            blockstore.insert_shreds(shreds, None, false).unwrap();
-            RepairService::update_epoch_slots(
-                Pubkey::default(),
-                root,
-                0,
-                &mut completed_slots,
-                &mut old_incomplete_slots,
-                &cluster_info,
-                &completed_slots_receiver,
-            );
-            expected.insert(num_slots + 2);
-            assert_eq!(completed_slots, expected);
-            writer.join().unwrap();
-        }
-        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
-    }
-
-    #[test]
-    fn test_stash_old_incomplete_slots() {
-        let mut cache: BTreeSet<Slot> = BTreeSet::new();
-        let mut stash: BTreeSet<Slot> = BTreeSet::new();
-
-        // When cache is empty.
-        RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
-        assert_eq!(stash.len(), 0);
-
-        // Insert some slots in cache ( < MAX_COMPLETED_SLOT_CACHE_LEN + 1)
-        cache.insert(101);
-        cache.insert(102);
-        cache.insert(104);
-        cache.insert(105);
-
-        // Not enough slots in cache. So stash should remain empty.
-        RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
-        assert_eq!(stash.len(), 0);
-        assert_eq!(cache.len(), 4);
-
-        // Insert slots in cache ( = MAX_COMPLETED_SLOT_CACHE_LEN)
-        let mut cache: BTreeSet<Slot> = BTreeSet::new();
-        (0..MAX_COMPLETED_SLOT_CACHE_LEN as u64)
-            .into_iter()
-            .for_each(|slot| {
-                cache.insert(slot);
-            });
-
-        // Not enough slots in cache. So stash should remain empty.
-        RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
-        assert_eq!(stash.len(), 0);
-        assert_eq!(cache.len(), MAX_COMPLETED_SLOT_CACHE_LEN);
-
-        // Insert 1 more to cross the threshold
-        cache.insert(MAX_COMPLETED_SLOT_CACHE_LEN as u64);
-        RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
-        // Stash is still empty, as no missing slots
-        assert_eq!(stash.len(), 0);
-        // It removed some entries from cache
-        assert_eq!(cache.len(), MAX_COMPLETED_SLOT_CACHE_LEN - 1);
-
-        // Insert more slots to create a missing slot
-        let mut cache: BTreeSet<Slot> = BTreeSet::new();
-        cache.insert(0);
-        (2..=MAX_COMPLETED_SLOT_CACHE_LEN as u64 + 2)
-            .into_iter()
-            .for_each(|slot| {
-                cache.insert(slot);
-            });
-        RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
-
-        // Stash is not empty
-        assert!(stash.contains(&1));
-        // It removed some entries from cache
-        assert_eq!(cache.len(), MAX_COMPLETED_SLOT_CACHE_LEN - 1);
-
-        // Test multiple missing slots at dispersed locations
-        let mut cache: BTreeSet<Slot> = BTreeSet::new();
-        (0..MAX_COMPLETED_SLOT_CACHE_LEN as u64 * 2)
-            .into_iter()
-            .for_each(|slot| {
-                cache.insert(slot);
-            });
-
-        cache.remove(&10);
-        cache.remove(&11);
-
-        cache.remove(&28);
-        cache.remove(&29);
-
-        cache.remove(&148);
-        cache.remove(&149);
-        cache.remove(&150);
-        cache.remove(&151);
-
-        RepairService::stash_old_incomplete_slots(&mut cache, &mut stash);
-
-        // Stash is not empty
-        assert!(stash.contains(&10));
-        assert!(stash.contains(&11));
-        assert!(stash.contains(&28));
-        assert!(stash.contains(&29));
-        assert!(stash.contains(&148));
-        assert!(stash.contains(&149));
-        assert!(stash.contains(&150));
-        assert!(stash.contains(&151));
-
-        assert!(!stash.contains(&147));
-        assert!(!stash.contains(&152));
-        // It removed some entries from cache
-        assert_eq!(cache.len(), MAX_COMPLETED_SLOT_CACHE_LEN - 1);
-        (MAX_COMPLETED_SLOT_CACHE_LEN + 1..MAX_COMPLETED_SLOT_CACHE_LEN * 2)
-            .into_iter()
-            .for_each(|slot| {
-                let slot: u64 = slot as u64;
-                assert!(cache.contains(&slot));
-            });
-    }
-
-    #[test]
-    fn test_prune_incomplete_slot_stash() {
-        // Prune empty stash
-        let mut stash: BTreeSet<Slot> = BTreeSet::new();
-        RepairService::prune_incomplete_slot_stash(&mut stash, 0);
-        assert!(stash.is_empty());
-
-        // Prune stash with slots < DEFAULT_SLOTS_PER_EPOCH
-        stash.insert(0);
-        stash.insert(10);
-        stash.insert(11);
-        stash.insert(50);
-        assert_eq!(stash.len(), 4);
-        RepairService::prune_incomplete_slot_stash(&mut stash, 100);
-        assert_eq!(stash.len(), 4);
-
-        // Prune stash with slots > DEFAULT_SLOTS_PER_EPOCH, but < 1.5 * DEFAULT_SLOTS_PER_EPOCH
-        stash.insert(DEFAULT_SLOTS_PER_EPOCH + 50);
-        assert_eq!(stash.len(), 5);
-        RepairService::prune_incomplete_slot_stash(&mut stash, DEFAULT_SLOTS_PER_EPOCH + 100);
-        assert_eq!(stash.len(), 5);
-
-        // Prune stash with slots > 1.5 * DEFAULT_SLOTS_PER_EPOCH
-        stash.insert(DEFAULT_SLOTS_PER_EPOCH + DEFAULT_SLOTS_PER_EPOCH / 2);
-        assert_eq!(stash.len(), 6);
-        RepairService::prune_incomplete_slot_stash(
-            &mut stash,
-            DEFAULT_SLOTS_PER_EPOCH + DEFAULT_SLOTS_PER_EPOCH / 2 + 1,
-        );
-        assert_eq!(stash.len(), 2);
-    }
-
-    #[test]
-    fn test_find_incomplete_slots() {
-        let blockstore_path = get_tmp_ledger_path!();
-        {
-            let blockstore = Blockstore::open(&blockstore_path).unwrap();
-            let num_entries_per_slot = 100;
-            let (mut shreds, _) = make_slot_entries(0, 0, num_entries_per_slot);
-            assert!(shreds.len() > 1);
-            let (shreds4, _) = make_slot_entries(4, 0, num_entries_per_slot);
-            shreds.extend(shreds4);
-            blockstore.insert_shreds(shreds, None, false).unwrap();
-
-            // Nothing is incomplete
-            assert!(RepairService::find_incomplete_slots(&blockstore, 0).is_empty());
-
-            // Insert a slot 5 that chains to an incomplete orphan slot 3
-            let (shreds5, _) = make_slot_entries(5, 3, num_entries_per_slot);
-            blockstore.insert_shreds(shreds5, None, false).unwrap();
-            assert_eq!(
-                RepairService::find_incomplete_slots(&blockstore, 0),
-                vec![3].into_iter().collect()
-            );
-
-            // Insert another incomplete orphan slot 2 that is the parent of slot 3.
-            // Both should be incomplete
-            let (shreds3, _) = make_slot_entries(3, 2, num_entries_per_slot);
-            blockstore
-                .insert_shreds(shreds3[1..].to_vec(), None, false)
-                .unwrap();
-            assert_eq!(
-                RepairService::find_incomplete_slots(&blockstore, 0),
-                vec![2, 3].into_iter().collect()
-            );
-
-            // Insert a incomplete slot 6 that chains to the root 0,
-            // should also be incomplete
-            let (shreds6, _) = make_slot_entries(6, 0, num_entries_per_slot);
-            blockstore
-                .insert_shreds(shreds6[1..].to_vec(), None, false)
-                .unwrap();
-            assert_eq!(
-                RepairService::find_incomplete_slots(&blockstore, 0),
-                vec![2, 3, 6].into_iter().collect()
-            );
-
-            // Complete slot 3, should no longer be marked incomplete
-            blockstore
-                .insert_shreds(shreds3[..].to_vec(), None, false)
-                .unwrap();
-
-            assert_eq!(
-                RepairService::find_incomplete_slots(&blockstore, 0),
-                vec![2, 6].into_iter().collect()
-            );
-        }
-
-        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
+    pub fn test_update_lowest_slot() {
+        let node_info = Node::new_localhost_with_pubkey(&Pubkey::default());
+        let cluster_info = RwLock::new(ClusterInfo::new_with_invalid_keypair(
+            node_info.info.clone(),
+        ));
+        RepairService::update_lowest_slot(&Pubkey::default(), 5, &cluster_info);
+        let lowest = cluster_info
+            .read()
+            .unwrap()
+            .get_lowest_slot_for_node(&Pubkey::default(), None)
+            .unwrap()
+            .0
+            .clone();
+        assert_eq!(lowest.lowest, 5);
     }
 }

core/src/retransmit_stage.rs

@@ -2,10 +2,9 @@
 
 use crate::{
     cluster_info::{compute_retransmit_peers, ClusterInfo, DATA_PLANE_FANOUT},
-    packet::Packets,
+    cluster_slots::ClusterSlots,
     repair_service::RepairStrategy,
     result::{Error, Result},
-    streamer::PacketReceiver,
     window_service::{should_retransmit_and_persist, WindowService},
 };
 use crossbeam_channel::Receiver as CrossbeamReceiver;
@@ -17,7 +16,9 @@ use solana_ledger::{
 };
 use solana_measure::measure::Measure;
 use solana_metrics::inc_new_counter_error;
+use solana_perf::packet::Packets;
 use solana_sdk::epoch_schedule::EpochSchedule;
+use solana_streamer::streamer::PacketReceiver;
 use std::{
     cmp,
     net::UdpSocket,
@@ -214,6 +215,7 @@ impl RetransmitStage {
         epoch_schedule: EpochSchedule,
         cfg: Option<Arc<AtomicBool>>,
         shred_version: u16,
+        cluster_slots: Arc<ClusterSlots>,
     ) -> Self {
         let (retransmit_sender, retransmit_receiver) = channel();
 
@@ -256,6 +258,7 @@ impl RetransmitStage {
                 );
                 rv && is_connected
             },
+            cluster_slots,
         );
 
         let thread_hdls = t_retransmit;
@@ -278,11 +281,11 @@ impl RetransmitStage {
 mod tests {
     use super::*;
     use crate::contact_info::ContactInfo;
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-    use crate::packet::{self, Meta, Packet, Packets};
     use solana_ledger::blockstore_processor::{process_blockstore, ProcessOptions};
     use solana_ledger::create_new_tmp_ledger;
+    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_net_utils::find_available_port_in_range;
+    use solana_perf::packet::{Meta, Packet, Packets};
     use solana_sdk::pubkey::Pubkey;
     use std::net::{IpAddr, Ipv4Addr};
 
@@ -333,7 +336,7 @@ mod tests {
         // it should send this over the sockets.
         retransmit_sender.send(packets).unwrap();
         let mut packets = Packets::new(vec![]);
-        packet::recv_from(&mut packets, &me_retransmit, 1).unwrap();
+        solana_streamer::packet::recv_from(&mut packets, &me_retransmit, 1).unwrap();
         assert_eq!(packets.packets.len(), 1);
         assert_eq!(packets.packets[0].meta.repair, false);
 
@@ -349,7 +352,7 @@ mod tests {
         let packets = Packets::new(vec![repair, Packet::default()]);
         retransmit_sender.send(packets).unwrap();
         let mut packets = Packets::new(vec![]);
-        packet::recv_from(&mut packets, &me_retransmit, 1).unwrap();
+        solana_streamer::packet::recv_from(&mut packets, &me_retransmit, 1).unwrap();
         assert_eq!(packets.packets.len(), 1);
         assert_eq!(packets.packets[0].meta.repair, false);
     }

core/src/rewards_recorder_service.rs

@@ -1,7 +1,7 @@
 use crossbeam_channel::{Receiver, RecvTimeoutError, Sender};
-use solana_client::rpc_response::RpcReward;
 use solana_ledger::blockstore::Blockstore;
 use solana_sdk::{clock::Slot, pubkey::Pubkey};
+use solana_transaction_status::Reward;
 use std::{
     sync::{
         atomic::{AtomicBool, Ordering},
@@ -49,7 +49,7 @@ impl RewardsRecorderService {
         let (slot, rewards) = rewards_receiver.recv_timeout(Duration::from_secs(1))?;
         let rpc_rewards = rewards
             .into_iter()
-            .map(|(pubkey, lamports)| RpcReward {
+            .map(|(pubkey, lamports)| Reward {
                 pubkey: pubkey.to_string(),
                 lamports,
             })

core/src/rpc.rs

@@ -2,7 +2,7 @@
 
 use crate::{
     cluster_info::ClusterInfo, commitment::BlockCommitmentCache, contact_info::ContactInfo,
-    packet::PACKET_DATA_SIZE, storage_stage::StorageState, validator::ValidatorExit,
+    storage_stage::StorageState, validator::ValidatorExit,
 };
 use bincode::serialize;
 use jsonrpc_core::{Error, Metadata, Result};
@@ -12,7 +12,8 @@ use solana_faucet::faucet::request_airdrop_transaction;
 use solana_ledger::{
     bank_forks::BankForks, blockstore::Blockstore, rooted_slot_iterator::RootedSlotIterator,
 };
-use solana_runtime::{bank::Bank, status_cache::SignatureConfirmationStatus};
+use solana_perf::packet::PACKET_DATA_SIZE;
+use solana_runtime::bank::Bank;
 use solana_sdk::{
     clock::{Slot, UnixTimestamp},
     commitment_config::{CommitmentConfig, CommitmentLevel},
@@ -22,8 +23,9 @@ use solana_sdk::{
     pubkey::Pubkey,
     signature::Signature,
     timing::slot_duration_from_slots_per_year,
-    transaction::Transaction,
+    transaction::{self, Transaction},
 };
+use solana_transaction_status::{ConfirmedBlock, TransactionEncoding, TransactionStatus};
 use solana_vote_program::vote_state::{VoteState, MAX_LOCKOUT_HISTORY};
 use std::{
     collections::HashMap,
@@ -194,11 +196,9 @@ impl JsonRpcRequestProcessor {
         match signature {
             Err(e) => Err(e),
             Ok(sig) => {
-                let status = bank.get_signature_confirmation_status(&sig);
+                let status = bank.get_signature_status(&sig);
                 match status {
-                    Some(SignatureConfirmationStatus { status, .. }) => {
-                        new_response(bank, status.is_ok())
-                    }
+                    Some(status) => new_response(bank, status.is_ok()),
                     None => new_response(bank, false),
                 }
             }
@@ -215,25 +215,6 @@ impl JsonRpcRequestProcessor {
         }
     }
 
-    pub fn get_signature_confirmation_status(
-        &self,
-        signature: Signature,
-        commitment: Option<CommitmentConfig>,
-    ) -> Option<RpcSignatureConfirmation> {
-        self.bank(commitment)
-            .get_signature_confirmation_status(&signature)
-            .map(
-                |SignatureConfirmationStatus {
-                     confirmations,
-                     status,
-                     ..
-                 }| RpcSignatureConfirmation {
-                    confirmations,
-                    status,
-                },
-            )
-    }
-
     fn get_slot(&self, commitment: Option<CommitmentConfig>) -> Result<u64> {
         Ok(self.bank(commitment).slot())
     }
@@ -365,8 +346,8 @@ impl JsonRpcRequestProcessor {
     pub fn get_confirmed_block(
         &self,
         slot: Slot,
-        encoding: Option<RpcTransactionEncoding>,
-    ) -> Result<Option<RpcConfirmedBlock>> {
+        encoding: Option<TransactionEncoding>,
+    ) -> Result<Option<ConfirmedBlock>> {
         if self.config.enable_rpc_transaction_history {
             Ok(self.blockstore.get_confirmed_block(slot, encoding).ok())
         } else {
@@ -416,25 +397,75 @@ impl JsonRpcRequestProcessor {
             .unwrap_or(None))
     }
 
+    pub fn get_signature_confirmation_status(
+        &self,
+        signature: Signature,
+        commitment: Option<CommitmentConfig>,
+    ) -> Option<RpcSignatureConfirmation> {
+        self.get_transaction_status(signature, &self.bank(commitment))
+            .map(
+                |TransactionStatus {
+                     status,
+                     confirmations,
+                     ..
+                 }| RpcSignatureConfirmation {
+                    confirmations: confirmations.unwrap_or(MAX_LOCKOUT_HISTORY + 1),
+                    status,
+                },
+            )
+    }
+
     pub fn get_signature_status(
+        &self,
+        signature: Signature,
+        commitment: Option<CommitmentConfig>,
+    ) -> Option<transaction::Result<()>> {
+        self.bank(commitment)
+            .get_signature_status_slot(&signature)
+            .map(|(_, status)| status)
+    }
+
+    pub fn get_signature_statuses(
         &self,
         signatures: Vec<Signature>,
         commitment: Option<CommitmentConfig>,
-    ) -> Result<Vec<Option<RpcTransactionStatus>>> {
-        let mut statuses: Vec<Option<RpcTransactionStatus>> = vec![];
+    ) -> RpcResponse<Vec<Option<TransactionStatus>>> {
+        let mut statuses: Vec<Option<TransactionStatus>> = vec![];
 
         let bank = self.bank(commitment);
 
         for signature in signatures {
-            let status = bank.get_signature_confirmation_status(&signature).map(
-                |SignatureConfirmationStatus { slot, status, .. }| RpcTransactionStatus {
-                    slot,
-                    status,
-                },
-            );
+            let status = self.get_transaction_status(signature, &bank);
             statuses.push(status);
         }
-        Ok(statuses)
+        Ok(Response {
+            context: RpcResponseContext { slot: bank.slot() },
+            value: statuses,
+        })
+    }
+
+    fn get_transaction_status(
+        &self,
+        signature: Signature,
+        bank: &Arc<Bank>,
+    ) -> Option<TransactionStatus> {
+        bank.get_signature_status_slot(&signature)
+            .map(|(slot, status)| {
+                let r_block_commitment_cache = self.block_commitment_cache.read().unwrap();
+
+                let confirmations = if r_block_commitment_cache.root() >= slot {
+                    None
+                } else {
+                    r_block_commitment_cache
+                        .get_confirmation_count(slot)
+                        .or(Some(0))
+                };
+                TransactionStatus {
+                    slot,
+                    status,
+                    confirmations,
+                }
+            })
     }
 }
 
@@ -558,13 +589,29 @@ pub trait RpcSol {
     #[rpc(meta, name = "getFeeRateGovernor")]
     fn get_fee_rate_governor(&self, meta: Self::Metadata) -> RpcResponse<RpcFeeRateGovernor>;
 
+    #[rpc(meta, name = "getSignatureConfirmation")]
+    fn get_signature_confirmation(
+        &self,
+        meta: Self::Metadata,
+        signature_str: String,
+        commitment: Option<CommitmentConfig>,
+    ) -> Result<Option<RpcSignatureConfirmation>>;
+
     #[rpc(meta, name = "getSignatureStatus")]
     fn get_signature_status(
+        &self,
+        meta: Self::Metadata,
+        signature_str: String,
+        commitment: Option<CommitmentConfig>,
+    ) -> Result<Option<transaction::Result<()>>>;
+
+    #[rpc(meta, name = "getSignatureStatuses")]
+    fn get_signature_statuses(
         &self,
         meta: Self::Metadata,
         signature_strs: Vec<String>,
         commitment: Option<CommitmentConfig>,
-    ) -> Result<Vec<Option<RpcTransactionStatus>>>;
+    ) -> RpcResponse<Vec<Option<TransactionStatus>>>;
 
     #[rpc(meta, name = "getSlot")]
     fn get_slot(&self, meta: Self::Metadata, commitment: Option<CommitmentConfig>) -> Result<u64>;
@@ -631,22 +678,6 @@ pub trait RpcSol {
     #[rpc(meta, name = "validatorExit")]
     fn validator_exit(&self, meta: Self::Metadata) -> Result<bool>;
 
-    #[rpc(meta, name = "getNumBlocksSinceSignatureConfirmation")]
-    fn get_num_blocks_since_signature_confirmation(
-        &self,
-        meta: Self::Metadata,
-        signature_str: String,
-        commitment: Option<CommitmentConfig>,
-    ) -> Result<Option<usize>>;
-
-    #[rpc(meta, name = "getSignatureConfirmation")]
-    fn get_signature_confirmation(
-        &self,
-        meta: Self::Metadata,
-        signature_str: String,
-        commitment: Option<CommitmentConfig>,
-    ) -> Result<Option<RpcSignatureConfirmation>>;
-
     #[rpc(meta, name = "getIdentity")]
     fn get_identity(&self, meta: Self::Metadata) -> Result<RpcIdentity>;
 
@@ -661,8 +692,8 @@ pub trait RpcSol {
         &self,
         meta: Self::Metadata,
         slot: Slot,
-        encoding: Option<RpcTransactionEncoding>,
-    ) -> Result<Option<RpcConfirmedBlock>>;
+        encoding: Option<TransactionEncoding>,
+    ) -> Result<Option<ConfirmedBlock>>;
 
     #[rpc(meta, name = "getBlockTime")]
     fn get_block_time(&self, meta: Self::Metadata, slot: Slot) -> Result<Option<UnixTimestamp>>;
@@ -905,36 +936,6 @@ impl RpcSol for RpcSolImpl {
             .get_fee_rate_governor()
     }
 
-    fn get_signature_status(
-        &self,
-        meta: Self::Metadata,
-        signature_strs: Vec<String>,
-        commitment: Option<CommitmentConfig>,
-    ) -> Result<Vec<Option<RpcTransactionStatus>>> {
-        let mut signatures: Vec<Signature> = vec![];
-        for signature_str in signature_strs {
-            signatures.push(verify_signature(&signature_str)?);
-        }
-        meta.request_processor
-            .read()
-            .unwrap()
-            .get_signature_status(signatures, commitment)
-    }
-
-    fn get_slot(&self, meta: Self::Metadata, commitment: Option<CommitmentConfig>) -> Result<u64> {
-        meta.request_processor.read().unwrap().get_slot(commitment)
-    }
-
-    fn get_num_blocks_since_signature_confirmation(
-        &self,
-        meta: Self::Metadata,
-        signature_str: String,
-        commitment: Option<CommitmentConfig>,
-    ) -> Result<Option<usize>> {
-        self.get_signature_confirmation(meta, signature_str, commitment)
-            .map(|res| res.map(|x| x.confirmations))
-    }
-
     fn get_signature_confirmation(
         &self,
         meta: Self::Metadata,
@@ -953,6 +954,40 @@ impl RpcSol for RpcSolImpl {
             .get_signature_confirmation_status(signature, commitment))
     }
 
+    fn get_signature_status(
+        &self,
+        meta: Self::Metadata,
+        signature_str: String,
+        commitment: Option<CommitmentConfig>,
+    ) -> Result<Option<transaction::Result<()>>> {
+        let signature = verify_signature(&signature_str)?;
+        Ok(meta
+            .request_processor
+            .read()
+            .unwrap()
+            .get_signature_status(signature, commitment))
+    }
+
+    fn get_signature_statuses(
+        &self,
+        meta: Self::Metadata,
+        signature_strs: Vec<String>,
+        commitment: Option<CommitmentConfig>,
+    ) -> RpcResponse<Vec<Option<TransactionStatus>>> {
+        let mut signatures: Vec<Signature> = vec![];
+        for signature_str in signature_strs {
+            signatures.push(verify_signature(&signature_str)?);
+        }
+        meta.request_processor
+            .read()
+            .unwrap()
+            .get_signature_statuses(signatures, commitment)
+    }
+
+    fn get_slot(&self, meta: Self::Metadata, commitment: Option<CommitmentConfig>) -> Result<u64> {
+        meta.request_processor.read().unwrap().get_slot(commitment)
+    }
+
     fn get_transaction_count(
         &self,
         meta: Self::Metadata,
@@ -1039,7 +1074,9 @@ impl RpcSol for RpcSolImpl {
                 .request_processor
                 .read()
                 .unwrap()
-                .get_signature_confirmation_status(signature, commitment.clone())
+                .get_signature_statuses(vec![signature], commitment.clone())?
+                .value[0]
+                .clone()
                 .map(|x| x.status);
 
             if signature_status == Some(Ok(())) {
@@ -1181,8 +1218,8 @@ impl RpcSol for RpcSolImpl {
         &self,
         meta: Self::Metadata,
         slot: Slot,
-        encoding: Option<RpcTransactionEncoding>,
-    ) -> Result<Option<RpcConfirmedBlock>> {
+        encoding: Option<TransactionEncoding>,
+    ) -> Result<Option<ConfirmedBlock>> {
         meta.request_processor
             .read()
             .unwrap()
@@ -1210,17 +1247,17 @@ impl RpcSol for RpcSolImpl {
 pub mod tests {
     use super::*;
     use crate::{
-        commitment::BlockCommitment,
-        contact_info::ContactInfo,
-        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        commitment::BlockCommitment, contact_info::ContactInfo,
         replay_stage::tests::create_test_transactions_and_populate_blockstore,
     };
     use bincode::deserialize;
     use jsonrpc_core::{MetaIoHandler, Output, Response, Value};
-    use solana_client::rpc_response::{RpcEncodedTransaction, RpcTransactionWithStatusMeta};
     use solana_ledger::{
-        blockstore::entries_to_test_shreds, blockstore_processor::fill_blockstore_slot_with_ticks,
-        entry::next_entry_mut, get_tmp_ledger_path,
+        blockstore::entries_to_test_shreds,
+        blockstore_processor::fill_blockstore_slot_with_ticks,
+        entry::next_entry_mut,
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        get_tmp_ledger_path,
     };
     use solana_sdk::{
         fee_calculator::DEFAULT_BURN_PERCENT,
@@ -1231,6 +1268,7 @@ pub mod tests {
         system_transaction,
         transaction::{self, TransactionError},
     };
+    use solana_transaction_status::{EncodedTransaction, TransactionWithStatusMeta};
     use solana_vote_program::{
         vote_instruction,
         vote_state::{Vote, VoteInit, MAX_LOCKOUT_HISTORY},
@@ -1268,18 +1306,6 @@ pub mod tests {
     ) -> RpcHandler {
         let (bank_forks, alice, leader_vote_keypair) = new_bank_forks();
         let bank = bank_forks.read().unwrap().working_bank();
-
-        let commitment_slot0 = BlockCommitment::new([8; MAX_LOCKOUT_HISTORY]);
-        let commitment_slot1 = BlockCommitment::new([9; MAX_LOCKOUT_HISTORY]);
-        let mut block_commitment: HashMap<u64, BlockCommitment> = HashMap::new();
-        block_commitment
-            .entry(0)
-            .or_insert(commitment_slot0.clone());
-        block_commitment
-            .entry(1)
-            .or_insert(commitment_slot1.clone());
-        let block_commitment_cache =
-            Arc::new(RwLock::new(BlockCommitmentCache::new(block_commitment, 42)));
         let ledger_path = get_tmp_ledger_path!();
         let blockstore = Blockstore::open(&ledger_path).unwrap();
         let blockstore = Arc::new(blockstore);
@@ -1295,6 +1321,24 @@ pub mod tests {
             blockstore.clone(),
         );
 
+        let mut commitment_slot0 = BlockCommitment::default();
+        commitment_slot0.increase_confirmation_stake(2, 9);
+        let mut commitment_slot1 = BlockCommitment::default();
+        commitment_slot1.increase_confirmation_stake(1, 9);
+        let mut block_commitment: HashMap<u64, BlockCommitment> = HashMap::new();
+        block_commitment
+            .entry(0)
+            .or_insert(commitment_slot0.clone());
+        block_commitment
+            .entry(1)
+            .or_insert(commitment_slot1.clone());
+        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new(
+            block_commitment,
+            10,
+            bank.clone(),
+            0,
+        )));
+
         // Add timestamp vote to blockstore
         let vote = Vote {
             slots: vec![1],
@@ -1796,6 +1840,76 @@ pub mod tests {
 
     #[test]
     fn test_rpc_get_signature_status() {
+        let bob_pubkey = Pubkey::new_rand();
+        let RpcHandler {
+            io,
+            meta,
+            blockhash,
+            alice,
+            ..
+        } = start_rpc_handler_with_tx(&bob_pubkey);
+
+        let tx = system_transaction::transfer(&alice, &bob_pubkey, 20, blockhash);
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getSignatureStatus","params":["{}"]}}"#,
+            tx.signatures[0]
+        );
+        let res = io.handle_request_sync(&req, meta.clone());
+        let expected_res: Option<transaction::Result<()>> = Some(Ok(()));
+        let expected = json!({
+            "jsonrpc": "2.0",
+            "result": expected_res,
+            "id": 1
+        });
+        let expected: Response =
+            serde_json::from_value(expected).expect("expected response deserialization");
+        let result: Response = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(expected, result);
+
+        // Test getSignatureStatus request on unprocessed tx
+        let tx = system_transaction::transfer(&alice, &bob_pubkey, 10, blockhash);
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getSignatureStatus","params":["{}"]}}"#,
+            tx.signatures[0]
+        );
+        let res = io.handle_request_sync(&req, meta.clone());
+        let expected_res: Option<String> = None;
+        let expected = json!({
+            "jsonrpc": "2.0",
+            "result": expected_res,
+            "id": 1
+        });
+        let expected: Response =
+            serde_json::from_value(expected).expect("expected response deserialization");
+        let result: Response = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(expected, result);
+
+        // Test getSignatureStatus request on a TransactionError
+        let tx = system_transaction::transfer(&alice, &bob_pubkey, std::u64::MAX, blockhash);
+        let req = format!(
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getSignatureStatus","params":["{}"]}}"#,
+            tx.signatures[0]
+        );
+        let res = io.handle_request_sync(&req, meta);
+        let expected_res: Option<transaction::Result<()>> = Some(Err(
+            TransactionError::InstructionError(0, InstructionError::CustomError(1)),
+        ));
+        let expected = json!({
+            "jsonrpc": "2.0",
+            "result": expected_res,
+            "id": 1
+        });
+        let expected: Response =
+            serde_json::from_value(expected).expect("expected response deserialization");
+        let result: Response = serde_json::from_str(&res.expect("actual response"))
+            .expect("actual response deserialization");
+        assert_eq!(expected, result);
+    }
+
+    #[test]
+    fn test_rpc_get_signature_statuses() {
         let bob_pubkey = Pubkey::new_rand();
         let RpcHandler {
             io,
@@ -1807,33 +1921,35 @@ pub mod tests {
         } = start_rpc_handler_with_tx(&bob_pubkey);
 
         let req = format!(
-            r#"{{"jsonrpc":"2.0","id":1,"method":"getSignatureStatus","params":[["{}"]]}}"#,
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getSignatureStatuses","params":[["{}"]]}}"#,
             confirmed_block_signatures[0]
         );
         let res = io.handle_request_sync(&req, meta.clone());
         let expected_res: transaction::Result<()> = Ok(());
         let json: Value = serde_json::from_str(&res.unwrap()).unwrap();
-        let result: Vec<Option<RpcTransactionStatus>> =
-            serde_json::from_value(json["result"].clone())
+        let result: Option<TransactionStatus> =
+            serde_json::from_value(json["result"]["value"][0].clone())
                 .expect("actual response deserialization");
-        assert_eq!(expected_res, result[0].as_ref().unwrap().status);
+        let result = result.as_ref().unwrap();
+        assert_eq!(expected_res, result.status);
+        assert_eq!(None, result.confirmations);
 
         // Test getSignatureStatus request on unprocessed tx
         let tx = system_transaction::transfer(&alice, &bob_pubkey, 10, blockhash);
         let req = format!(
-            r#"{{"jsonrpc":"2.0","id":1,"method":"getSignatureStatus","params":[["{}"]]}}"#,
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getSignatureStatuses","params":[["{}"]]}}"#,
             tx.signatures[0]
         );
         let res = io.handle_request_sync(&req, meta.clone());
         let json: Value = serde_json::from_str(&res.unwrap()).unwrap();
-        let result: Vec<Option<RpcTransactionStatus>> =
-            serde_json::from_value(json["result"].clone())
+        let result: Option<TransactionStatus> =
+            serde_json::from_value(json["result"]["value"][0].clone())
                 .expect("actual response deserialization");
-        assert!(result[0].is_none());
+        assert!(result.is_none());
 
         // Test getSignatureStatus request on a TransactionError
         let req = format!(
-            r#"{{"jsonrpc":"2.0","id":1,"method":"getSignatureStatus","params":[["{}"]]}}"#,
+            r#"{{"jsonrpc":"2.0","id":1,"method":"getSignatureStatuses","params":[["{}"]]}}"#,
             confirmed_block_signatures[1]
         );
         let res = io.handle_request_sync(&req, meta.clone());
@@ -1842,10 +1958,10 @@ pub mod tests {
             InstructionError::CustomError(1),
         ));
         let json: Value = serde_json::from_str(&res.unwrap()).unwrap();
-        let result: Vec<Option<RpcTransactionStatus>> =
-            serde_json::from_value(json["result"].clone())
+        let result: Option<TransactionStatus> =
+            serde_json::from_value(json["result"]["value"][0].clone())
                 .expect("actual response deserialization");
-        assert_eq!(expected_res, result[0].as_ref().unwrap().status);
+        assert_eq!(expected_res, result.as_ref().unwrap().status);
     }
 
     #[test]
@@ -2165,6 +2281,8 @@ pub mod tests {
     fn test_rpc_processor_get_block_commitment() {
         let exit = Arc::new(AtomicBool::new(false));
         let validator_exit = create_validator_exit(&exit);
+        let bank_forks = new_bank_forks().0;
+
         let commitment_slot0 = BlockCommitment::new([8; MAX_LOCKOUT_HISTORY]);
         let commitment_slot1 = BlockCommitment::new([9; MAX_LOCKOUT_HISTORY]);
         let mut block_commitment: HashMap<u64, BlockCommitment> = HashMap::new();
@@ -2174,8 +2292,12 @@ pub mod tests {
         block_commitment
             .entry(1)
             .or_insert(commitment_slot1.clone());
-        let block_commitment_cache =
-            Arc::new(RwLock::new(BlockCommitmentCache::new(block_commitment, 42)));
+        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new(
+            block_commitment,
+            42,
+            bank_forks.read().unwrap().working_bank(),
+            0,
+        )));
         let ledger_path = get_tmp_ledger_path!();
         let blockstore = Blockstore::open(&ledger_path).unwrap();
 
@@ -2183,7 +2305,7 @@ pub mod tests {
         config.enable_validator_exit = true;
         let request_processor = JsonRpcRequestProcessor::new(
             config,
-            new_bank_forks().0,
+            bank_forks,
             block_commitment_cache,
             Arc::new(blockstore),
             StorageState::default(),
@@ -2247,7 +2369,7 @@ pub mod tests {
                 .get_block_commitment(0)
                 .map(|block_commitment| block_commitment.commitment)
         );
-        assert_eq!(total_stake, 42);
+        assert_eq!(total_stake, 10);
 
         let req =
             format!(r#"{{"jsonrpc":"2.0","id":1,"method":"getBlockCommitment","params":[2]}}"#);
@@ -2265,7 +2387,7 @@ pub mod tests {
                 panic!("Expected single response");
             };
         assert_eq!(commitment_response.commitment, None);
-        assert_eq!(commitment_response.total_stake, 42);
+        assert_eq!(commitment_response.total_stake, 10);
     }
 
     #[test]
@@ -2284,15 +2406,15 @@ pub mod tests {
         let res = io.handle_request_sync(&req, meta.clone());
         let result: Value = serde_json::from_str(&res.expect("actual response"))
             .expect("actual response deserialization");
-        let confirmed_block: Option<RpcConfirmedBlock> =
+        let confirmed_block: Option<ConfirmedBlock> =
             serde_json::from_value(result["result"].clone()).unwrap();
         let confirmed_block = confirmed_block.unwrap();
         assert_eq!(confirmed_block.transactions.len(), 3);
 
-        for RpcTransactionWithStatusMeta { transaction, meta } in
+        for TransactionWithStatusMeta { transaction, meta } in
             confirmed_block.transactions.into_iter()
         {
-            if let RpcEncodedTransaction::Json(transaction) = transaction {
+            if let EncodedTransaction::Json(transaction) = transaction {
                 if transaction.signatures[0] == confirmed_block_signatures[0].to_string() {
                     assert_eq!(transaction.message.recent_blockhash, blockhash.to_string());
                     assert_eq!(meta.unwrap().status, Ok(()));
@@ -2316,15 +2438,15 @@ pub mod tests {
         let res = io.handle_request_sync(&req, meta);
         let result: Value = serde_json::from_str(&res.expect("actual response"))
             .expect("actual response deserialization");
-        let confirmed_block: Option<RpcConfirmedBlock> =
+        let confirmed_block: Option<ConfirmedBlock> =
             serde_json::from_value(result["result"].clone()).unwrap();
         let confirmed_block = confirmed_block.unwrap();
         assert_eq!(confirmed_block.transactions.len(), 3);
 
-        for RpcTransactionWithStatusMeta { transaction, meta } in
+        for TransactionWithStatusMeta { transaction, meta } in
             confirmed_block.transactions.into_iter()
         {
-            if let RpcEncodedTransaction::Binary(transaction) = transaction {
+            if let EncodedTransaction::Binary(transaction) = transaction {
                 let decoded_transaction: Transaction =
                     deserialize(&bs58::decode(&transaction).into_vec().unwrap()).unwrap();
                 if decoded_transaction.signatures[0] == confirmed_block_signatures[0] {

core/src/rpc_pubsub.rs

@@ -5,7 +5,7 @@ use jsonrpc_core::{Error, ErrorCode, Result};
 use jsonrpc_derive::rpc;
 use jsonrpc_pubsub::{typed::Subscriber, Session, SubscriptionId};
 use solana_client::rpc_response::{Response as RpcResponse, RpcAccount, RpcKeyedAccount};
-use solana_sdk::{pubkey::Pubkey, signature::Signature, transaction};
+use solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature, transaction};
 use std::sync::{atomic, Arc};
 
 // Suppress needless_return due to
@@ -102,6 +102,18 @@ pub trait RpcSolPubSub {
         name = "slotUnsubscribe"
     )]
     fn slot_unsubscribe(&self, meta: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool>;
+
+    // Get notification when a new root is set
+    #[pubsub(subscription = "rootNotification", subscribe, name = "rootSubscribe")]
+    fn root_subscribe(&self, meta: Self::Metadata, subscriber: Subscriber<Slot>);
+
+    // Unsubscribe from slot notification subscription.
+    #[pubsub(
+        subscription = "rootNotification",
+        unsubscribe,
+        name = "rootUnsubscribe"
+    )]
+    fn root_unsubscribe(&self, meta: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool>;
 }
 
 #[derive(Default)]
@@ -274,17 +286,41 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
             })
         }
     }
+
+    fn root_subscribe(&self, _meta: Self::Metadata, subscriber: Subscriber<Slot>) {
+        info!("root_subscribe");
+        let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
+        let sub_id = SubscriptionId::Number(id as u64);
+        info!("root_subscribe: id={:?}", sub_id);
+        self.subscriptions.add_root_subscription(sub_id, subscriber);
+    }
+
+    fn root_unsubscribe(&self, _meta: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool> {
+        info!("root_unsubscribe");
+        if self.subscriptions.remove_root_subscription(&id) {
+            Ok(true)
+        } else {
+            Err(Error {
+                code: ErrorCode::InvalidParams,
+                message: "Invalid Request: Subscription id does not exist".into(),
+                data: None,
+            })
+        }
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-    use crate::rpc_subscriptions::tests::robust_poll_or_panic;
+    use crate::{
+        commitment::{BlockCommitment, BlockCommitmentCache},
+        rpc_subscriptions::tests::robust_poll_or_panic,
+    };
     use jsonrpc_core::{futures::sync::mpsc, Response};
     use jsonrpc_pubsub::{PubSubHandler, Session};
     use solana_budget_program::{self, budget_instruction};
     use solana_ledger::bank_forks::BankForks;
+    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_runtime::bank::Bank;
     use solana_sdk::{
         pubkey::Pubkey,
@@ -292,7 +328,12 @@ mod tests {
         system_program, system_transaction,
         transaction::{self, Transaction},
     };
-    use std::{sync::RwLock, thread::sleep, time::Duration};
+    use std::{
+        collections::HashMap,
+        sync::{atomic::AtomicBool, RwLock},
+        thread::sleep,
+        time::Duration,
+    };
 
     fn process_transaction_and_notify(
         bank_forks: &Arc<RwLock<BankForks>>,
@@ -325,8 +366,13 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
-
-        let rpc = RpcSolPubSubImpl::default();
+        let rpc = RpcSolPubSubImpl {
+            subscriptions: Arc::new(RpcSubscriptions::new(
+                &Arc::new(AtomicBool::new(false)),
+                Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            )),
+            ..RpcSolPubSubImpl::default()
+        };
 
         // Test signature subscriptions
         let tx = system_transaction::transfer(&alice, &bob_pubkey, 20, blockhash);
@@ -338,7 +384,7 @@ mod tests {
         process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions).unwrap();
 
         // Test signature confirmation notification
-        let response = robust_poll_or_panic(receiver);
+        let (response, _) = robust_poll_or_panic(receiver);
         let expected_res: Option<transaction::Result<()>> = Some(Ok(()));
         let expected = json!({
            "jsonrpc": "2.0",
@@ -424,7 +470,13 @@ mod tests {
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
 
-        let rpc = RpcSolPubSubImpl::default();
+        let rpc = RpcSolPubSubImpl {
+            subscriptions: Arc::new(RpcSubscriptions::new(
+                &Arc::new(AtomicBool::new(false)),
+                Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            )),
+            ..RpcSolPubSubImpl::default()
+        };
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification");
         rpc.account_subscribe(
@@ -480,7 +532,7 @@ mod tests {
            }
         });
 
-        let response = robust_poll_or_panic(receiver);
+        let (response, _) = robust_poll_or_panic(receiver);
         assert_eq!(serde_json::to_string(&expected).unwrap(), response);
 
         let tx = system_transaction::transfer(&alice, &witness.pubkey(), 1, blockhash);
@@ -558,7 +610,13 @@ mod tests {
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
         let bob = Keypair::new();
 
-        let rpc = RpcSolPubSubImpl::default();
+        let mut rpc = RpcSolPubSubImpl::default();
+        let exit = Arc::new(AtomicBool::new(false));
+        let subscriptions = RpcSubscriptions::new(
+            &exit,
+            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+        );
+        rpc.subscriptions = Arc::new(subscriptions);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification");
         rpc.account_subscribe(session, subscriber, bob.pubkey().to_string(), Some(2));
@@ -589,7 +647,12 @@ mod tests {
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
         let bob = Keypair::new();
 
-        let rpc = RpcSolPubSubImpl::default();
+        let mut rpc = RpcSolPubSubImpl::default();
+        let exit = Arc::new(AtomicBool::new(false));
+        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests()));
+
+        let subscriptions = RpcSubscriptions::new(&exit, block_commitment_cache.clone());
+        rpc.subscriptions = Arc::new(subscriptions);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification");
         rpc.account_subscribe(session, subscriber, bob.pubkey().to_string(), Some(2));
@@ -607,10 +670,32 @@ mod tests {
         let bank0 = bank_forks.read().unwrap()[0].clone();
         let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
         bank_forks.write().unwrap().insert(bank1);
-        rpc.subscriptions.notify_subscribers(1, &bank_forks);
         let bank1 = bank_forks.read().unwrap()[1].clone();
+
+        let mut cache0 = BlockCommitment::default();
+        cache0.increase_confirmation_stake(1, 10);
+        let mut block_commitment = HashMap::new();
+        block_commitment.entry(0).or_insert(cache0.clone());
+        let mut new_block_commitment =
+            BlockCommitmentCache::new(block_commitment, 10, bank1.clone(), 0);
+        let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
+        std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
+        drop(w_block_commitment_cache);
+
+        rpc.subscriptions.notify_subscribers(1, &bank_forks);
         let bank2 = Bank::new_from_parent(&bank1, &Pubkey::default(), 2);
         bank_forks.write().unwrap().insert(bank2);
+        let bank2 = bank_forks.read().unwrap()[2].clone();
+
+        let mut cache0 = BlockCommitment::default();
+        cache0.increase_confirmation_stake(2, 10);
+        let mut block_commitment = HashMap::new();
+        block_commitment.entry(0).or_insert(cache0.clone());
+        let mut new_block_commitment = BlockCommitmentCache::new(block_commitment, 10, bank2, 0);
+        let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
+        std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
+        drop(w_block_commitment_cache);
+
         rpc.subscriptions.notify_subscribers(2, &bank_forks);
         let expected = json!({
            "jsonrpc": "2.0",
@@ -629,7 +714,7 @@ mod tests {
                "subscription": 0,
            }
         });
-        let response = robust_poll_or_panic(receiver);
+        let (response, _) = robust_poll_or_panic(receiver);
         assert_eq!(serde_json::to_string(&expected).unwrap(), response);
     }
 
@@ -642,7 +727,7 @@ mod tests {
 
         rpc.subscriptions.notify_slot(0, 0, 0);
         // Test slot confirmation notification
-        let response = robust_poll_or_panic(receiver);
+        let (response, _) = robust_poll_or_panic(receiver);
         let expected_res = SlotInfo {
             parent: 0,
             slot: 0,
@@ -664,7 +749,7 @@ mod tests {
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification");
         rpc.slot_subscribe(session, subscriber);
         rpc.subscriptions.notify_slot(0, 0, 0);
-        let response = robust_poll_or_panic(receiver);
+        let (response, _) = robust_poll_or_panic(receiver);
         let expected_res = SlotInfo {
             parent: 0,
             slot: 0,

core/src/rpc_pubsub_service.rs

@@ -1,14 +1,20 @@
 //! The `pubsub` module implements a threaded subscription service on client RPC request
 
-use crate::rpc_pubsub::{RpcSolPubSub, RpcSolPubSubImpl};
-use crate::rpc_subscriptions::RpcSubscriptions;
+use crate::{
+    rpc_pubsub::{RpcSolPubSub, RpcSolPubSubImpl},
+    rpc_subscriptions::RpcSubscriptions,
+};
 use jsonrpc_pubsub::{PubSubHandler, Session};
 use jsonrpc_ws_server::{RequestContext, ServerBuilder};
-use std::net::SocketAddr;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
-use std::thread::{self, sleep, Builder, JoinHandle};
-use std::time::Duration;
+use std::{
+    net::SocketAddr,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    thread::{self, sleep, Builder, JoinHandle},
+    time::Duration,
+};
 
 pub struct PubSubService {
     thread_hdl: JoinHandle<()>,
@@ -66,13 +72,20 @@ impl PubSubService {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::net::{IpAddr, Ipv4Addr};
+    use crate::commitment::BlockCommitmentCache;
+    use std::{
+        net::{IpAddr, Ipv4Addr},
+        sync::RwLock,
+    };
 
     #[test]
     fn test_pubsub_new() {
         let pubsub_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0);
         let exit = Arc::new(AtomicBool::new(false));
-        let subscriptions = Arc::new(RpcSubscriptions::new(&exit));
+        let subscriptions = Arc::new(RpcSubscriptions::new(
+            &exit,
+            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+        ));
         let pubsub_service = PubSubService::new(&subscriptions, pubsub_addr, &exit);
         let thread = pubsub_service.thread_hdl.thread();
         assert_eq!(thread.name().unwrap(), "solana-pubsub");

core/src/rpc_service.rs

@@ -85,9 +85,19 @@ impl RpcRequestMiddleware {
     }
 
     fn get(&self, path: &str) -> RequestMiddlewareAction {
-        let filename = self.ledger_path.join(
-            path.split_at(1).1, // Drop leading '/' from path
-        );
+        let stem = path.split_at(1).1; // Drop leading '/' from path
+        let filename = {
+            match path {
+                "/genesis.tar.bz2" => self.ledger_path.join(stem),
+                _ => self
+                    .snapshot_config
+                    .as_ref()
+                    .unwrap()
+                    .snapshot_package_output_path
+                    .join(stem),
+            }
+        };
+
         info!("get {} -> {:?}", path, filename);
 
         RequestMiddlewareAction::Respond {
@@ -238,12 +248,11 @@ impl JsonRpcService {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{
-        contact_info::ContactInfo,
+    use crate::{contact_info::ContactInfo, rpc::tests::create_validator_exit};
+    use solana_ledger::{
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
-        rpc::tests::create_validator_exit,
+        get_tmp_ledger_path,
     };
-    use solana_ledger::get_tmp_ledger_path;
     use solana_runtime::bank::Bank;
     use solana_sdk::signature::Signer;
     use std::net::{IpAddr, Ipv4Addr, SocketAddr};

core/src/rpc_subscriptions.rs

@@ -1,5 +1,6 @@
 //! The `pubsub` module implements a threaded subscription service on client RPC request
 
+use crate::commitment::BlockCommitmentCache;
 use core::hash::Hash;
 use jsonrpc_core::futures::Future;
 use jsonrpc_pubsub::{
@@ -14,11 +15,14 @@ use solana_sdk::{
     account::Account, clock::Slot, pubkey::Pubkey, signature::Signature, transaction,
 };
 use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::mpsc::{Receiver, RecvTimeoutError, SendError, Sender};
+use std::sync::{
+    atomic::{AtomicBool, Ordering},
+    mpsc::{Receiver, RecvTimeoutError, SendError, Sender},
+};
 use std::thread::{Builder, JoinHandle};
 use std::time::Duration;
 use std::{
+    cmp::min,
     collections::{HashMap, HashSet},
     iter,
     sync::{Arc, Mutex, RwLock},
@@ -38,12 +42,14 @@ pub struct SlotInfo {
 
 enum NotificationEntry {
     Slot(SlotInfo),
+    Root(Slot),
     Bank((Slot, Arc<RwLock<BankForks>>)),
 }
 
 impl std::fmt::Debug for NotificationEntry {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         match self {
+            NotificationEntry::Root(root) => write!(f, "Root({})", root),
             NotificationEntry::Slot(slot_info) => write!(f, "Slot({:?})", slot_info),
             NotificationEntry::Bank((current_slot, _)) => {
                 write!(f, "Bank({{current_slot: {:?}}})", current_slot)
@@ -64,6 +70,7 @@ type RpcSignatureSubscriptions = RwLock<
     >,
 >;
 type RpcSlotSubscriptions = RwLock<HashMap<SubscriptionId, Sink<SlotInfo>>>;
+type RpcRootSubscriptions = RwLock<HashMap<SubscriptionId, Sink<Slot>>>;
 
 fn add_subscription<K, S>(
     subscriptions: &mut HashMap<K, HashMap<SubscriptionId, (Sink<S>, Confirmations)>>,
@@ -77,11 +84,7 @@ fn add_subscription<K, S>(
 {
     let sink = subscriber.assign_id(sub_id.clone()).unwrap();
     let confirmations = confirmations.unwrap_or(0);
-    let confirmations = if confirmations > MAX_LOCKOUT_HISTORY {
-        MAX_LOCKOUT_HISTORY
-    } else {
-        confirmations
-    };
+    let confirmations = min(confirmations, MAX_LOCKOUT_HISTORY + 1);
     if let Some(current_hashmap) = subscriptions.get_mut(&hashmap_key) {
         current_hashmap.insert(sub_id, (sink, confirmations));
         return;
@@ -117,8 +120,8 @@ where
 fn check_confirmations_and_notify<K, S, B, F, X>(
     subscriptions: &HashMap<K, HashMap<SubscriptionId, (Sink<Response<S>>, Confirmations)>>,
     hashmap_key: &K,
-    current_slot: Slot,
     bank_forks: &Arc<RwLock<BankForks>>,
+    block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
     bank_method: B,
     filter_results: F,
     notifier: &RpcNotifier,
@@ -130,6 +133,10 @@ where
     F: Fn(X, u64) -> Box<dyn Iterator<Item = S>>,
     X: Clone + Serialize,
 {
+    let mut confirmation_slots: HashMap<usize, Slot> = HashMap::new();
+    let r_block_commitment_cache = block_commitment_cache.read().unwrap();
+    let current_slot = r_block_commitment_cache.slot();
+    let root = r_block_commitment_cache.root();
     let current_ancestors = bank_forks
         .read()
         .unwrap()
@@ -137,27 +144,29 @@ where
         .unwrap()
         .ancestors
         .clone();
+    for (slot, _) in current_ancestors.iter() {
+        if let Some(confirmations) = r_block_commitment_cache.get_confirmation_count(*slot) {
+            confirmation_slots.entry(confirmations).or_insert(*slot);
+        }
+    }
+    drop(r_block_commitment_cache);
 
     let mut notified_set: HashSet<SubscriptionId> = HashSet::new();
     if let Some(hashmap) = subscriptions.get(hashmap_key) {
         for (sub_id, (sink, confirmations)) in hashmap.iter() {
-            let desired_slot: Vec<u64> = current_ancestors
-                .iter()
-                .filter(|(_, &v)| v == *confirmations)
-                .map(|(k, _)| k)
-                .cloned()
-                .collect();
-            let root: Vec<u64> = current_ancestors
-                .iter()
-                .filter(|(_, &v)| v == 32)
-                .map(|(k, _)| k)
-                .cloned()
-                .collect();
-            let root = if root.len() == 1 { root[0] } else { 0 };
-            if desired_slot.len() == 1 {
-                let slot = desired_slot[0];
-                let desired_bank = bank_forks.read().unwrap().get(slot).unwrap().clone();
-                let results = bank_method(&desired_bank, hashmap_key);
+            let desired_slot = if *confirmations == 0 {
+                Some(&current_slot)
+            } else if *confirmations == MAX_LOCKOUT_HISTORY + 1 {
+                Some(&root)
+            } else {
+                confirmation_slots.get(confirmations)
+            };
+            if let Some(&slot) = desired_slot {
+                let results = {
+                    let bank_forks = bank_forks.read().unwrap();
+                    let desired_bank = bank_forks.get(slot).unwrap();
+                    bank_method(&desired_bank, hashmap_key)
+                };
                 for result in filter_results(results, root) {
                     notifier.notify(
                         Response {
@@ -224,6 +233,7 @@ pub struct RpcSubscriptions {
     program_subscriptions: Arc<RpcProgramSubscriptions>,
     signature_subscriptions: Arc<RpcSignatureSubscriptions>,
     slot_subscriptions: Arc<RpcSlotSubscriptions>,
+    root_subscriptions: Arc<RpcRootSubscriptions>,
     notification_sender: Arc<Mutex<Sender<NotificationEntry>>>,
     t_cleanup: Option<JoinHandle<()>>,
     notifier_runtime: Option<Runtime>,
@@ -232,7 +242,10 @@ pub struct RpcSubscriptions {
 
 impl Default for RpcSubscriptions {
     fn default() -> Self {
-        Self::new(&Arc::new(AtomicBool::new(false)))
+        Self::new(
+            &Arc::new(AtomicBool::new(false)),
+            Arc::new(RwLock::new(BlockCommitmentCache::default())),
+        )
     }
 }
 
@@ -245,7 +258,10 @@ impl Drop for RpcSubscriptions {
 }
 
 impl RpcSubscriptions {
-    pub fn new(exit: &Arc<AtomicBool>) -> Self {
+    pub fn new(
+        exit: &Arc<AtomicBool>,
+        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+    ) -> Self {
         let (notification_sender, notification_receiver): (
             Sender<NotificationEntry>,
             Receiver<NotificationEntry>,
@@ -255,6 +271,7 @@ impl RpcSubscriptions {
         let program_subscriptions = Arc::new(RpcProgramSubscriptions::default());
         let signature_subscriptions = Arc::new(RpcSignatureSubscriptions::default());
         let slot_subscriptions = Arc::new(RpcSlotSubscriptions::default());
+        let root_subscriptions = Arc::new(RpcRootSubscriptions::default());
         let notification_sender = Arc::new(Mutex::new(notification_sender));
 
         let exit_clone = exit.clone();
@@ -262,6 +279,7 @@ impl RpcSubscriptions {
         let program_subscriptions_clone = program_subscriptions.clone();
         let signature_subscriptions_clone = signature_subscriptions.clone();
         let slot_subscriptions_clone = slot_subscriptions.clone();
+        let root_subscriptions_clone = root_subscriptions.clone();
 
         let notifier_runtime = RuntimeBuilder::new()
             .core_threads(1)
@@ -281,6 +299,8 @@ impl RpcSubscriptions {
                     program_subscriptions_clone,
                     signature_subscriptions_clone,
                     slot_subscriptions_clone,
+                    root_subscriptions_clone,
+                    block_commitment_cache,
                 );
             })
             .unwrap();
@@ -290,6 +310,7 @@ impl RpcSubscriptions {
             program_subscriptions,
             signature_subscriptions,
             slot_subscriptions,
+            root_subscriptions,
             notification_sender,
             notifier_runtime: Some(notifier_runtime),
             t_cleanup: Some(t_cleanup),
@@ -299,8 +320,8 @@ impl RpcSubscriptions {
 
     fn check_account(
         pubkey: &Pubkey,
-        current_slot: Slot,
         bank_forks: &Arc<RwLock<BankForks>>,
+        block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
         account_subscriptions: Arc<RpcAccountSubscriptions>,
         notifier: &RpcNotifier,
     ) {
@@ -308,8 +329,8 @@ impl RpcSubscriptions {
         check_confirmations_and_notify(
             &subscriptions,
             pubkey,
-            current_slot,
             bank_forks,
+            block_commitment_cache,
             Bank::get_account_modified_since_parent,
             filter_account_result,
             notifier,
@@ -318,8 +339,8 @@ impl RpcSubscriptions {
 
     fn check_program(
         program_id: &Pubkey,
-        current_slot: Slot,
         bank_forks: &Arc<RwLock<BankForks>>,
+        block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
         program_subscriptions: Arc<RpcProgramSubscriptions>,
         notifier: &RpcNotifier,
     ) {
@@ -327,8 +348,8 @@ impl RpcSubscriptions {
         check_confirmations_and_notify(
             &subscriptions,
             program_id,
-            current_slot,
             bank_forks,
+            block_commitment_cache,
             Bank::get_program_accounts_modified_since_parent,
             filter_program_results,
             notifier,
@@ -337,8 +358,8 @@ impl RpcSubscriptions {
 
     fn check_signature(
         signature: &Signature,
-        current_slot: Slot,
         bank_forks: &Arc<RwLock<BankForks>>,
+        block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
         signature_subscriptions: Arc<RpcSignatureSubscriptions>,
         notifier: &RpcNotifier,
     ) {
@@ -346,8 +367,8 @@ impl RpcSubscriptions {
         let notified_ids = check_confirmations_and_notify(
             &subscriptions,
             signature,
-            current_slot,
             bank_forks,
+            block_commitment_cache,
             Bank::get_signature_status_processed_since_parent,
             filter_signature_result,
             notifier,
@@ -447,6 +468,24 @@ impl RpcSubscriptions {
         self.enqueue_notification(NotificationEntry::Slot(SlotInfo { slot, parent, root }));
     }
 
+    pub fn add_root_subscription(&self, sub_id: SubscriptionId, subscriber: Subscriber<Slot>) {
+        let sink = subscriber.assign_id(sub_id.clone()).unwrap();
+        let mut subscriptions = self.root_subscriptions.write().unwrap();
+        subscriptions.insert(sub_id, sink);
+    }
+
+    pub fn remove_root_subscription(&self, id: &SubscriptionId) -> bool {
+        let mut subscriptions = self.root_subscriptions.write().unwrap();
+        subscriptions.remove(id).is_some()
+    }
+
+    pub fn notify_roots(&self, mut rooted_slots: Vec<Slot>) {
+        rooted_slots.sort();
+        rooted_slots.into_iter().for_each(|root| {
+            self.enqueue_notification(NotificationEntry::Root(root));
+        });
+    }
+
     fn enqueue_notification(&self, notification_entry: NotificationEntry) {
         match self
             .notification_sender
@@ -472,6 +511,8 @@ impl RpcSubscriptions {
         program_subscriptions: Arc<RpcProgramSubscriptions>,
         signature_subscriptions: Arc<RpcSignatureSubscriptions>,
         slot_subscriptions: Arc<RpcSlotSubscriptions>,
+        root_subscriptions: Arc<RpcRootSubscriptions>,
+        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
     ) {
         loop {
             if exit.load(Ordering::Relaxed) {
@@ -485,7 +526,13 @@ impl RpcSubscriptions {
                             notifier.notify(slot_info, sink);
                         }
                     }
-                    NotificationEntry::Bank((current_slot, bank_forks)) => {
+                    NotificationEntry::Root(root) => {
+                        let subscriptions = root_subscriptions.read().unwrap();
+                        for (_, sink) in subscriptions.iter() {
+                            notifier.notify(root, sink);
+                        }
+                    }
+                    NotificationEntry::Bank((_current_slot, bank_forks)) => {
                         let pubkeys: Vec<_> = {
                             let subs = account_subscriptions.read().unwrap();
                             subs.keys().cloned().collect()
@@ -493,8 +540,8 @@ impl RpcSubscriptions {
                         for pubkey in &pubkeys {
                             Self::check_account(
                                 pubkey,
-                                current_slot,
                                 &bank_forks,
+                                &block_commitment_cache,
                                 account_subscriptions.clone(),
                                 &notifier,
                             );
@@ -507,8 +554,8 @@ impl RpcSubscriptions {
                         for program_id in &programs {
                             Self::check_program(
                                 program_id,
-                                current_slot,
                                 &bank_forks,
+                                &block_commitment_cache,
                                 program_subscriptions.clone(),
                                 &notifier,
                             );
@@ -521,8 +568,8 @@ impl RpcSubscriptions {
                         for signature in &signatures {
                             Self::check_signature(
                                 signature,
-                                current_slot,
                                 &bank_forks,
+                                &block_commitment_cache,
                                 signature_subscriptions.clone(),
                                 &notifier,
                             );
@@ -563,34 +610,41 @@ impl RpcSubscriptions {
 #[cfg(test)]
 pub(crate) mod tests {
     use super::*;
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-    use jsonrpc_core::futures;
+    use crate::commitment::BlockCommitment;
+    use jsonrpc_core::futures::{self, stream::Stream};
     use jsonrpc_pubsub::typed::Subscriber;
     use solana_budget_program;
+    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_sdk::{
         signature::{Keypair, Signer},
         system_transaction,
     };
-    use tokio::prelude::{Async, Stream};
+    use std::{fmt::Debug, sync::mpsc::channel, time::Instant};
+    use tokio::{prelude::FutureExt, runtime::Runtime, timer::Delay};
 
-    pub(crate) fn robust_poll<T>(
-        mut receiver: futures::sync::mpsc::Receiver<T>,
-    ) -> Result<T, RecvTimeoutError> {
-        const INITIAL_DELAY_MS: u64 = RECEIVE_DELAY_MILLIS * 2;
+    pub(crate) fn robust_poll_or_panic<T: Debug + Send + 'static>(
+        receiver: futures::sync::mpsc::Receiver<T>,
+    ) -> (T, futures::sync::mpsc::Receiver<T>) {
+        let (inner_sender, inner_receiver) = channel();
+        let mut rt = Runtime::new().unwrap();
+        rt.spawn(futures::lazy(|| {
+            let recv_timeout = receiver
+                .into_future()
+                .timeout(Duration::from_millis(RECEIVE_DELAY_MILLIS))
+                .map(move |result| match result {
+                    (Some(value), receiver) => {
+                        inner_sender.send((value, receiver)).expect("send error")
+                    }
+                    (None, _) => panic!("unexpected end of stream"),
+                })
+                .map_err(|err| panic!("stream error {:?}", err));
 
-        std::thread::sleep(Duration::from_millis(INITIAL_DELAY_MS));
-        for _i in 0..5 {
-            let found = receiver.poll();
-            if let Ok(Async::Ready(Some(result))) = found {
-                return Ok(result);
-            }
-            std::thread::sleep(Duration::from_millis(RECEIVE_DELAY_MILLIS));
-        }
-        Err(RecvTimeoutError::Timeout)
-    }
-
-    pub(crate) fn robust_poll_or_panic<T>(receiver: futures::sync::mpsc::Receiver<T>) -> T {
-        robust_poll(receiver).unwrap_or_else(|err| panic!("expected response! {}", err))
+            const INITIAL_DELAY_MS: u64 = RECEIVE_DELAY_MILLIS * 2;
+            Delay::new(Instant::now() + Duration::from_millis(INITIAL_DELAY_MS))
+                .and_then(|_| recv_timeout)
+                .map_err(|err| panic!("timer error {:?}", err))
+        }));
+        inner_receiver.recv().expect("recv error")
     }
 
     #[test]
@@ -624,7 +678,10 @@ pub(crate) mod tests {
             Subscriber::new_test("accountNotification");
         let sub_id = SubscriptionId::Number(0 as u64);
         let exit = Arc::new(AtomicBool::new(false));
-        let subscriptions = RpcSubscriptions::new(&exit);
+        let subscriptions = RpcSubscriptions::new(
+            &exit,
+            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+        );
         subscriptions.add_account_subscription(alice.pubkey(), None, sub_id.clone(), subscriber);
 
         assert!(subscriptions
@@ -634,7 +691,7 @@ pub(crate) mod tests {
             .contains_key(&alice.pubkey()));
 
         subscriptions.notify_subscribers(0, &bank_forks);
-        let response = robust_poll_or_panic(transport_receiver);
+        let (response, _) = robust_poll_or_panic(transport_receiver);
         let expected = json!({
            "jsonrpc": "2.0",
            "method": "accountNotification",
@@ -693,7 +750,10 @@ pub(crate) mod tests {
             Subscriber::new_test("programNotification");
         let sub_id = SubscriptionId::Number(0 as u64);
         let exit = Arc::new(AtomicBool::new(false));
-        let subscriptions = RpcSubscriptions::new(&exit);
+        let subscriptions = RpcSubscriptions::new(
+            &exit,
+            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+        );
         subscriptions.add_program_subscription(
             solana_budget_program::id(),
             None,
@@ -708,7 +768,7 @@ pub(crate) mod tests {
             .contains_key(&solana_budget_program::id()));
 
         subscriptions.notify_subscribers(0, &bank_forks);
-        let response = robust_poll_or_panic(transport_receiver);
+        let (response, _) = robust_poll_or_panic(transport_receiver);
         let expected = json!({
            "jsonrpc": "2.0",
            "method": "programNotification",
@@ -773,27 +833,41 @@ pub(crate) mod tests {
             .unwrap()
             .process_transaction(&processed_tx)
             .unwrap();
+        let bank1 = bank_forks[1].clone();
 
         let bank_forks = Arc::new(RwLock::new(bank_forks));
 
-        let exit = Arc::new(AtomicBool::new(false));
-        let subscriptions = RpcSubscriptions::new(&exit);
+        let mut cache0 = BlockCommitment::default();
+        cache0.increase_confirmation_stake(1, 10);
+        let cache1 = BlockCommitment::default();
 
-        let (past_bank_sub, _id_receiver, past_bank_recv) =
+        let mut block_commitment = HashMap::new();
+        block_commitment.entry(0).or_insert(cache0.clone());
+        block_commitment.entry(1).or_insert(cache1.clone());
+        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 10, bank1, 0);
+
+        let exit = Arc::new(AtomicBool::new(false));
+        let subscriptions =
+            RpcSubscriptions::new(&exit, Arc::new(RwLock::new(block_commitment_cache)));
+
+        let (past_bank_sub1, _id_receiver, past_bank_recv1) =
+            Subscriber::new_test("signatureNotification");
+        let (past_bank_sub2, _id_receiver, past_bank_recv2) =
             Subscriber::new_test("signatureNotification");
         let (processed_sub, _id_receiver, processed_recv) =
             Subscriber::new_test("signatureNotification");
+
         subscriptions.add_signature_subscription(
             past_bank_tx.signatures[0],
             Some(0),
             SubscriptionId::Number(1 as u64),
-            Subscriber::new_test("signatureNotification").0,
+            past_bank_sub1,
         );
         subscriptions.add_signature_subscription(
             past_bank_tx.signatures[0],
             Some(1),
             SubscriptionId::Number(2 as u64),
-            past_bank_sub,
+            past_bank_sub2,
         );
         subscriptions.add_signature_subscription(
             processed_tx.signatures[0],
@@ -818,41 +892,46 @@ pub(crate) mod tests {
         subscriptions.notify_subscribers(1, &bank_forks);
         let expected_res: Option<transaction::Result<()>> = Some(Ok(()));
 
-        let expected = json!({
-            "jsonrpc": "2.0",
-            "method": "signatureNotification",
-            "params": {
-                "result": {
-                    "context": { "slot": 0 },
-                    "value": expected_res,
-                },
-                "subscription": 2,
-            }
-        });
-        let response = robust_poll_or_panic(past_bank_recv);
-        assert_eq!(serde_json::to_string(&expected).unwrap(), response);
+        struct Notification {
+            slot: Slot,
+            id: u64,
+        }
 
-        let expected = json!({
-            "jsonrpc": "2.0",
-            "method": "signatureNotification",
-            "params": {
-                "result": {
-                    "context": { "slot": 1 },
-                    "value": expected_res,
-                },
-                "subscription": 3,
-            }
-        });
-        let response = robust_poll_or_panic(processed_recv);
-        assert_eq!(serde_json::to_string(&expected).unwrap(), response);
+        let expected_notification = |exp: Notification| -> String {
+            let json = json!({
+                "jsonrpc": "2.0",
+                "method": "signatureNotification",
+                "params": {
+                    "result": {
+                        "context": { "slot": exp.slot },
+                        "value": &expected_res,
+                    },
+                    "subscription": exp.id,
+                }
+            });
+            serde_json::to_string(&json).unwrap()
+        };
 
-        let sig_subs = subscriptions.signature_subscriptions.read().unwrap();
+        // Expect to receive a notification from bank 1 because this subscription is
+        // looking for 0 confirmations and so checks the current bank
+        let expected = expected_notification(Notification { slot: 1, id: 1 });
+        let (response, _) = robust_poll_or_panic(past_bank_recv1);
+        assert_eq!(expected, response);
+
+        // Expect to receive a notification from bank 0 because this subscription is
+        // looking for 1 confirmation and so checks the past bank
+        let expected = expected_notification(Notification { slot: 0, id: 2 });
+        let (response, _) = robust_poll_or_panic(past_bank_recv2);
+        assert_eq!(expected, response);
+
+        let expected = expected_notification(Notification { slot: 1, id: 3 });
+        let (response, _) = robust_poll_or_panic(processed_recv);
+        assert_eq!(expected, response);
 
         // Subscription should be automatically removed after notification
+        let sig_subs = subscriptions.signature_subscriptions.read().unwrap();
         assert!(!sig_subs.contains_key(&processed_tx.signatures[0]));
-
-        // Only one notification is expected for signature processed in previous bank
-        assert_eq!(sig_subs.get(&past_bank_tx.signatures[0]).unwrap().len(), 1);
+        assert!(!sig_subs.contains_key(&past_bank_tx.signatures[0]));
 
         // Unprocessed signature subscription should not be removed
         assert_eq!(
@@ -867,7 +946,10 @@ pub(crate) mod tests {
             Subscriber::new_test("slotNotification");
         let sub_id = SubscriptionId::Number(0 as u64);
         let exit = Arc::new(AtomicBool::new(false));
-        let subscriptions = RpcSubscriptions::new(&exit);
+        let subscriptions = RpcSubscriptions::new(
+            &exit,
+            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+        );
         subscriptions.add_slot_subscription(sub_id.clone(), subscriber);
 
         assert!(subscriptions
@@ -877,7 +959,7 @@ pub(crate) mod tests {
             .contains_key(&sub_id));
 
         subscriptions.notify_slot(0, 0, 0);
-        let response = robust_poll_or_panic(transport_receiver);
+        let (response, _) = robust_poll_or_panic(transport_receiver);
         let expected_res = SlotInfo {
             parent: 0,
             slot: 0,
@@ -899,6 +981,46 @@ pub(crate) mod tests {
             .contains_key(&sub_id));
     }
 
+    #[test]
+    fn test_check_root_subscribe() {
+        let (subscriber, _id_receiver, mut transport_receiver) =
+            Subscriber::new_test("rootNotification");
+        let sub_id = SubscriptionId::Number(0 as u64);
+        let exit = Arc::new(AtomicBool::new(false));
+        let subscriptions = RpcSubscriptions::new(
+            &exit,
+            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+        );
+        subscriptions.add_root_subscription(sub_id.clone(), subscriber);
+
+        assert!(subscriptions
+            .root_subscriptions
+            .read()
+            .unwrap()
+            .contains_key(&sub_id));
+
+        subscriptions.notify_roots(vec![2, 1, 3]);
+
+        for expected_root in 1..=3 {
+            let (response, receiver) = robust_poll_or_panic(transport_receiver);
+            transport_receiver = receiver;
+            let expected_res_str =
+                serde_json::to_string(&serde_json::to_value(expected_root).unwrap()).unwrap();
+            let expected = format!(
+                r#"{{"jsonrpc":"2.0","method":"rootNotification","params":{{"result":{},"subscription":0}}}}"#,
+                expected_res_str
+            );
+            assert_eq!(expected, response);
+        }
+
+        subscriptions.remove_root_subscription(&sub_id);
+        assert!(!subscriptions
+            .root_subscriptions
+            .read()
+            .unwrap()
+            .contains_key(&sub_id));
+    }
+
     #[test]
     fn test_add_and_remove_subscription() {
         let mut subscriptions: HashMap<u64, HashMap<SubscriptionId, (Sink<()>, Confirmations)>> =

core/src/serve_repair.rs

@@ -1,23 +1,26 @@
-use crate::packet::limited_deserialize;
-use crate::streamer::{PacketReceiver, PacketSender};
 use crate::{
     cluster_info::{ClusterInfo, ClusterInfoError},
+    cluster_slots::ClusterSlots,
     contact_info::ContactInfo,
-    packet::Packet,
+    repair_service::RepairStats,
     result::{Error, Result},
+    weighted_shuffle::weighted_best,
 };
 use bincode::serialize;
-use rand::{thread_rng, Rng};
 use solana_ledger::blockstore::Blockstore;
+use solana_measure::measure::Measure;
 use solana_measure::thread_mem_usage;
 use solana_metrics::{datapoint_debug, inc_new_counter_debug};
-use solana_perf::packet::{Packets, PacketsRecycler};
+use solana_perf::packet::{limited_deserialize, Packet, Packets, PacketsRecycler};
 use solana_sdk::{
     clock::Slot,
+    pubkey::Pubkey,
     signature::{Keypair, Signer},
     timing::duration_as_ms,
 };
+use solana_streamer::streamer::{PacketReceiver, PacketSender};
 use std::{
+    collections::HashMap,
     net::SocketAddr,
     sync::atomic::{AtomicBool, Ordering},
     sync::{Arc, RwLock},
@@ -45,6 +48,17 @@ impl RepairType {
     }
 }
 
+#[derive(Default)]
+pub struct ServeRepairStats {
+    pub total_packets: usize,
+    pub dropped_packets: usize,
+    pub processed: usize,
+    pub self_repair: usize,
+    pub window_index: usize,
+    pub highest_window_index: usize,
+    pub orphan: usize,
+}
+
 /// Window protocol messages
 #[derive(Serialize, Deserialize, Debug)]
 enum RepairProtocol {
@@ -61,6 +75,8 @@ pub struct ServeRepair {
     cluster_info: Arc<RwLock<ClusterInfo>>,
 }
 
+type RepairCache = HashMap<Slot, (Vec<ContactInfo>, Vec<(u64, usize)>)>;
+
 impl ServeRepair {
     /// Without a valid keypair gossip will not function. Only useful for tests.
     pub fn new_with_invalid_keypair(contact_info: ContactInfo) -> Self {
@@ -103,6 +119,7 @@ impl ServeRepair {
         from_addr: &SocketAddr,
         blockstore: Option<&Arc<Blockstore>>,
         request: RepairProtocol,
+        stats: &mut ServeRepairStats,
     ) -> Option<Packets> {
         let now = Instant::now();
 
@@ -110,18 +127,14 @@ impl ServeRepair {
         let my_id = me.read().unwrap().keypair.pubkey();
         let from = Self::get_repair_sender(&request);
         if from.id == my_id {
-            warn!(
-                "{}: Ignored received repair request from ME {}",
-                my_id, from.id,
-            );
-            inc_new_counter_debug!("serve_repair-handle-repair--eq", 1);
+            stats.self_repair += 1;
             return None;
         }
 
         let (res, label) = {
             match &request {
                 RepairProtocol::WindowIndex(from, slot, shred_index) => {
-                    inc_new_counter_debug!("serve_repair-request-window-index", 1);
+                    stats.window_index += 1;
                     (
                         Self::run_window_request(
                             recycler,
@@ -137,7 +150,7 @@ impl ServeRepair {
                 }
 
                 RepairProtocol::HighestWindowIndex(_, slot, highest_index) => {
-                    inc_new_counter_debug!("serve_repair-request-highest-window-index", 1);
+                    stats.highest_window_index += 1;
                     (
                         Self::run_highest_window_request(
                             recycler,
@@ -150,7 +163,7 @@ impl ServeRepair {
                     )
                 }
                 RepairProtocol::Orphan(_, slot) => {
-                    inc_new_counter_debug!("serve_repair-request-orphan", 1);
+                    stats.orphan += 1;
                     (
                         Self::run_orphan(
                             recycler,
@@ -184,15 +197,71 @@ impl ServeRepair {
         blockstore: Option<&Arc<Blockstore>>,
         requests_receiver: &PacketReceiver,
         response_sender: &PacketSender,
+        stats: &mut ServeRepairStats,
+        max_packets: &mut usize,
     ) -> Result<()> {
         //TODO cache connections
         let timeout = Duration::new(1, 0);
-        let reqs = requests_receiver.recv_timeout(timeout)?;
+        let mut reqs_v = vec![requests_receiver.recv_timeout(timeout)?];
+        let mut total_packets = reqs_v[0].packets.len();
 
-        Self::handle_packets(obj, &recycler, blockstore, reqs, response_sender);
+        let mut dropped_packets = 0;
+        while let Ok(more) = requests_receiver.try_recv() {
+            total_packets += more.packets.len();
+            if total_packets < *max_packets {
+                // Drop the rest in the channel in case of dos
+                reqs_v.push(more);
+            } else {
+                dropped_packets += more.packets.len();
+            }
+        }
+
+        stats.dropped_packets += dropped_packets;
+        stats.total_packets += total_packets;
+
+        let mut time = Measure::start("repair::handle_packets");
+        for reqs in reqs_v {
+            Self::handle_packets(obj, &recycler, blockstore, reqs, response_sender, stats);
+        }
+        time.stop();
+        if total_packets >= *max_packets {
+            if time.as_ms() > 1000 {
+                *max_packets = (*max_packets * 9) / 10;
+            } else {
+                *max_packets = (*max_packets * 10) / 9;
+            }
+        }
         Ok(())
     }
 
+    fn report_reset_stats(me: &Arc<RwLock<Self>>, stats: &mut ServeRepairStats) {
+        if stats.self_repair > 0 {
+            let my_id = me.read().unwrap().keypair.pubkey();
+            warn!(
+                "{}: Ignored received repair requests from ME: {}",
+                my_id, stats.self_repair,
+            );
+            inc_new_counter_debug!("serve_repair-handle-repair--eq", stats.self_repair);
+        }
+
+        inc_new_counter_info!("serve_repair-total_packets", stats.total_packets);
+        inc_new_counter_info!("serve_repair-dropped_packets", stats.dropped_packets);
+
+        debug!(
+            "repair_listener: total_packets: {} passed: {}",
+            stats.total_packets, stats.processed
+        );
+
+        inc_new_counter_debug!("serve_repair-request-window-index", stats.window_index);
+        inc_new_counter_debug!(
+            "serve_repair-request-highest-window-index",
+            stats.highest_window_index
+        );
+        inc_new_counter_debug!("serve_repair-request-orphan", stats.orphan);
+
+        *stats = ServeRepairStats::default();
+    }
+
     pub fn listen(
         me: Arc<RwLock<Self>>,
         blockstore: Option<Arc<Blockstore>>,
@@ -204,22 +273,33 @@ impl ServeRepair {
         let recycler = PacketsRecycler::default();
         Builder::new()
             .name("solana-repair-listen".to_string())
-            .spawn(move || loop {
-                let result = Self::run_listen(
-                    &me,
-                    &recycler,
-                    blockstore.as_ref(),
-                    &requests_receiver,
-                    &response_sender,
-                );
-                match result {
-                    Err(Error::RecvTimeoutError(_)) | Ok(_) => {}
-                    Err(err) => info!("repair listener error: {:?}", err),
-                };
-                if exit.load(Ordering::Relaxed) {
-                    return;
+            .spawn(move || {
+                let mut last_print = Instant::now();
+                let mut stats = ServeRepairStats::default();
+                let mut max_packets = 1024;
+                loop {
+                    let result = Self::run_listen(
+                        &me,
+                        &recycler,
+                        blockstore.as_ref(),
+                        &requests_receiver,
+                        &response_sender,
+                        &mut stats,
+                        &mut max_packets,
+                    );
+                    match result {
+                        Err(Error::RecvTimeoutError(_)) | Ok(_) => {}
+                        Err(err) => info!("repair listener error: {:?}", err),
+                    };
+                    if exit.load(Ordering::Relaxed) {
+                        return;
+                    }
+                    if last_print.elapsed().as_secs() > 2 {
+                        Self::report_reset_stats(&me, &mut stats);
+                        last_print = Instant::now();
+                    }
+                    thread_mem_usage::datapoint("solana-repair-listen");
                 }
-                thread_mem_usage::datapoint("solana-repair-listen");
             })
             .unwrap()
     }
@@ -230,6 +310,7 @@ impl ServeRepair {
         blockstore: Option<&Arc<Blockstore>>,
         packets: Packets,
         response_sender: &PacketSender,
+        stats: &mut ServeRepairStats,
     ) {
         // iter over the packets, collect pulls separately and process everything else
         let allocated = thread_mem_usage::Allocatedp::default();
@@ -239,7 +320,9 @@ impl ServeRepair {
             limited_deserialize(&packet.data[..packet.meta.size])
                 .into_iter()
                 .for_each(|request| {
-                    let rsp = Self::handle_repair(me, recycler, &from_addr, blockstore, request);
+                    stats.processed += 1;
+                    let rsp =
+                        Self::handle_repair(me, recycler, &from_addr, blockstore, request, stats);
                     if let Some(rsp) = rsp {
                         let _ignore_disconnect = response_sender.send(rsp);
                     }
@@ -269,44 +352,50 @@ impl ServeRepair {
         Ok(out)
     }
 
-    pub fn repair_request(&self, repair_request: &RepairType) -> Result<(SocketAddr, Vec<u8>)> {
+    pub fn repair_request(
+        &self,
+        cluster_slots: &ClusterSlots,
+        repair_request: &RepairType,
+        cache: &mut RepairCache,
+        repair_stats: &mut RepairStats,
+    ) -> Result<(SocketAddr, Vec<u8>)> {
         // find a peer that appears to be accepting replication and has the desired slot, as indicated
         // by a valid tvu port location
-        let valid: Vec<_> = self
-            .cluster_info
-            .read()
-            .unwrap()
-            .repair_peers(repair_request.slot());
-        if valid.is_empty() {
-            return Err(ClusterInfoError::NoPeers.into());
+        if cache.get(&repair_request.slot()).is_none() {
+            let repair_peers: Vec<_> = self
+                .cluster_info
+                .read()
+                .unwrap()
+                .repair_peers(repair_request.slot());
+            if repair_peers.is_empty() {
+                return Err(ClusterInfoError::NoPeers.into());
+            }
+            let weights = cluster_slots.compute_weights(repair_request.slot(), &repair_peers);
+            cache.insert(repair_request.slot(), (repair_peers, weights));
         }
-        let n = thread_rng().gen::<usize>() % valid.len();
-        let addr = valid[n].serve_repair; // send the request to the peer's serve_repair port
-        let out = self.map_repair_request(repair_request)?;
-
+        let (repair_peers, weights) = cache.get(&repair_request.slot()).unwrap();
+        let n = weighted_best(&weights, Pubkey::new_rand().to_bytes());
+        let addr = repair_peers[n].serve_repair; // send the request to the peer's serve_repair port
+        let out = self.map_repair_request(repair_request, repair_stats)?;
         Ok((addr, out))
     }
 
-    pub fn map_repair_request(&self, repair_request: &RepairType) -> Result<Vec<u8>> {
+    pub fn map_repair_request(
+        &self,
+        repair_request: &RepairType,
+        repair_stats: &mut RepairStats,
+    ) -> Result<Vec<u8>> {
         match repair_request {
             RepairType::Shred(slot, shred_index) => {
-                datapoint_debug!(
-                    "serve_repair-repair",
-                    ("repair-slot", *slot, i64),
-                    ("repair-ix", *shred_index, i64)
-                );
+                repair_stats.shred.update(*slot);
                 Ok(self.window_index_request_bytes(*slot, *shred_index)?)
             }
             RepairType::HighestShred(slot, shred_index) => {
-                datapoint_info!(
-                    "serve_repair-repair_highest",
-                    ("repair-highest-slot", *slot, i64),
-                    ("repair-highest-ix", *shred_index, i64)
-                );
+                repair_stats.highest_shred.update(*slot);
                 Ok(self.window_highest_index_request_bytes(*slot, *shred_index)?)
             }
             RepairType::Orphan(slot) => {
-                datapoint_info!("serve_repair-repair_orphan", ("repair-orphan", *slot, i64));
+                repair_stats.orphan.update(*slot);
                 Ok(self.orphan_bytes(*slot)?)
             }
         }
@@ -563,10 +652,16 @@ mod tests {
 
     #[test]
     fn window_index_request() {
+        let cluster_slots = ClusterSlots::default();
         let me = ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp());
         let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(me)));
         let serve_repair = ServeRepair::new(cluster_info.clone());
-        let rv = serve_repair.repair_request(&RepairType::Shred(0, 0));
+        let rv = serve_repair.repair_request(
+            &cluster_slots,
+            &RepairType::Shred(0, 0),
+            &mut HashMap::new(),
+            &mut RepairStats::default(),
+        );
         assert_matches!(rv, Err(Error::ClusterInfoError(ClusterInfoError::NoPeers)));
 
         let serve_repair_addr = socketaddr!([127, 0, 0, 1], 1243);
@@ -587,7 +682,12 @@ mod tests {
         };
         cluster_info.write().unwrap().insert_info(nxt.clone());
         let rv = serve_repair
-            .repair_request(&RepairType::Shred(0, 0))
+            .repair_request(
+                &cluster_slots,
+                &RepairType::Shred(0, 0),
+                &mut HashMap::new(),
+                &mut RepairStats::default(),
+            )
             .unwrap();
         assert_eq!(nxt.serve_repair, serve_repair_addr);
         assert_eq!(rv.0, nxt.serve_repair);
@@ -614,7 +714,12 @@ mod tests {
         while !one || !two {
             //this randomly picks an option, so eventually it should pick both
             let rv = serve_repair
-                .repair_request(&RepairType::Shred(0, 0))
+                .repair_request(
+                    &cluster_slots,
+                    &RepairType::Shred(0, 0),
+                    &mut HashMap::new(),
+                    &mut RepairStats::default(),
+                )
                 .unwrap();
             if rv.0 == serve_repair_addr {
                 one = true;

core/src/serve_repair_service.rs

@@ -1,7 +1,7 @@
 use crate::serve_repair::ServeRepair;
-use crate::streamer;
 use solana_ledger::blockstore::Blockstore;
 use solana_perf::recycler::Recycler;
+use solana_streamer::streamer;
 use std::net::UdpSocket;
 use std::sync::atomic::AtomicBool;
 use std::sync::mpsc::channel;

core/src/shred_fetch_stage.rs

@@ -1,7 +1,5 @@
 //! The `shred_fetch_stage` pulls shreds from UDP sockets and sends it to a channel.
 
-use crate::packet::{Packet, PacketsRecycler};
-use crate::streamer::{self, PacketReceiver, PacketSender};
 use bv::BitVec;
 use solana_ledger::bank_forks::BankForks;
 use solana_ledger::blockstore::MAX_DATA_SHREDS_PER_SLOT;
@@ -9,9 +7,10 @@ use solana_ledger::shred::{
     OFFSET_OF_SHRED_INDEX, OFFSET_OF_SHRED_SLOT, SIZE_OF_SHRED_INDEX, SIZE_OF_SHRED_SLOT,
 };
 use solana_perf::cuda_runtime::PinnedVec;
-use solana_perf::packet::limited_deserialize;
+use solana_perf::packet::{limited_deserialize, Packet, PacketsRecycler};
 use solana_perf::recycler::Recycler;
 use solana_sdk::clock::Slot;
+use solana_streamer::streamer::{self, PacketReceiver, PacketSender};
 use std::collections::HashMap;
 use std::net::UdpSocket;
 use std::sync::atomic::AtomicBool;
@@ -102,9 +101,7 @@ impl ShredFetchStage {
                     last_root = bank_forks_r.root();
                     let working_bank = bank_forks_r.working_bank();
                     last_slot = working_bank.slot();
-                    let root_bank = bank_forks_r
-                        .get(bank_forks_r.root())
-                        .expect("Root bank should exist");
+                    let root_bank = bank_forks_r.root_bank();
                     slots_per_epoch = root_bank.get_slots_in_epoch(root_bank.epoch());
                 }
             }

core/src/sigverify_shreds.rs

@@ -1,11 +1,11 @@
 #![allow(clippy::implicit_hasher)]
-use crate::packet::{limited_deserialize, Packets};
 use crate::sigverify;
 use crate::sigverify_stage::SigVerifier;
 use solana_ledger::bank_forks::BankForks;
 use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
 use solana_ledger::shred::{OFFSET_OF_SHRED_SLOT, SIZE_OF_SHRED_SLOT};
 use solana_ledger::sigverify_shreds::verify_shreds_gpu;
+use solana_perf::packet::{limited_deserialize, Packets};
 use solana_perf::recycler_cache::RecyclerCache;
 use std::collections::{HashMap, HashSet};
 use std::sync::{Arc, RwLock};
@@ -72,9 +72,9 @@ impl SigVerifier for ShredSigVerifier {
 #[cfg(test)]
 pub mod tests {
     use super::*;
-    use crate::genesis_utils::create_genesis_config_with_leader;
-    use crate::packet::Packet;
+    use solana_ledger::genesis_utils::create_genesis_config_with_leader;
     use solana_ledger::shred::{Shred, Shredder};
+    use solana_perf::packet::Packet;
     use solana_runtime::bank::Bank;
     use solana_sdk::signature::{Keypair, Signer};
 

core/src/sigverify_stage.rs

@@ -5,14 +5,14 @@
 //! transaction. All processing is done on the CPU by default and on a GPU
 //! if perf-libs are available
 
-use crate::packet::Packets;
 use crate::sigverify;
-use crate::streamer::{self, PacketReceiver, StreamerError};
 use crossbeam_channel::{SendError, Sender as CrossbeamSender};
 use solana_measure::measure::Measure;
 use solana_metrics::datapoint_debug;
+use solana_perf::packet::Packets;
 use solana_perf::perf_libs;
 use solana_sdk::timing;
+use solana_streamer::streamer::{self, PacketReceiver, StreamerError};
 use std::sync::mpsc::{Receiver, RecvTimeoutError};
 use std::sync::{Arc, Mutex};
 use std::thread::{self, Builder, JoinHandle};

core/src/snapshot_packager_service.rs

@@ -137,10 +137,12 @@ mod tests {
             .collect();
 
         // Create some fake snapshot
-        fs::create_dir_all(&snapshots_dir).unwrap();
         let snapshots_paths: Vec<_> = (0..5)
             .map(|i| {
-                let fake_snapshot_path = snapshots_dir.join(format!("fake_snapshot_{}", i));
+                let snapshot_file_name = format!("{}", i);
+                let snapshots_dir = snapshots_dir.join(&snapshot_file_name);
+                fs::create_dir_all(&snapshots_dir).unwrap();
+                let fake_snapshot_path = snapshots_dir.join(&snapshot_file_name);
                 let mut fake_snapshot_file = OpenOptions::new()
                     .read(true)
                     .write(true)
@@ -157,7 +159,9 @@ mod tests {
         let link_snapshots_dir = tempfile::tempdir_in(&temp_dir).unwrap();
         for snapshots_path in snapshots_paths {
             let snapshot_file_name = snapshots_path.file_name().unwrap();
-            let link_path = link_snapshots_dir.path().join(snapshot_file_name);
+            let link_snapshots_dir = link_snapshots_dir.path().join(snapshot_file_name);
+            fs::create_dir_all(&link_snapshots_dir).unwrap();
+            let link_path = link_snapshots_dir.join(snapshot_file_name);
             fs::hard_link(&snapshots_path, &link_path).unwrap();
         }
 

core/src/storage_stage.rs

@@ -4,6 +4,7 @@
 
 use crate::{
     cluster_info::ClusterInfo,
+    commitment::BlockCommitmentCache,
     contact_info::ContactInfo,
     result::{Error, Result},
 };
@@ -11,9 +12,7 @@ use rand::{Rng, SeedableRng};
 use rand_chacha::ChaChaRng;
 use solana_chacha_cuda::chacha_cuda::chacha_cbc_encrypt_file_many_keys;
 use solana_ledger::{bank_forks::BankForks, blockstore::Blockstore};
-use solana_runtime::{
-    bank::Bank, status_cache::SignatureConfirmationStatus, storage_utils::archiver_accounts,
-};
+use solana_runtime::{bank::Bank, storage_utils::archiver_accounts};
 use solana_sdk::{
     account::Account,
     account_utils::StateMut,
@@ -30,6 +29,7 @@ use solana_storage_program::{
     storage_instruction,
     storage_instruction::proof_validation,
 };
+use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
 use std::{
     cmp,
     collections::HashMap,
@@ -185,6 +185,7 @@ impl StorageStage {
         exit: &Arc<AtomicBool>,
         bank_forks: &Arc<RwLock<BankForks>>,
         cluster_info: &Arc<RwLock<ClusterInfo>>,
+        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
     ) -> Self {
         let (instruction_sender, instruction_receiver) = channel();
 
@@ -256,6 +257,7 @@ impl StorageStage {
                                     &keypair,
                                     &storage_keypair,
                                     &transactions_socket,
+                                    &block_commitment_cache,
                                 )
                                 .unwrap_or_else(|err| {
                                     info!("failed to send storage transaction: {:?}", err)
@@ -289,6 +291,7 @@ impl StorageStage {
         keypair: &Arc<Keypair>,
         storage_keypair: &Arc<Keypair>,
         transactions_socket: &UdpSocket,
+        block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
     ) -> io::Result<()> {
         let working_bank = bank_forks.read().unwrap().working_bank();
         let blockhash = working_bank.confirmed_last_blockhash().0;
@@ -323,8 +326,13 @@ impl StorageStage {
                 cluster_info.read().unwrap().my_data().tpu,
             )?;
             sleep(Duration::from_millis(100));
-            if Self::poll_for_signature_confirmation(bank_forks, &transaction.signatures[0], 0)
-                .is_ok()
+            if Self::poll_for_signature_confirmation(
+                bank_forks,
+                block_commitment_cache,
+                &transaction.signatures[0],
+                0,
+            )
+            .is_ok()
             {
                 break;
             };
@@ -334,23 +342,24 @@ impl StorageStage {
 
     fn poll_for_signature_confirmation(
         bank_forks: &Arc<RwLock<BankForks>>,
+        block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
         signature: &Signature,
         min_confirmed_blocks: usize,
     ) -> Result<()> {
         let mut now = Instant::now();
         let mut confirmed_blocks = 0;
         loop {
-            let response = bank_forks
-                .read()
-                .unwrap()
-                .working_bank()
-                .get_signature_confirmation_status(signature);
-            if let Some(SignatureConfirmationStatus {
-                confirmations,
-                status,
-                ..
-            }) = response
-            {
+            let working_bank = bank_forks.read().unwrap().working_bank();
+            let response = working_bank.get_signature_status_slot(signature);
+            if let Some((slot, status)) = response {
+                let confirmations = if working_bank.src.roots().contains(&slot) {
+                    MAX_LOCKOUT_HISTORY + 1
+                } else {
+                    let r_block_commitment_cache = block_commitment_cache.read().unwrap();
+                    r_block_commitment_cache
+                        .get_confirmation_count(slot)
+                        .unwrap_or(0)
+                };
                 if status.is_ok() {
                     if confirmed_blocks != confirmations {
                         now = Instant::now();
@@ -652,15 +661,21 @@ pub fn test_cluster_info(id: &Pubkey) -> Arc<RwLock<ClusterInfo>> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use rayon::prelude::*;
+    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_runtime::bank::Bank;
-    use solana_sdk::hash::Hasher;
-    use solana_sdk::signature::{Keypair, Signer};
-    use std::cmp::{max, min};
-    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
-    use std::sync::mpsc::channel;
-    use std::sync::{Arc, RwLock};
+    use solana_sdk::{
+        hash::Hasher,
+        signature::{Keypair, Signer},
+    };
+    use std::{
+        cmp::{max, min},
+        sync::{
+            atomic::{AtomicBool, AtomicUsize, Ordering},
+            mpsc::channel,
+            Arc, RwLock,
+        },
+    };
 
     #[test]
     fn test_storage_stage_none_ledger() {
@@ -675,6 +690,7 @@ mod tests {
             &[bank.clone()],
             vec![0],
         )));
+        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
         let (_slot_sender, slot_receiver) = channel();
         let storage_state = StorageState::new(
             &bank.last_blockhash(),
@@ -690,6 +706,7 @@ mod tests {
             &exit.clone(),
             &bank_forks,
             &cluster_info,
+            block_commitment_cache,
         );
         exit.store(true, Ordering::Relaxed);
         storage_stage.join().unwrap();

core/src/tpu.rs

@@ -3,16 +3,18 @@
 
 use crate::{
     banking_stage::BankingStage,
-    broadcast_stage::{BroadcastStage, BroadcastStageType},
+    broadcast_stage::{BroadcastStage, BroadcastStageType, RetransmitSlotsReceiver},
     cluster_info::ClusterInfo,
-    cluster_info_vote_listener::ClusterInfoVoteListener,
+    cluster_info_vote_listener::{ClusterInfoVoteListener, VoteTracker},
     fetch_stage::FetchStage,
     poh_recorder::{PohRecorder, WorkingBankEntry},
     sigverify::TransactionSigVerifier,
     sigverify_stage::{DisabledSigVerifier, SigVerifyStage},
 };
 use crossbeam_channel::unbounded;
-use solana_ledger::{blockstore::Blockstore, blockstore_processor::TransactionStatusSender};
+use solana_ledger::{
+    bank_forks::BankForks, blockstore::Blockstore, blockstore_processor::TransactionStatusSender,
+};
 use std::{
     net::UdpSocket,
     sync::{
@@ -37,6 +39,7 @@ impl Tpu {
         cluster_info: &Arc<RwLock<ClusterInfo>>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
         entry_receiver: Receiver<WorkingBankEntry>,
+        retransmit_slots_receiver: RetransmitSlotsReceiver,
         transactions_sockets: Vec<UdpSocket>,
         tpu_forwards_sockets: Vec<UdpSocket>,
         broadcast_sockets: Vec<UdpSocket>,
@@ -46,6 +49,8 @@ impl Tpu {
         broadcast_type: &BroadcastStageType,
         exit: &Arc<AtomicBool>,
         shred_version: u16,
+        vote_tracker: Arc<VoteTracker>,
+        bank_forks: Arc<RwLock<BankForks>>,
     ) -> Self {
         let (packet_sender, packet_receiver) = channel();
         let fetch_stage = FetchStage::new_with_sender(
@@ -72,6 +77,8 @@ impl Tpu {
             sigverify_disabled,
             verified_vote_sender,
             &poh_recorder,
+            vote_tracker,
+            bank_forks,
         );
 
         let banking_stage = BankingStage::new(
@@ -86,6 +93,7 @@ impl Tpu {
             broadcast_sockets,
             cluster_info.clone(),
             entry_receiver,
+            retransmit_slots_receiver,
             &exit,
             blockstore,
             shred_version,

core/src/transaction_status_service.rs

@@ -1,10 +1,10 @@
 use crossbeam_channel::{Receiver, RecvTimeoutError};
-use solana_client::rpc_response::RpcTransactionStatusMeta;
 use solana_ledger::{blockstore::Blockstore, blockstore_processor::TransactionStatusBatch};
 use solana_runtime::{
     bank::{Bank, HashAgeKind},
     nonce_utils,
 };
+use solana_transaction_status::TransactionStatusMeta;
 use std::{
     sync::{
         atomic::{AtomicBool, Ordering},
@@ -73,7 +73,7 @@ impl TransactionStatusService {
                 blockstore
                     .write_transaction_status(
                         (slot, transaction.signatures[0]),
-                        &RpcTransactionStatusMeta {
+                        &TransactionStatusMeta {
                             status,
                             fee,
                             pre_balances,

core/src/tvu.rs

@@ -2,9 +2,12 @@
 //! validation pipeline in software.
 
 use crate::{
+    accounts_cleanup_service::AccountsCleanupService,
     accounts_hash_verifier::AccountsHashVerifier,
-    blockstream_service::BlockstreamService,
+    broadcast_stage::RetransmitSlotsSender,
     cluster_info::ClusterInfo,
+    cluster_info_vote_listener::VoteTracker,
+    cluster_slots::ClusterSlots,
     commitment::BlockCommitmentCache,
     ledger_cleanup_service::LedgerCleanupService,
     poh_recorder::PohRecorder,
@@ -32,7 +35,6 @@ use solana_sdk::{
 use std::collections::HashSet;
 use std::{
     net::UdpSocket,
-    path::PathBuf,
     sync::{
         atomic::AtomicBool,
         mpsc::{channel, Receiver},
@@ -46,8 +48,8 @@ pub struct Tvu {
     sigverify_stage: SigVerifyStage,
     retransmit_stage: RetransmitStage,
     replay_stage: ReplayStage,
-    blockstream_service: Option<BlockstreamService>,
     ledger_cleanup_service: Option<LedgerCleanupService>,
+    accounts_cleanup_service: AccountsCleanupService,
     storage_stage: StorageStage,
     accounts_hash_verifier: AccountsHashVerifier,
 }
@@ -79,14 +81,13 @@ impl Tvu {
     #[allow(clippy::new_ret_no_self, clippy::too_many_arguments)]
     pub fn new(
         vote_account: &Pubkey,
-        voting_keypair: Option<Arc<Keypair>>,
+        authorized_voter_keypairs: Vec<Arc<Keypair>>,
         storage_keypair: &Arc<Keypair>,
         bank_forks: &Arc<RwLock<BankForks>>,
         cluster_info: &Arc<RwLock<ClusterInfo>>,
         sockets: Sockets,
         blockstore: Arc<Blockstore>,
         storage_state: &StorageState,
-        blockstream_unix_socket: Option<&PathBuf>,
         ledger_signal_receiver: Receiver<bool>,
         subscriptions: &Arc<RpcSubscriptions>,
         poh_recorder: &Arc<Mutex<PohRecorder>>,
@@ -98,6 +99,8 @@ impl Tvu {
         transaction_status_sender: Option<TransactionStatusSender>,
         rewards_recorder_sender: Option<RewardsRecorderSender>,
         snapshot_package_sender: Option<SnapshotPackageSender>,
+        vote_tracker: Arc<VoteTracker>,
+        retransmit_slots_sender: RetransmitSlotsSender,
         tvu_config: TvuConfig,
     ) -> Self {
         let keypair: Arc<Keypair> = cluster_info
@@ -143,6 +146,7 @@ impl Tvu {
             )
         };
 
+        let cluster_slots = Arc::new(ClusterSlots::default());
         let retransmit_stage = RetransmitStage::new(
             bank_forks.clone(),
             leader_schedule_cache,
@@ -156,9 +160,9 @@ impl Tvu {
             *bank_forks.read().unwrap().working_bank().epoch_schedule(),
             cfg,
             tvu_config.shred_version,
+            cluster_slots.clone(),
         );
 
-        let (blockstream_slot_sender, blockstream_slot_receiver) = channel();
         let (ledger_cleanup_slot_sender, ledger_cleanup_slot_receiver) = channel();
 
         let (accounts_hash_sender, accounts_hash_receiver) = channel();
@@ -175,14 +179,13 @@ impl Tvu {
         let replay_stage_config = ReplayStageConfig {
             my_pubkey: keypair.pubkey(),
             vote_account: *vote_account,
-            voting_keypair,
+            authorized_voter_keypairs,
             exit: exit.clone(),
             subscriptions: subscriptions.clone(),
             leader_schedule_cache: leader_schedule_cache.clone(),
-            slot_full_senders: vec![blockstream_slot_sender],
             latest_root_senders: vec![ledger_cleanup_slot_sender],
             accounts_hash_sender: Some(accounts_hash_sender),
-            block_commitment_cache,
+            block_commitment_cache: block_commitment_cache.clone(),
             transaction_status_sender,
             rewards_recorder_sender,
         };
@@ -194,20 +197,11 @@ impl Tvu {
             cluster_info.clone(),
             ledger_signal_receiver,
             poh_recorder.clone(),
+            vote_tracker,
+            cluster_slots,
+            retransmit_slots_sender,
         );
 
-        let blockstream_service = if let Some(blockstream_unix_socket) = blockstream_unix_socket {
-            let blockstream_service = BlockstreamService::new(
-                blockstream_slot_receiver,
-                blockstore.clone(),
-                blockstream_unix_socket,
-                &exit,
-            );
-            Some(blockstream_service)
-        } else {
-            None
-        };
-
         let ledger_cleanup_service = tvu_config.max_ledger_slots.map(|max_ledger_slots| {
             LedgerCleanupService::new(
                 ledger_cleanup_slot_receiver,
@@ -217,6 +211,8 @@ impl Tvu {
             )
         });
 
+        let accounts_cleanup_service = AccountsCleanupService::new(bank_forks.clone(), &exit);
+
         let storage_stage = StorageStage::new(
             storage_state,
             root_bank_receiver,
@@ -226,6 +222,7 @@ impl Tvu {
             &exit,
             &bank_forks,
             &cluster_info,
+            block_commitment_cache,
         );
 
         Tvu {
@@ -233,8 +230,8 @@ impl Tvu {
             sigverify_stage,
             retransmit_stage,
             replay_stage,
-            blockstream_service,
             ledger_cleanup_service,
+            accounts_cleanup_service,
             storage_stage,
             accounts_hash_verifier,
         }
@@ -245,12 +242,10 @@ impl Tvu {
         self.fetch_stage.join()?;
         self.sigverify_stage.join()?;
         self.storage_stage.join()?;
-        if self.blockstream_service.is_some() {
-            self.blockstream_service.unwrap().join()?;
-        }
         if self.ledger_cleanup_service.is_some() {
             self.ledger_cleanup_service.unwrap().join()?;
         }
+        self.accounts_cleanup_service.join()?;
         self.replay_stage.join()?;
         self.accounts_hash_verifier.join()?;
         Ok(())
@@ -262,8 +257,8 @@ pub mod tests {
     use super::*;
     use crate::banking_stage::create_test_recorder;
     use crate::cluster_info::{ClusterInfo, Node};
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_ledger::create_new_tmp_ledger;
+    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_runtime::bank::Bank;
     use std::sync::atomic::Ordering;
 
@@ -292,13 +287,14 @@ pub mod tests {
         let bank = bank_forks.working_bank();
         let (exit, poh_recorder, poh_service, _entry_receiver) =
             create_test_recorder(&bank, &blockstore, None);
-        let voting_keypair = Keypair::new();
+        let vote_keypair = Keypair::new();
         let storage_keypair = Arc::new(Keypair::new());
         let leader_schedule_cache = Arc::new(LeaderScheduleCache::new_from_bank(&bank));
         let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
+        let (retransmit_slots_sender, _retransmit_slots_receiver) = unbounded();
         let tvu = Tvu::new(
-            &voting_keypair.pubkey(),
-            Some(Arc::new(voting_keypair)),
+            &vote_keypair.pubkey(),
+            vec![Arc::new(vote_keypair)],
             &storage_keypair,
             &Arc::new(RwLock::new(bank_forks)),
             &cref1,
@@ -312,9 +308,11 @@ pub mod tests {
             },
             blockstore,
             &StorageState::default(),
-            None,
             l_receiver,
-            &Arc::new(RpcSubscriptions::new(&exit)),
+            &Arc::new(RpcSubscriptions::new(
+                &exit,
+                Arc::new(RwLock::new(BlockCommitmentCache::default())),
+            )),
             &poh_recorder,
             &leader_schedule_cache,
             &exit,
@@ -324,6 +322,8 @@ pub mod tests {
             None,
             None,
             None,
+            Arc::new(VoteTracker::new(&bank)),
+            retransmit_slots_sender,
             TvuConfig::default(),
         );
         exit.store(true, Ordering::Relaxed);

core/src/validator.rs

@@ -3,6 +3,7 @@
 use crate::{
     broadcast_stage::BroadcastStageType,
     cluster_info::{ClusterInfo, Node},
+    cluster_info_vote_listener::VoteTracker,
     commitment::BlockCommitmentCache,
     contact_info::ContactInfo,
     gossip_service::{discover_cluster, GossipService},
@@ -29,6 +30,7 @@ use solana_ledger::{
     blockstore::{Blockstore, CompletedSlotsReceiver},
     blockstore_processor::{self, BankForksInfo},
     create_new_tmp_ledger,
+    hardened_unpack::open_genesis_config,
     leader_schedule::FixedSchedule,
     leader_schedule_cache::LeaderScheduleCache,
 };
@@ -63,7 +65,6 @@ pub struct ValidatorConfig {
     pub expected_genesis_hash: Option<Hash>,
     pub expected_shred_version: Option<u16>,
     pub voting_disabled: bool,
-    pub blockstream_unix_socket: Option<PathBuf>,
     pub storage_slots_per_turn: u64,
     pub account_paths: Vec<PathBuf>,
     pub rpc_config: JsonRpcConfig,
@@ -90,7 +91,6 @@ impl Default for ValidatorConfig {
             expected_genesis_hash: None,
             expected_shred_version: None,
             voting_disabled: false,
-            blockstream_unix_socket: None,
             storage_slots_per_turn: DEFAULT_SLOTS_PER_TURN,
             max_ledger_slots: None,
             account_paths: Vec::new(),
@@ -151,7 +151,7 @@ impl Validator {
         keypair: &Arc<Keypair>,
         ledger_path: &Path,
         vote_account: &Pubkey,
-        authorized_voter: &Arc<Keypair>,
+        mut authorized_voter_keypairs: Vec<Arc<Keypair>>,
         storage_keypair: &Arc<Keypair>,
         entrypoint_info_option: Option<&ContactInfo>,
         poh_verify: bool,
@@ -162,7 +162,15 @@ impl Validator {
 
         warn!("identity: {}", id);
         warn!("vote account: {}", vote_account);
-        warn!("authorized voter: {}", authorized_voter.pubkey());
+
+        if config.voting_disabled {
+            warn!("voting disabled");
+            authorized_voter_keypairs.clear();
+        } else {
+            for authorized_voter_keypair in &authorized_voter_keypairs {
+                warn!("authorized voter: {}", authorized_voter_keypair.pubkey());
+            }
+        }
         report_target_features();
 
         info!("entrypoint: {:?}", entrypoint_info_option);
@@ -181,12 +189,7 @@ impl Validator {
             completed_slots_receiver,
             leader_schedule_cache,
             snapshot_hash,
-        ) = new_banks_from_blockstore(
-            config.expected_genesis_hash,
-            ledger_path,
-            poh_verify,
-            config,
-        );
+        ) = new_banks_from_blockstore(config, ledger_path, poh_verify);
 
         let leader_schedule_cache = Arc::new(leader_schedule_cache);
         let exit = Arc::new(AtomicBool::new(false));
@@ -239,7 +242,7 @@ impl Validator {
 
         let blockstore = Arc::new(blockstore);
 
-        let subscriptions = Arc::new(RpcSubscriptions::new(&exit));
+        let subscriptions = Arc::new(RpcSubscriptions::new(&exit, block_commitment_cache.clone()));
 
         let rpc_service = config.rpc_ports.map(|(rpc_port, rpc_pubsub_port)| {
             if ContactInfo::is_valid_address(&node.info.rpc) {
@@ -386,13 +389,12 @@ impl Validator {
             "New shred signal for the TVU should be the same as the clear bank signal."
         );
 
+        let vote_tracker = Arc::new({ VoteTracker::new(bank_forks.read().unwrap().root_bank()) });
+
+        let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
         let tvu = Tvu::new(
             vote_account,
-            if config.voting_disabled {
-                None
-            } else {
-                Some(authorized_voter.clone())
-            },
+            authorized_voter_keypairs,
             storage_keypair,
             &bank_forks,
             &cluster_info,
@@ -423,7 +425,6 @@ impl Validator {
             },
             blockstore.clone(),
             &storage_state,
-            config.blockstream_unix_socket.as_ref(),
             ledger_signal_receiver,
             &subscriptions,
             &poh_recorder,
@@ -435,6 +436,8 @@ impl Validator {
             transaction_status_sender.clone(),
             rewards_recorder_sender,
             snapshot_package_sender,
+            vote_tracker.clone(),
+            retransmit_slots_sender,
             TvuConfig {
                 max_ledger_slots: config.max_ledger_slots,
                 sigverify_disabled: config.dev_sigverify_disabled,
@@ -454,6 +457,7 @@ impl Validator {
             &cluster_info,
             &poh_recorder,
             entry_receiver,
+            retransmit_slots_receiver,
             node.sockets.tpu,
             node.sockets.tpu_forwards,
             node.sockets.broadcast,
@@ -463,6 +467,8 @@ impl Validator {
             &config.broadcast_stage_type,
             &exit,
             node.info.shred_version,
+            vote_tracker,
+            bank_forks,
         );
 
         datapoint_info!("validator-new", ("id", id.to_string(), String));
@@ -551,10 +557,9 @@ impl Validator {
 
 #[allow(clippy::type_complexity)]
 fn new_banks_from_blockstore(
-    expected_genesis_hash: Option<Hash>,
+    config: &ValidatorConfig,
     blockstore_path: &Path,
     poh_verify: bool,
-    config: &ValidatorConfig,
 ) -> (
     GenesisConfig,
     BankForks,
@@ -565,10 +570,7 @@ fn new_banks_from_blockstore(
     LeaderScheduleCache,
     Option<(Slot, Hash)>,
 ) {
-    let genesis_config = GenesisConfig::load(blockstore_path).unwrap_or_else(|err| {
-        error!("Failed to load genesis from {:?}: {}", blockstore_path, err);
-        process::exit(1);
-    });
+    let genesis_config = open_genesis_config(blockstore_path);
 
     // This needs to be limited otherwise the state in the VoteAccount data
     // grows too large
@@ -580,7 +582,7 @@ fn new_banks_from_blockstore(
     let genesis_hash = genesis_config.hash();
     info!("genesis hash: {}", genesis_hash);
 
-    if let Some(expected_genesis_hash) = expected_genesis_hash {
+    if let Some(expected_genesis_hash) = config.expected_genesis_hash {
         if genesis_hash != expected_genesis_hash {
             error!("genesis hash mismatch: expected {}", expected_genesis_hash);
             error!(
@@ -672,7 +674,7 @@ pub struct TestValidatorOptions {
 
 impl Default for TestValidatorOptions {
     fn default() -> Self {
-        use crate::genesis_utils::BOOTSTRAP_VALIDATOR_LAMPORTS;
+        use solana_ledger::genesis_utils::BOOTSTRAP_VALIDATOR_LAMPORTS;
         TestValidatorOptions {
             fees: 0,
             bootstrap_validator_lamports: BOOTSTRAP_VALIDATOR_LAMPORTS,
@@ -686,7 +688,9 @@ impl TestValidator {
     }
 
     pub fn run_with_options(options: TestValidatorOptions) -> Self {
-        use crate::genesis_utils::{create_genesis_config_with_leader_ex, GenesisConfigInfo};
+        use solana_ledger::genesis_utils::{
+            create_genesis_config_with_leader_ex, GenesisConfigInfo,
+        };
         use solana_sdk::fee_calculator::FeeRateGovernor;
 
         let TestValidatorOptions {
@@ -728,7 +732,7 @@ impl TestValidator {
             &node_keypair,
             &ledger_path,
             &leader_voting_keypair.pubkey(),
-            &leader_voting_keypair,
+            vec![leader_voting_keypair.clone()],
             &storage_keypair,
             None,
             true,
@@ -756,16 +760,19 @@ fn report_target_features() {
         }
     );
 
-    // Validator binaries built on a machine with AVX support will generate invalid opcodes
-    // when run on machines without AVX causing a non-obvious process abort.  Instead detect
-    // the mismatch and error cleanly.
-    #[target_feature(enable = "avx")]
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
     {
-        if is_x86_feature_detected!("avx") {
-            info!("AVX detected");
-        } else {
-            error!("Your machine does not have AVX support, please rebuild from source on your machine");
-            process::exit(1);
+        // Validator binaries built on a machine with AVX support will generate invalid opcodes
+        // when run on machines without AVX causing a non-obvious process abort.  Instead detect
+        // the mismatch and error cleanly.
+        #[target_feature(enable = "avx")]
+        {
+            if is_x86_feature_detected!("avx") {
+                info!("AVX detected");
+            } else {
+                error!("Your machine does not have AVX support, please rebuild from source on your machine");
+                process::exit(1);
+            }
         }
     }
 }
@@ -803,7 +810,7 @@ fn get_stake_percent_in_gossip(bank: &Arc<Bank>, cluster_info: &Arc<RwLock<Clust
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::genesis_utils::create_genesis_config_with_leader;
+    use solana_ledger::genesis_utils::create_genesis_config_with_leader;
     use std::fs::remove_dir_all;
 
     #[test]
@@ -833,7 +840,7 @@ mod tests {
             &Arc::new(validator_keypair),
             &validator_ledger_path,
             &voting_keypair.pubkey(),
-            &voting_keypair,
+            vec![voting_keypair.clone()],
             &storage_keypair,
             Some(&leader_node.info),
             true,
@@ -858,7 +865,7 @@ mod tests {
                         .genesis_config;
                 let (validator_ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
                 ledger_paths.push(validator_ledger_path.clone());
-                let voting_keypair = Arc::new(Keypair::new());
+                let vote_account_keypair = Arc::new(Keypair::new());
                 let storage_keypair = Arc::new(Keypair::new());
                 let config = ValidatorConfig {
                     rpc_ports: Some((
@@ -871,8 +878,8 @@ mod tests {
                     validator_node,
                     &Arc::new(validator_keypair),
                     &validator_ledger_path,
-                    &voting_keypair.pubkey(),
-                    &voting_keypair,
+                    &vote_account_keypair.pubkey(),
+                    vec![vote_account_keypair.clone()],
                     &storage_keypair,
                     Some(&leader_node.info),
                     true,
@@ -881,7 +888,7 @@ mod tests {
             })
             .collect();
 
-        // Each validator can exit in parallel to speed many sequential calls to `join`
+        // Each validator can exit in parallel to speed many sequential calls to join`
         validators.iter_mut().for_each(|v| v.exit());
         // While join is called sequentially, the above exit call notified all the
         // validators to exit from all their threads

core/src/verified_vote_packets.rs

@@ -0,0 +1,163 @@
+use crate::{
+    cluster_info_vote_listener::VerifiedVotePacketsReceiver, crds_value::CrdsValueLabel,
+    result::Result,
+};
+use solana_perf::packet::Packets;
+use std::{collections::HashMap, ops::Deref, time::Duration};
+
+#[derive(Default)]
+pub struct VerifiedVotePackets(HashMap<CrdsValueLabel, (u64, Packets)>);
+
+impl Deref for VerifiedVotePackets {
+    type Target = HashMap<CrdsValueLabel, (u64, Packets)>;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl VerifiedVotePackets {
+    pub fn get_and_process_vote_packets(
+        &mut self,
+        vote_packets_receiver: &VerifiedVotePacketsReceiver,
+        last_update_version: &mut u64,
+    ) -> Result<()> {
+        let timer = Duration::from_millis(200);
+        let vote_packets = vote_packets_receiver.recv_timeout(timer)?;
+        *last_update_version += 1;
+        for (label, packet) in vote_packets {
+            self.0.insert(label, (*last_update_version, packet));
+        }
+        while let Ok(vote_packets) = vote_packets_receiver.try_recv() {
+            for (label, packet) in vote_packets {
+                self.0.insert(label, (*last_update_version, packet));
+            }
+        }
+        Ok(())
+    }
+
+    pub fn get_latest_votes(&self, last_update_version: u64) -> (u64, Vec<Packets>) {
+        let mut new_update_version = last_update_version;
+        let msgs: Vec<_> = self
+            .iter()
+            .filter_map(|(_, (msg_update_version, msg))| {
+                if *msg_update_version > last_update_version {
+                    new_update_version = std::cmp::max(*msg_update_version, new_update_version);
+                    Some(msg)
+                } else {
+                    None
+                }
+            })
+            .cloned()
+            .collect();
+        (new_update_version, msgs)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::result::Error;
+    use crossbeam_channel::{unbounded, RecvTimeoutError};
+    use solana_perf::packet::{Meta, Packet};
+    use solana_sdk::pubkey::Pubkey;
+
+    #[test]
+    fn test_get_latest_votes() {
+        let pubkey = Pubkey::new_rand();
+        let label1 = CrdsValueLabel::Vote(0 as u8, pubkey);
+        let label2 = CrdsValueLabel::Vote(1 as u8, pubkey);
+        let mut verified_vote_packets = VerifiedVotePackets(HashMap::new());
+
+        let data = Packet {
+            meta: Meta {
+                repair: true,
+                ..Meta::default()
+            },
+            ..Packet::default()
+        };
+
+        let none_empty_packets = Packets::new(vec![data, Packet::default()]);
+
+        verified_vote_packets
+            .0
+            .insert(label1, (2, none_empty_packets));
+        verified_vote_packets
+            .0
+            .insert(label2, (1, Packets::default()));
+
+        // Both updates have timestamps greater than 0, so both should be returned
+        let (new_update_version, updates) = verified_vote_packets.get_latest_votes(0);
+        assert_eq!(new_update_version, 2);
+        assert_eq!(updates.len(), 2);
+
+        // Only the nonempty packet had a timestamp greater than 1
+        let (new_update_version, updates) = verified_vote_packets.get_latest_votes(1);
+        assert_eq!(new_update_version, 2);
+        assert_eq!(updates.len(), 1);
+        assert!(updates[0].packets.len() > 0);
+
+        // If the given timestamp is greater than all timestamps in any update,
+        // returned timestamp should be the same as the given timestamp, and
+        // no updates should be returned
+        let (new_update_version, updates) = verified_vote_packets.get_latest_votes(3);
+        assert_eq!(new_update_version, 3);
+        assert!(updates.is_empty());
+    }
+
+    #[test]
+    fn test_get_and_process_vote_packets() {
+        let (s, r) = unbounded();
+        let pubkey = Pubkey::new_rand();
+        let label1 = CrdsValueLabel::Vote(0 as u8, pubkey);
+        let label2 = CrdsValueLabel::Vote(1 as u8, pubkey);
+        let mut update_version = 0;
+        s.send(vec![(label1.clone(), Packets::default())]).unwrap();
+        s.send(vec![(label2.clone(), Packets::default())]).unwrap();
+
+        let data = Packet {
+            meta: Meta {
+                repair: true,
+                ..Meta::default()
+            },
+            ..Packet::default()
+        };
+
+        let later_packets = Packets::new(vec![data, Packet::default()]);
+        s.send(vec![(label1.clone(), later_packets.clone())])
+            .unwrap();
+        let mut verified_vote_packets = VerifiedVotePackets(HashMap::new());
+        verified_vote_packets
+            .get_and_process_vote_packets(&r, &mut update_version)
+            .unwrap();
+
+        // Test timestamps for same batch are the same
+        let update_version1 = verified_vote_packets.get(&label1).unwrap().0;
+        assert_eq!(
+            update_version1,
+            verified_vote_packets.get(&label2).unwrap().0
+        );
+
+        // Test the later value overwrote the earlier one for this label
+        assert!(verified_vote_packets.get(&label1).unwrap().1.packets.len() > 1);
+        assert_eq!(
+            verified_vote_packets.get(&label2).unwrap().1.packets.len(),
+            0
+        );
+
+        // Test timestamp for next batch overwrites the original
+        s.send(vec![(label2.clone(), Packets::default())]).unwrap();
+        verified_vote_packets
+            .get_and_process_vote_packets(&r, &mut update_version)
+            .unwrap();
+        let update_version2 = verified_vote_packets.get(&label2).unwrap().0;
+        assert!(update_version2 > update_version1);
+
+        // Test empty doesn't bump the version
+        let before = update_version;
+        assert_matches!(
+            verified_vote_packets.get_and_process_vote_packets(&r, &mut update_version),
+            Err(Error::CrossbeamRecvTimeoutError(RecvTimeoutError::Timeout))
+        );
+        assert_eq!(before, update_version);
+    }
+}

core/src/window_service.rs

@@ -1,11 +1,12 @@
 //! `window_service` handles the data plane incoming shreds, storing them in
 //!   blockstore and retransmitting where required
 //!
-use crate::cluster_info::ClusterInfo;
-use crate::packet::Packets;
-use crate::repair_service::{RepairService, RepairStrategy};
-use crate::result::{Error, Result};
-use crate::streamer::PacketSender;
+use crate::{
+    cluster_info::ClusterInfo,
+    cluster_slots::ClusterSlots,
+    repair_service::{RepairService, RepairStrategy},
+    result::{Error, Result},
+};
 use crossbeam_channel::{
     unbounded, Receiver as CrossbeamReceiver, RecvTimeoutError, Sender as CrossbeamSender,
 };
@@ -13,14 +14,18 @@ use rayon::iter::IntoParallelRefMutIterator;
 use rayon::iter::ParallelIterator;
 use rayon::ThreadPool;
 use solana_ledger::bank_forks::BankForks;
-use solana_ledger::blockstore::{self, Blockstore, MAX_DATA_SHREDS_PER_SLOT};
+use solana_ledger::blockstore::{
+    self, Blockstore, BlockstoreInsertionMetrics, MAX_DATA_SHREDS_PER_SLOT,
+};
 use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
 use solana_ledger::shred::Shred;
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_error};
+use solana_perf::packet::Packets;
 use solana_rayon_threadlimit::get_thread_count;
 use solana_runtime::bank::Bank;
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::timing::duration_as_ms;
+use solana_streamer::streamer::PacketSender;
 use std::net::UdpSocket;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
@@ -107,6 +112,7 @@ fn run_insert<F>(
     blockstore: &Arc<Blockstore>,
     leader_schedule_cache: &Arc<LeaderScheduleCache>,
     handle_duplicate: F,
+    metrics: &mut BlockstoreInsertionMetrics,
 ) -> Result<()>
 where
     F: Fn(Shred) -> (),
@@ -118,14 +124,13 @@ where
         shreds.append(&mut more_shreds)
     }
 
-    let blockstore_insert_metrics = blockstore.insert_shreds_handle_duplicate(
+    blockstore.insert_shreds_handle_duplicate(
         shreds,
         Some(leader_schedule_cache),
         false,
         &handle_duplicate,
+        metrics,
     )?;
-    blockstore_insert_metrics.report_metrics("recv-window-insert-shreds");
-
     Ok(())
 }
 
@@ -252,6 +257,7 @@ impl WindowService {
         repair_strategy: RepairStrategy,
         leader_schedule_cache: &Arc<LeaderScheduleCache>,
         shred_filter: F,
+        cluster_slots: Arc<ClusterSlots>,
     ) -> WindowService
     where
         F: 'static
@@ -270,6 +276,7 @@ impl WindowService {
             repair_socket,
             cluster_info.clone(),
             repair_strategy,
+            cluster_slots,
         );
 
         let (insert_sender, insert_receiver) = unbounded();
@@ -353,6 +360,8 @@ impl WindowService {
                 let handle_duplicate = |shred| {
                     let _ = duplicate_sender.send(shred);
                 };
+                let mut metrics = BlockstoreInsertionMetrics::default();
+                let mut last_print = Instant::now();
                 loop {
                     if exit.load(Ordering::Relaxed) {
                         break;
@@ -363,11 +372,18 @@ impl WindowService {
                         &blockstore,
                         &leader_schedule_cache,
                         &handle_duplicate,
+                        &mut metrics,
                     ) {
                         if Self::should_exit_on_error(e, &mut handle_timeout, &handle_error) {
                             break;
                         }
                     }
+
+                    if last_print.elapsed().as_secs() > 2 {
+                        metrics.report_metrics("recv-window-insert-shreds");
+                        metrics = BlockstoreInsertionMetrics::default();
+                        last_print = Instant::now();
+                    }
                 }
             })
             .unwrap()
@@ -476,20 +492,18 @@ impl WindowService {
 mod test {
     use super::*;
     use crate::{
-        cluster_info::ClusterInfo,
-        contact_info::ContactInfo,
-        genesis_utils::create_genesis_config_with_leader,
-        packet::{Packet, Packets},
-        repair_service::RepairSlotRange,
+        cluster_info::ClusterInfo, contact_info::ContactInfo, repair_service::RepairSlotRange,
     };
     use rand::thread_rng;
     use solana_ledger::shred::DataShredHeader;
     use solana_ledger::{
         blockstore::{make_many_slot_entries, Blockstore},
         entry::{create_ticks, Entry},
+        genesis_utils::create_genesis_config_with_leader,
         get_tmp_ledger_path,
         shred::Shredder,
     };
+    use solana_perf::packet::Packet;
     use solana_sdk::{
         clock::Slot,
         epoch_schedule::MINIMUM_SLOTS_PER_EPOCH,
@@ -622,6 +636,7 @@ mod test {
         let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
             ContactInfo::new_localhost(&Pubkey::default(), 0),
         )));
+        let cluster_slots = Arc::new(ClusterSlots::default());
         let repair_sock = Arc::new(UdpSocket::bind(socketaddr_any!()).unwrap());
         let window = WindowService::new(
             blockstore,
@@ -633,6 +648,7 @@ mod test {
             RepairStrategy::RepairRange(RepairSlotRange { start: 0, end: 0 }),
             &Arc::new(LeaderScheduleCache::default()),
             |_, _, _, _| true,
+            cluster_slots,
         );
         window
     }