Budget for gossip traffic (#9550)

(cherry picked from commit 65a9658b13)

.appveyor.yml

@@ -1,42 +0,0 @@
-version: '{build}'
-
-branches:
-  only:
-  - master
-  - /^v[0-9.]+\.[0-9.]+/
-
-cache:
-  - '%USERPROFILE%\.cargo'
-  - '%APPVEYOR_BUILD_FOLDER%\target'
-
-clone_folder: d:\projects\solana
-
-build_script:
-  - bash ci/publish-tarball.sh
-
-notifications:
-  - provider: Slack
-    incoming_webhook:
-      secure: GJsBey+F5apAtUm86MHVJ68Uqa6WN1SImcuIc4TsTZrDhA8K1QWUNw9FFQPybUWDyOcS5dly3kubnUqlGt9ux6Ad2efsfRIQYWv0tOVXKeY=
-    channel: ci-status
-    on_build_success: false
-    on_build_failure: true
-    on_build_status_changed: true
-
-deploy:
-  - provider: S3
-    access_key_id:
-      secure: fTbJl6JpFebR40J7cOWZ2mXBa3kIvEiXgzxAj6L3N7A=
-    secret_access_key:
-      secure: vItsBXb2rEFLvkWtVn/Rcxu5a5+2EwC+b7GsA0waJy9hXh6XuBAD0lnHd9re3g/4
-    bucket: release.solana.com
-    region: us-west-1
-    set_public: true
-
-  - provider: GitHub
-    auth_token:
-      secure: 81fEmPZ0cV1wLtNuUrcmtgxKF6ROQF1+/ft5m+fHX21z6PoeCbaNo8cTyLioWBj7
-    draft: false
-    prerelease: false
-    on:
-      appveyor_repo_tag: true

.buildkite/env/secrets.ejson

@@ -7,9 +7,6 @@
       "GITHUB_TOKEN": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:Vq2dkGTOzfEpRht0BAGHFp/hDogMvXJe:tFXHg1epVt2mq9hkuc5sRHe+KAnVREi/p8S+IZu67XRyzdiA/nGak1k860FXYuuzuaE0QWekaEc=]",
       "INFLUX_DATABASE": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:5KI9WBkXx3R/W4m256mU5MJOE7N8aAT9:Cb8QFELZ9I60t5zhJ9h55Kcs]",
       "INFLUX_PASSWORD": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:hQRMpLCrav+OYkNphkeM4hagdVoZv5Iw:AUO76rr6+gF1OLJA8ZLSG8wHKXgYCPNk6gRCV8rBhZBJ4KwDaxpvOhMl7bxxXG6jol7v4aRa/Lk=]",
-      "INFLUX_USERNAME": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:R7BNmQjfeqoGDAFTJu9bYTGHol2NgnYN:Q2tOT/EBcFvhFk+DKLKmVU7tLCpVC3Ui]",
-      "SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:Egc2dMrHDU0NcZ71LwGv/V66shUhwYUE:04VoIb8CKy7KYhQ5W4cEW9SDKZltxWBL5Hob106lMBbUOD/yUvKYcG3Ep8JfTMwO3K8zowW5HpU/IdGoilX0XWLiJJ6t+p05WWK0TA16nOEtwrEG+UK8wm3sN+xCO20i4jDhpNpgg3FYFHT5rKTHW8+zaBTNUX/SFxkN67Lm+92IM28CXYE43SU1WV6H99hGFFVpTK5JVM3JuYU1ex/dHRE+xCzTr4MYUB/F+nGoNFW8HUDV/y0e1jxT9to3x0SmnytEEuk+5RUzFuEt9cKNFeNml3fOCi4qL+sfj/Y5pjH9xDiUxsvH/8NL35jbLP244aFHgWcp]",
-      "SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_apple_darwin": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:NeOxSoWCvXB9AL4H6OK26l/7bmsKd/oz:Ijfoxtvk2CHlN1ZXHup3Gg/914kbbAkEGWJfvozA8UIe+aUzUObMyTrKkVOeNAH8Q8YH9tNzk7RRnrTcpnzeCCBLlWcVEeruMxHox3mPRzmSeDLxtbzCl9VePlRO3T7jg90K5hW+ZAkd5J/WJNzpAcmr93ts/of3MbvGHSujId/efCTzJEcP6JInnBb8Vrj7TlgKbzUlnqpq1+NjYPSXN3maKa9pKeo2JWxZlGBMoy6QWUUY5GbYEylw9smwh1LJcHZjlaZNMuOl4gNKtaSr38IXQkAXaRUJDPAmPras00YObKzXU8RkTrP4EoP/jx5LPR7f]",
-      "SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_pc_windows_msvc": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:7t+56twjW+jR7fpFNNeRFLPd7E4lbmyN:JuviDpkQrfVcNUGRGsa2e/UhvH6tTYyk1s4cHHE5xZH1NByL7Kpqx36VG/+o1AUGEeSQdsBnKgzYdMoFYbO8o50DoRPc86QIEVXCupD6J9avxLFtQgOWgJp+/mCdUVXlqXiFs/vQgS/L4psrcKdF6WHd77BeUr6ll8DjH+9m5FC9Rcai2pXno6VbPpunHQ0oUdYzhFR64+LiRacBaefQ9igZ+nSEWDLqbaZSyfm9viWkijoVFTq8gAgdXXEh7g0QdxVE5T6bPristJhT6jWBhWunPUCDNFFErWIsbRGctepl4pbCWqh2hNTw9btSgVfeY6uGCOsdy9E=]"
+      "INFLUX_USERNAME": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:R7BNmQjfeqoGDAFTJu9bYTGHol2NgnYN:Q2tOT/EBcFvhFk+DKLKmVU7tLCpVC3Ui]"
     }
 }

.gitignore

@@ -1,5 +1,6 @@
 /docs/html/
 /docs/src/tests.ok
+/docs/src/cli/usage.md
 /docs/src/.gitbook/assets/*.svg
 /farf/
 /solana-release/

.mergify.yml

@@ -19,20 +19,13 @@ pull_request_rules:
       label:
         add:
           - automerge
-  - name: v0.23 backport
-    conditions:
-      - base=master
-      - label=v0.23
-    actions:
-      backport:
-        branches:
-          - v0.23
   - name: v1.0 backport
     conditions:
       - base=master
       - label=v1.0
     actions:
       backport:
+        ignore_conflicts: true
         branches:
           - v1.0
   - name: v1.1 backport
@@ -41,6 +34,7 @@ pull_request_rules:
       - label=v1.1
     actions:
       backport:
+        ignore_conflicts: true
         branches:
           - v1.1
   - name: v1.2 backport
@@ -49,5 +43,6 @@ pull_request_rules:
       - label=v1.2
     actions:
       backport:
+        ignore_conflicts: true
         branches:
           - v1.2

.travis.yml

@@ -1,5 +1,6 @@
 os:
   - osx
+  - windows
 
 language: rust
 rust:

CONTRIBUTING.md

@@ -45,7 +45,7 @@ $ git pull --rebase upstream master
 
 If there are no functional changes, PRs can be very large and that's no
 problem. If, however, your changes are making meaningful changes or additions,
-then about 1000 lines of changes is about the most you should ask a Solana
+then about 1,000 lines of changes is about the most you should ask a Solana
 maintainer to review.
 
 ### Should I send small PRs as I develop large, new components?

Cargo.toml

@@ -3,6 +3,7 @@ members = [
     "bench-exchange",
     "bench-streamer",
     "bench-tps",
+    "accounts-bench",
     "banking-bench",
     "chacha",
     "chacha-cuda",
@@ -10,6 +11,8 @@ members = [
     "cli-config",
     "client",
     "core",
+    "dos",
+    "download-utils",
     "faucet",
     "perf",
     "validator",
@@ -24,6 +27,7 @@ members = [
     "logger",
     "log-analyzer",
     "merkle-tree",
+    "streamer",
     "measure",
     "metrics",
     "net-shaper",
@@ -48,7 +52,10 @@ members = [
     "sdk",
     "sdk-c",
     "scripts",
+    "stake-accounts",
+    "stake-monitor",
     "sys-tuner",
+    "transaction-status",
     "upload-perf",
     "net-utils",
     "vote-signer",

README.md

@@ -9,46 +9,7 @@ Blockchain Rebuilt for Scale
 Solana™ is a new blockchain architecture built from the ground up for scale. The architecture supports
 up to 710 thousand transactions per second on a gigabit network.
 
-Documentation
-===
-
-Before you jump into the code, review the documentation [Solana: Blockchain Rebuilt for Scale](https://docs.solana.com).
-
-(The _latest_ development version of the docs is [available here](https://docs.solana.com/v/master).)
-
-Release Binaries
-===
-Official release binaries are available at [Github Releases](https://github.com/solana-labs/solana/releases).
-
-Additionally we provide pre-release binaries for the latest code on the edge and
-beta channels.  Note that these pre-release binaries may be less stable than an
-official release.
-
-### Edge channel
-#### Linux (x86_64-unknown-linux-gnu)
-* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-unknown-linux-gnu.tar.bz2)
-* [solana-install-init](http://release.solana.com/edge/solana-install-init-x86_64-unknown-linux-gnu) as a stand-alone executable
-#### mac OS (x86_64-apple-darwin)
-* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-apple-darwin.tar.bz2)
-* [solana-install-init](http://release.solana.com/edge/solana-install-init-x86_64-apple-darwin) as a stand-alone executable
-#### Windows (x86_64-pc-windows-msvc)
-* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-pc-windows-msvc.tar.bz2)
-* [solana-install-init.exe](http://release.solana.com/edge/solana-install-init-x86_64-pc-windows-msvc.exe) as a stand-alone executable
-#### All platforms
-* [solana-metrics.tar.bz2](http://release.solana.com.s3.amazonaws.com/edge/solana-metrics.tar.bz2)
-
-### Beta channel
-#### Linux (x86_64-unknown-linux-gnu)
-* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-unknown-linux-gnu.tar.bz2)
-* [solana-install-init](http://release.solana.com/beta/solana-install-init-x86_64-unknown-linux-gnu) as a stand-alone executable
-#### mac OS (x86_64-apple-darwin)
-* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-apple-darwin.tar.bz2)
-* [solana-install-init](http://release.solana.com/beta/solana-install-init-x86_64-apple-darwin) as a stand-alone executable
-#### Windows (x86_64-pc-windows-msvc)
-* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-pc-windows-msvc.tar.bz2)
-* [solana-install-init.exe](http://release.solana.com/beta/solana-install-init-x86_64-pc-windows-msvc.exe) as a stand-alone executable
-#### All platforms
-* [solana-metrics.tar.bz2](http://release.solana.com.s3.amazonaws.com/beta/solana-metrics.tar.bz2)
+Read all about it at [Solana: Blockchain Rebuilt for Scale](https://docs.solana.com/v/master).
 
 Developing
 ===

accounts-bench/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.com>"]
+edition = "2018"
+name = "solana-accounts-bench"
+version = "1.1.5"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+
+[dependencies]
+log = "0.4.6"
+rayon = "1.3.0"
+solana-logger = { path = "../logger", version = "1.1.5" }
+solana-runtime = { path = "../runtime", version = "1.1.5" }
+solana-measure = { path = "../measure", version = "1.1.5" }
+solana-sdk = { path = "../sdk", version = "1.1.5" }
+rand = "0.6.5"
+clap = "2.33.0"
+crossbeam-channel = "0.4"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accounts-bench/src/main.rs

@@ -0,0 +1,103 @@
+use clap::{value_t, App, Arg};
+use rayon::prelude::*;
+use solana_measure::measure::Measure;
+use solana_runtime::accounts::{create_test_accounts, update_accounts, Accounts};
+use solana_sdk::pubkey::Pubkey;
+use std::collections::HashMap;
+use std::fs;
+use std::path::PathBuf;
+
+fn main() {
+    solana_logger::setup();
+
+    let matches = App::new("crate")
+        .about("about")
+        .version("version")
+        .arg(
+            Arg::with_name("num_slots")
+                .long("num_slots")
+                .takes_value(true)
+                .value_name("SLOTS")
+                .help("Number of slots to store to."),
+        )
+        .arg(
+            Arg::with_name("num_accounts")
+                .long("num_accounts")
+                .takes_value(true)
+                .value_name("NUM_ACCOUNTS")
+                .help("Total number of accounts"),
+        )
+        .arg(
+            Arg::with_name("iterations")
+                .long("iterations")
+                .takes_value(true)
+                .value_name("ITERATIONS")
+                .help("Number of bench iterations"),
+        )
+        .arg(
+            Arg::with_name("clean")
+                .long("clean")
+                .takes_value(false)
+                .help("Run clean"),
+        )
+        .get_matches();
+
+    let num_slots = value_t!(matches, "num_slots", usize).unwrap_or(4);
+    let num_accounts = value_t!(matches, "num_accounts", usize).unwrap_or(10_000);
+    let iterations = value_t!(matches, "iterations", usize).unwrap_or(20);
+    let clean = matches.is_present("clean");
+    println!("clean: {:?}", clean);
+
+    let path = PathBuf::from("farf/accounts-bench");
+    if fs::remove_dir_all(path.clone()).is_err() {
+        println!("Warning: Couldn't remove {:?}", path);
+    }
+    let accounts = Accounts::new(vec![path]);
+    println!("Creating {} accounts", num_accounts);
+    let mut create_time = Measure::start("create accounts");
+    let pubkeys: Vec<_> = (0..num_slots)
+        .into_par_iter()
+        .map(|slot| {
+            let mut pubkeys: Vec<Pubkey> = vec![];
+            create_test_accounts(
+                &accounts,
+                &mut pubkeys,
+                num_accounts / num_slots,
+                slot as u64,
+            );
+            pubkeys
+        })
+        .collect();
+    let pubkeys: Vec<_> = pubkeys.into_iter().flatten().collect();
+    create_time.stop();
+    println!(
+        "created {} accounts in {} slots {}",
+        (num_accounts / num_slots) * num_slots,
+        num_slots,
+        create_time
+    );
+    let mut ancestors: HashMap<u64, usize> = vec![(0, 0)].into_iter().collect();
+    for i in 1..num_slots {
+        ancestors.insert(i as u64, i - 1);
+        accounts.add_root(i as u64);
+    }
+    for x in 0..iterations {
+        if clean {
+            let mut time = Measure::start("clean");
+            accounts.accounts_db.clean_accounts();
+            time.stop();
+            println!("{}", time);
+            for slot in 0..num_slots {
+                update_accounts(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64);
+                accounts.add_root((x * num_slots + slot) as u64);
+            }
+        } else {
+            let mut pubkeys: Vec<Pubkey> = vec![];
+            let mut time = Measure::start("hash");
+            let hash = accounts.accounts_db.update_accounts_hash(0, &ancestors);
+            time.stop();
+            println!("hash: {} {}", hash, time);
+            create_test_accounts(&accounts, &mut pubkeys, 1, 0);
+        }
+    }
+}

archiver-lib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-archiver-lib"
-version = "1.0.6"
+version = "1.1.5"
 description = "Solana Archiver Library"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,30 +10,34 @@ edition = "2018"
 
 [dependencies]
 bincode = "1.2.1"
-crossbeam-channel = "0.3"
+crossbeam-channel = "0.4"
 ed25519-dalek = "=1.0.0-pre.1"
 log = "0.4.8"
 rand = "0.6.5"
 rand_chacha = "0.1.1"
-solana-client = { path = "../client", version = "1.0.6" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.6" }
+solana-client = { path = "../client", version = "1.1.5" }
+solana-storage-program = { path = "../programs/storage", version = "1.1.5" }
 thiserror = "1.0"
-serde = "1.0.104"
-serde_json = "1.0.46"
+serde = "1.0.105"
+serde_json = "1.0.48"
 serde_derive = "1.0.103"
-solana-net-utils = { path = "../net-utils", version = "1.0.6" }
-solana-chacha = { path = "../chacha", version = "1.0.6" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.6" }
-solana-ledger = { path = "../ledger", version = "1.0.6" }
-solana-logger = { path = "../logger", version = "1.0.6" }
-solana-perf = { path = "../perf", version = "1.0.6" }
-solana-sdk = { path = "../sdk", version = "1.0.6" }
-solana-core = { path = "../core", version = "1.0.6" }
-solana-archiver-utils = { path = "../archiver-utils", version = "1.0.6" }
-solana-metrics = { path = "../metrics", version = "1.0.6" }
+solana-net-utils = { path = "../net-utils", version = "1.1.5" }
+solana-chacha = { path = "../chacha", version = "1.1.5" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.1.5" }
+solana-ledger = { path = "../ledger", version = "1.1.5" }
+solana-logger = { path = "../logger", version = "1.1.5" }
+solana-perf = { path = "../perf", version = "1.1.5" }
+solana-sdk = { path = "../sdk", version = "1.1.5" }
+solana-core = { path = "../core", version = "1.1.5" }
+solana-streamer = { path = "../streamer", version = "1.1.5" }
+solana-archiver-utils = { path = "../archiver-utils", version = "1.1.5" }
+solana-metrics = { path = "../metrics", version = "1.1.5" }
 
 [dev-dependencies]
-hex = "0.4.0"
+hex = "0.4.2"
 
 [lib]
 name = "solana_archiver_lib"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

archiver-lib/src/archiver.rs

@@ -10,16 +10,15 @@ use solana_client::{
 };
 use solana_core::{
     cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE},
+    cluster_slots::ClusterSlots,
     contact_info::ContactInfo,
     gossip_service::GossipService,
-    packet::{limited_deserialize, PACKET_DATA_SIZE},
     repair_service,
-    repair_service::{RepairService, RepairSlotRange, RepairStrategy},
+    repair_service::{RepairService, RepairSlotRange, RepairStats, RepairStrategy},
     serve_repair::ServeRepair,
     shred_fetch_stage::ShredFetchStage,
     sigverify_stage::{DisabledSigVerifier, SigVerifyStage},
     storage_stage::NUM_STORAGE_SAMPLES,
-    streamer::{receiver, responder, PacketReceiver},
     window_service::WindowService,
 };
 use solana_ledger::{
@@ -27,6 +26,7 @@ use solana_ledger::{
 };
 use solana_net_utils::bind_in_range;
 use solana_perf::packet::Packets;
+use solana_perf::packet::{limited_deserialize, PACKET_DATA_SIZE};
 use solana_perf::recycler::Recycler;
 use solana_sdk::packet::Packet;
 use solana_sdk::{
@@ -45,6 +45,7 @@ use solana_storage_program::{
     storage_contract::StorageContract,
     storage_instruction::{self, StorageAccountType},
 };
+use solana_streamer::streamer::{receiver, responder, PacketReceiver};
 use std::{
     io::{self, ErrorKind},
     net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
@@ -187,7 +188,7 @@ impl Archiver {
         let mut cluster_info = ClusterInfo::new(node.info.clone(), keypair.clone());
         cluster_info.set_entrypoint(cluster_entrypoint.clone());
         let cluster_info = Arc::new(RwLock::new(cluster_info));
-
+        let cluster_slots = Arc::new(ClusterSlots::default());
         // Note for now, this ledger will not contain any of the existing entries
         // in the ledger located at ledger_path, and will only append on newly received
         // entries after being passed to window_service
@@ -235,6 +236,7 @@ impl Archiver {
             shred_forward_sockets,
             repair_socket.clone(),
             &shred_fetch_sender,
+            None,
             &exit,
         );
         let (slot_sender, slot_receiver) = channel();
@@ -261,6 +263,7 @@ impl Archiver {
                     repair_socket,
                     shred_fetch_receiver,
                     slot_sender,
+                    cluster_slots,
                 ) {
                     Ok(window_service) => window_service,
                     Err(e) => {
@@ -399,6 +402,7 @@ impl Archiver {
     }
 
     // Find a segment to replicate and download it.
+    #[allow(clippy::too_many_arguments)]
     fn setup(
         meta: &mut ArchiverMeta,
         cluster_info: Arc<RwLock<ClusterInfo>>,
@@ -409,6 +413,7 @@ impl Archiver {
         repair_socket: Arc<UdpSocket>,
         shred_fetch_receiver: PacketReceiver,
         slot_sender: Sender<u64>,
+        cluster_slots: Arc<ClusterSlots>,
     ) -> Result<WindowService> {
         let slots_per_segment =
             match Self::get_segment_config(&cluster_info, meta.client_commitment) {
@@ -466,6 +471,7 @@ impl Archiver {
             RepairStrategy::RepairRange(repair_slot_range),
             &Arc::new(LeaderScheduleCache::default()),
             |_, _, _, _| true,
+            cluster_slots,
         );
         info!("waiting for ledger download");
         Self::wait_for_segment_download(
@@ -838,13 +844,14 @@ impl Archiver {
                 repair_service::MAX_REPAIR_LENGTH,
                 &repair_slot_range,
             );
+            let mut repair_stats = RepairStats::default();
             //iter over the repairs and send them
             if let Ok(repairs) = repairs {
                 let reqs: Vec<_> = repairs
                     .into_iter()
                     .filter_map(|repair_request| {
                         serve_repair
-                            .map_repair_request(&repair_request)
+                            .map_repair_request(&repair_request, &mut repair_stats)
                             .map(|result| ((archiver_info.gossip, result), repair_request))
                             .ok()
                     })

archiver-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-archiver-utils"
-version = "1.0.6"
+version = "1.1.5"
 description = "Solana Archiver Utils"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,15 +11,18 @@ edition = "2018"
 [dependencies]
 log = "0.4.8"
 rand = "0.6.5"
-solana-chacha = { path = "../chacha", version = "1.0.6" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.6" }
-solana-ledger = { path = "../ledger", version = "1.0.6" }
-solana-logger = { path = "../logger", version = "1.0.6" }
-solana-perf = { path = "../perf", version = "1.0.6" }
-solana-sdk = { path = "../sdk", version = "1.0.6" }
+solana-chacha = { path = "../chacha", version = "1.1.5" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.1.5" }
+solana-ledger = { path = "../ledger", version = "1.1.5" }
+solana-logger = { path = "../logger", version = "1.1.5" }
+solana-perf = { path = "../perf", version = "1.1.5" }
+solana-sdk = { path = "../sdk", version = "1.1.5" }
 
 [dev-dependencies]
-hex = "0.4.0"
+hex = "0.4.2"
 
 [lib]
 name = "solana_archiver_utils"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

archiver/Cargo.toml

@@ -2,19 +2,22 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-archiver"
-version = "1.0.6"
+version = "1.1.5"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-console = "0.9.2"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.6" }
-solana-core = { path = "../core", version = "1.0.6" }
-solana-logger = { path = "../logger", version = "1.0.6" }
-solana-metrics = { path = "../metrics", version = "1.0.6" }
-solana-archiver-lib = { path = "../archiver-lib", version = "1.0.6" }
-solana-net-utils = { path = "../net-utils", version = "1.0.6" }
-solana-sdk = { path = "../sdk", version = "1.0.6" }
+console = "0.10.0"
+solana-clap-utils = { path = "../clap-utils", version = "1.1.5" }
+solana-core = { path = "../core", version = "1.1.5" }
+solana-logger = { path = "../logger", version = "1.1.5" }
+solana-metrics = { path = "../metrics", version = "1.1.5" }
+solana-archiver-lib = { path = "../archiver-lib", version = "1.1.5" }
+solana-net-utils = { path = "../net-utils", version = "1.1.5" }
+solana-sdk = { path = "../sdk", version = "1.1.5" }
 
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

archiver/src/main.rs

@@ -28,7 +28,7 @@ fn main() {
         .arg(
             Arg::with_name("identity_keypair")
                 .short("i")
-                .long("identity-keypair")
+                .long("identity")
                 .value_name("PATH")
                 .takes_value(true)
                 .validator(is_keypair_or_ask_keyword)

banking-bench/Cargo.toml

@@ -2,19 +2,24 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.0.6"
+version = "1.1.5"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 log = "0.4.6"
-rayon = "1.2.0"
-solana-core = { path = "../core", version = "1.0.6" }
-solana-ledger = { path = "../ledger", version = "1.0.6" }
-solana-logger = { path = "../logger", version = "1.0.6" }
-solana-runtime = { path = "../runtime", version = "1.0.6" }
-solana-measure = { path = "../measure", version = "1.0.6" }
-solana-sdk = { path = "../sdk", version = "1.0.6" }
+rayon = "1.3.0"
+solana-core = { path = "../core", version = "1.1.5" }
+solana-streamer = { path = "../streamer", version = "1.1.5" }
+solana-perf = { path = "../perf", version = "1.1.5" }
+solana-ledger = { path = "../ledger", version = "1.1.5" }
+solana-logger = { path = "../logger", version = "1.1.5" }
+solana-runtime = { path = "../runtime", version = "1.1.5" }
+solana-measure = { path = "../measure", version = "1.1.5" }
+solana-sdk = { path = "../sdk", version = "1.1.5" }
 rand = "0.6.5"
-crossbeam-channel = "0.3"
+crossbeam-channel = "0.4"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -2,29 +2,36 @@ use crossbeam_channel::unbounded;
 use log::*;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
-use solana_core::banking_stage::{create_test_recorder, BankingStage};
-use solana_core::cluster_info::ClusterInfo;
-use solana_core::cluster_info::Node;
-use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-use solana_core::packet::to_packets_chunked;
-use solana_core::poh_recorder::PohRecorder;
-use solana_core::poh_recorder::WorkingBankEntry;
-use solana_ledger::bank_forks::BankForks;
-use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
+use solana_core::{
+    banking_stage::{create_test_recorder, BankingStage},
+    cluster_info::ClusterInfo,
+    cluster_info::Node,
+    poh_recorder::PohRecorder,
+    poh_recorder::WorkingBankEntry,
+};
+use solana_ledger::{
+    bank_forks::BankForks,
+    blockstore::Blockstore,
+    genesis_utils::{create_genesis_config, GenesisConfigInfo},
+    get_tmp_ledger_path,
+};
 use solana_measure::measure::Measure;
+use solana_perf::packet::to_packets_chunked;
 use solana_runtime::bank::Bank;
-use solana_sdk::hash::Hash;
-use solana_sdk::pubkey::Pubkey;
-use solana_sdk::signature::Keypair;
-use solana_sdk::signature::Signature;
-use solana_sdk::system_transaction;
-use solana_sdk::timing::{duration_as_us, timestamp};
-use solana_sdk::transaction::Transaction;
-use std::sync::atomic::Ordering;
-use std::sync::mpsc::Receiver;
-use std::sync::{Arc, Mutex, RwLock};
-use std::thread::sleep;
-use std::time::{Duration, Instant};
+use solana_sdk::{
+    hash::Hash,
+    pubkey::Pubkey,
+    signature::Keypair,
+    signature::Signature,
+    system_transaction,
+    timing::{duration_as_us, timestamp},
+    transaction::Transaction,
+};
+use std::{
+    sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex, RwLock},
+    thread::sleep,
+    time::{Duration, Instant},
+};
 
 fn check_txs(
     receiver: &Arc<Receiver<WorkingBankEntry>>,

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.0.6"
+version = "1.1.5"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,25 +10,28 @@ publish = false
 
 [dependencies]
 clap = "2.32.0"
-itertools = "0.8.2"
+itertools = "0.9.0"
 log = "0.4.8"
 num-derive = "0.3"
 num-traits = "0.2"
 rand = "0.6.5"
-rayon = "1.2.0"
-serde_json = "1.0.46"
+rayon = "1.3.0"
+serde_json = "1.0.48"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.6" }
-solana-core = { path = "../core", version = "1.0.6" }
-solana-genesis = { path = "../genesis", version = "1.0.6" }
-solana-client = { path = "../client", version = "1.0.6" }
-solana-faucet = { path = "../faucet", version = "1.0.6" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.0.6" }
-solana-logger = { path = "../logger", version = "1.0.6" }
-solana-metrics = { path = "../metrics", version = "1.0.6" }
-solana-net-utils = { path = "../net-utils", version = "1.0.6" }
-solana-runtime = { path = "../runtime", version = "1.0.6" }
-solana-sdk = { path = "../sdk", version = "1.0.6" }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.5" }
+solana-core = { path = "../core", version = "1.1.5" }
+solana-genesis = { path = "../genesis", version = "1.1.5" }
+solana-client = { path = "../client", version = "1.1.5" }
+solana-faucet = { path = "../faucet", version = "1.1.5" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.1.5" }
+solana-logger = { path = "../logger", version = "1.1.5" }
+solana-metrics = { path = "../metrics", version = "1.1.5" }
+solana-net-utils = { path = "../net-utils", version = "1.1.5" }
+solana-runtime = { path = "../runtime", version = "1.1.5" }
+solana-sdk = { path = "../sdk", version = "1.1.5" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.0.6" }
+solana-local-cluster = { path = "../local-cluster", version = "1.1.5" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/Cargo.toml

@@ -2,14 +2,17 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.0.6"
+version = "1.1.5"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.6" }
-solana-core = { path = "../core", version = "1.0.6" }
-solana-logger = { path = "../logger", version = "1.0.6" }
-solana-net-utils = { path = "../net-utils", version = "1.0.6" }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.5" }
+solana-streamer = { path = "../streamer", version = "1.1.5" }
+solana-logger = { path = "../logger", version = "1.1.5" }
+solana-net-utils = { path = "../net-utils", version = "1.1.5" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/src/main.rs

@@ -1,6 +1,6 @@
 use clap::{crate_description, crate_name, App, Arg};
-use solana_core::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
-use solana_core::streamer::{receiver, PacketReceiver};
+use solana_streamer::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
+use solana_streamer::streamer::{receiver, PacketReceiver};
 use std::cmp::max;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.0.6"
+version = "1.1.5"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,27 +11,30 @@ homepage = "https://solana.com/"
 bincode = "1.2.1"
 clap = "2.33.0"
 log = "0.4.8"
-rayon = "1.2.0"
-serde_json = "1.0.46"
+rayon = "1.3.0"
+serde_json = "1.0.48"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.6" }
-solana-core = { path = "../core", version = "1.0.6" }
-solana-genesis = { path = "../genesis", version = "1.0.6" }
-solana-client = { path = "../client", version = "1.0.6" }
-solana-faucet = { path = "../faucet", version = "1.0.6" }
-solana-librapay = { path = "../programs/librapay", version = "1.0.6", optional = true }
-solana-logger = { path = "../logger", version = "1.0.6" }
-solana-metrics = { path = "../metrics", version = "1.0.6" }
-solana-measure = { path = "../measure", version = "1.0.6" }
-solana-net-utils = { path = "../net-utils", version = "1.0.6" }
-solana-runtime = { path = "../runtime", version = "1.0.6" }
-solana-sdk = { path = "../sdk", version = "1.0.6" }
-solana-move-loader-program = { path = "../programs/move_loader", version = "1.0.6", optional = true }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.5" }
+solana-core = { path = "../core", version = "1.1.5" }
+solana-genesis = { path = "../genesis", version = "1.1.5" }
+solana-client = { path = "../client", version = "1.1.5" }
+solana-faucet = { path = "../faucet", version = "1.1.5" }
+solana-librapay = { path = "../programs/librapay", version = "1.1.5", optional = true }
+solana-logger = { path = "../logger", version = "1.1.5" }
+solana-metrics = { path = "../metrics", version = "1.1.5" }
+solana-measure = { path = "../measure", version = "1.1.5" }
+solana-net-utils = { path = "../net-utils", version = "1.1.5" }
+solana-runtime = { path = "../runtime", version = "1.1.5" }
+solana-sdk = { path = "../sdk", version = "1.1.5" }
+solana-move-loader-program = { path = "../programs/move_loader", version = "1.1.5", optional = true }
 
 [dev-dependencies]
-serial_test = "0.3.2"
+serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.0.6" }
+solana-local-cluster = { path = "../local-cluster", version = "1.1.5" }
 
 [features]
 move = ["solana-librapay", "solana-move-loader-program"]
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

chacha-cuda/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha-cuda"
-version = "1.0.6"
+version = "1.1.5"
 description = "Solana Chacha Cuda APIs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,15 +10,18 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-solana-archiver-utils = { path = "../archiver-utils", version = "1.0.6" }
-solana-chacha = { path = "../chacha", version = "1.0.6" }
-solana-ledger = { path = "../ledger", version = "1.0.6" }
-solana-logger = { path = "../logger", version = "1.0.6" }
-solana-perf = { path = "../perf", version = "1.0.6" }
-solana-sdk = { path = "../sdk", version = "1.0.6" }
+solana-archiver-utils = { path = "../archiver-utils", version = "1.1.5" }
+solana-chacha = { path = "../chacha", version = "1.1.5" }
+solana-ledger = { path = "../ledger", version = "1.1.5" }
+solana-logger = { path = "../logger", version = "1.1.5" }
+solana-perf = { path = "../perf", version = "1.1.5" }
+solana-sdk = { path = "../sdk", version = "1.1.5" }
 
 [dev-dependencies]
 hex-literal = "0.2.1"
 
 [lib]
 name = "solana_chacha_cuda"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

chacha-sys/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha-sys"
-version = "1.0.6"
+version = "1.1.5"
 description = "Solana chacha-sys"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,3 +10,6 @@ edition = "2018"
 
 [build-dependencies]
 cc = "1.0.49"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

chacha/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha"
-version = "1.0.6"
+version = "1.1.5"
 description = "Solana Chacha APIs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,14 +12,17 @@ edition = "2018"
 log = "0.4.8"
 rand = "0.6.5"
 rand_chacha = "0.1.1"
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.6" }
-solana-ledger = { path = "../ledger", version = "1.0.6" }
-solana-logger = { path = "../logger", version = "1.0.6" }
-solana-perf = { path = "../perf", version = "1.0.6" }
-solana-sdk = { path = "../sdk", version = "1.0.6" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.1.5" }
+solana-ledger = { path = "../ledger", version = "1.1.5" }
+solana-logger = { path = "../logger", version = "1.1.5" }
+solana-perf = { path = "../perf", version = "1.1.5" }
+solana-sdk = { path = "../sdk", version = "1.1.5" }
 
 [dev-dependencies]
 hex-literal = "0.2.1"
 
 [lib]
 name = "solana_chacha"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

ci/_

@@ -5,7 +5,13 @@
 # |source| me
 #
 
+base_dir=$(realpath --strip "$(dirname "$0")/..")
+
 _() {
-  echo "--- $*"
+  if [[ $(pwd) = $base_dir ]]; then
+    echo "--- $*"
+  else
+    echo "--- $* (wd: $(pwd))"
+  fi
   "$@"
 }

ci/buildkite-secondary.yml

@@ -18,3 +18,6 @@ steps:
   - command: "ci/publish-docs.sh"
     timeout_in_minutes: 15
     name: "publish docs"
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
+    name: "move"
+    timeout_in_minutes: 20

ci/buildkite-tests.yml

@@ -0,0 +1,26 @@
+# These steps are conditionally triggered by ci/buildkite.yml when files
+# other than those in docs/ are modified
+
+steps:
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh"
+    name: "coverage"
+    timeout_in_minutes: 30
+  - wait
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh"
+    name: "stable"
+    timeout_in_minutes: 60
+    artifact_paths: "log-*.txt"
+  - wait
+  - command: "ci/test-stable-perf.sh"
+    name: "stable-perf"
+    timeout_in_minutes: 40
+    artifact_paths: "log-*.txt"
+    agents:
+      - "queue=cuda"
+  - command: "ci/test-bench.sh"
+    name: "bench"
+    timeout_in_minutes: 30
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-local-cluster.sh"
+    name: "local-cluster"
+    timeout_in_minutes: 45
+    artifact_paths: "log-*.txt"

ci/buildkite.yml

@@ -1,42 +1,25 @@
 # Build steps that run on pushes and pull requests.
+# If files other than those in docs/ were modified, this will be followed up by
+# ci/buildkite-tests.yml
 #
 # Release tags use buildkite-release.yml instead
+
 steps:
-  - command: "ci/shellcheck.sh"
-    name: "shellcheck"
-    timeout_in_minutes: 5
   - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-checks.sh"
     name: "checks"
     timeout_in_minutes: 20
+  - command: "ci/shellcheck.sh"
+    name: "shellcheck"
+    timeout_in_minutes: 5
+
   - wait
-  - command: "ci/test-stable-perf.sh"
-    name: "stable-perf"
-    timeout_in_minutes: 40
-    artifact_paths: "log-*.txt"
-    agents:
-      - "queue=cuda"
-  - command: "ci/test-bench.sh"
-    name: "bench"
-    timeout_in_minutes: 30
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh"
-    name: "stable"
-    timeout_in_minutes: 60
-    artifact_paths: "log-*.txt"
-    agents:
-      - "queue=rpc-test-capable"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
-    name: "move"
-    timeout_in_minutes: 20
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-local-cluster.sh"
-    name: "local-cluster"
-    timeout_in_minutes: 30
-    artifact_paths: "log-*.txt"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh"
-    name: "coverage"
-    timeout_in_minutes: 30
-    agents:
-      - "queue=rpc-test-capable"
+
+  - command: "ci/maybe-trigger-tests.sh"
+    name: "maybe-trigger-tests"
+    timeout_in_minutes: 2
+
   - wait
+
   - trigger: "solana-secondary"
     branches: "!pull/*"
     async: true

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.41.1
+FROM solanalabs/rust:1.42.0
 ARG date
 
 RUN set -x \

ci/docker-rust-nightly/README.md

@@ -15,6 +15,8 @@ To update the pinned version:
 1. Run `ci/docker-rust-nightly/build.sh` to rebuild the nightly image locally,
    or potentially `ci/docker-rust-nightly/build.sh YYYY-MM-DD` if there's a
    specific YYYY-MM-DD that is desired (default is today's build).
+   Check https://rust-lang.github.io/rustup-components-history/ for build
+   status
 1. Update `ci/rust-version.sh` to reflect the new nightly `YYY-MM-DD`
 1. Run `SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-coverage.sh`
    to confirm the new nightly image builds.  Fix any issues as needed

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.41.1
+FROM rust:1.42.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0
@@ -32,6 +32,7 @@ RUN set -x \
  && cargo install cargo-audit \
  && cargo install svgbob_cli \
  && cargo install mdbook \
+ && cargo install mdbook-linkcheck \
  && rustc --version \
  && cargo --version \
  && curl -OL https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP \

ci/localnet-sanity.sh

@@ -178,7 +178,7 @@ startNodes() {
 
       (
         set -x
-        $solana_cli --keypair config/bootstrap-validator/identity-keypair.json \
+        $solana_cli --keypair config/bootstrap-validator/identity.json \
           --url http://127.0.0.1:8899 genesis-hash
       ) | tee genesis-hash.log
       maybeExpectedGenesisHash="--expected-genesis-hash $(tail -n1 genesis-hash.log)"

ci/maybe-trigger-tests.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -e
+cd "$(dirname "$0")/.."
+
+annotate() {
+  ${BUILDKITE:-false} && {
+    buildkite-agent annotate "$@"
+  }
+}
+
+# Skip if only the docs have been modified
+ci/affects-files.sh \
+  \!^docs/ \
+|| {
+  annotate --style info \
+    "Skipping all further tests as only docs/ files were modified"
+  exit 0
+}
+
+annotate --style info "Triggering tests"
+buildkite-agent pipeline upload ci/buildkite-tests.yml

ci/publish-tarball.sh

@@ -45,7 +45,7 @@ linux)
   TARGET=x86_64-unknown-linux-gnu
   ;;
 windows)
-  TARGET=x86_64-pc-windows-msvc
+  TARGET=x86_64-pc-windows-gnu
   ;;
 *)
   echo CI_OS_NAME unset
@@ -73,15 +73,6 @@ echo --- Creating release tarball
   source ci/rust-version.sh stable
   scripts/cargo-install-all.sh +"$rust_stable" --use-move solana-release
 
-  # Reduce the Windows archive size until
-  # https://github.com/appveyor/ci/issues/2997 is fixed
-  if [[ -n $APPVEYOR ]]; then
-    rm -f \
-      solana-release/bin/solana-validator.exe \
-      solana-release/bin/solana-bench-exchange.exe \
-
-  fi
-
   tar cvf solana-release-$TARGET.tar solana-release
   bzip2 solana-release-$TARGET.tar
   cp solana-release/bin/solana-install-init solana-install-init-$TARGET
@@ -104,9 +95,8 @@ fi
 source ci/upload-ci-artifact.sh
 
 for file in solana-release-$TARGET.tar.bz2 solana-release-$TARGET.yml solana-install-init-"$TARGET"* $MAYBE_TARBALLS; do
-  upload-ci-artifact "$file"
-
   if [[ -n $DO_NOT_PUBLISH_TAR ]]; then
+    upload-ci-artifact "$file"
     echo "Skipped $file due to DO_NOT_PUBLISH_TAR"
     continue
   fi

ci/rust-version.sh

@@ -16,13 +16,13 @@
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.41.1
+  stable_version=1.42.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2020-02-27
+  nightly_version=2020-03-12
 fi
 
 

ci/test-bench.sh

@@ -67,8 +67,9 @@ _ cargo +$rust_nightly bench --manifest-path core/Cargo.toml ${V:+--verbose} \
 _ cargo +$rust_nightly bench --manifest-path programs/bpf/Cargo.toml ${V:+--verbose} --features=bpf_c \
   -- -Z unstable-options --format=json --nocapture | tee -a "$BENCH_FILE"
 
-# Run banking bench. Doesn't require nightly, but use since it is already built.
+# Run banking/accounts bench. Doesn't require nightly, but use since it is already built.
 _ cargo +$rust_nightly run --release --manifest-path banking-bench/Cargo.toml ${V:+--verbose} | tee -a "$BENCH_FILE"
+_ cargo +$rust_nightly run --release --manifest-path accounts-bench/Cargo.toml ${V:+--verbose} -- --num_accounts 10000 --num_slots 4 | tee -a "$BENCH_FILE"
 
 # `solana-upload-perf` disabled as it can take over 30 minutes to complete for some
 # reason

ci/test-checks.sh

@@ -22,7 +22,7 @@ _ cargo +"$rust_stable" clippy --all --exclude solana-sdk-c -- --deny=warnings
 _ cargo +"$rust_stable" clippy --manifest-path sdk-c/Cargo.toml -- --deny=warnings
 
 _ cargo +"$rust_stable" audit --version
-_ cargo +"$rust_stable" audit --ignore RUSTSEC-2020-0002
+_ cargo +"$rust_stable" audit --ignore RUSTSEC-2020-0002 --ignore RUSTSEC-2020-0008
 _ ci/nits.sh
 _ ci/order-crates-for-publishing.py
 _ docs/build.sh

ci/test-stable.sh

@@ -38,10 +38,15 @@ test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
 # Clear the BPF sysroot files, they are not automatically rebuilt
 rm -rf target/xargo # Issue #3105
 
+# Limit compiler jobs to reduce memory usage
+# on machines with 1gb/thread of memory
+NPROC=$(nproc)
+NPROC=$((NPROC>16 ? 16 : NPROC))
+
 echo "Executing $testName"
 case $testName in
 test-stable)
-  _ cargo +"$rust_stable" test --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
+  _ cargo +"$rust_stable" test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
   _ cargo +"$rust_stable" test --manifest-path bench-tps/Cargo.toml --features=move ${V:+--verbose} test_bench_tps_local_cluster_move -- --nocapture
   ;;
 test-stable-perf)

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.0.6"
+version = "1.1.5"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.0.6" }
-solana-sdk = { path = "../sdk", version = "1.0.6" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.1.5" }
+solana-sdk = { path = "../sdk", version = "1.1.5" }
 thiserror = "1.0.11"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"
@@ -20,3 +20,6 @@ chrono = "0.4"
 
 [lib]
 name = "solana_clap_utils"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

clap-utils/src/input_parsers.rs

@@ -1,6 +1,6 @@
 use crate::keypair::{
-    keypair_from_seed_phrase, pubkey_from_path, signer_from_path, ASK_KEYWORD,
-    SKIP_SEED_PHRASE_VALIDATION_ARG,
+    keypair_from_seed_phrase, pubkey_from_path, resolve_signer_from_path, signer_from_path,
+    ASK_KEYWORD, SKIP_SEED_PHRASE_VALIDATION_ARG,
 };
 use chrono::DateTime;
 use clap::ArgMatches;
@@ -62,6 +62,21 @@ pub fn keypair_of(matches: &ArgMatches<'_>, name: &str) -> Option<Keypair> {
     }
 }
 
+pub fn keypairs_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<Keypair>> {
+    matches.values_of(name).map(|values| {
+        values
+            .filter_map(|value| {
+                if value == ASK_KEYWORD {
+                    let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
+                    keypair_from_seed_phrase(name, skip_validation, true).ok()
+                } else {
+                    read_keypair_file(value).ok()
+                }
+            })
+            .collect()
+    })
+}
+
 // Return a pubkey for an argument that can itself be parsed into a pubkey,
 // or is a filename that can be read as a keypair
 pub fn pubkey_of(matches: &ArgMatches<'_>, name: &str) -> Option<Pubkey> {
@@ -101,7 +116,7 @@ pub fn pubkeys_sigs_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<(Pubk
 pub fn signer_of(
     matches: &ArgMatches<'_>,
     name: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<(Option<Box<dyn Signer>>, Option<Pubkey>), Box<dyn std::error::Error>> {
     if let Some(location) = matches.value_of(name) {
         let signer = signer_from_path(matches, location, name, wallet_manager)?;
@@ -115,7 +130,7 @@ pub fn signer_of(
 pub fn pubkey_of_signer(
     matches: &ArgMatches<'_>,
     name: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<Option<Pubkey>, Box<dyn std::error::Error>> {
     if let Some(location) = matches.value_of(name) {
         Ok(Some(pubkey_from_path(
@@ -129,6 +144,35 @@ pub fn pubkey_of_signer(
     }
 }
 
+pub fn pubkeys_of_multiple_signers(
+    matches: &ArgMatches<'_>,
+    name: &str,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<Option<Vec<Pubkey>>, Box<dyn std::error::Error>> {
+    if let Some(pubkey_matches) = matches.values_of(name) {
+        let mut pubkeys: Vec<Pubkey> = vec![];
+        for signer in pubkey_matches {
+            pubkeys.push(pubkey_from_path(matches, signer, name, wallet_manager)?);
+        }
+        Ok(Some(pubkeys))
+    } else {
+        Ok(None)
+    }
+}
+
+pub fn resolve_signer(
+    matches: &ArgMatches<'_>,
+    name: &str,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<Option<String>, Box<dyn std::error::Error>> {
+    Ok(resolve_signer_from_path(
+        matches,
+        matches.value_of(name).unwrap(),
+        name,
+        wallet_manager,
+    )?)
+}
+
 pub fn lamports_of_sol(matches: &ArgMatches<'_>, name: &str) -> Option<u64> {
     value_of(matches, name).map(sol_to_lamports)
 }

clap-utils/src/input_validators.rs

@@ -46,18 +46,27 @@ pub fn is_pubkey_or_keypair(string: String) -> Result<(), String> {
     is_pubkey(string.clone()).or_else(|_| is_keypair(string))
 }
 
-// Return an error if string cannot be parsed as pubkey or keypair file or keypair ask keyword
-pub fn is_pubkey_or_keypair_or_ask_keyword(string: String) -> Result<(), String> {
-    is_pubkey(string.clone()).or_else(|_| is_keypair_or_ask_keyword(string))
-}
-
-pub fn is_valid_signer(string: String) -> Result<(), String> {
+// Return an error if string cannot be parsed as a pubkey string, or a valid Signer that can
+// produce a pubkey()
+pub fn is_valid_pubkey(string: String) -> Result<(), String> {
     match parse_keypair_path(&string) {
         KeypairUrl::Filepath(path) => is_keypair(path),
         _ => Ok(()),
     }
 }
 
+// Return an error if string cannot be parsed as a valid Signer. This is an alias of
+// `is_valid_pubkey`, and does accept pubkey strings, even though a Pubkey is not by itself
+// sufficient to sign a transaction.
+//
+// In the current offline-signing implementation, a pubkey is the valid input for a signer field
+// when paired with an offline `--signer` argument to provide a Presigner (pubkey + signature).
+// Clap validators can't check multiple fields at once, so the verification that a `--signer` is
+// also provided and correct happens in parsing, not in validation.
+pub fn is_valid_signer(string: String) -> Result<(), String> {
+    is_valid_pubkey(string)
+}
+
 // Return an error if string cannot be parsed as pubkey=signature string
 pub fn is_pubkey_sig(string: String) -> Result<(), String> {
     let mut signer = string.split('=');

clap-utils/src/keypair.rs

@@ -1,16 +1,20 @@
-use crate::{input_parsers::pubkeys_sigs_of, offline::SIGNER_ARG, ArgConstant};
+use crate::{
+    input_parsers::pubkeys_sigs_of,
+    offline::{SIGNER_ARG, SIGN_ONLY_ARG},
+    ArgConstant,
+};
 use bip39::{Language, Mnemonic, Seed};
 use clap::ArgMatches;
 use rpassword::prompt_password_stderr;
 use solana_remote_wallet::{
     remote_keypair::generate_remote_keypair,
-    remote_wallet::{RemoteWalletError, RemoteWalletManager},
+    remote_wallet::{maybe_wallet_manager, RemoteWalletError, RemoteWalletManager},
 };
 use solana_sdk::{
     pubkey::Pubkey,
     signature::{
         keypair_from_seed, keypair_from_seed_phrase_and_passphrase, read_keypair,
-        read_keypair_file, Keypair, Presigner, Signature, Signer,
+        read_keypair_file, Keypair, NullSigner, Presigner, Signature, Signer,
     },
 };
 use std::{
@@ -60,7 +64,7 @@ pub fn signer_from_path(
     matches: &ArgMatches,
     path: &str,
     keypair_name: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<Box<dyn Signer>, Box<dyn error::Error>> {
     match parse_keypair_path(path) {
         KeypairUrl::Ask => {
@@ -84,6 +88,9 @@ pub fn signer_from_path(
             Ok(Box::new(read_keypair(&mut stdin)?))
         }
         KeypairUrl::Usb(path) => {
+            if wallet_manager.is_none() {
+                *wallet_manager = maybe_wallet_manager()?;
+            }
             if let Some(wallet_manager) = wallet_manager {
                 Ok(Box::new(generate_remote_keypair(
                     path,
@@ -101,10 +108,12 @@ pub fn signer_from_path(
                 .and_then(|presigners| presigner_from_pubkey_sigs(&pubkey, presigners));
             if let Some(presigner) = presigner {
                 Ok(Box::new(presigner))
+            } else if matches.is_present(SIGN_ONLY_ARG.name) {
+                Ok(Box::new(NullSigner::new(&pubkey)))
             } else {
                 Err(std::io::Error::new(
                     std::io::ErrorKind::Other,
-                    "missing signature for supplied pubkey".to_string(),
+                    format!("missing signature for supplied pubkey: {}", pubkey),
                 )
                 .into())
             }
@@ -116,7 +125,7 @@ pub fn pubkey_from_path(
     matches: &ArgMatches,
     path: &str,
     keypair_name: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<Pubkey, Box<dyn error::Error>> {
     match parse_keypair_path(path) {
         KeypairUrl::Pubkey(pubkey) => Ok(pubkey),
@@ -124,6 +133,54 @@ pub fn pubkey_from_path(
     }
 }
 
+pub fn resolve_signer_from_path(
+    matches: &ArgMatches,
+    path: &str,
+    keypair_name: &str,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<Option<String>, Box<dyn error::Error>> {
+    match parse_keypair_path(path) {
+        KeypairUrl::Ask => {
+            let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
+            // This method validates the seed phrase, but returns `None` because there is no path
+            // on disk or to a device
+            keypair_from_seed_phrase(keypair_name, skip_validation, false).map(|_| None)
+        }
+        KeypairUrl::Filepath(path) => match read_keypair_file(&path) {
+            Err(e) => Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                format!("could not find keypair file: {} error: {}", path, e),
+            )
+            .into()),
+            Ok(_) => Ok(Some(path.to_string())),
+        },
+        KeypairUrl::Stdin => {
+            let mut stdin = std::io::stdin();
+            // This method validates the keypair from stdin, but returns `None` because there is no
+            // path on disk or to a device
+            read_keypair(&mut stdin).map(|_| None)
+        }
+        KeypairUrl::Usb(path) => {
+            if wallet_manager.is_none() {
+                *wallet_manager = maybe_wallet_manager()?;
+            }
+            if let Some(wallet_manager) = wallet_manager {
+                let path = generate_remote_keypair(
+                    path,
+                    wallet_manager,
+                    matches.is_present("confirm_key"),
+                    keypair_name,
+                )
+                .map(|keypair| keypair.path)?;
+                Ok(Some(path))
+            } else {
+                Err(RemoteWalletError::NoDeviceFound.into())
+            }
+        }
+        _ => Ok(Some(path.to_string())),
+    }
+}
+
 // Keyword used to indicate that the user should be asked for a keypair seed phrase
 pub const ASK_KEYWORD: &str = "ASK";
 

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.6"
+version = "1.1.5"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,7 +11,10 @@ homepage = "https://solana.com/"
 [dependencies]
 dirs = "2.0.2"
 lazy_static = "1.4.0"
-serde = "1.0.104"
+serde = "1.0.105"
 serde_derive = "1.0.103"
 serde_yaml = "0.8.11"
 url = "2.1.1"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

cli-config/src/config.rs

@@ -1,10 +1,6 @@
 // Wallet settings that can be configured for long-term use
 use serde_derive::{Deserialize, Serialize};
-use std::{
-    fs::{create_dir_all, File},
-    io::{self, Write},
-    path::Path,
-};
+use std::io;
 use url::Url;
 
 lazy_static! {
@@ -30,7 +26,7 @@ impl Default for Config {
             keypair_path.extend(&[".config", "solana", "id.json"]);
             keypair_path.to_str().unwrap().to_string()
         };
-        let json_rpc_url = "http://127.0.0.1:8899".to_string();
+        let json_rpc_url = "https://api.mainnet-beta.solana.com".to_string();
 
         // Empty websocket_url string indicates the client should
         // `Config::compute_websocket_url(&json_rpc_url)`
@@ -46,23 +42,11 @@ impl Default for Config {
 
 impl Config {
     pub fn load(config_file: &str) -> Result<Self, io::Error> {
-        let file = File::open(config_file.to_string())?;
-        let config = serde_yaml::from_reader(file)
-            .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
-        Ok(config)
+        crate::load_config_file(config_file)
     }
 
     pub fn save(&self, config_file: &str) -> Result<(), io::Error> {
-        let serialized = serde_yaml::to_string(self)
-            .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
-
-        if let Some(outdir) = Path::new(&config_file).parent() {
-            create_dir_all(outdir)?;
-        }
-        let mut file = File::create(config_file)?;
-        file.write_all(&serialized.into_bytes())?;
-
-        Ok(())
+        crate::save_config_file(self, config_file)
     }
 
     pub fn compute_websocket_url(json_rpc_url: &str) -> String {
@@ -76,17 +60,38 @@ impl Config {
         ws_url
             .set_scheme(if is_secure { "wss" } else { "ws" })
             .expect("unable to set scheme");
-        let ws_port = match json_rpc_url.port() {
-            Some(port) => port + 1,
-            None => {
-                if is_secure {
-                    8901
-                } else {
-                    8900
-                }
-            }
-        };
-        ws_url.set_port(Some(ws_port)).expect("unable to set port");
+        if let Some(port) = json_rpc_url.port() {
+            ws_url.set_port(Some(port + 1)).expect("unable to set port");
+        }
         ws_url.to_string()
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn compute_websocket_url() {
+        assert_eq!(
+            Config::compute_websocket_url(&"http://devnet.solana.com"),
+            "ws://devnet.solana.com/".to_string()
+        );
+
+        assert_eq!(
+            Config::compute_websocket_url(&"https://devnet.solana.com"),
+            "wss://devnet.solana.com/".to_string()
+        );
+
+        assert_eq!(
+            Config::compute_websocket_url(&"http://example.com:8899"),
+            "ws://example.com:8900/".to_string()
+        );
+        assert_eq!(
+            Config::compute_websocket_url(&"https://example.com:1234"),
+            "wss://example.com:1235/".to_string()
+        );
+
+        assert_eq!(Config::compute_websocket_url(&"garbage"), String::new());
+    }
+}

cli-config/src/lib.rs

@@ -3,3 +3,37 @@ extern crate lazy_static;
 
 mod config;
 pub use config::{Config, CONFIG_FILE};
+
+use std::{
+    fs::{create_dir_all, File},
+    io::{self, Write},
+    path::Path,
+};
+
+pub fn load_config_file<T, P>(config_file: P) -> Result<T, io::Error>
+where
+    T: serde::de::DeserializeOwned,
+    P: AsRef<Path>,
+{
+    let file = File::open(config_file)?;
+    let config = serde_yaml::from_reader(file)
+        .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
+    Ok(config)
+}
+
+pub fn save_config_file<T, P>(config: &T, config_file: P) -> Result<(), io::Error>
+where
+    T: serde::ser::Serialize,
+    P: AsRef<Path>,
+{
+    let serialized = serde_yaml::to_string(config)
+        .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
+
+    if let Some(outdir) = config_file.as_ref().parent() {
+        create_dir_all(outdir)?;
+    }
+    let mut file = File::create(config_file)?;
+    file.write_all(&serialized.into_bytes())?;
+
+    Ok(())
+}

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.6"
+version = "1.1.5"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,45 +11,49 @@ homepage = "https://solana.com/"
 [dependencies]
 bincode = "1.2.1"
 bs58 = "0.3.0"
-chrono = { version = "0.4.10", features = ["serde"] }
+chrono = { version = "0.4.11", features = ["serde"] }
 clap = "2.33.0"
 criterion-stats = "0.3.0"
 ctrlc = { version = "3.1.4", features = ["termination"] }
-console = "0.9.2"
+console = "0.10.0"
 dirs = "2.0.2"
 log = "0.4.8"
+Inflector = "0.11.4"
 indicatif = "0.14.0"
 humantime = "2.0.0"
 num-traits = "0.2"
 pretty-hex = "0.1.1"
-reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
-serde = "1.0.104"
+reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+serde = "1.0.105"
 serde_derive = "1.0.103"
-serde_json = "1.0.46"
-solana-budget-program = { path = "../programs/budget", version = "1.0.6" }
-solana-clap-utils = { path = "../clap-utils", version = "1.0.6" }
-solana-cli-config = { path = "../cli-config", version = "1.0.6" }
-solana-client = { path = "../client", version = "1.0.6" }
-solana-config-program = { path = "../programs/config", version = "1.0.6" }
-solana-faucet = { path = "../faucet", version = "1.0.6" }
-solana-logger = { path = "../logger", version = "1.0.6" }
-solana-net-utils = { path = "../net-utils", version = "1.0.6" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.0.6" }
-solana-runtime = { path = "../runtime", version = "1.0.6" }
-solana-sdk = { path = "../sdk", version = "1.0.6" }
-solana-stake-program = { path = "../programs/stake", version = "1.0.6" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.6" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.6" }
-solana-vote-signer = { path = "../vote-signer", version = "1.0.6" }
-titlecase = "1.1.0"
-thiserror = "1.0.11"
+serde_json = "1.0.48"
+solana-budget-program = { path = "../programs/budget", version = "1.1.5" }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.5" }
+solana-cli-config = { path = "../cli-config", version = "1.1.5" }
+solana-client = { path = "../client", version = "1.1.5" }
+solana-config-program = { path = "../programs/config", version = "1.1.5" }
+solana-faucet = { path = "../faucet", version = "1.1.5" }
+solana-logger = { path = "../logger", version = "1.1.5" }
+solana-net-utils = { path = "../net-utils", version = "1.1.5" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.1.5" }
+solana-runtime = { path = "../runtime", version = "1.1.5" }
+solana-sdk = { path = "../sdk", version = "1.1.5" }
+solana-stake-program = { path = "../programs/stake", version = "1.1.5" }
+solana-storage-program = { path = "../programs/storage", version = "1.1.5" }
+solana-transaction-status = { path = "../transaction-status", version = "1.1.5" }
+solana-vote-program = { path = "../programs/vote", version = "1.1.5" }
+solana-vote-signer = { path = "../vote-signer", version = "1.1.5" }
+thiserror = "1.0.13"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.0.6" }
-solana-budget-program = { path = "../programs/budget", version = "1.0.6" }
+solana-core = { path = "../core", version = "1.1.5" }
+solana-budget-program = { path = "../programs/budget", version = "1.1.5" }
 tempfile = "3.1.0"
 
 [[bin]]
 name = "solana"
 path = "src/main.rs"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

cli/src/cli_output.rs

@@ -0,0 +1,841 @@
+use crate::{cli::build_balance_message, display::writeln_name_value};
+use chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc};
+use console::{style, Emoji};
+use inflector::cases::titlecase::to_title_case;
+use serde::Serialize;
+use serde_json::{Map, Value};
+use solana_client::rpc_response::{RpcEpochInfo, RpcKeyedAccount, RpcVoteAccountInfo};
+use solana_sdk::{
+    clock::{self, Epoch, Slot, UnixTimestamp},
+    stake_history::StakeHistoryEntry,
+};
+use solana_stake_program::stake_state::{Authorized, Lockup};
+use solana_vote_program::{
+    authorized_voters::AuthorizedVoters,
+    vote_state::{BlockTimestamp, Lockout},
+};
+use std::{collections::BTreeMap, fmt, time::Duration};
+
+static WARNING: Emoji = Emoji("⚠️", "!");
+
+#[derive(PartialEq)]
+pub enum OutputFormat {
+    Display,
+    Json,
+    JsonCompact,
+}
+
+impl OutputFormat {
+    pub fn formatted_print<T>(&self, item: &T)
+    where
+        T: Serialize + fmt::Display,
+    {
+        match self {
+            OutputFormat::Display => {
+                println!("{}", item);
+            }
+            OutputFormat::Json => {
+                println!("{}", serde_json::to_string_pretty(item).unwrap());
+            }
+            OutputFormat::JsonCompact => {
+                println!("{}", serde_json::to_value(item).unwrap());
+            }
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct CliAccount {
+    #[serde(flatten)]
+    pub keyed_account: RpcKeyedAccount,
+    #[serde(skip_serializing)]
+    pub use_lamports_unit: bool,
+}
+
+impl fmt::Display for CliAccount {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f)?;
+        writeln_name_value(f, "Public Key:", &self.keyed_account.pubkey)?;
+        writeln_name_value(
+            f,
+            "Balance:",
+            &build_balance_message(
+                self.keyed_account.account.lamports,
+                self.use_lamports_unit,
+                true,
+            ),
+        )?;
+        writeln_name_value(f, "Owner:", &self.keyed_account.account.owner)?;
+        writeln_name_value(
+            f,
+            "Executable:",
+            &self.keyed_account.account.executable.to_string(),
+        )?;
+        writeln_name_value(
+            f,
+            "Rent Epoch:",
+            &self.keyed_account.account.rent_epoch.to_string(),
+        )?;
+        Ok(())
+    }
+}
+
+#[derive(Default, Serialize, Deserialize)]
+pub struct CliBlockProduction {
+    pub epoch: Epoch,
+    pub start_slot: Slot,
+    pub end_slot: Slot,
+    pub total_slots: usize,
+    pub total_blocks_produced: usize,
+    pub total_slots_skipped: usize,
+    pub leaders: Vec<CliBlockProductionEntry>,
+    pub individual_slot_status: Vec<CliSlotStatus>,
+    #[serde(skip_serializing)]
+    pub verbose: bool,
+}
+
+impl fmt::Display for CliBlockProduction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f)?;
+        writeln!(
+            f,
+            "{}",
+            style(format!(
+                "  {:<44}  {:>15}  {:>15}  {:>15}  {:>23}",
+                "Identity Pubkey",
+                "Leader Slots",
+                "Blocks Produced",
+                "Skipped Slots",
+                "Skipped Slot Percentage",
+            ))
+            .bold()
+        )?;
+        for leader in &self.leaders {
+            writeln!(
+                f,
+                "  {:<44}  {:>15}  {:>15}  {:>15}  {:>22.2}%",
+                leader.identity_pubkey,
+                leader.leader_slots,
+                leader.blocks_produced,
+                leader.skipped_slots,
+                leader.skipped_slots as f64 / leader.leader_slots as f64 * 100.
+            )?;
+        }
+        writeln!(f)?;
+        writeln!(
+            f,
+            "  {:<44}  {:>15}  {:>15}  {:>15}  {:>22.2}%",
+            format!("Epoch {} total:", self.epoch),
+            self.total_slots,
+            self.total_blocks_produced,
+            self.total_slots_skipped,
+            self.total_slots_skipped as f64 / self.total_slots as f64 * 100.
+        )?;
+        writeln!(
+            f,
+            "  (using data from {} slots: {} to {})",
+            self.total_slots, self.start_slot, self.end_slot
+        )?;
+        if self.verbose {
+            writeln!(f)?;
+            writeln!(f)?;
+            writeln!(
+                f,
+                "{}",
+                style(format!("  {:<15} {:<44}", "Slot", "Identity Pubkey")).bold(),
+            )?;
+            for status in &self.individual_slot_status {
+                if status.skipped {
+                    writeln!(
+                        f,
+                        "{}",
+                        style(format!(
+                            "  {:<15} {:<44} SKIPPED",
+                            status.slot, status.leader
+                        ))
+                        .red()
+                    )?;
+                } else {
+                    writeln!(
+                        f,
+                        "{}",
+                        style(format!("  {:<15} {:<44}", status.slot, status.leader))
+                    )?;
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+#[derive(Default, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliBlockProductionEntry {
+    pub identity_pubkey: String,
+    pub leader_slots: u64,
+    pub blocks_produced: u64,
+    pub skipped_slots: u64,
+}
+
+#[derive(Default, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliSlotStatus {
+    pub slot: Slot,
+    pub leader: String,
+    pub skipped: bool,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliEpochInfo {
+    #[serde(flatten)]
+    pub epoch_info: RpcEpochInfo,
+}
+
+impl From<RpcEpochInfo> for CliEpochInfo {
+    fn from(epoch_info: RpcEpochInfo) -> Self {
+        Self { epoch_info }
+    }
+}
+
+impl fmt::Display for CliEpochInfo {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f)?;
+        writeln_name_value(f, "Slot:", &self.epoch_info.absolute_slot.to_string())?;
+        writeln_name_value(f, "Epoch:", &self.epoch_info.epoch.to_string())?;
+        let start_slot = self.epoch_info.absolute_slot - self.epoch_info.slot_index;
+        let end_slot = start_slot + self.epoch_info.slots_in_epoch;
+        writeln_name_value(
+            f,
+            "Epoch Slot Range:",
+            &format!("[{}..{})", start_slot, end_slot),
+        )?;
+        writeln_name_value(
+            f,
+            "Epoch Completed Percent:",
+            &format!(
+                "{:>3.3}%",
+                self.epoch_info.slot_index as f64 / self.epoch_info.slots_in_epoch as f64 * 100_f64
+            ),
+        )?;
+        let remaining_slots_in_epoch = self.epoch_info.slots_in_epoch - self.epoch_info.slot_index;
+        writeln_name_value(
+            f,
+            "Epoch Completed Slots:",
+            &format!(
+                "{}/{} ({} remaining)",
+                self.epoch_info.slot_index,
+                self.epoch_info.slots_in_epoch,
+                remaining_slots_in_epoch
+            ),
+        )?;
+        writeln_name_value(
+            f,
+            "Epoch Completed Time:",
+            &format!(
+                "{}/{} ({} remaining)",
+                slot_to_human_time(self.epoch_info.slot_index),
+                slot_to_human_time(self.epoch_info.slots_in_epoch),
+                slot_to_human_time(remaining_slots_in_epoch)
+            ),
+        )
+    }
+}
+
+fn slot_to_human_time(slot: Slot) -> String {
+    humantime::format_duration(Duration::from_secs(
+        slot * clock::DEFAULT_TICKS_PER_SLOT / clock::DEFAULT_TICKS_PER_SECOND,
+    ))
+    .to_string()
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliValidators {
+    pub total_active_stake: u64,
+    pub total_current_stake: u64,
+    pub total_deliquent_stake: u64,
+    pub current_validators: Vec<CliValidator>,
+    pub delinquent_validators: Vec<CliValidator>,
+    #[serde(skip_serializing)]
+    pub use_lamports_unit: bool,
+}
+
+impl fmt::Display for CliValidators {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        fn write_vote_account(
+            f: &mut fmt::Formatter,
+            validator: &CliValidator,
+            total_active_stake: u64,
+            use_lamports_unit: bool,
+            delinquent: bool,
+        ) -> fmt::Result {
+            fn non_zero_or_dash(v: u64) -> String {
+                if v == 0 {
+                    "-".into()
+                } else {
+                    format!("{}", v)
+                }
+            }
+
+            writeln!(
+                f,
+                "{} {:<44}  {:<44}  {:>9}%   {:>8}  {:>10}  {:>7}  {}",
+                if delinquent {
+                    WARNING.to_string()
+                } else {
+                    " ".to_string()
+                },
+                validator.identity_pubkey,
+                validator.vote_account_pubkey,
+                validator.commission,
+                non_zero_or_dash(validator.last_vote),
+                non_zero_or_dash(validator.root_slot),
+                validator.credits,
+                if validator.activated_stake > 0 {
+                    format!(
+                        "{} ({:.2}%)",
+                        build_balance_message(validator.activated_stake, use_lamports_unit, true),
+                        100. * validator.activated_stake as f64 / total_active_stake as f64
+                    )
+                } else {
+                    "-".into()
+                },
+            )
+        }
+        writeln_name_value(
+            f,
+            "Active Stake:",
+            &build_balance_message(self.total_active_stake, self.use_lamports_unit, true),
+        )?;
+        if self.total_deliquent_stake > 0 {
+            writeln_name_value(
+                f,
+                "Current Stake:",
+                &format!(
+                    "{} ({:0.2}%)",
+                    &build_balance_message(self.total_current_stake, self.use_lamports_unit, true),
+                    100. * self.total_current_stake as f64 / self.total_active_stake as f64
+                ),
+            )?;
+            writeln_name_value(
+                f,
+                "Delinquent Stake:",
+                &format!(
+                    "{} ({:0.2}%)",
+                    &build_balance_message(
+                        self.total_deliquent_stake,
+                        self.use_lamports_unit,
+                        true
+                    ),
+                    100. * self.total_deliquent_stake as f64 / self.total_active_stake as f64
+                ),
+            )?;
+        }
+        writeln!(f)?;
+        writeln!(
+            f,
+            "{}",
+            style(format!(
+                "  {:<44}  {:<44}  {}  {}  {}  {:>7}  {}",
+                "Identity Pubkey",
+                "Vote Account Pubkey",
+                "Commission",
+                "Last Vote",
+                "Root Block",
+                "Credits",
+                "Active Stake",
+            ))
+            .bold()
+        )?;
+        for validator in &self.current_validators {
+            write_vote_account(
+                f,
+                validator,
+                self.total_active_stake,
+                self.use_lamports_unit,
+                false,
+            )?;
+        }
+        for validator in &self.delinquent_validators {
+            write_vote_account(
+                f,
+                validator,
+                self.total_active_stake,
+                self.use_lamports_unit,
+                true,
+            )?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliValidator {
+    pub identity_pubkey: String,
+    pub vote_account_pubkey: String,
+    pub commission: u8,
+    pub last_vote: u64,
+    pub root_slot: u64,
+    pub credits: u64,
+    pub activated_stake: u64,
+}
+
+impl CliValidator {
+    pub fn new(vote_account: &RpcVoteAccountInfo, current_epoch: Epoch) -> Self {
+        Self {
+            identity_pubkey: vote_account.node_pubkey.to_string(),
+            vote_account_pubkey: vote_account.vote_pubkey.to_string(),
+            commission: vote_account.commission,
+            last_vote: vote_account.last_vote,
+            root_slot: vote_account.root_slot,
+            credits: vote_account
+                .epoch_credits
+                .iter()
+                .find_map(|(epoch, credits, _)| {
+                    if *epoch == current_epoch {
+                        Some(*credits)
+                    } else {
+                        None
+                    }
+                })
+                .unwrap_or(0),
+            activated_stake: vote_account.activated_stake,
+        }
+    }
+}
+
+#[derive(Default, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliNonceAccount {
+    pub balance: u64,
+    pub minimum_balance_for_rent_exemption: u64,
+    pub nonce: Option<String>,
+    pub lamports_per_signature: Option<u64>,
+    pub authority: Option<String>,
+    #[serde(skip_serializing)]
+    pub use_lamports_unit: bool,
+}
+
+impl fmt::Display for CliNonceAccount {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(
+            f,
+            "Balance: {}",
+            build_balance_message(self.balance, self.use_lamports_unit, true)
+        )?;
+        writeln!(
+            f,
+            "Minimum Balance Required: {}",
+            build_balance_message(
+                self.minimum_balance_for_rent_exemption,
+                self.use_lamports_unit,
+                true
+            )
+        )?;
+        let nonce = self.nonce.as_deref().unwrap_or("uninitialized");
+        writeln!(f, "Nonce: {}", nonce)?;
+        if let Some(fees) = self.lamports_per_signature {
+            writeln!(f, "Fee: {} lamports per signature", fees)?;
+        } else {
+            writeln!(f, "Fees: uninitialized")?;
+        }
+        let authority = self.authority.as_deref().unwrap_or("uninitialized");
+        writeln!(f, "Authority: {}", authority)
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct CliStakeVec(Vec<CliKeyedStakeState>);
+
+impl CliStakeVec {
+    pub fn new(list: Vec<CliKeyedStakeState>) -> Self {
+        Self(list)
+    }
+}
+
+impl fmt::Display for CliStakeVec {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        for state in &self.0 {
+            writeln!(f)?;
+            write!(f, "{}", state)?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliKeyedStakeState {
+    pub stake_pubkey: String,
+    #[serde(flatten)]
+    pub stake_state: CliStakeState,
+}
+
+impl fmt::Display for CliKeyedStakeState {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f, "Stake Pubkey: {}", self.stake_pubkey)?;
+        write!(f, "{}", self.stake_state)
+    }
+}
+
+#[derive(Default, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliStakeState {
+    pub stake_type: CliStakeType,
+    pub total_stake: u64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub delegated_stake: Option<u64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub delegated_vote_account_address: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub activation_epoch: Option<Epoch>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub deactivation_epoch: Option<Epoch>,
+    #[serde(flatten, skip_serializing_if = "Option::is_none")]
+    pub authorized: Option<CliAuthorized>,
+    #[serde(flatten, skip_serializing_if = "Option::is_none")]
+    pub lockup: Option<CliLockup>,
+    #[serde(skip_serializing)]
+    pub use_lamports_unit: bool,
+}
+
+impl fmt::Display for CliStakeState {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        fn show_authorized(f: &mut fmt::Formatter, authorized: &CliAuthorized) -> fmt::Result {
+            writeln!(f, "Stake Authority: {}", authorized.staker)?;
+            writeln!(f, "Withdraw Authority: {}", authorized.withdrawer)?;
+            Ok(())
+        }
+        fn show_lockup(f: &mut fmt::Formatter, lockup: &CliLockup) -> fmt::Result {
+            writeln!(
+                f,
+                "Lockup Timestamp: {} (UnixTimestamp: {})",
+                DateTime::<Utc>::from_utc(
+                    NaiveDateTime::from_timestamp(lockup.unix_timestamp, 0),
+                    Utc
+                )
+                .to_rfc3339_opts(SecondsFormat::Secs, true),
+                lockup.unix_timestamp
+            )?;
+            writeln!(f, "Lockup Epoch: {}", lockup.epoch)?;
+            writeln!(f, "Lockup Custodian: {}", lockup.custodian)?;
+            Ok(())
+        }
+
+        match self.stake_type {
+            CliStakeType::RewardsPool => writeln!(f, "Stake account is a rewards pool")?,
+            CliStakeType::Uninitialized => writeln!(f, "Stake account is uninitialized")?,
+            CliStakeType::Initialized => {
+                writeln!(
+                    f,
+                    "Total Stake: {}",
+                    build_balance_message(self.total_stake, self.use_lamports_unit, true)
+                )?;
+                writeln!(f, "Stake account is undelegated")?;
+                show_authorized(f, self.authorized.as_ref().unwrap())?;
+                show_lockup(f, self.lockup.as_ref().unwrap())?;
+            }
+            CliStakeType::Stake => {
+                writeln!(
+                    f,
+                    "Total Stake: {}",
+                    build_balance_message(self.total_stake, self.use_lamports_unit, true)
+                )?;
+                writeln!(
+                    f,
+                    "Delegated Stake: {}",
+                    build_balance_message(
+                        self.delegated_stake.unwrap(),
+                        self.use_lamports_unit,
+                        true
+                    )
+                )?;
+                if let Some(delegated_vote_account_address) = &self.delegated_vote_account_address {
+                    writeln!(
+                        f,
+                        "Delegated Vote Account Address: {}",
+                        delegated_vote_account_address
+                    )?;
+                }
+                writeln!(
+                    f,
+                    "Stake activates starting from epoch: {}",
+                    self.activation_epoch.unwrap()
+                )?;
+                if let Some(deactivation_epoch) = self.deactivation_epoch {
+                    writeln!(
+                        f,
+                        "Stake deactivates starting from epoch: {}",
+                        deactivation_epoch
+                    )?;
+                }
+                show_authorized(f, self.authorized.as_ref().unwrap())?;
+                show_lockup(f, self.lockup.as_ref().unwrap())?;
+            }
+        }
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub enum CliStakeType {
+    Stake,
+    RewardsPool,
+    Uninitialized,
+    Initialized,
+}
+
+impl Default for CliStakeType {
+    fn default() -> Self {
+        Self::Uninitialized
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliStakeHistory {
+    pub entries: Vec<CliStakeHistoryEntry>,
+    #[serde(skip_serializing)]
+    pub use_lamports_unit: bool,
+}
+
+impl fmt::Display for CliStakeHistory {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(f)?;
+        writeln!(
+            f,
+            "{}",
+            style(format!(
+                "  {:<5}  {:>20}  {:>20}  {:>20}",
+                "Epoch", "Effective Stake", "Activating Stake", "Deactivating Stake",
+            ))
+            .bold()
+        )?;
+        for entry in &self.entries {
+            writeln!(
+                f,
+                "  {:>5}  {:>20}  {:>20}  {:>20} {}",
+                entry.epoch,
+                build_balance_message(entry.effective_stake, self.use_lamports_unit, false),
+                build_balance_message(entry.activating_stake, self.use_lamports_unit, false),
+                build_balance_message(entry.deactivating_stake, self.use_lamports_unit, false),
+                if self.use_lamports_unit {
+                    "lamports"
+                } else {
+                    "SOL"
+                }
+            )?;
+        }
+        Ok(())
+    }
+}
+
+impl From<&(Epoch, StakeHistoryEntry)> for CliStakeHistoryEntry {
+    fn from((epoch, entry): &(Epoch, StakeHistoryEntry)) -> Self {
+        Self {
+            epoch: *epoch,
+            effective_stake: entry.effective,
+            activating_stake: entry.activating,
+            deactivating_stake: entry.deactivating,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliStakeHistoryEntry {
+    pub epoch: Epoch,
+    pub effective_stake: u64,
+    pub activating_stake: u64,
+    pub deactivating_stake: u64,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliAuthorized {
+    pub staker: String,
+    pub withdrawer: String,
+}
+
+impl From<&Authorized> for CliAuthorized {
+    fn from(authorized: &Authorized) -> Self {
+        Self {
+            staker: authorized.staker.to_string(),
+            withdrawer: authorized.withdrawer.to_string(),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliLockup {
+    pub unix_timestamp: UnixTimestamp,
+    pub epoch: Epoch,
+    pub custodian: String,
+}
+
+impl From<&Lockup> for CliLockup {
+    fn from(lockup: &Lockup) -> Self {
+        Self {
+            unix_timestamp: lockup.unix_timestamp,
+            epoch: lockup.epoch,
+            custodian: lockup.custodian.to_string(),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct CliValidatorInfoVec(Vec<CliValidatorInfo>);
+
+impl CliValidatorInfoVec {
+    pub fn new(list: Vec<CliValidatorInfo>) -> Self {
+        Self(list)
+    }
+}
+
+impl fmt::Display for CliValidatorInfoVec {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        if self.0.is_empty() {
+            writeln!(f, "No validator info accounts found")?;
+        }
+        for validator_info in &self.0 {
+            writeln!(f)?;
+            write!(f, "{}", validator_info)?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliValidatorInfo {
+    pub identity_pubkey: String,
+    pub info_pubkey: String,
+    pub info: Map<String, Value>,
+}
+
+impl fmt::Display for CliValidatorInfo {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln_name_value(f, "Validator Identity Pubkey:", &self.identity_pubkey)?;
+        writeln_name_value(f, "  Info Pubkey:", &self.info_pubkey)?;
+        for (key, value) in self.info.iter() {
+            writeln_name_value(
+                f,
+                &format!("  {}:", to_title_case(key)),
+                &value.as_str().unwrap_or("?"),
+            )?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliVoteAccount {
+    pub account_balance: u64,
+    pub validator_identity: String,
+    #[serde(flatten)]
+    pub authorized_voters: CliAuthorizedVoters,
+    pub authorized_withdrawer: String,
+    pub credits: u64,
+    pub commission: u8,
+    pub root_slot: Option<Slot>,
+    pub recent_timestamp: BlockTimestamp,
+    pub votes: Vec<CliLockout>,
+    pub epoch_voting_history: Vec<CliEpochVotingHistory>,
+    #[serde(skip_serializing)]
+    pub use_lamports_unit: bool,
+}
+
+impl fmt::Display for CliVoteAccount {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(
+            f,
+            "Account Balance: {}",
+            build_balance_message(self.account_balance, self.use_lamports_unit, true)
+        )?;
+        writeln!(f, "Validator Identity: {}", self.validator_identity)?;
+        writeln!(f, "Authorized Voters: {}", self.authorized_voters)?;
+        writeln!(f, "Authorized Withdrawer: {}", self.authorized_withdrawer)?;
+        writeln!(f, "Credits: {}", self.credits)?;
+        writeln!(f, "Commission: {}%", self.commission)?;
+        writeln!(
+            f,
+            "Root Slot: {}",
+            match self.root_slot {
+                Some(slot) => slot.to_string(),
+                None => "~".to_string(),
+            }
+        )?;
+        writeln!(f, "Recent Timestamp: {:?}", self.recent_timestamp)?;
+        if !self.votes.is_empty() {
+            writeln!(f, "Recent Votes:")?;
+            for vote in &self.votes {
+                writeln!(
+                    f,
+                    "- slot: {}\n  confirmation count: {}",
+                    vote.slot, vote.confirmation_count
+                )?;
+            }
+            writeln!(f, "Epoch Voting History:")?;
+            for epoch_info in &self.epoch_voting_history {
+                writeln!(
+                    f,
+                    "- epoch: {}\n  slots in epoch: {}\n  credits earned: {}",
+                    epoch_info.epoch, epoch_info.slots_in_epoch, epoch_info.credits_earned,
+                )?;
+            }
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliAuthorizedVoters {
+    authorized_voters: BTreeMap<Epoch, String>,
+}
+
+impl fmt::Display for CliAuthorizedVoters {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{:?}", self.authorized_voters)
+    }
+}
+
+impl From<&AuthorizedVoters> for CliAuthorizedVoters {
+    fn from(authorized_voters: &AuthorizedVoters) -> Self {
+        let mut voter_map: BTreeMap<Epoch, String> = BTreeMap::new();
+        for (epoch, voter) in authorized_voters.iter() {
+            voter_map.insert(*epoch, voter.to_string());
+        }
+        Self {
+            authorized_voters: voter_map,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliEpochVotingHistory {
+    pub epoch: Epoch,
+    pub slots_in_epoch: u64,
+    pub credits_earned: u64,
+}
+
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct CliLockout {
+    pub slot: Slot,
+    pub confirmation_count: u32,
+}
+
+impl From<&Lockout> for CliLockout {
+    fn from(lockout: &Lockout) -> Self {
+        Self {
+            slot: lockout.slot,
+            confirmation_count: lockout.confirmation_count,
+        }
+    }
+}

cli/src/cluster_query.rs

@@ -1,10 +1,12 @@
 use crate::{
-    cli::{
-        build_balance_message, check_account_for_fee, CliCommand, CliCommandInfo, CliConfig,
-        CliError, ProcessResult,
+    cli::{check_account_for_fee, CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult},
+    cli_output::{
+        CliBlockProduction, CliBlockProductionEntry, CliEpochInfo, CliKeyedStakeState,
+        CliSlotStatus, CliStakeVec, CliValidator, CliValidators,
     },
     display::println_name_value,
 };
+use chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc};
 use clap::{value_t, value_t_or_exit, App, Arg, ArgMatches, SubCommand};
 use console::{style, Emoji};
 use indicatif::{ProgressBar, ProgressStyle};
@@ -12,7 +14,6 @@ use solana_clap_utils::{input_parsers::*, input_validators::*, keypair::signer_f
 use solana_client::{
     pubsub_client::{PubsubClient, SlotInfoMessage},
     rpc_client::RpcClient,
-    rpc_response::RpcVoteAccountInfo,
 };
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
@@ -22,6 +23,7 @@ use solana_sdk::{
     epoch_schedule::Epoch,
     hash::Hash,
     message::Message,
+    native_token::lamports_to_sol,
     pubkey::Pubkey,
     signature::{Keypair, Signer},
     system_instruction,
@@ -40,7 +42,6 @@ use std::{
 
 static CHECK_MARK: Emoji = Emoji("✅ ", "");
 static CROSS_MARK: Emoji = Emoji("❌ ", "");
-static WARNING: Emoji = Emoji("⚠️", "!");
 
 pub trait ClusterQuerySubCommands {
     fn cluster_query_subcommands(self) -> Self;
@@ -55,8 +56,8 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                     Arg::with_name("node_pubkey")
                         .index(1)
                         .takes_value(true)
-                        .value_name("PUBKEY")
-                        .validator(is_pubkey_or_keypair)
+                        .value_name("VALIDATOR_PUBKEY")
+                        .validator(is_valid_pubkey)
                         .required(true)
                         .help("Identity pubkey of the validator"),
                 )
@@ -67,6 +68,20 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .takes_value(true)
                         .validator(is_url)
                         .help("JSON RPC URL for validator, which is useful for validators with a private RPC service")
+                )
+                .arg(
+                    Arg::with_name("confirmed")
+                        .long("confirmed")
+                        .takes_value(false)
+                        .help(
+                            "Return information at maximum-lockout commitment level",
+                        ),
+                )
+                .arg(
+                    Arg::with_name("follow")
+                        .long("follow")
+                        .takes_value(false)
+                        .help("Continue reporting progress even after the validator has caught up"),
                 ),
         )
         .subcommand(
@@ -117,6 +132,28 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                     ),
             ),
         )
+        .subcommand(
+            SubCommand::with_name("epoch").about("Get current epoch")
+            .arg(
+                Arg::with_name("confirmed")
+                    .long("confirmed")
+                    .takes_value(false)
+                    .help(
+                        "Return epoch at maximum-lockout commitment level",
+                    ),
+            ),
+        )
+        .subcommand(
+            SubCommand::with_name("total-supply").about("Get total number of SOL")
+            .arg(
+                Arg::with_name("confirmed")
+                    .long("confirmed")
+                    .takes_value(false)
+                    .help(
+                        "Return count at maximum-lockout commitment level",
+                    ),
+            ),
+        )
         .subcommand(
             SubCommand::with_name("transaction-count").about("Get current transaction count")
             .alias("get-transaction-count")
@@ -208,10 +245,10 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("vote_account_pubkeys")
                         .index(1)
-                        .value_name("VOTE ACCOUNT PUBKEYS")
+                        .value_name("VOTE_ACCOUNT_PUBKEYS")
                         .takes_value(true)
                         .multiple(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Only show stake accounts delegated to the provided vote accounts"),
                 )
                 .arg(
@@ -243,13 +280,24 @@ impl ClusterQuerySubCommands for App<'_, '_> {
     }
 }
 
-pub fn parse_catchup(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let node_pubkey = pubkey_of(matches, "node_pubkey").unwrap();
+pub fn parse_catchup(
+    matches: &ArgMatches<'_>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let node_pubkey = pubkey_of_signer(matches, "node_pubkey", wallet_manager)?.unwrap();
     let node_json_rpc_url = value_t!(matches, "node_json_rpc_url", String).ok();
+    let commitment_config = if matches.is_present("confirmed") {
+        CommitmentConfig::default()
+    } else {
+        CommitmentConfig::recent()
+    };
+    let follow = matches.is_present("follow");
     Ok(CliCommandInfo {
         command: CliCommand::Catchup {
             node_pubkey,
             node_json_rpc_url,
+            commitment_config,
+            follow,
         },
         signers: vec![],
     })
@@ -258,7 +306,7 @@ pub fn parse_catchup(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliErro
 pub fn parse_cluster_ping(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
     let lamports = value_t_or_exit!(matches, "lamports", u64);
     let interval = Duration::from_secs(value_t_or_exit!(matches, "interval", u64));
@@ -322,6 +370,30 @@ pub fn parse_get_slot(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliErr
     })
 }
 
+pub fn parse_get_epoch(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+    let commitment_config = if matches.is_present("confirmed") {
+        CommitmentConfig::default()
+    } else {
+        CommitmentConfig::recent()
+    };
+    Ok(CliCommandInfo {
+        command: CliCommand::GetEpoch { commitment_config },
+        signers: vec![],
+    })
+}
+
+pub fn parse_total_supply(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+    let commitment_config = if matches.is_present("confirmed") {
+        CommitmentConfig::default()
+    } else {
+        CommitmentConfig::recent()
+    };
+    Ok(CliCommandInfo {
+        command: CliCommand::TotalSupply { commitment_config },
+        signers: vec![],
+    })
+}
+
 pub fn parse_get_transaction_count(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let commitment_config = if matches.is_present("confirmed") {
         CommitmentConfig::default()
@@ -334,9 +406,13 @@ pub fn parse_get_transaction_count(matches: &ArgMatches<'_>) -> Result<CliComman
     })
 }
 
-pub fn parse_show_stakes(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+pub fn parse_show_stakes(
+    matches: &ArgMatches<'_>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
     let use_lamports_unit = matches.is_present("lamports");
-    let vote_account_pubkeys = pubkeys_of(matches, "vote_account_pubkeys");
+    let vote_account_pubkeys =
+        pubkeys_of_multiple_signers(matches, "vote_account_pubkeys", wallet_manager)?;
 
     Ok(CliCommandInfo {
         command: CliCommand::ShowStakes {
@@ -377,20 +453,37 @@ pub fn process_catchup(
     rpc_client: &RpcClient,
     node_pubkey: &Pubkey,
     node_json_rpc_url: &Option<String>,
+    commitment_config: CommitmentConfig,
+    follow: bool,
 ) -> ProcessResult {
-    let cluster_nodes = rpc_client.get_cluster_nodes()?;
+    let sleep_interval = 5;
+
+    let progress_bar = new_spinner_progress_bar();
+    progress_bar.set_message("Connecting...");
 
     let node_client = if let Some(node_json_rpc_url) = node_json_rpc_url {
         RpcClient::new(node_json_rpc_url.to_string())
     } else {
-        RpcClient::new_socket(
-            cluster_nodes
+        let rpc_addr = loop {
+            let cluster_nodes = rpc_client.get_cluster_nodes()?;
+            if let Some(contact_info) = cluster_nodes
                 .iter()
                 .find(|contact_info| contact_info.pubkey == node_pubkey.to_string())
-                .ok_or_else(|| format!("Contact information not found for {}", node_pubkey))?
-                .rpc
-                .ok_or_else(|| format!("RPC service not found for {}", node_pubkey))?,
-        )
+            {
+                if let Some(rpc_addr) = contact_info.rpc {
+                    break rpc_addr;
+                }
+                progress_bar.set_message(&format!("RPC service not found for {}", node_pubkey));
+            } else {
+                progress_bar.set_message(&format!(
+                    "Contact information not found for {}",
+                    node_pubkey
+                ));
+            }
+            sleep(Duration::from_secs(sleep_interval as u64));
+        };
+
+        RpcClient::new_socket(rpc_addr)
     };
 
     let reported_node_pubkey = node_client.get_identity()?;
@@ -406,16 +499,12 @@ pub fn process_catchup(
         return Err("Both RPC URLs reference the same node, unable to monitor for catchup.  Try a different --url".into());
     }
 
-    let progress_bar = new_spinner_progress_bar();
-    progress_bar.set_message("Connecting...");
-
     let mut previous_rpc_slot = std::u64::MAX;
     let mut previous_slot_distance = 0;
-    let sleep_interval = 5;
     loop {
-        let rpc_slot = rpc_client.get_slot_with_commitment(CommitmentConfig::recent())?;
-        let node_slot = node_client.get_slot_with_commitment(CommitmentConfig::recent())?;
-        if node_slot > std::cmp::min(previous_rpc_slot, rpc_slot) {
+        let rpc_slot = rpc_client.get_slot_with_commitment(commitment_config)?;
+        let node_slot = node_client.get_slot_with_commitment(commitment_config)?;
+        if !follow && node_slot > std::cmp::min(previous_rpc_slot, rpc_slot) {
             progress_bar.finish_and_clear();
             return Ok(format!(
                 "{} has caught up (us:{} them:{})",
@@ -429,7 +518,7 @@ pub fn process_catchup(
             slot_distance,
             node_slot,
             rpc_slot,
-            if previous_rpc_slot == std::u64::MAX {
+            if slot_distance == 0 || previous_rpc_slot == std::u64::MAX {
                 "".to_string()
             } else {
                 let slots_per_second =
@@ -504,54 +593,24 @@ pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
 
 pub fn process_get_block_time(rpc_client: &RpcClient, slot: Slot) -> ProcessResult {
     let timestamp = rpc_client.get_block_time(slot)?;
-    Ok(timestamp.to_string())
-}
-
-fn slot_to_human_time(slot: Slot) -> String {
-    humantime::format_duration(Duration::from_secs(
-        slot * clock::DEFAULT_TICKS_PER_SLOT / clock::DEFAULT_TICKS_PER_SECOND,
-    ))
-    .to_string()
+    let result = format!(
+        "{} (UnixTimestamp: {})",
+        DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(timestamp, 0), Utc)
+            .to_rfc3339_opts(SecondsFormat::Secs, true),
+        timestamp
+    );
+    Ok(result)
 }
 
 pub fn process_get_epoch_info(
     rpc_client: &RpcClient,
+    config: &CliConfig,
     commitment_config: CommitmentConfig,
 ) -> ProcessResult {
-    let epoch_info = rpc_client.get_epoch_info_with_commitment(commitment_config.clone())?;
-    println!();
-    println_name_value("Slot:", &epoch_info.absolute_slot.to_string());
-    println_name_value("Epoch:", &epoch_info.epoch.to_string());
-    let start_slot = epoch_info.absolute_slot - epoch_info.slot_index;
-    let end_slot = start_slot + epoch_info.slots_in_epoch;
-    println_name_value(
-        "Epoch Slot Range:",
-        &format!("[{}..{})", start_slot, end_slot),
-    );
-    println_name_value(
-        "Epoch Completed Percent:",
-        &format!(
-            "{:>3.3}%",
-            epoch_info.slot_index as f64 / epoch_info.slots_in_epoch as f64 * 100_f64
-        ),
-    );
-    let remaining_slots_in_epoch = epoch_info.slots_in_epoch - epoch_info.slot_index;
-    println_name_value(
-        "Epoch Completed Slots:",
-        &format!(
-            "{}/{} ({} remaining)",
-            epoch_info.slot_index, epoch_info.slots_in_epoch, remaining_slots_in_epoch
-        ),
-    );
-    println_name_value(
-        "Epoch Completed Time:",
-        &format!(
-            "{}/{} ({} remaining)",
-            slot_to_human_time(epoch_info.slot_index),
-            slot_to_human_time(epoch_info.slots_in_epoch),
-            slot_to_human_time(remaining_slots_in_epoch)
-        ),
-    );
+    let epoch_info: CliEpochInfo = rpc_client
+        .get_epoch_info_with_commitment(commitment_config.clone())?
+        .into();
+    config.output_format.formatted_print(&epoch_info);
     Ok("".to_string())
 }
 
@@ -568,6 +627,14 @@ pub fn process_get_slot(
     Ok(slot.to_string())
 }
 
+pub fn process_get_epoch(
+    rpc_client: &RpcClient,
+    commitment_config: CommitmentConfig,
+) -> ProcessResult {
+    let epoch_info = rpc_client.get_epoch_info_with_commitment(commitment_config.clone())?;
+    Ok(epoch_info.epoch.to_string())
+}
+
 pub fn parse_show_block_production(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let epoch = value_t!(matches, "epoch", Epoch).ok();
     let slot_limit = value_t!(matches, "slot_limit", u64).ok();
@@ -636,9 +703,9 @@ pub fn process_show_block_production(
     let start_slot_index = (start_slot - first_slot_in_epoch) as usize;
     let end_slot_index = (end_slot - first_slot_in_epoch) as usize;
     let total_slots = end_slot_index - start_slot_index + 1;
-    let total_blocks = confirmed_blocks.len();
-    assert!(total_blocks <= total_slots);
-    let total_slots_skipped = total_slots - total_blocks;
+    let total_blocks_produced = confirmed_blocks.len();
+    assert!(total_blocks_produced <= total_slots);
+    let total_slots_skipped = total_slots - total_blocks_produced;
     let mut leader_slot_count = HashMap::new();
     let mut leader_skipped_slots = HashMap::new();
 
@@ -662,7 +729,7 @@ pub fn process_show_block_production(
 
     progress_bar.set_message(&format!(
         "Processing {} slots containing {} blocks and {} empty slots...",
-        total_slots, total_blocks, total_slots_skipped
+        total_slots, total_blocks_produced, total_slots_skipped
     ));
 
     let mut confirmed_blocks_index = 0;
@@ -681,74 +748,63 @@ pub fn process_show_block_production(
                     continue;
                 }
                 if slot_of_next_confirmed_block == slot {
-                    individual_slot_status
-                        .push(style(format!("  {:<15} {:<44}", slot, leader)).to_string());
+                    individual_slot_status.push(CliSlotStatus {
+                        slot,
+                        leader: (*leader).to_string(),
+                        skipped: false,
+                    });
                     break;
                 }
             }
             *skipped_slots += 1;
-            individual_slot_status.push(
-                style(format!("  {:<15} {:<44} SKIPPED", slot, leader))
-                    .red()
-                    .to_string(),
-            );
+            individual_slot_status.push(CliSlotStatus {
+                slot,
+                leader: (*leader).to_string(),
+                skipped: true,
+            });
             break;
         }
     }
 
     progress_bar.finish_and_clear();
-    println!(
-        "\n{}",
-        style(format!(
-            "  {:<44}  {:>15}  {:>15}  {:>15}  {:>23}",
-            "Identity Pubkey",
-            "Leader Slots",
-            "Blocks Produced",
-            "Skipped Slots",
-            "Skipped Slot Percentage",
-        ))
-        .bold()
-    );
 
-    let mut table = vec![];
-    for (leader, leader_slots) in leader_slot_count.iter() {
-        let skipped_slots = leader_skipped_slots.get(leader).unwrap();
-        let blocks_produced = leader_slots - skipped_slots;
-        table.push(format!(
-            "  {:<44}  {:>15}  {:>15}  {:>15}  {:>22.2}%",
-            leader,
-            leader_slots,
-            blocks_produced,
-            skipped_slots,
-            *skipped_slots as f64 / *leader_slots as f64 * 100.
-        ));
-    }
-    table.sort();
-
-    println!(
-        "{}\n\n  {:<44}  {:>15}  {:>15}  {:>15}  {:>22.2}%",
-        table.join("\n"),
-        format!("Epoch {} total:", epoch),
+    let mut leaders: Vec<CliBlockProductionEntry> = leader_slot_count
+        .iter()
+        .map(|(leader, leader_slots)| {
+            let skipped_slots = leader_skipped_slots.get(leader).unwrap();
+            let blocks_produced = leader_slots - skipped_slots;
+            CliBlockProductionEntry {
+                identity_pubkey: (**leader).to_string(),
+                leader_slots: *leader_slots,
+                blocks_produced,
+                skipped_slots: *skipped_slots,
+            }
+        })
+        .collect();
+    leaders.sort_by(|a, b| a.identity_pubkey.partial_cmp(&b.identity_pubkey).unwrap());
+    let block_production = CliBlockProduction {
+        epoch,
+        start_slot,
+        end_slot,
         total_slots,
-        total_blocks,
+        total_blocks_produced,
         total_slots_skipped,
-        total_slots_skipped as f64 / total_slots as f64 * 100.
-    );
-    println!(
-        "  (using data from {} slots: {} to {})",
-        total_slots, start_slot, end_slot
-    );
-
-    if config.verbose {
-        println!(
-            "\n\n{}\n{}",
-            style(format!("  {:<15} {:<44}", "Slot", "Identity Pubkey")).bold(),
-            individual_slot_status.join("\n")
-        );
-    }
+        leaders,
+        individual_slot_status,
+        verbose: config.verbose,
+    };
+    config.output_format.formatted_print(&block_production);
     Ok("".to_string())
 }
 
+pub fn process_total_supply(
+    rpc_client: &RpcClient,
+    commitment_config: CommitmentConfig,
+) -> ProcessResult {
+    let total_supply = rpc_client.total_supply_with_commitment(commitment_config.clone())?;
+    Ok(format!("{} SOL", lamports_to_sol(total_supply)))
+}
+
 pub fn process_get_transaction_count(
     rpc_client: &RpcClient,
     commitment_config: CommitmentConfig,
@@ -1019,10 +1075,11 @@ pub fn process_show_gossip(rpc_client: &RpcClient) -> ProcessResult {
 
 pub fn process_show_stakes(
     rpc_client: &RpcClient,
+    config: &CliConfig,
     use_lamports_unit: bool,
     vote_account_pubkeys: Option<&[Pubkey]>,
 ) -> ProcessResult {
-    use crate::stake::print_stake_state;
+    use crate::stake::build_stake_state;
     use solana_stake_program::stake_state::StakeState;
 
     let progress_bar = new_spinner_progress_bar();
@@ -1030,13 +1087,20 @@ pub fn process_show_stakes(
     let all_stake_accounts = rpc_client.get_program_accounts(&solana_stake_program::id())?;
     progress_bar.finish_and_clear();
 
+    let mut stake_accounts: Vec<CliKeyedStakeState> = vec![];
     for (stake_pubkey, stake_account) in all_stake_accounts {
         if let Ok(stake_state) = stake_account.state() {
             match stake_state {
                 StakeState::Initialized(_) => {
                     if vote_account_pubkeys.is_none() {
-                        println!("\nstake pubkey: {}", stake_pubkey);
-                        print_stake_state(stake_account.lamports, &stake_state, use_lamports_unit);
+                        stake_accounts.push(CliKeyedStakeState {
+                            stake_pubkey: stake_pubkey.to_string(),
+                            stake_state: build_stake_state(
+                                stake_account.lamports,
+                                &stake_state,
+                                use_lamports_unit,
+                            ),
+                        });
                     }
                 }
                 StakeState::Stake(_, stake) => {
@@ -1045,19 +1109,29 @@ pub fn process_show_stakes(
                             .unwrap()
                             .contains(&stake.delegation.voter_pubkey)
                     {
-                        println!("\nstake pubkey: {}", stake_pubkey);
-                        print_stake_state(stake_account.lamports, &stake_state, use_lamports_unit);
+                        stake_accounts.push(CliKeyedStakeState {
+                            stake_pubkey: stake_pubkey.to_string(),
+                            stake_state: build_stake_state(
+                                stake_account.lamports,
+                                &stake_state,
+                                use_lamports_unit,
+                            ),
+                        });
                     }
                 }
                 _ => {}
             }
         }
     }
+    config
+        .output_format
+        .formatted_print(&CliStakeVec::new(stake_accounts));
     Ok("".to_string())
 }
 
 pub fn process_show_validators(
     rpc_client: &RpcClient,
+    config: &CliConfig,
     use_lamports_unit: bool,
     commitment_config: CommitmentConfig,
 ) -> ProcessResult {
@@ -1067,122 +1141,36 @@ pub fn process_show_validators(
         .current
         .iter()
         .chain(vote_accounts.delinquent.iter())
-        .fold(0, |acc, vote_account| acc + vote_account.activated_stake)
-        as f64;
+        .fold(0, |acc, vote_account| acc + vote_account.activated_stake);
 
     let total_deliquent_stake = vote_accounts
         .delinquent
         .iter()
-        .fold(0, |acc, vote_account| acc + vote_account.activated_stake)
-        as f64;
+        .fold(0, |acc, vote_account| acc + vote_account.activated_stake);
     let total_current_stake = total_active_stake - total_deliquent_stake;
 
-    println_name_value(
-        "Active Stake:",
-        &build_balance_message(total_active_stake as u64, use_lamports_unit, true),
-    );
-    if total_deliquent_stake > 0. {
-        println_name_value(
-            "Current Stake:",
-            &format!(
-                "{} ({:0.2}%)",
-                &build_balance_message(total_current_stake as u64, use_lamports_unit, true),
-                100. * total_current_stake / total_active_stake
-            ),
-        );
-        println_name_value(
-            "Delinquent Stake:",
-            &format!(
-                "{} ({:0.2}%)",
-                &build_balance_message(total_deliquent_stake as u64, use_lamports_unit, true),
-                100. * total_deliquent_stake / total_active_stake
-            ),
-        );
-    }
-    println!();
-
-    println!(
-        "{}",
-        style(format!(
-            "  {:<44}  {:<44}  {}  {}  {}  {:>7}  {}",
-            "Identity Pubkey",
-            "Vote Account Pubkey",
-            "Commission",
-            "Last Vote",
-            "Root Block",
-            "Credits",
-            "Active Stake",
-        ))
-        .bold()
-    );
-
-    fn print_vote_account(
-        vote_account: RpcVoteAccountInfo,
-        current_epoch: Epoch,
-        total_active_stake: f64,
-        use_lamports_unit: bool,
-        delinquent: bool,
-    ) {
-        fn non_zero_or_dash(v: u64) -> String {
-            if v == 0 {
-                "-".into()
-            } else {
-                format!("{}", v)
-            }
-        }
-
-        println!(
-            "{} {:<44}  {:<44}  {:>9}%   {:>8}  {:>10}  {:>7}  {}",
-            if delinquent {
-                WARNING.to_string()
-            } else {
-                " ".to_string()
-            },
-            vote_account.node_pubkey,
-            vote_account.vote_pubkey,
-            vote_account.commission,
-            non_zero_or_dash(vote_account.last_vote),
-            non_zero_or_dash(vote_account.root_slot),
-            vote_account
-                .epoch_credits
-                .iter()
-                .find_map(|(epoch, credits, _)| if *epoch == current_epoch {
-                    Some(*credits)
-                } else {
-                    None
-                })
-                .unwrap_or(0),
-            if vote_account.activated_stake > 0 {
-                format!(
-                    "{} ({:.2}%)",
-                    build_balance_message(vote_account.activated_stake, use_lamports_unit, true),
-                    100. * vote_account.activated_stake as f64 / total_active_stake
-                )
-            } else {
-                "-".into()
-            },
-        );
-    }
-
-    for vote_account in vote_accounts.current.into_iter() {
-        print_vote_account(
-            vote_account,
-            epoch_info.epoch,
-            total_active_stake,
-            use_lamports_unit,
-            false,
-        );
-    }
-    for vote_account in vote_accounts.delinquent.into_iter() {
-        print_vote_account(
-            vote_account,
-            epoch_info.epoch,
-            total_active_stake,
-            use_lamports_unit,
-            true,
-        );
-    }
+    let mut current = vote_accounts.current;
+    current.sort_by(|a, b| b.activated_stake.cmp(&a.activated_stake));
+    let current_validators: Vec<CliValidator> = current
+        .iter()
+        .map(|vote_account| CliValidator::new(vote_account, epoch_info.epoch))
+        .collect();
+    let mut delinquent = vote_accounts.delinquent;
+    delinquent.sort_by(|a, b| b.activated_stake.cmp(&a.activated_stake));
+    let delinquent_validators: Vec<CliValidator> = delinquent
+        .iter()
+        .map(|vote_account| CliValidator::new(vote_account, epoch_info.epoch))
+        .collect();
 
+    let cli_validators = CliValidators {
+        total_active_stake,
+        total_current_stake,
+        total_deliquent_stake,
+        current_validators,
+        delinquent_validators,
+        use_lamports_unit,
+    };
+    config.output_format.formatted_print(&cli_validators);
     Ok("".to_string())
 }
 
@@ -1209,7 +1197,7 @@ mod tests {
             .clone()
             .get_matches_from(vec!["test", "cluster-version"]);
         assert_eq!(
-            parse_command(&test_cluster_version, &default_keypair_file, None).unwrap(),
+            parse_command(&test_cluster_version, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::ClusterVersion,
                 signers: vec![],
@@ -1218,7 +1206,7 @@ mod tests {
 
         let test_fees = test_commands.clone().get_matches_from(vec!["test", "fees"]);
         assert_eq!(
-            parse_command(&test_fees, &default_keypair_file, None).unwrap(),
+            parse_command(&test_fees, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::Fees,
                 signers: vec![],
@@ -1231,7 +1219,7 @@ mod tests {
                 .clone()
                 .get_matches_from(vec!["test", "block-time", &slot.to_string()]);
         assert_eq!(
-            parse_command(&test_get_block_time, &default_keypair_file, None).unwrap(),
+            parse_command(&test_get_block_time, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::GetBlockTime { slot },
                 signers: vec![],
@@ -1242,7 +1230,7 @@ mod tests {
             .clone()
             .get_matches_from(vec!["test", "epoch-info"]);
         assert_eq!(
-            parse_command(&test_get_epoch_info, &default_keypair_file, None).unwrap(),
+            parse_command(&test_get_epoch_info, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::GetEpochInfo {
                     commitment_config: CommitmentConfig::recent(),
@@ -1255,7 +1243,7 @@ mod tests {
             .clone()
             .get_matches_from(vec!["test", "genesis-hash"]);
         assert_eq!(
-            parse_command(&test_get_genesis_hash, &default_keypair_file, None).unwrap(),
+            parse_command(&test_get_genesis_hash, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::GetGenesisHash,
                 signers: vec![],
@@ -1264,7 +1252,7 @@ mod tests {
 
         let test_get_slot = test_commands.clone().get_matches_from(vec!["test", "slot"]);
         assert_eq!(
-            parse_command(&test_get_slot, &default_keypair_file, None).unwrap(),
+            parse_command(&test_get_slot, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::GetSlot {
                     commitment_config: CommitmentConfig::recent(),
@@ -1273,11 +1261,37 @@ mod tests {
             }
         );
 
+        let test_get_epoch = test_commands
+            .clone()
+            .get_matches_from(vec!["test", "epoch"]);
+        assert_eq!(
+            parse_command(&test_get_epoch, &default_keypair_file, &mut None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::GetEpoch {
+                    commitment_config: CommitmentConfig::recent(),
+                },
+                signers: vec![],
+            }
+        );
+
+        let test_total_supply = test_commands
+            .clone()
+            .get_matches_from(vec!["test", "total-supply"]);
+        assert_eq!(
+            parse_command(&test_total_supply, &default_keypair_file, &mut None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::TotalSupply {
+                    commitment_config: CommitmentConfig::recent(),
+                },
+                signers: vec![],
+            }
+        );
+
         let test_transaction_count = test_commands
             .clone()
             .get_matches_from(vec!["test", "transaction-count"]);
         assert_eq!(
-            parse_command(&test_transaction_count, &default_keypair_file, None).unwrap(),
+            parse_command(&test_transaction_count, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::GetTransactionCount {
                     commitment_config: CommitmentConfig::recent(),
@@ -1298,7 +1312,7 @@ mod tests {
             "--confirmed",
         ]);
         assert_eq!(
-            parse_command(&test_ping, &default_keypair_file, None).unwrap(),
+            parse_command(&test_ping, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::Ping {
                     lamports: 1,

cli/src/display.rs

@@ -1,6 +1,11 @@
 use crate::cli::SettingType;
 use console::style;
-use solana_sdk::transaction::Transaction;
+use solana_sdk::{
+    hash::Hash, native_token::lamports_to_sol, program_utils::limited_deserialize,
+    transaction::Transaction,
+};
+use solana_transaction_status::RpcTransactionStatusMeta;
+use std::fmt;
 
 // Pretty print a "name value"
 pub fn println_name_value(name: &str, value: &str) {
@@ -12,6 +17,15 @@ pub fn println_name_value(name: &str, value: &str) {
     println!("{} {}", style(name).bold(), styled_value);
 }
 
+pub fn writeln_name_value(f: &mut fmt::Formatter, name: &str, value: &str) -> fmt::Result {
+    let styled_value = if value == "" {
+        style("(not set)").italic()
+    } else {
+        style(value)
+    };
+    writeln!(f, "{} {}", style(name).bold(), styled_value)
+}
+
 pub fn println_name_value_or(name: &str, value: &str, setting_type: SettingType) {
     let description = match setting_type {
         SettingType::Explicit => "",
@@ -27,13 +41,128 @@ pub fn println_name_value_or(name: &str, value: &str, setting_type: SettingType)
     );
 }
 
-pub fn println_signers(tx: &Transaction) {
+pub fn println_signers(
+    blockhash: &Hash,
+    signers: &[String],
+    absent: &[String],
+    bad_sig: &[String],
+) {
     println!();
-    println!("Blockhash: {}", tx.message.recent_blockhash);
-    println!("Signers (Pubkey=Signature):");
-    tx.signatures
-        .iter()
-        .zip(tx.message.account_keys.clone())
-        .for_each(|(signature, pubkey)| println!("  {:?}={:?}", pubkey, signature));
+    println!("Blockhash: {}", blockhash);
+    if !signers.is_empty() {
+        println!("Signers (Pubkey=Signature):");
+        signers.iter().for_each(|signer| println!("  {}", signer))
+    }
+    if !absent.is_empty() {
+        println!("Absent Signers (Pubkey):");
+        absent.iter().for_each(|pubkey| println!("  {}", pubkey))
+    }
+    if !bad_sig.is_empty() {
+        println!("Bad Signatures (Pubkey):");
+        bad_sig.iter().for_each(|pubkey| println!("  {}", pubkey))
+    }
     println!();
 }
+
+pub fn println_transaction(
+    transaction: &Transaction,
+    transaction_status: &Option<RpcTransactionStatusMeta>,
+    prefix: &str,
+) {
+    let message = &transaction.message;
+    println!("{}Recent Blockhash: {:?}", prefix, message.recent_blockhash);
+    for (signature_index, signature) in transaction.signatures.iter().enumerate() {
+        println!("{}Signature {}: {:?}", prefix, signature_index, signature);
+    }
+    println!("{}{:?}", prefix, message.header);
+    for (account_index, account) in message.account_keys.iter().enumerate() {
+        println!("{}Account {}: {:?}", prefix, account_index, account);
+    }
+    for (instruction_index, instruction) in message.instructions.iter().enumerate() {
+        let program_pubkey = message.account_keys[instruction.program_id_index as usize];
+        println!("{}Instruction {}", prefix, instruction_index);
+        println!(
+            "{}  Program: {} ({})",
+            prefix, program_pubkey, instruction.program_id_index
+        );
+        for (account_index, account) in instruction.accounts.iter().enumerate() {
+            let account_pubkey = message.account_keys[*account as usize];
+            println!(
+                "{}  Account {}: {} ({})",
+                prefix, account_index, account_pubkey, account
+            );
+        }
+
+        let mut raw = true;
+        if program_pubkey == solana_vote_program::id() {
+            if let Ok(vote_instruction) = limited_deserialize::<
+                solana_vote_program::vote_instruction::VoteInstruction,
+            >(&instruction.data)
+            {
+                println!("{}  {:?}", prefix, vote_instruction);
+                raw = false;
+            }
+        } else if program_pubkey == solana_stake_program::id() {
+            if let Ok(stake_instruction) = limited_deserialize::<
+                solana_stake_program::stake_instruction::StakeInstruction,
+            >(&instruction.data)
+            {
+                println!("{}  {:?}", prefix, stake_instruction);
+                raw = false;
+            }
+        } else if program_pubkey == solana_sdk::system_program::id() {
+            if let Ok(system_instruction) = limited_deserialize::<
+                solana_sdk::system_instruction::SystemInstruction,
+            >(&instruction.data)
+            {
+                println!("{}  {:?}", prefix, system_instruction);
+                raw = false;
+            }
+        }
+
+        if raw {
+            println!("{}  Data: {:?}", prefix, instruction.data);
+        }
+    }
+
+    if let Some(transaction_status) = transaction_status {
+        println!(
+            "{}Status: {}",
+            prefix,
+            match &transaction_status.status {
+                Ok(_) => "Ok".into(),
+                Err(err) => err.to_string(),
+            }
+        );
+        println!("{}  Fee: {}", prefix, transaction_status.fee);
+        assert_eq!(
+            transaction_status.pre_balances.len(),
+            transaction_status.post_balances.len()
+        );
+        for (i, (pre, post)) in transaction_status
+            .pre_balances
+            .iter()
+            .zip(transaction_status.post_balances.iter())
+            .enumerate()
+        {
+            if pre == post {
+                println!(
+                    "{}  Account {} balance: {} SOL",
+                    prefix,
+                    i,
+                    lamports_to_sol(*pre)
+                );
+            } else {
+                println!(
+                    "{}  Account {} balance: {} SOL -> {} SOL",
+                    prefix,
+                    i,
+                    lamports_to_sol(*pre),
+                    lamports_to_sol(*post)
+                );
+            }
+        }
+    } else {
+        println!("{}Status: Unavailable", prefix);
+    }
+}

cli/src/lib.rs

@@ -1,4 +1,8 @@
+#[macro_use]
+extern crate serde_derive;
+
 pub mod cli;
+pub mod cli_output;
 pub mod cluster_query;
 pub mod display;
 pub mod nonce;

cli/src/main.rs

@@ -2,14 +2,16 @@ use clap::{crate_description, crate_name, AppSettings, Arg, ArgGroup, ArgMatches
 use console::style;
 
 use solana_clap_utils::{
-    input_validators::is_url, keypair::SKIP_SEED_PHRASE_VALIDATION_ARG, DisplayError,
+    input_validators::is_url, keypair::SKIP_SEED_PHRASE_VALIDATION_ARG, offline::SIGN_ONLY_ARG,
+    DisplayError,
 };
 use solana_cli::{
     cli::{app, parse_command, process_command, CliCommandInfo, CliConfig, CliSigners},
+    cli_output::OutputFormat,
     display::{println_name_value, println_name_value_or},
 };
 use solana_cli_config::{Config, CONFIG_FILE};
-use solana_remote_wallet::remote_wallet::{maybe_wallet_manager, RemoteWalletManager};
+use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use std::{error, sync::Arc};
 
 fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error>> {
@@ -33,7 +35,7 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
                     if let Some(field) = subcommand_matches.value_of("specific_setting") {
                         let (field_name, value, setting_type) = match field {
                             "json_rpc_url" => ("RPC URL", json_rpc_url, url_setting_type),
-                            "websocket_url" => ("WS URL", websocket_url, ws_setting_type),
+                            "websocket_url" => ("WebSocket URL", websocket_url, ws_setting_type),
                             "keypair" => ("Key Path", keypair_path, keypair_setting_type),
                             _ => unreachable!(),
                         };
@@ -41,7 +43,7 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
                     } else {
                         println_name_value("Config File:", config_file);
                         println_name_value_or("RPC URL:", &json_rpc_url, url_setting_type);
-                        println_name_value_or("WS URL:", &websocket_url, ws_setting_type);
+                        println_name_value_or("WebSocket URL:", &websocket_url, ws_setting_type);
                         println_name_value_or("Keypair Path:", &keypair_path, keypair_setting_type);
                     }
                 } else {
@@ -82,7 +84,7 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
 
                     println_name_value("Config File:", config_file);
                     println_name_value_or("RPC URL:", &json_rpc_url, url_setting_type);
-                    println_name_value_or("WS URL:", &websocket_url, ws_setting_type);
+                    println_name_value_or("WebSocket URL:", &websocket_url, ws_setting_type);
                     println_name_value_or("Keypair Path:", &keypair_path, keypair_setting_type);
                 } else {
                     println!(
@@ -101,7 +103,7 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
 
 pub fn parse_args<'a>(
     matches: &ArgMatches<'_>,
-    wallet_manager: Option<Arc<RemoteWalletManager>>,
+    mut wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<(CliConfig<'a>, CliSigners), Box<dyn error::Error>> {
     let config = if let Some(config_file) = matches.value_of("config_file") {
         Config::load(config_file).unwrap_or_default()
@@ -124,7 +126,16 @@ pub fn parse_args<'a>(
     );
 
     let CliCommandInfo { command, signers } =
-        parse_command(&matches, &default_signer_path, wallet_manager.as_ref())?;
+        parse_command(&matches, &default_signer_path, &mut wallet_manager)?;
+
+    let output_format = matches
+        .value_of("output_format")
+        .map(|value| match value {
+            "json" => OutputFormat::Json,
+            "json-compact" => OutputFormat::JsonCompact,
+            _ => unreachable!(),
+        })
+        .unwrap_or(OutputFormat::Display);
 
     Ok((
         CliConfig {
@@ -135,6 +146,7 @@ pub fn parse_args<'a>(
             keypair_path: default_signer_path,
             rpc_client: None,
             verbose: matches.is_present("verbose"),
+            output_format,
         },
         signers,
     ))
@@ -151,7 +163,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         let arg = Arg::with_name("config_file")
             .short("C")
             .long("config")
-            .value_name("PATH")
+            .value_name("FILEPATH")
             .takes_value(true)
             .global(true)
             .help("Configuration file to use");
@@ -184,17 +196,25 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         Arg::with_name("keypair")
             .short("k")
             .long("keypair")
-            .value_name("PATH")
+            .value_name("KEYPAIR")
             .global(true)
             .takes_value(true)
-            .help("/path/to/id.json or usb://remote/wallet/path"),
+            .help("Filepath or URL to a keypair"),
     )
     .arg(
         Arg::with_name("verbose")
             .long("verbose")
             .short("v")
             .global(true)
-            .help("Show extra information header"),
+            .help("Show additional information"),
+    )
+    .arg(
+        Arg::with_name("output_format")
+            .long("output")
+            .global(true)
+            .takes_value(true)
+            .possible_values(&["json", "json-compact"])
+            .help("Return information in specified output format. Supports: json, json-compact"),
     )
     .arg(
         Arg::with_name(SKIP_SEED_PHRASE_VALIDATION_ARG.name)
@@ -237,12 +257,18 @@ fn main() -> Result<(), Box<dyn error::Error>> {
 
 fn do_main(matches: &ArgMatches<'_>) -> Result<(), Box<dyn error::Error>> {
     if parse_settings(&matches)? {
-        let wallet_manager = maybe_wallet_manager()?;
+        let mut wallet_manager = None;
 
-        let (mut config, signers) = parse_args(&matches, wallet_manager)?;
+        let (mut config, signers) = parse_args(&matches, &mut wallet_manager)?;
         config.signers = signers.iter().map(|s| s.as_ref()).collect();
         let result = process_command(&config)?;
-        println!("{}", result);
+        let (_, submatches) = matches.subcommand();
+        let sign_only = submatches
+            .map(|m| m.is_present(SIGN_ONLY_ARG.name))
+            .unwrap_or(false);
+        if !sign_only {
+            println!("{}", result);
+        }
     };
     Ok(())
 }

cli/src/nonce.rs

@@ -1,7 +1,10 @@
-use crate::cli::{
-    build_balance_message, check_account_for_fee, check_unique_pubkeys, generate_unique_signers,
-    log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult,
-    SignerIndex,
+use crate::{
+    cli::{
+        check_account_for_fee, check_unique_pubkeys, generate_unique_signers,
+        log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError,
+        ProcessResult, SignerIndex,
+    },
+    cli_output::CliNonceAccount,
 };
 use clap::{App, Arg, ArgMatches, SubCommand};
 use solana_clap_utils::{
@@ -21,9 +24,8 @@ use solana_sdk::{
     },
     pubkey::Pubkey,
     system_instruction::{
-        advance_nonce_account, authorize_nonce_account, create_address_with_seed,
-        create_nonce_account, create_nonce_account_with_seed, withdraw_nonce_account, NonceError,
-        SystemError,
+        advance_nonce_account, authorize_nonce_account, create_nonce_account,
+        create_nonce_account_with_seed, withdraw_nonce_account, NonceError, SystemError,
     },
     system_program,
     transaction::Transaction,
@@ -74,7 +76,7 @@ pub fn nonce_arg<'a, 'b>() -> Arg<'a, 'b> {
         .takes_value(true)
         .value_name("PUBKEY")
         .requires(BLOCKHASH_ARG.name)
-        .validator(is_pubkey)
+        .validator(is_valid_pubkey)
         .help(NONCE_ARG.help)
 }
 
@@ -82,7 +84,7 @@ pub fn nonce_authority_arg<'a, 'b>() -> Arg<'a, 'b> {
     Arg::with_name(NONCE_AUTHORITY_ARG.name)
         .long(NONCE_AUTHORITY_ARG.long)
         .takes_value(true)
-        .value_name("KEYPAIR or PUBKEY or REMOTE WALLET PATH")
+        .value_name("KEYPAIR")
         .validator(is_valid_signer)
         .help(NONCE_AUTHORITY_ARG.help)
 }
@@ -93,30 +95,23 @@ impl NonceSubCommands for App<'_, '_> {
             SubCommand::with_name("authorize-nonce-account")
                 .about("Assign account authority to a new entity")
                 .arg(
-                    Arg::with_name("nonce_account_keypair")
+                    Arg::with_name("nonce_account_pubkey")
                         .index(1)
-                        .value_name("NONCE_ACCOUNT")
+                        .value_name("NONCE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Address of the nonce account"),
                 )
                 .arg(
                     Arg::with_name("new_authority")
                         .index(2)
-                        .value_name("NEW_AUTHORITY_PUBKEY")
+                        .value_name("AUTHORITY_PUBKEY")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Account to be granted authority of the nonce account"),
                 )
-                .arg(
-                    Arg::with_name("seed")
-                        .long("seed")
-                        .value_name("SEED STRING")
-                        .takes_value(true)
-                        .help("Seed for address generation; if specified, the resulting account will be at a derived address of the NONCE_ACCOUNT pubkey")
-                )
                 .arg(nonce_authority_arg()),
         )
         .subcommand(
@@ -125,10 +120,10 @@ impl NonceSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("nonce_account_keypair")
                         .index(1)
-                        .value_name("NONCE ACCOUNT")
+                        .value_name("ACCOUNT_KEYPAIR")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_signer)
                         .help("Keypair of the nonce account to fund"),
                 )
                 .arg(
@@ -144,9 +139,16 @@ impl NonceSubCommands for App<'_, '_> {
                     Arg::with_name(NONCE_AUTHORITY_ARG.name)
                         .long(NONCE_AUTHORITY_ARG.long)
                         .takes_value(true)
-                        .value_name("BASE58_PUBKEY")
-                        .validator(is_pubkey_or_keypair)
+                        .value_name("PUBKEY")
+                        .validator(is_valid_pubkey)
                         .help("Assign noncing authority to another entity"),
+                )
+                .arg(
+                    Arg::with_name("seed")
+                        .long("seed")
+                        .value_name("STRING")
+                        .takes_value(true)
+                        .help("Seed for address generation; if specified, the resulting account will be at a derived address of the NONCE_ACCOUNT pubkey")
                 ),
         )
         .subcommand(
@@ -156,10 +158,10 @@ impl NonceSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("nonce_account_pubkey")
                         .index(1)
-                        .value_name("NONCE ACCOUNT")
+                        .value_name("NONCE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Address of the nonce account to display"),
                 ),
         )
@@ -167,12 +169,12 @@ impl NonceSubCommands for App<'_, '_> {
             SubCommand::with_name("new-nonce")
                 .about("Generate a new nonce, rendering the existing nonce useless")
                 .arg(
-                    Arg::with_name("nonce_account_keypair")
+                    Arg::with_name("nonce_account_pubkey")
                         .index(1)
-                        .value_name("NONCE ACCOUNT")
+                        .value_name("NONCE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Address of the nonce account"),
                 )
                 .arg(nonce_authority_arg()),
@@ -184,10 +186,10 @@ impl NonceSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("nonce_account_pubkey")
                         .index(1)
-                        .value_name("NONCE ACCOUNT")
+                        .value_name("NONCE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Address of the nonce account to display"),
                 )
                 .arg(
@@ -199,24 +201,24 @@ impl NonceSubCommands for App<'_, '_> {
         )
         .subcommand(
             SubCommand::with_name("withdraw-from-nonce-account")
-                .about("Withdraw lamports from the nonce account")
+                .about("Withdraw SOL from the nonce account")
                 .arg(
-                    Arg::with_name("nonce_account_keypair")
+                    Arg::with_name("nonce_account_pubkey")
                         .index(1)
-                        .value_name("NONCE ACCOUNT")
+                        .value_name("NONCE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_keypair_or_ask_keyword)
-                        .help("Nonce account from to withdraw from"),
+                        .validator(is_valid_pubkey)
+                        .help("Nonce account to withdraw from"),
                 )
                 .arg(
                     Arg::with_name("destination_account_pubkey")
                         .index(2)
-                        .value_name("DESTINATION ACCOUNT")
+                        .value_name("RECIPIENT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("The account to which the lamports should be transferred"),
+                        .validator(is_valid_pubkey)
+                        .help("The account to which the SOL should be transferred"),
                 )
                 .arg(
                     Arg::with_name("amount")
@@ -277,10 +279,10 @@ pub fn data_from_state(state: &State) -> Result<&Data, CliNonceError> {
 pub fn parse_authorize_nonce_account(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let nonce_account = pubkey_of(matches, "nonce_account_keypair").unwrap();
-    let new_authority = pubkey_of(matches, "new_authority").unwrap();
+    let nonce_account = pubkey_of_signer(matches, "nonce_account_pubkey", wallet_manager)?.unwrap();
+    let new_authority = pubkey_of_signer(matches, "new_authority", wallet_manager)?.unwrap();
     let (nonce_authority, nonce_authority_pubkey) =
         signer_of(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
 
@@ -305,13 +307,13 @@ pub fn parse_authorize_nonce_account(
 pub fn parse_nonce_create_account(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
     let (nonce_account, nonce_account_pubkey) =
         signer_of(matches, "nonce_account_keypair", wallet_manager)?;
     let seed = matches.value_of("seed").map(|s| s.to_string());
     let lamports = lamports_of_sol(matches, "amount").unwrap();
-    let nonce_authority = pubkey_of(matches, NONCE_AUTHORITY_ARG.name);
+    let nonce_authority = pubkey_of_signer(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
 
     let payer_provided = None;
     let signer_info = generate_unique_signers(
@@ -332,8 +334,12 @@ pub fn parse_nonce_create_account(
     })
 }
 
-pub fn parse_get_nonce(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let nonce_account_pubkey = pubkey_of(matches, "nonce_account_pubkey").unwrap();
+pub fn parse_get_nonce(
+    matches: &ArgMatches<'_>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let nonce_account_pubkey =
+        pubkey_of_signer(matches, "nonce_account_pubkey", wallet_manager)?.unwrap();
 
     Ok(CliCommandInfo {
         command: CliCommand::GetNonce(nonce_account_pubkey),
@@ -344,9 +350,9 @@ pub fn parse_get_nonce(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliEr
 pub fn parse_new_nonce(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let nonce_account = pubkey_of(matches, "nonce_account_keypair").unwrap();
+    let nonce_account = pubkey_of_signer(matches, "nonce_account_pubkey", wallet_manager)?.unwrap();
     let (nonce_authority, nonce_authority_pubkey) =
         signer_of(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
 
@@ -367,8 +373,12 @@ pub fn parse_new_nonce(
     })
 }
 
-pub fn parse_show_nonce_account(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let nonce_account_pubkey = pubkey_of(matches, "nonce_account_pubkey").unwrap();
+pub fn parse_show_nonce_account(
+    matches: &ArgMatches<'_>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let nonce_account_pubkey =
+        pubkey_of_signer(matches, "nonce_account_pubkey", wallet_manager)?.unwrap();
     let use_lamports_unit = matches.is_present("lamports");
 
     Ok(CliCommandInfo {
@@ -383,10 +393,11 @@ pub fn parse_show_nonce_account(matches: &ArgMatches<'_>) -> Result<CliCommandIn
 pub fn parse_withdraw_from_nonce_account(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let nonce_account = pubkey_of(matches, "nonce_account_keypair").unwrap();
-    let destination_account_pubkey = pubkey_of(matches, "destination_account_pubkey").unwrap();
+    let nonce_account = pubkey_of_signer(matches, "nonce_account_pubkey", wallet_manager)?.unwrap();
+    let destination_account_pubkey =
+        pubkey_of_signer(matches, "destination_account_pubkey", wallet_manager)?.unwrap();
     let lamports = lamports_of_sol(matches, "amount").unwrap();
     let (nonce_authority, nonce_authority_pubkey) =
         signer_of(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
@@ -451,7 +462,7 @@ pub fn process_authorize_nonce_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
     log_instruction_custom_error::<NonceError>(result)
 }
 
@@ -465,7 +476,7 @@ pub fn process_create_nonce_account(
 ) -> ProcessResult {
     let nonce_account_pubkey = config.signers[nonce_account].pubkey();
     let nonce_account_address = if let Some(seed) = seed.clone() {
-        create_address_with_seed(&nonce_account_pubkey, &seed, &system_program::id())?
+        Pubkey::create_with_seed(&nonce_account_pubkey, &seed, &system_program::id())?
     } else {
         nonce_account_pubkey
     };
@@ -528,7 +539,7 @@ pub fn process_create_nonce_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
     log_instruction_custom_error::<SystemError>(result)
 }
 
@@ -569,45 +580,33 @@ pub fn process_new_nonce(
         &fee_calculator,
         &tx.message,
     )?;
-    let result =
-        rpc_client.send_and_confirm_transaction(&mut tx, &[config.signers[0], nonce_authority]);
+    let result = rpc_client
+        .send_and_confirm_transaction_with_spinner(&mut tx, &[config.signers[0], nonce_authority]);
     log_instruction_custom_error::<SystemError>(result)
 }
 
 pub fn process_show_nonce_account(
     rpc_client: &RpcClient,
+    config: &CliConfig,
     nonce_account_pubkey: &Pubkey,
     use_lamports_unit: bool,
 ) -> ProcessResult {
     let nonce_account = get_account(rpc_client, nonce_account_pubkey)?;
     let print_account = |data: Option<&nonce::state::Data>| {
-        println!(
-            "Balance: {}",
-            build_balance_message(nonce_account.lamports, use_lamports_unit, true)
-        );
-        println!(
-            "Minimum Balance Required: {}",
-            build_balance_message(
-                rpc_client.get_minimum_balance_for_rent_exemption(State::size())?,
-                use_lamports_unit,
-                true
-            )
-        );
-        match data {
-            Some(ref data) => {
-                println!("Nonce: {}", data.blockhash);
-                println!(
-                    "Fee: {} lamports per signature",
-                    data.fee_calculator.lamports_per_signature
-                );
-                println!("Authority: {}", data.authority);
-            }
-            None => {
-                println!("Nonce: uninitialized");
-                println!("Fees: uninitialized");
-                println!("Authority: uninitialized");
-            }
+        let mut nonce_account = CliNonceAccount {
+            balance: nonce_account.lamports,
+            minimum_balance_for_rent_exemption: rpc_client
+                .get_minimum_balance_for_rent_exemption(State::size())?,
+            use_lamports_unit,
+            ..CliNonceAccount::default()
+        };
+        if let Some(ref data) = data {
+            nonce_account.nonce = Some(data.blockhash.to_string());
+            nonce_account.lamports_per_signature = Some(data.fee_calculator.lamports_per_signature);
+            nonce_account.authority = Some(data.authority.to_string());
         }
+
+        config.output_format.formatted_print(&nonce_account);
         Ok("".to_string())
     };
     match state_from_account(&nonce_account)? {
@@ -642,7 +641,7 @@ pub fn process_withdraw_from_nonce_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
     log_instruction_custom_error::<NonceError>(result)
 }
 
@@ -689,7 +688,12 @@ mod tests {
             &Pubkey::default().to_string(),
         ]);
         assert_eq!(
-            parse_command(&test_authorize_nonce_account, &default_keypair_file, None).unwrap(),
+            parse_command(
+                &test_authorize_nonce_account,
+                &default_keypair_file,
+                &mut None
+            )
+            .unwrap(),
             CliCommandInfo {
                 command: CliCommand::AuthorizeNonceAccount {
                     nonce_account: nonce_account_pubkey,
@@ -710,7 +714,12 @@ mod tests {
             &authority_keypair_file,
         ]);
         assert_eq!(
-            parse_command(&test_authorize_nonce_account, &default_keypair_file, None).unwrap(),
+            parse_command(
+                &test_authorize_nonce_account,
+                &default_keypair_file,
+                &mut None
+            )
+            .unwrap(),
             CliCommandInfo {
                 command: CliCommand::AuthorizeNonceAccount {
                     nonce_account: read_keypair_file(&keypair_file).unwrap().pubkey(),
@@ -732,7 +741,7 @@ mod tests {
             "50",
         ]);
         assert_eq!(
-            parse_command(&test_create_nonce_account, &default_keypair_file, None).unwrap(),
+            parse_command(&test_create_nonce_account, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::CreateNonceAccount {
                     nonce_account: 1,
@@ -757,7 +766,7 @@ mod tests {
             &authority_keypair_file,
         ]);
         assert_eq!(
-            parse_command(&test_create_nonce_account, &default_keypair_file, None).unwrap(),
+            parse_command(&test_create_nonce_account, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::CreateNonceAccount {
                     nonce_account: 1,
@@ -779,7 +788,7 @@ mod tests {
             &nonce_account_string,
         ]);
         assert_eq!(
-            parse_command(&test_get_nonce, &default_keypair_file, None).unwrap(),
+            parse_command(&test_get_nonce, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::GetNonce(nonce_account_keypair.pubkey()),
                 signers: vec![],
@@ -793,7 +802,7 @@ mod tests {
                 .get_matches_from(vec!["test", "new-nonce", &keypair_file]);
         let nonce_account = read_keypair_file(&keypair_file).unwrap();
         assert_eq!(
-            parse_command(&test_new_nonce, &default_keypair_file, None).unwrap(),
+            parse_command(&test_new_nonce, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::NewNonce {
                     nonce_account: nonce_account.pubkey(),
@@ -813,7 +822,7 @@ mod tests {
         ]);
         let nonce_account = read_keypair_file(&keypair_file).unwrap();
         assert_eq!(
-            parse_command(&test_new_nonce, &default_keypair_file, None).unwrap(),
+            parse_command(&test_new_nonce, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::NewNonce {
                     nonce_account: nonce_account.pubkey(),
@@ -833,7 +842,7 @@ mod tests {
             &nonce_account_string,
         ]);
         assert_eq!(
-            parse_command(&test_show_nonce_account, &default_keypair_file, None).unwrap(),
+            parse_command(&test_show_nonce_account, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::ShowNonceAccount {
                     nonce_account_pubkey: nonce_account_keypair.pubkey(),
@@ -855,7 +864,7 @@ mod tests {
             parse_command(
                 &test_withdraw_from_nonce_account,
                 &default_keypair_file,
-                None
+                &mut None
             )
             .unwrap(),
             CliCommandInfo {
@@ -869,31 +878,6 @@ mod tests {
             }
         );
 
-        let test_withdraw_from_nonce_account = test_commands.clone().get_matches_from(vec![
-            "test",
-            "withdraw-from-nonce-account",
-            &keypair_file,
-            &nonce_account_string,
-            "42",
-        ]);
-        assert_eq!(
-            parse_command(
-                &test_withdraw_from_nonce_account,
-                &default_keypair_file,
-                None
-            )
-            .unwrap(),
-            CliCommandInfo {
-                command: CliCommand::WithdrawFromNonceAccount {
-                    nonce_account: read_keypair_file(&keypair_file).unwrap().pubkey(),
-                    nonce_authority: 0,
-                    destination_account_pubkey: nonce_account_pubkey,
-                    lamports: 42000000000
-                },
-                signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
-            }
-        );
-
         // Test WithdrawFromNonceAccount Subcommand with authority
         let test_withdraw_from_nonce_account = test_commands.clone().get_matches_from(vec![
             "test",
@@ -908,7 +892,7 @@ mod tests {
             parse_command(
                 &test_withdraw_from_nonce_account,
                 &default_keypair_file,
-                None
+                &mut None
             )
             .unwrap(),
             CliCommandInfo {

cli/src/offline/mod.rs

@@ -6,10 +6,16 @@ use serde_json::Value;
 use solana_clap_utils::{
     input_parsers::{pubkey_of, value_of},
     input_validators::{is_hash, is_pubkey_sig},
+    keypair::presigner_from_pubkey_sigs,
     offline::{BLOCKHASH_ARG, SIGNER_ARG, SIGN_ONLY_ARG},
 };
 use solana_client::rpc_client::RpcClient;
-use solana_sdk::{fee_calculator::FeeCalculator, hash::Hash, pubkey::Pubkey, signature::Signature};
+use solana_sdk::{
+    fee_calculator::FeeCalculator,
+    hash::Hash,
+    pubkey::Pubkey,
+    signature::{Presigner, Signature},
+};
 use std::str::FromStr;
 
 fn blockhash_arg<'a, 'b>() -> Arg<'a, 'b> {
@@ -33,7 +39,7 @@ fn signer_arg<'a, 'b>() -> Arg<'a, 'b> {
     Arg::with_name(SIGNER_ARG.name)
         .long(SIGNER_ARG.long)
         .takes_value(true)
-        .value_name("BASE58_PUBKEY=BASE58_SIG")
+        .value_name("PUBKEY=SIGNATURE")
         .validator(is_pubkey_sig)
         .requires(BLOCKHASH_ARG.name)
         .multiple(true)
@@ -52,12 +58,29 @@ impl OfflineArgs for App<'_, '_> {
     }
 }
 
-pub fn parse_sign_only_reply_string(reply: &str) -> (Hash, Vec<(Pubkey, Signature)>) {
+pub struct SignOnly {
+    pub blockhash: Hash,
+    pub present_signers: Vec<(Pubkey, Signature)>,
+    pub absent_signers: Vec<Pubkey>,
+    pub bad_signers: Vec<Pubkey>,
+}
+
+impl SignOnly {
+    pub fn has_all_signers(&self) -> bool {
+        self.absent_signers.is_empty() && self.bad_signers.is_empty()
+    }
+
+    pub fn presigner_of(&self, pubkey: &Pubkey) -> Option<Presigner> {
+        presigner_from_pubkey_sigs(pubkey, &self.present_signers)
+    }
+}
+
+pub fn parse_sign_only_reply_string(reply: &str) -> SignOnly {
     let object: Value = serde_json::from_str(&reply).unwrap();
     let blockhash_str = object.get("blockhash").unwrap().as_str().unwrap();
     let blockhash = blockhash_str.parse::<Hash>().unwrap();
     let signer_strings = object.get("signers").unwrap().as_array().unwrap();
-    let signers = signer_strings
+    let present_signers = signer_strings
         .iter()
         .map(|signer_string| {
             let mut signer = signer_string.as_str().unwrap().split('=');
@@ -66,5 +89,26 @@ pub fn parse_sign_only_reply_string(reply: &str) -> (Hash, Vec<(Pubkey, Signatur
             (key, sig)
         })
         .collect();
-    (blockhash, signers)
+    let signer_strings = object.get("absent").unwrap().as_array().unwrap();
+    let absent_signers = signer_strings
+        .iter()
+        .map(|val| {
+            let s = val.as_str().unwrap();
+            Pubkey::from_str(s).unwrap()
+        })
+        .collect();
+    let signer_strings = object.get("badSig").unwrap().as_array().unwrap();
+    let bad_signers = signer_strings
+        .iter()
+        .map(|val| {
+            let s = val.as_str().unwrap();
+            Pubkey::from_str(s).unwrap()
+        })
+        .collect();
+    SignOnly {
+        blockhash,
+        present_signers,
+        absent_signers,
+        bad_signers,
+    }
 }

cli/src/storage.rs

@@ -1,6 +1,7 @@
 use crate::cli::{
-    check_account_for_fee, check_unique_pubkeys, log_instruction_custom_error, CliCommand,
-    CliCommandInfo, CliConfig, CliError, ProcessResult, SignerIndex,
+    check_account_for_fee, check_unique_pubkeys, generate_unique_signers,
+    log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult,
+    SignerIndex,
 };
 use clap::{App, Arg, ArgMatches, SubCommand};
 use solana_clap_utils::{input_parsers::*, input_validators::*, keypair::signer_from_path};
@@ -25,18 +26,18 @@ impl StorageSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("storage_account_owner")
                         .index(1)
-                        .value_name("STORAGE ACCOUNT OWNER PUBKEY")
+                        .value_name("AUTHORITY_PUBKEY")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair),
+                        .validator(is_valid_pubkey),
                 )
                 .arg(
                     Arg::with_name("storage_account")
                         .index(2)
-                        .value_name("STORAGE ACCOUNT")
+                        .value_name("ACCOUNT_KEYPAIR")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_keypair_or_ask_keyword),
+                        .validator(is_valid_signer),
                 ),
         )
         .subcommand(
@@ -45,18 +46,18 @@ impl StorageSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("storage_account_owner")
                         .index(1)
-                        .value_name("STORAGE ACCOUNT OWNER PUBKEY")
+                        .value_name("AUTHORITY_PUBKEY")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair),
+                        .validator(is_valid_pubkey),
                 )
                 .arg(
                     Arg::with_name("storage_account")
                         .index(2)
-                        .value_name("STORAGE ACCOUNT")
+                        .value_name("ACCOUNT_KEYPAIR")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_keypair_or_ask_keyword),
+                        .validator(is_valid_signer),
                 ),
         )
         .subcommand(
@@ -65,19 +66,19 @@ impl StorageSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("node_account_pubkey")
                         .index(1)
-                        .value_name("NODE PUBKEY")
+                        .value_name("NODE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("The node account to credit the rewards to"),
                 )
                 .arg(
                     Arg::with_name("storage_account_pubkey")
                         .index(2)
-                        .value_name("STORAGE ACCOUNT PUBKEY")
+                        .value_name("STORAGE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Storage account address to redeem credits for"),
                 ),
         )
@@ -88,11 +89,11 @@ impl StorageSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("storage_account_pubkey")
                         .index(1)
-                        .value_name("STORAGE ACCOUNT PUBKEY")
+                        .value_name("STORAGE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("Storage account pubkey"),
+                        .validator(is_valid_pubkey)
+                        .help("Storage account address"),
                 ),
         )
     }
@@ -101,50 +102,68 @@ impl StorageSubCommands for App<'_, '_> {
 pub fn parse_storage_create_archiver_account(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let account_owner = pubkey_of(matches, "storage_account_owner").unwrap();
-    let storage_account = keypair_of(matches, "storage_account").unwrap();
+    let account_owner =
+        pubkey_of_signer(matches, "storage_account_owner", wallet_manager)?.unwrap();
+    let (storage_account, storage_account_pubkey) =
+        signer_of(matches, "storage_account", wallet_manager)?;
+
+    let payer_provided = None;
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, storage_account],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
+
     Ok(CliCommandInfo {
         command: CliCommand::CreateStorageAccount {
             account_owner,
-            storage_account: 1,
+            storage_account: signer_info.index_of(storage_account_pubkey).unwrap(),
             account_type: StorageAccountType::Archiver,
         },
-        signers: vec![
-            signer_from_path(matches, default_signer_path, "keypair", wallet_manager)?,
-            storage_account.into(),
-        ],
+        signers: signer_info.signers,
     })
 }
 
 pub fn parse_storage_create_validator_account(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let account_owner = pubkey_of(matches, "storage_account_owner").unwrap();
-    let storage_account = keypair_of(matches, "storage_account").unwrap();
+    let account_owner =
+        pubkey_of_signer(matches, "storage_account_owner", wallet_manager)?.unwrap();
+    let (storage_account, storage_account_pubkey) =
+        signer_of(matches, "storage_account", wallet_manager)?;
+
+    let payer_provided = None;
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, storage_account],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
+
     Ok(CliCommandInfo {
         command: CliCommand::CreateStorageAccount {
             account_owner,
-            storage_account: 1,
+            storage_account: signer_info.index_of(storage_account_pubkey).unwrap(),
             account_type: StorageAccountType::Validator,
         },
-        signers: vec![
-            signer_from_path(matches, default_signer_path, "keypair", wallet_manager)?,
-            storage_account.into(),
-        ],
+        signers: signer_info.signers,
     })
 }
 
 pub fn parse_storage_claim_reward(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let node_account_pubkey = pubkey_of(matches, "node_account_pubkey").unwrap();
-    let storage_account_pubkey = pubkey_of(matches, "storage_account_pubkey").unwrap();
+    let node_account_pubkey =
+        pubkey_of_signer(matches, "node_account_pubkey", wallet_manager)?.unwrap();
+    let storage_account_pubkey =
+        pubkey_of_signer(matches, "storage_account_pubkey", wallet_manager)?.unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::ClaimStorageReward {
             node_account_pubkey,
@@ -161,8 +180,10 @@ pub fn parse_storage_claim_reward(
 
 pub fn parse_storage_get_account_command(
     matches: &ArgMatches<'_>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let storage_account_pubkey = pubkey_of(matches, "storage_account_pubkey").unwrap();
+    let storage_account_pubkey =
+        pubkey_of_signer(matches, "storage_account_pubkey", wallet_manager)?.unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::ShowStorageAccount(storage_account_pubkey),
         signers: vec![],
@@ -221,7 +242,7 @@ pub fn process_create_storage_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
     log_instruction_custom_error::<SystemError>(result)
 }
 
@@ -245,8 +266,8 @@ pub fn process_claim_storage_reward(
         &fee_calculator,
         &tx.message,
     )?;
-    let signature_str = rpc_client.send_and_confirm_transaction(&mut tx, &signers)?;
-    Ok(signature_str)
+    let signature = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &signers)?;
+    Ok(signature.to_string())
 }
 
 pub fn process_show_storage_account(
@@ -309,7 +330,7 @@ mod tests {
             parse_command(
                 &test_create_archiver_storage_account,
                 &default_keypair_file,
-                None
+                &mut None
             )
             .unwrap(),
             CliCommandInfo {
@@ -341,7 +362,7 @@ mod tests {
             parse_command(
                 &test_create_validator_storage_account,
                 &default_keypair_file,
-                None
+                &mut None
             )
             .unwrap(),
             CliCommandInfo {
@@ -364,7 +385,7 @@ mod tests {
             &storage_account_string,
         ]);
         assert_eq!(
-            parse_command(&test_claim_storage_reward, &default_keypair_file, None).unwrap(),
+            parse_command(&test_claim_storage_reward, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::ClaimStorageReward {
                     node_account_pubkey: pubkey,

cli/src/validator_info.rs

@@ -1,6 +1,6 @@
 use crate::{
     cli::{check_account_for_fee, CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult},
-    display::println_name_value,
+    cli_output::{CliValidatorInfo, CliValidatorInfoVec},
 };
 use bincode::deserialize;
 use clap::{App, AppSettings, Arg, ArgMatches, SubCommand};
@@ -25,7 +25,6 @@ use solana_sdk::{
     transaction::Transaction,
 };
 use std::{error, sync::Arc};
-use titlecase::titlecase;
 
 pub const MAX_SHORT_FIELD_LENGTH: usize = 70;
 pub const MAX_LONG_FIELD_LENGTH: usize = 300;
@@ -229,7 +228,7 @@ impl ValidatorInfoSubCommands for App<'_, '_> {
 pub fn parse_validator_info_command(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
     let info_pubkey = pubkey_of(matches, "info_pubkey");
     // Prepare validator info
@@ -368,14 +367,18 @@ pub fn process_set_validator_info(
         &fee_calculator,
         &tx.message,
     )?;
-    let signature_str = rpc_client.send_and_confirm_transaction(&mut tx, &signers)?;
+    let signature_str = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &signers)?;
 
     println!("Success! Validator info published at: {:?}", info_pubkey);
     println!("{}", signature_str);
     Ok("".to_string())
 }
 
-pub fn process_get_validator_info(rpc_client: &RpcClient, pubkey: Option<Pubkey>) -> ProcessResult {
+pub fn process_get_validator_info(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    pubkey: Option<Pubkey>,
+) -> ProcessResult {
     let validator_info: Vec<(Pubkey, Account)> = if let Some(validator_info_pubkey) = pubkey {
         vec![(
             validator_info_pubkey,
@@ -394,23 +397,22 @@ pub fn process_get_validator_info(rpc_client: &RpcClient, pubkey: Option<Pubkey>
             .collect()
     };
 
+    let mut validator_info_list: Vec<CliValidatorInfo> = vec![];
     if validator_info.is_empty() {
         println!("No validator info accounts found");
     }
     for (validator_info_pubkey, validator_info_account) in validator_info.iter() {
         let (validator_pubkey, validator_info) =
             parse_validator_info(&validator_info_pubkey, &validator_info_account)?;
-        println!();
-        println_name_value("Validator Identity Pubkey:", &validator_pubkey.to_string());
-        println_name_value("  Info Pubkey:", &validator_info_pubkey.to_string());
-        for (key, value) in validator_info.iter() {
-            println_name_value(
-                &format!("  {}:", titlecase(key)),
-                &value.as_str().unwrap_or("?"),
-            );
-        }
+        validator_info_list.push(CliValidatorInfo {
+            identity_pubkey: validator_pubkey.to_string(),
+            info_pubkey: validator_info_pubkey.to_string(),
+            info: validator_info,
+        });
     }
-
+    config
+        .output_format
+        .formatted_print(&CliValidatorInfoVec::new(validator_info_list));
     Ok("".to_string())
 }
 

cli/src/vote.rs

@@ -1,22 +1,21 @@
-use crate::cli::{
-    build_balance_message, check_account_for_fee, check_unique_pubkeys, generate_unique_signers,
-    log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError, CliSignerInfo,
-    ProcessResult,
+use crate::{
+    cli::{
+        check_account_for_fee, check_unique_pubkeys, generate_unique_signers,
+        log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError,
+        ProcessResult, SignerIndex,
+    },
+    cli_output::{CliEpochVotingHistory, CliLockout, CliVoteAccount},
 };
 use clap::{value_t_or_exit, App, Arg, ArgMatches, SubCommand};
 use solana_clap_utils::{input_parsers::*, input_validators::*};
 use solana_client::rpc_client::RpcClient;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
-    account::Account,
-    commitment_config::CommitmentConfig,
-    message::Message,
-    pubkey::Pubkey,
-    system_instruction::{create_address_with_seed, SystemError},
-    transaction::Transaction,
+    account::Account, commitment_config::CommitmentConfig, message::Message, pubkey::Pubkey,
+    system_instruction::SystemError, transaction::Transaction,
 };
 use solana_vote_program::{
-    vote_instruction::{self, VoteError},
+    vote_instruction::{self, withdraw, VoteError},
     vote_state::{VoteAuthorize, VoteInit, VoteState},
 };
 use std::sync::Arc;
@@ -33,25 +32,25 @@ impl VoteSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("vote_account")
                         .index(1)
-                        .value_name("VOTE ACCOUNT KEYPAIR")
+                        .value_name("ACCOUNT_KEYPAIR")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_keypair_or_ask_keyword)
-                        .help("Vote account keypair to fund"),
+                        .validator(is_valid_signer)
+                        .help("Vote account keypair to create"),
                 )
                 .arg(
-                    Arg::with_name("identity_pubkey")
+                    Arg::with_name("identity_account")
                         .index(2)
-                        .value_name("VALIDATOR IDENTITY PUBKEY")
+                        .value_name("IDENTITY_KEYPAIR")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("Validator that will vote with this account"),
+                        .validator(is_valid_signer)
+                        .help("Keypair of validator that will vote with this account"),
                 )
                 .arg(
                     Arg::with_name("commission")
                         .long("commission")
-                        .value_name("NUM")
+                        .value_name("PERCENTAGE")
                         .takes_value(true)
                         .default_value("100")
                         .help("The commission taken on reward redemption (0-100)"),
@@ -59,78 +58,55 @@ impl VoteSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("authorized_voter")
                         .long("authorized-voter")
-                        .value_name("PUBKEY")
+                        .value_name("VOTER_PUBKEY")
                         .takes_value(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("Public key of the authorized voter (defaults to vote account)"),
+                        .validator(is_valid_pubkey)
+                        .help("Public key of the authorized voter [default: validator identity pubkey]"),
                 )
                 .arg(
                     Arg::with_name("authorized_withdrawer")
                         .long("authorized-withdrawer")
-                        .value_name("PUBKEY")
+                        .value_name("WITHDRAWER_PUBKEY")
                         .takes_value(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("Public key of the authorized withdrawer (defaults to cli config pubkey)"),
+                        .validator(is_valid_pubkey)
+                        .help("Public key of the authorized withdrawer [default: validator identity pubkey]"),
                 )
                 .arg(
                     Arg::with_name("seed")
                         .long("seed")
-                        .value_name("SEED STRING")
+                        .value_name("STRING")
                         .takes_value(true)
                         .help("Seed for address generation; if specified, the resulting account will be at a derived address of the VOTE ACCOUNT pubkey")
                 ),
         )
-        .subcommand(
-            SubCommand::with_name("vote-update-validator")
-                .about("Update the vote account's validator identity")
-                .arg(
-                    Arg::with_name("vote_account_pubkey")
-                        .index(1)
-                        .value_name("VOTE ACCOUNT PUBKEY")
-                        .takes_value(true)
-                        .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("Vote account to update"),
-                )
-                .arg(
-                    Arg::with_name("new_identity_pubkey")
-                        .index(2)
-                        .value_name("NEW VALIDATOR IDENTITY PUBKEY")
-                        .takes_value(true)
-                        .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("New validator that will vote with this account"),
-                )
-                .arg(
-                    Arg::with_name("authorized_voter")
-                        .index(3)
-                        .value_name("AUTHORIZED VOTER KEYPAIR")
-                        .takes_value(true)
-                        .required(true)
-                        .validator(is_keypair)
-                        .help("Authorized voter keypair"),
-                )
-        )
         .subcommand(
             SubCommand::with_name("vote-authorize-voter")
                 .about("Authorize a new vote signing keypair for the given vote account")
                 .arg(
                     Arg::with_name("vote_account_pubkey")
                         .index(1)
-                        .value_name("VOTE ACCOUNT PUBKEY")
+                        .value_name("VOTE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Vote account in which to set the authorized voter"),
                 )
                 .arg(
-                    Arg::with_name("new_authorized_pubkey")
+                    Arg::with_name("authorized")
                         .index(2)
-                        .value_name("NEW VOTER PUBKEY")
+                        .value_name("AUTHORIZED_KEYPAIR")
+                        .required(true)
+                        .validator(is_valid_signer)
+                        .help("Current authorized vote signer"),
+                )
+                .arg(
+                    Arg::with_name("new_authorized_pubkey")
+                        .index(3)
+                        .value_name("AUTHORIZED_PUBKEY")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("New vote signer to authorize"),
+                        .validator(is_valid_pubkey)
+                        .help("New authorized vote signer"),
                 ),
         )
         .subcommand(
@@ -139,22 +115,61 @@ impl VoteSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("vote_account_pubkey")
                         .index(1)
-                        .value_name("VOTE ACCOUNT PUBKEY")
+                        .value_name("VOTE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Vote account in which to set the authorized withdrawer"),
                 )
                 .arg(
-                    Arg::with_name("new_authorized_pubkey")
+                    Arg::with_name("authorized")
                         .index(2)
-                        .value_name("NEW WITHDRAWER PUBKEY")
+                        .value_name("AUTHORIZED_KEYPAIR")
+                        .required(true)
+                        .validator(is_valid_signer)
+                        .help("Current authorized withdrawer"),
+                )
+                .arg(
+                    Arg::with_name("new_authorized_pubkey")
+                        .index(3)
+                        .value_name("AUTHORIZED_PUBKEY")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("New withdrawer to authorize"),
+                        .validator(is_valid_pubkey)
+                        .help("New authorized withdrawer"),
                 ),
         )
+        .subcommand(
+            SubCommand::with_name("vote-update-validator")
+                .about("Update the vote account's validator identity")
+                .arg(
+                    Arg::with_name("vote_account_pubkey")
+                        .index(1)
+                        .value_name("VOTE_ACCOUNT_ADDRESS")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_pubkey)
+                        .help("Vote account to update"),
+                )
+                .arg(
+                    Arg::with_name("new_identity_account")
+                        .index(2)
+                        .value_name("IDENTITY_KEYPAIR")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_signer)
+                        .help("Keypair of new validator that will vote with this account"),
+                )
+                .arg(
+                    Arg::with_name("authorized_withdrawer")
+                        .index(3)
+                        .value_name("AUTHORIZED_KEYPAIR")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_signer)
+                        .help("Authorized withdrawer keypair"),
+                )
+        )
         .subcommand(
             SubCommand::with_name("vote-account")
                 .about("Show the contents of a vote account")
@@ -170,10 +185,10 @@ impl VoteSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("vote_account_pubkey")
                         .index(1)
-                        .value_name("VOTE ACCOUNT PUBKEY")
+                        .value_name("VOTE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Vote account pubkey"),
                 )
                 .arg(
@@ -183,24 +198,64 @@ impl VoteSubCommands for App<'_, '_> {
                         .help("Display balance in lamports instead of SOL"),
                 ),
         )
+        .subcommand(
+            SubCommand::with_name("withdraw-from-vote-account")
+                .about("Withdraw lamports from a vote account into a specified account")
+                .arg(
+                    Arg::with_name("vote_account_pubkey")
+                        .index(1)
+                        .value_name("VOTE_ACCOUNT_ADDRESS")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_pubkey)
+                        .help("Vote account from which to withdraw"),
+                )
+                .arg(
+                    Arg::with_name("destination_account_pubkey")
+                        .index(2)
+                        .value_name("RECIPIENT_ADDRESS")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_pubkey)
+                        .help("The recipient of withdrawn SOL"),
+                )
+                .arg(
+                    Arg::with_name("amount")
+                        .index(3)
+                        .value_name("AMOUNT")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_amount)
+                        .help("The amount to withdraw, in SOL"),
+                )
+                .arg(
+                    Arg::with_name("authorized_withdrawer")
+                        .long("authorized-withdrawer")
+                        .value_name("AUTHORIZED_KEYPAIR")
+                        .takes_value(true)
+                        .validator(is_valid_signer)
+                        .help("Authorized withdrawer [default: cli config keypair]"),
+                )
+        )
     }
 }
 
-pub fn parse_vote_create_account(
+pub fn parse_create_vote_account(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
     let (vote_account, _) = signer_of(matches, "vote_account", wallet_manager)?;
     let seed = matches.value_of("seed").map(|s| s.to_string());
-    let identity_pubkey = pubkey_of(matches, "identity_pubkey").unwrap();
+    let (identity_account, identity_pubkey) =
+        signer_of(matches, "identity_account", wallet_manager)?;
     let commission = value_t_or_exit!(matches, "commission", u8);
-    let authorized_voter = pubkey_of(matches, "authorized_voter");
-    let authorized_withdrawer = pubkey_of(matches, "authorized_withdrawer");
+    let authorized_voter = pubkey_of_signer(matches, "authorized_voter", wallet_manager)?;
+    let authorized_withdrawer = pubkey_of_signer(matches, "authorized_withdrawer", wallet_manager)?;
 
     let payer_provided = None;
-    let CliSignerInfo { signers } = generate_unique_signers(
-        vec![payer_provided, vote_account],
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, vote_account, identity_account],
         matches,
         default_signer_path,
         wallet_manager,
@@ -209,27 +264,30 @@ pub fn parse_vote_create_account(
     Ok(CliCommandInfo {
         command: CliCommand::CreateVoteAccount {
             seed,
-            node_pubkey: identity_pubkey,
+            identity_account: signer_info.index_of(identity_pubkey).unwrap(),
             authorized_voter,
             authorized_withdrawer,
             commission,
         },
-        signers,
+        signers: signer_info.signers,
     })
 }
 
 pub fn parse_vote_authorize(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
     vote_authorize: VoteAuthorize,
 ) -> Result<CliCommandInfo, CliError> {
-    let vote_account_pubkey = pubkey_of(matches, "vote_account_pubkey").unwrap();
-    let new_authorized_pubkey = pubkey_of(matches, "new_authorized_pubkey").unwrap();
+    let vote_account_pubkey =
+        pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
+    let new_authorized_pubkey =
+        pubkey_of_signer(matches, "new_authorized_pubkey", wallet_manager)?.unwrap();
+    let (authorized, _) = signer_of(matches, "authorized", wallet_manager)?;
 
-    let authorized_voter_provided = None;
-    let CliSignerInfo { signers } = generate_unique_signers(
-        vec![authorized_voter_provided],
+    let payer_provided = None;
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, authorized],
         matches,
         default_signer_path,
         wallet_manager,
@@ -241,22 +299,24 @@ pub fn parse_vote_authorize(
             new_authorized_pubkey,
             vote_authorize,
         },
-        signers,
+        signers: signer_info.signers,
     })
 }
 
 pub fn parse_vote_update_validator(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
-    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let vote_account_pubkey = pubkey_of(matches, "vote_account_pubkey").unwrap();
-    let new_identity_pubkey = pubkey_of(matches, "new_identity_pubkey").unwrap();
-    let (authorized_voter, _) = signer_of(matches, "authorized_voter", wallet_manager)?;
+    let vote_account_pubkey =
+        pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
+    let (new_identity_account, new_identity_pubkey) =
+        signer_of(matches, "new_identity_account", wallet_manager)?;
+    let (authorized_withdrawer, _) = signer_of(matches, "authorized_withdrawer", wallet_manager)?;
 
     let payer_provided = None;
-    let CliSignerInfo { signers } = generate_unique_signers(
-        vec![payer_provided, authorized_voter],
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, authorized_withdrawer, new_identity_account],
         matches,
         default_signer_path,
         wallet_manager,
@@ -265,16 +325,18 @@ pub fn parse_vote_update_validator(
     Ok(CliCommandInfo {
         command: CliCommand::VoteUpdateValidator {
             vote_account_pubkey,
-            new_identity_pubkey,
+            new_identity_account: signer_info.index_of(new_identity_pubkey).unwrap(),
         },
-        signers,
+        signers: signer_info.signers,
     })
 }
 
 pub fn parse_vote_get_account_command(
     matches: &ArgMatches<'_>,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let vote_account_pubkey = pubkey_of(matches, "vote_account_pubkey").unwrap();
+    let vote_account_pubkey =
+        pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
     let use_lamports_unit = matches.is_present("lamports");
     let commitment_config = if matches.is_present("confirmed") {
         CommitmentConfig::default()
@@ -291,11 +353,43 @@ pub fn parse_vote_get_account_command(
     })
 }
 
+pub fn parse_withdraw_from_vote_account(
+    matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let vote_account_pubkey =
+        pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
+    let destination_account_pubkey =
+        pubkey_of_signer(matches, "destination_account_pubkey", wallet_manager)?.unwrap();
+    let lamports = lamports_of_sol(matches, "amount").unwrap();
+    let (withdraw_authority, withdraw_authority_pubkey) =
+        signer_of(matches, "authorized_withdrawer", wallet_manager)?;
+
+    let payer_provided = None;
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, withdraw_authority],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
+
+    Ok(CliCommandInfo {
+        command: CliCommand::WithdrawFromVoteAccount {
+            vote_account_pubkey,
+            destination_account_pubkey,
+            withdraw_authority: signer_info.index_of(withdraw_authority_pubkey).unwrap(),
+            lamports,
+        },
+        signers: signer_info.signers,
+    })
+}
+
 pub fn process_create_vote_account(
     rpc_client: &RpcClient,
     config: &CliConfig,
     seed: &Option<String>,
-    identity_pubkey: &Pubkey,
+    identity_account: SignerIndex,
     authorized_voter: &Option<Pubkey>,
     authorized_withdrawer: &Option<Pubkey>,
     commission: u8,
@@ -303,7 +397,7 @@ pub fn process_create_vote_account(
     let vote_account = config.signers[1];
     let vote_account_pubkey = vote_account.pubkey();
     let vote_account_address = if let Some(seed) = seed {
-        create_address_with_seed(&vote_account_pubkey, &seed, &solana_vote_program::id())?
+        Pubkey::create_with_seed(&vote_account_pubkey, &seed, &solana_vote_program::id())?
     } else {
         vote_account_pubkey
     };
@@ -312,6 +406,8 @@ pub fn process_create_vote_account(
         (&vote_account_address, "vote_account".to_string()),
     )?;
 
+    let identity_account = config.signers[identity_account];
+    let identity_pubkey = identity_account.pubkey();
     check_unique_pubkeys(
         (&vote_account_address, "vote_account".to_string()),
         (&identity_pubkey, "identity_pubkey".to_string()),
@@ -334,9 +430,9 @@ pub fn process_create_vote_account(
         .max(1);
 
     let vote_init = VoteInit {
-        node_pubkey: *identity_pubkey,
-        authorized_voter: authorized_voter.unwrap_or(vote_account_pubkey),
-        authorized_withdrawer: authorized_withdrawer.unwrap_or(vote_account_pubkey),
+        node_pubkey: identity_pubkey,
+        authorized_voter: authorized_voter.unwrap_or(identity_pubkey),
+        authorized_withdrawer: authorized_withdrawer.unwrap_or(identity_pubkey),
         commission,
     };
 
@@ -368,7 +464,7 @@ pub fn process_create_vote_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
     log_instruction_custom_error::<SystemError>(result)
 }
 
@@ -379,16 +475,24 @@ pub fn process_vote_authorize(
     new_authorized_pubkey: &Pubkey,
     vote_authorize: VoteAuthorize,
 ) -> ProcessResult {
+    // If the `authorized_account` is also the fee payer, `config.signers` will only have one
+    // keypair in it
+    let authorized = if config.signers.len() == 2 {
+        config.signers[1]
+    } else {
+        config.signers[0]
+    };
+
     check_unique_pubkeys(
-        (vote_account_pubkey, "vote_account_pubkey".to_string()),
+        (&authorized.pubkey(), "authorized_account".to_string()),
         (new_authorized_pubkey, "new_authorized_pubkey".to_string()),
     )?;
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
     let ixs = vec![vote_instruction::authorize(
-        vote_account_pubkey,         // vote account to update
-        &config.signers[0].pubkey(), // current authorized voter
-        new_authorized_pubkey,       // new vote signer/withdrawer
-        vote_authorize,              // vote or withdraw
+        vote_account_pubkey,   // vote account to update
+        &authorized.pubkey(),  // current authorized
+        new_authorized_pubkey, // new vote signer/withdrawer
+        vote_authorize,        // vote or withdraw
     )];
 
     let message = Message::new_with_payer(&ixs, Some(&config.signers[0].pubkey()));
@@ -400,7 +504,8 @@ pub fn process_vote_authorize(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &[config.signers[0]]);
+    let result =
+        rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &[config.signers[0]]);
     log_instruction_custom_error::<VoteError>(result)
 }
 
@@ -408,18 +513,20 @@ pub fn process_vote_update_validator(
     rpc_client: &RpcClient,
     config: &CliConfig,
     vote_account_pubkey: &Pubkey,
-    new_identity_pubkey: &Pubkey,
+    new_identity_account: SignerIndex,
 ) -> ProcessResult {
-    let authorized_voter = config.signers[1];
+    let authorized_withdrawer = config.signers[1];
+    let new_identity_account = config.signers[new_identity_account];
+    let new_identity_pubkey = new_identity_account.pubkey();
     check_unique_pubkeys(
         (vote_account_pubkey, "vote_account_pubkey".to_string()),
-        (new_identity_pubkey, "new_identity_pubkey".to_string()),
+        (&new_identity_pubkey, "new_identity_account".to_string()),
     )?;
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
-    let ixs = vec![vote_instruction::update_node(
+    let ixs = vec![vote_instruction::update_validator_identity(
         vote_account_pubkey,
-        &authorized_voter.pubkey(),
-        new_identity_pubkey,
+        &authorized_withdrawer.pubkey(),
+        &new_identity_pubkey,
     )];
 
     let message = Message::new_with_payer(&ixs, Some(&config.signers[0].pubkey()));
@@ -431,7 +538,7 @@ pub fn process_vote_update_validator(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
     log_instruction_custom_error::<VoteError>(result)
 }
 
@@ -465,7 +572,7 @@ fn get_vote_account(
 
 pub fn process_show_vote_account(
     rpc_client: &RpcClient,
-    _config: &CliConfig,
+    config: &CliConfig,
     vote_account_pubkey: &Pubkey,
     use_lamports_unit: bool,
     commitment_config: CommitmentConfig,
@@ -475,48 +582,73 @@ pub fn process_show_vote_account(
 
     let epoch_schedule = rpc_client.get_epoch_schedule()?;
 
-    println!(
-        "Account Balance: {}",
-        build_balance_message(vote_account.lamports, use_lamports_unit, true)
-    );
-    println!("Validator Identity: {}", vote_state.node_pubkey);
-    println!("Authorized Voter: {:?}", vote_state.authorized_voters());
-    println!(
-        "Authorized Withdrawer: {}",
-        vote_state.authorized_withdrawer
-    );
-    println!("Credits: {}", vote_state.credits());
-    println!("Commission: {}%", vote_state.commission);
-    println!(
-        "Root Slot: {}",
-        match vote_state.root_slot {
-            Some(slot) => slot.to_string(),
-            None => "~".to_string(),
-        }
-    );
-    println!("Recent Timestamp: {:?}", vote_state.last_timestamp);
+    let mut votes: Vec<CliLockout> = vec![];
+    let mut epoch_voting_history: Vec<CliEpochVotingHistory> = vec![];
     if !vote_state.votes.is_empty() {
-        println!("recent votes:");
         for vote in &vote_state.votes {
-            println!(
-                "- slot: {}\n  confirmation count: {}",
-                vote.slot, vote.confirmation_count
-            );
+            votes.push(vote.into());
         }
-
-        println!("Epoch Voting History:");
         for (epoch, credits, prev_credits) in vote_state.epoch_credits() {
             let credits_earned = credits - prev_credits;
             let slots_in_epoch = epoch_schedule.get_slots_in_epoch(*epoch);
-            println!(
-                "- epoch: {}\n  slots in epoch: {}\n  credits earned: {}",
-                epoch, slots_in_epoch, credits_earned,
-            );
+            epoch_voting_history.push(CliEpochVotingHistory {
+                epoch: *epoch,
+                slots_in_epoch,
+                credits_earned,
+            });
         }
     }
+
+    let vote_account_data = CliVoteAccount {
+        account_balance: vote_account.lamports,
+        validator_identity: vote_state.node_pubkey.to_string(),
+        authorized_voters: vote_state.authorized_voters().into(),
+        authorized_withdrawer: vote_state.authorized_withdrawer.to_string(),
+        credits: vote_state.credits(),
+        commission: vote_state.commission,
+        root_slot: vote_state.root_slot,
+        recent_timestamp: vote_state.last_timestamp.clone(),
+        votes,
+        epoch_voting_history,
+        use_lamports_unit,
+    };
+
+    config.output_format.formatted_print(&vote_account_data);
     Ok("".to_string())
 }
 
+pub fn process_withdraw_from_vote_account(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    vote_account_pubkey: &Pubkey,
+    withdraw_authority: SignerIndex,
+    lamports: u64,
+    destination_account_pubkey: &Pubkey,
+) -> ProcessResult {
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
+    let withdraw_authority = config.signers[withdraw_authority];
+
+    let ix = withdraw(
+        vote_account_pubkey,
+        &withdraw_authority.pubkey(),
+        lamports,
+        destination_account_pubkey,
+    );
+
+    let message = Message::new_with_payer(&[ix], Some(&config.signers[0].pubkey()));
+    let mut transaction = Transaction::new_unsigned(message);
+    transaction.try_sign(&config.signers, recent_blockhash)?;
+    check_account_for_fee(
+        rpc_client,
+        &config.signers[0].pubkey(),
+        &fee_calculator,
+        &transaction.message,
+    )?;
+    let result =
+        rpc_client.send_and_confirm_transaction_with_spinner(&mut transaction, &config.signers);
+    log_instruction_custom_error::<VoteError>(result)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -547,10 +679,11 @@ mod tests {
             "test",
             "vote-authorize-voter",
             &pubkey_string,
+            &default_keypair_file,
             &pubkey2_string,
         ]);
         assert_eq!(
-            parse_command(&test_authorize_voter, &default_keypair_file, None).unwrap(),
+            parse_command(&test_authorize_voter, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::VoteAuthorize {
                     vote_account_pubkey: pubkey,
@@ -561,33 +694,61 @@ mod tests {
             }
         );
 
+        let authorized_keypair = Keypair::new();
+        let (authorized_keypair_file, mut tmp_file) = make_tmp_file();
+        write_keypair(&authorized_keypair, tmp_file.as_file_mut()).unwrap();
+
+        let test_authorize_voter = test_commands.clone().get_matches_from(vec![
+            "test",
+            "vote-authorize-voter",
+            &pubkey_string,
+            &authorized_keypair_file,
+            &pubkey2_string,
+        ]);
+        assert_eq!(
+            parse_command(&test_authorize_voter, &default_keypair_file, &mut None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::VoteAuthorize {
+                    vote_account_pubkey: pubkey,
+                    new_authorized_pubkey: pubkey2,
+                    vote_authorize: VoteAuthorize::Voter
+                },
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    read_keypair_file(&authorized_keypair_file).unwrap().into(),
+                ],
+            }
+        );
+
         let (keypair_file, mut tmp_file) = make_tmp_file();
         let keypair = Keypair::new();
         write_keypair(&keypair, tmp_file.as_file_mut()).unwrap();
         // Test CreateVoteAccount SubCommand
-        let node_pubkey = Pubkey::new_rand();
-        let node_pubkey_string = format!("{}", node_pubkey);
+        let (identity_keypair_file, mut tmp_file) = make_tmp_file();
+        let identity_keypair = Keypair::new();
+        write_keypair(&identity_keypair, tmp_file.as_file_mut()).unwrap();
         let test_create_vote_account = test_commands.clone().get_matches_from(vec![
             "test",
             "create-vote-account",
             &keypair_file,
-            &node_pubkey_string,
+            &identity_keypair_file,
             "--commission",
             "10",
         ]);
         assert_eq!(
-            parse_command(&test_create_vote_account, &default_keypair_file, None).unwrap(),
+            parse_command(&test_create_vote_account, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::CreateVoteAccount {
                     seed: None,
-                    node_pubkey,
+                    identity_account: 2,
                     authorized_voter: None,
                     authorized_withdrawer: None,
                     commission: 10,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
-                    Box::new(keypair)
+                    Box::new(keypair),
+                    read_keypair_file(&identity_keypair_file).unwrap().into(),
                 ],
             }
         );
@@ -600,21 +761,22 @@ mod tests {
             "test",
             "create-vote-account",
             &keypair_file,
-            &node_pubkey_string,
+            &identity_keypair_file,
         ]);
         assert_eq!(
-            parse_command(&test_create_vote_account2, &default_keypair_file, None).unwrap(),
+            parse_command(&test_create_vote_account2, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::CreateVoteAccount {
                     seed: None,
-                    node_pubkey,
+                    identity_account: 2,
                     authorized_voter: None,
                     authorized_withdrawer: None,
                     commission: 100,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
-                    Box::new(keypair)
+                    Box::new(keypair),
+                    read_keypair_file(&identity_keypair_file).unwrap().into(),
                 ],
             }
         );
@@ -629,23 +791,24 @@ mod tests {
             "test",
             "create-vote-account",
             &keypair_file,
-            &node_pubkey_string,
+            &identity_keypair_file,
             "--authorized-voter",
             &authed.to_string(),
         ]);
         assert_eq!(
-            parse_command(&test_create_vote_account3, &default_keypair_file, None).unwrap(),
+            parse_command(&test_create_vote_account3, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::CreateVoteAccount {
                     seed: None,
-                    node_pubkey,
+                    identity_account: 2,
                     authorized_voter: Some(authed),
                     authorized_withdrawer: None,
                     commission: 100
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
-                    Box::new(keypair)
+                    Box::new(keypair),
+                    read_keypair_file(&identity_keypair_file).unwrap().into(),
                 ],
             }
         );
@@ -658,23 +821,24 @@ mod tests {
             "test",
             "create-vote-account",
             &keypair_file,
-            &node_pubkey_string,
+            &identity_keypair_file,
             "--authorized-withdrawer",
             &authed.to_string(),
         ]);
         assert_eq!(
-            parse_command(&test_create_vote_account4, &default_keypair_file, None).unwrap(),
+            parse_command(&test_create_vote_account4, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::CreateVoteAccount {
                     seed: None,
-                    node_pubkey,
+                    identity_account: 2,
                     authorized_voter: None,
                     authorized_withdrawer: Some(authed),
                     commission: 100
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
-                    Box::new(keypair)
+                    Box::new(keypair),
+                    read_keypair_file(&identity_keypair_file).unwrap().into(),
                 ],
             }
         );
@@ -683,19 +847,80 @@ mod tests {
             "test",
             "vote-update-validator",
             &pubkey_string,
-            &pubkey2_string,
+            &identity_keypair_file,
             &keypair_file,
         ]);
         assert_eq!(
-            parse_command(&test_update_validator, &default_keypair_file, None).unwrap(),
+            parse_command(&test_update_validator, &default_keypair_file, &mut None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::VoteUpdateValidator {
                     vote_account_pubkey: pubkey,
-                    new_identity_pubkey: pubkey2,
+                    new_identity_account: 2,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
-                    Box::new(read_keypair_file(&keypair_file).unwrap())
+                    Box::new(read_keypair_file(&keypair_file).unwrap()),
+                    read_keypair_file(&identity_keypair_file).unwrap().into(),
+                ],
+            }
+        );
+
+        // Test WithdrawFromVoteAccount subcommand
+        let test_withdraw_from_vote_account = test_commands.clone().get_matches_from(vec![
+            "test",
+            "withdraw-from-vote-account",
+            &keypair_file,
+            &pubkey_string,
+            "42",
+        ]);
+        assert_eq!(
+            parse_command(
+                &test_withdraw_from_vote_account,
+                &default_keypair_file,
+                &mut None
+            )
+            .unwrap(),
+            CliCommandInfo {
+                command: CliCommand::WithdrawFromVoteAccount {
+                    vote_account_pubkey: read_keypair_file(&keypair_file).unwrap().pubkey(),
+                    destination_account_pubkey: pubkey,
+                    withdraw_authority: 0,
+                    lamports: 42_000_000_000
+                },
+                signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
+            }
+        );
+
+        // Test WithdrawFromVoteAccount subcommand with authority
+        let withdraw_authority = Keypair::new();
+        let (withdraw_authority_file, mut tmp_file) = make_tmp_file();
+        write_keypair(&withdraw_authority, tmp_file.as_file_mut()).unwrap();
+        let test_withdraw_from_vote_account = test_commands.clone().get_matches_from(vec![
+            "test",
+            "withdraw-from-vote-account",
+            &keypair_file,
+            &pubkey_string,
+            "42",
+            "--authorized-withdrawer",
+            &withdraw_authority_file,
+        ]);
+        assert_eq!(
+            parse_command(
+                &test_withdraw_from_vote_account,
+                &default_keypair_file,
+                &mut None
+            )
+            .unwrap(),
+            CliCommandInfo {
+                command: CliCommand::WithdrawFromVoteAccount {
+                    vote_account_pubkey: read_keypair_file(&keypair_file).unwrap().pubkey(),
+                    destination_account_pubkey: pubkey,
+                    withdraw_authority: 1,
+                    lamports: 42_000_000_000
+                },
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    read_keypair_file(&withdraw_authority_file).unwrap().into()
                 ],
             }
         );

cli/tests/nonce.rs

@@ -1,12 +1,18 @@
-use solana_cli::cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig};
+use solana_cli::{
+    cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
+    nonce,
+    offline::{
+        blockhash_query::{self, BlockhashQuery},
+        parse_sign_only_reply_string,
+    },
+};
 use solana_client::rpc_client::RpcClient;
-use solana_core::validator::TestValidator;
+use solana_core::validator::{TestValidator, TestValidatorOptions};
 use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
     hash::Hash,
     pubkey::Pubkey,
     signature::{keypair_from_seed, Keypair, Signer},
-    system_instruction::create_address_with_seed,
     system_program,
 };
 use std::{fs::remove_dir_all, sync::mpsc::channel, thread::sleep, time::Duration};
@@ -123,7 +129,7 @@ fn full_battery_tests(
     config_nonce.signers = vec![&nonce_keypair];
 
     let nonce_account = if let Some(seed) = seed.as_ref() {
-        create_address_with_seed(
+        Pubkey::create_with_seed(
             &config_nonce.signers[0].pubkey(),
             seed,
             &system_program::id(),
@@ -247,3 +253,127 @@ fn full_battery_tests(
     check_balance(800, &rpc_client, &nonce_account);
     check_balance(200, &rpc_client, &payee_pubkey);
 }
+
+#[test]
+fn test_create_account_with_seed() {
+    let TestValidator {
+        server,
+        leader_data,
+        alice: mint_keypair,
+        ledger_path,
+        ..
+    } = TestValidator::run_with_options(TestValidatorOptions {
+        fees: 1,
+        bootstrap_validator_lamports: 42_000,
+        ..TestValidatorOptions::default()
+    });
+
+    let (sender, receiver) = channel();
+    run_local_faucet(mint_keypair, sender, None);
+    let faucet_addr = receiver.recv().unwrap();
+
+    let offline_nonce_authority_signer = keypair_from_seed(&[1u8; 32]).unwrap();
+    let online_nonce_creator_signer = keypair_from_seed(&[2u8; 32]).unwrap();
+    let to_address = Pubkey::new(&[3u8; 32]);
+
+    // Setup accounts
+    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    request_and_confirm_airdrop(
+        &rpc_client,
+        &faucet_addr,
+        &offline_nonce_authority_signer.pubkey(),
+        42,
+    )
+    .unwrap();
+    request_and_confirm_airdrop(
+        &rpc_client,
+        &faucet_addr,
+        &online_nonce_creator_signer.pubkey(),
+        4242,
+    )
+    .unwrap();
+    check_balance(42, &rpc_client, &offline_nonce_authority_signer.pubkey());
+    check_balance(4242, &rpc_client, &online_nonce_creator_signer.pubkey());
+    check_balance(0, &rpc_client, &to_address);
+
+    // Create nonce account
+    let creator_pubkey = online_nonce_creator_signer.pubkey();
+    let authority_pubkey = offline_nonce_authority_signer.pubkey();
+    let seed = authority_pubkey.to_string()[0..32].to_string();
+    let nonce_address =
+        Pubkey::create_with_seed(&creator_pubkey, &seed, &system_program::id()).unwrap();
+    check_balance(0, &rpc_client, &nonce_address);
+
+    let mut creator_config = CliConfig::default();
+    creator_config.json_rpc_url =
+        format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    creator_config.signers = vec![&online_nonce_creator_signer];
+    creator_config.command = CliCommand::CreateNonceAccount {
+        nonce_account: 0,
+        seed: Some(seed),
+        nonce_authority: Some(authority_pubkey),
+        lamports: 241,
+    };
+    process_command(&creator_config).unwrap();
+    check_balance(241, &rpc_client, &nonce_address);
+    check_balance(42, &rpc_client, &offline_nonce_authority_signer.pubkey());
+    check_balance(4000, &rpc_client, &online_nonce_creator_signer.pubkey());
+    check_balance(0, &rpc_client, &to_address);
+
+    // Fetch nonce hash
+    let nonce_hash = nonce::get_account(&rpc_client, &nonce_address)
+        .and_then(|ref a| nonce::data_from_account(a))
+        .unwrap()
+        .blockhash;
+
+    // Test by creating transfer TX with nonce, fully offline
+    let mut authority_config = CliConfig::default();
+    authority_config.json_rpc_url = String::default();
+    authority_config.signers = vec![&offline_nonce_authority_signer];
+    // Verify we cannot contact the cluster
+    authority_config.command = CliCommand::ClusterVersion;
+    process_command(&authority_config).unwrap_err();
+    authority_config.command = CliCommand::Transfer {
+        lamports: 10,
+        to: to_address,
+        from: 0,
+        sign_only: true,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::None(nonce_hash),
+        nonce_account: Some(nonce_address),
+        nonce_authority: 0,
+        fee_payer: 0,
+    };
+    let sign_only_reply = process_command(&authority_config).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_only_reply);
+    let authority_presigner = sign_only.presigner_of(&authority_pubkey).unwrap();
+    assert_eq!(sign_only.blockhash, nonce_hash);
+
+    // And submit it
+    let mut submit_config = CliConfig::default();
+    submit_config.json_rpc_url =
+        format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    submit_config.signers = vec![&authority_presigner];
+    submit_config.command = CliCommand::Transfer {
+        lamports: 10,
+        to: to_address,
+        from: 0,
+        sign_only: false,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::FeeCalculator(
+            blockhash_query::Source::NonceAccount(nonce_address),
+            sign_only.blockhash,
+        ),
+        nonce_account: Some(nonce_address),
+        nonce_authority: 0,
+        fee_payer: 0,
+    };
+    process_command(&submit_config).unwrap();
+    check_balance(241, &rpc_client, &nonce_address);
+    check_balance(31, &rpc_client, &offline_nonce_authority_signer.pubkey());
+    check_balance(4000, &rpc_client, &online_nonce_creator_signer.pubkey());
+    check_balance(10, &rpc_client, &to_address);
+
+    server.close().unwrap();
+    remove_dir_all(ledger_path).unwrap();
+}

cli/tests/pay.rs

@@ -1,6 +1,5 @@
 use chrono::prelude::*;
 use serde_json::Value;
-use solana_clap_utils::keypair::presigner_from_pubkey_sigs;
 use solana_cli::{
     cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig, PayCommand},
     nonce,
@@ -335,9 +334,11 @@ fn test_offline_pay_tx() {
     check_balance(50, &rpc_client, &config_online.signers[0].pubkey());
     check_balance(0, &rpc_client, &bob_pubkey);
 
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner =
-        presigner_from_pubkey_sigs(&config_offline.signers[0].pubkey(), &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only
+        .presigner_of(&config_offline.signers[0].pubkey())
+        .unwrap();
     let online_pubkey = config_online.signers[0].pubkey();
     config_online.signers = vec![&offline_presigner];
     config_online.command = CliCommand::Pay(PayCommand {

cli/tests/stake.rs

@@ -1,4 +1,3 @@
-use solana_clap_utils::keypair::presigner_from_pubkey_sigs;
 use solana_cli::{
     cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
     nonce,
@@ -15,7 +14,6 @@ use solana_sdk::{
     nonce::State as NonceState,
     pubkey::Pubkey,
     signature::{keypair_from_seed, Keypair, Signer},
-    system_instruction::create_address_with_seed,
 };
 use solana_stake_program::{
     stake_instruction::LockupArgs,
@@ -69,7 +67,7 @@ fn test_stake_delegation_force() {
     config.signers = vec![&default_signer, &vote_keypair];
     config.command = CliCommand::CreateVoteAccount {
         seed: None,
-        node_pubkey: config.signers[0].pubkey(),
+        identity_account: 0,
         authorized_voter: None,
         authorized_withdrawer: None,
         commission: 0,
@@ -161,7 +159,7 @@ fn test_seed_stake_delegation_and_deactivation() {
     .unwrap();
     check_balance(100_000, &rpc_client, &config_validator.signers[0].pubkey());
 
-    let stake_address = create_address_with_seed(
+    let stake_address = Pubkey::create_with_seed(
         &config_validator.signers[0].pubkey(),
         "hi there",
         &solana_stake_program::id(),
@@ -388,9 +386,11 @@ fn test_offline_stake_delegation_and_deactivation() {
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner =
-        presigner_from_pubkey_sigs(&config_offline.signers[0].pubkey(), &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only
+        .presigner_of(&config_offline.signers[0].pubkey())
+        .unwrap();
     config_payer.signers = vec![&offline_presigner];
     config_payer.command = CliCommand::DelegateStake {
         stake_account_pubkey: stake_keypair.pubkey(),
@@ -417,9 +417,11 @@ fn test_offline_stake_delegation_and_deactivation() {
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner =
-        presigner_from_pubkey_sigs(&config_offline.signers[0].pubkey(), &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only
+        .presigner_of(&config_offline.signers[0].pubkey())
+        .unwrap();
     config_payer.signers = vec![&offline_presigner];
     config_payer.command = CliCommand::DeactivateStake {
         stake_account_pubkey: stake_keypair.pubkey(),
@@ -623,9 +625,7 @@ fn test_stake_authorize() {
     config.signers.pop();
     config.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: online_authority_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 0,
+        new_authorizations: vec![(StakeAuthorize::Staker, online_authority_pubkey, 0)],
         sign_only: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
@@ -641,13 +641,40 @@ fn test_stake_authorize() {
     };
     assert_eq!(current_authority, online_authority_pubkey);
 
-    // Assign new offline stake authority
+    // Assign new online stake and withdraw authorities
+    let online_authority2 = Keypair::new();
+    let online_authority2_pubkey = online_authority2.pubkey();
+    let withdraw_authority = Keypair::new();
+    let withdraw_authority_pubkey = withdraw_authority.pubkey();
     config.signers.push(&online_authority);
     config.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: offline_authority_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 1,
+        new_authorizations: vec![
+            (StakeAuthorize::Staker, online_authority2_pubkey, 1),
+            (StakeAuthorize::Withdrawer, withdraw_authority_pubkey, 0),
+        ],
+        sign_only: false,
+        blockhash_query: BlockhashQuery::default(),
+        nonce_account: None,
+        nonce_authority: 0,
+        fee_payer: 0,
+    };
+    process_command(&config).unwrap();
+    let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
+    let stake_state: StakeState = stake_account.state().unwrap();
+    let (current_staker, current_withdrawer) = match stake_state {
+        StakeState::Initialized(meta) => (meta.authorized.staker, meta.authorized.withdrawer),
+        _ => panic!("Unexpected stake state!"),
+    };
+    assert_eq!(current_staker, online_authority2_pubkey);
+    assert_eq!(current_withdrawer, withdraw_authority_pubkey);
+
+    // Assign new offline stake authority
+    config.signers.pop();
+    config.signers.push(&online_authority2);
+    config.command = CliCommand::StakeAuthorize {
+        stake_account_pubkey,
+        new_authorizations: vec![(StakeAuthorize::Staker, offline_authority_pubkey, 1)],
         sign_only: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
@@ -669,9 +696,7 @@ fn test_stake_authorize() {
     let (blockhash, _) = rpc_client.get_recent_blockhash().unwrap();
     config_offline.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: nonced_authority_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 0,
+        new_authorizations: vec![(StakeAuthorize::Staker, nonced_authority_pubkey, 0)],
         sign_only: true,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
@@ -679,15 +704,13 @@ fn test_stake_authorize() {
         fee_payer: 0,
     };
     let sign_reply = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sign_reply);
-    let offline_presigner =
-        presigner_from_pubkey_sigs(&offline_authority_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_reply);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_authority_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: nonced_authority_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 0,
+        new_authorizations: vec![(StakeAuthorize::Staker, nonced_authority_pubkey, 0)],
         sign_only: false,
         blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
@@ -729,9 +752,7 @@ fn test_stake_authorize() {
     config_offline.signers.push(&nonced_authority);
     config_offline.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: online_authority_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 1,
+        new_authorizations: vec![(StakeAuthorize::Staker, online_authority_pubkey, 1)],
         sign_only: true,
         blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_account.pubkey()),
@@ -739,22 +760,19 @@ fn test_stake_authorize() {
         fee_payer: 0,
     };
     let sign_reply = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sign_reply);
-    assert_eq!(blockhash, nonce_hash);
-    let offline_presigner =
-        presigner_from_pubkey_sigs(&offline_authority_pubkey, &signers).unwrap();
-    let nonced_authority_presigner =
-        presigner_from_pubkey_sigs(&nonced_authority_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_reply);
+    assert!(sign_only.has_all_signers());
+    assert_eq!(sign_only.blockhash, nonce_hash);
+    let offline_presigner = sign_only.presigner_of(&offline_authority_pubkey).unwrap();
+    let nonced_authority_presigner = sign_only.presigner_of(&nonced_authority_pubkey).unwrap();
     config.signers = vec![&offline_presigner, &nonced_authority_presigner];
     config.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: online_authority_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 1,
+        new_authorizations: vec![(StakeAuthorize::Staker, online_authority_pubkey, 1)],
         sign_only: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
-            blockhash,
+            sign_only.blockhash,
         ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
@@ -793,6 +811,7 @@ fn test_stake_authorize_with_fee_payer() {
     } = TestValidator::run_with_options(TestValidatorOptions {
         fees: SIG_FEE,
         bootstrap_validator_lamports: 42_000,
+        ..TestValidatorOptions::default()
     });
     let (sender, receiver) = channel();
     run_local_faucet(alice, sender, None);
@@ -816,6 +835,7 @@ fn test_stake_authorize_with_fee_payer() {
     let mut config_offline = CliConfig::default();
     let offline_signer = Keypair::new();
     config_offline.signers = vec![&offline_signer];
+    config_offline.json_rpc_url = String::new();
     let offline_pubkey = config_offline.signers[0].pubkey();
     // Verify we're offline
     config_offline.command = CliCommand::ClusterVersion;
@@ -856,9 +876,7 @@ fn test_stake_authorize_with_fee_payer() {
     config.signers = vec![&default_signer, &payer_keypair];
     config.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: offline_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 0,
+        new_authorizations: vec![(StakeAuthorize::Staker, offline_pubkey, 0)],
         sign_only: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
@@ -876,9 +894,7 @@ fn test_stake_authorize_with_fee_payer() {
     let (blockhash, _) = rpc_client.get_recent_blockhash().unwrap();
     config_offline.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: payer_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 0,
+        new_authorizations: vec![(StakeAuthorize::Staker, payer_pubkey, 0)],
         sign_only: true,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
@@ -886,14 +902,13 @@ fn test_stake_authorize_with_fee_payer() {
         fee_payer: 0,
     };
     let sign_reply = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sign_reply);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_reply);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: payer_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 0,
+        new_authorizations: vec![(StakeAuthorize::Staker, payer_pubkey, 0)],
         sign_only: false,
         blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
@@ -924,6 +939,7 @@ fn test_stake_split() {
     } = TestValidator::run_with_options(TestValidatorOptions {
         fees: 1,
         bootstrap_validator_lamports: 42_000,
+        ..TestValidatorOptions::default()
     });
     let (sender, receiver) = channel();
     run_local_faucet(alice, sender, None);
@@ -1023,8 +1039,9 @@ fn test_stake_split() {
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner, &split_account];
     config.command = CliCommand::SplitStake {
         stake_account_pubkey: stake_account_pubkey,
@@ -1032,7 +1049,7 @@ fn test_stake_split() {
         sign_only: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
-            blockhash,
+            sign_only.blockhash,
         ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
@@ -1070,6 +1087,7 @@ fn test_stake_set_lockup() {
     } = TestValidator::run_with_options(TestValidatorOptions {
         fees: 1,
         bootstrap_validator_lamports: 42_000,
+        ..TestValidatorOptions::default()
     });
     let (sender, receiver) = channel();
     run_local_faucet(alice, sender, None);
@@ -1275,8 +1293,9 @@ fn test_stake_set_lockup() {
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::StakeSetLockup {
         stake_account_pubkey,
@@ -1285,7 +1304,7 @@ fn test_stake_set_lockup() {
         sign_only: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account_pubkey),
-            blockhash,
+            sign_only.blockhash,
         ),
         nonce_account: Some(nonce_account_pubkey),
         nonce_authority: 0,
@@ -1392,9 +1411,10 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         from: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
-    let stake_presigner = presigner_from_pubkey_sigs(&stake_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
+    let stake_presigner = sign_only.presigner_of(&stake_pubkey).unwrap();
     config.signers = vec![&offline_presigner, &stake_presigner];
     config.command = CliCommand::CreateStakeAccount {
         stake_account: 1,
@@ -1406,7 +1426,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         sign_only: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_pubkey),
-            blockhash,
+            sign_only.blockhash,
         ),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
@@ -1438,8 +1458,8 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::WithdrawStake {
         stake_account_pubkey: stake_pubkey,
@@ -1449,7 +1469,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         sign_only: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_pubkey),
-            blockhash,
+            sign_only.blockhash,
         ),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
@@ -1482,9 +1502,9 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         from: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
-    let stake_presigner = presigner_from_pubkey_sigs(&stake_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
+    let stake_presigner = sign_only.presigner_of(&stake_pubkey).unwrap();
     config.signers = vec![&offline_presigner, &stake_presigner];
     config.command = CliCommand::CreateStakeAccount {
         stake_account: 1,
@@ -1496,7 +1516,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         sign_only: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_pubkey),
-            blockhash,
+            sign_only.blockhash,
         ),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
@@ -1505,7 +1525,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     };
     process_command(&config).unwrap();
     let seed_address =
-        create_address_with_seed(&stake_pubkey, seed, &solana_stake_program::id()).unwrap();
+        Pubkey::create_with_seed(&stake_pubkey, seed, &solana_stake_program::id()).unwrap();
     check_balance(50_000, &rpc_client, &seed_address);
 
     server.close().unwrap();

cli/tests/transfer.rs

@@ -1,4 +1,3 @@
-use solana_clap_utils::keypair::presigner_from_pubkey_sigs;
 use solana_cli::{
     cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
     nonce,
@@ -13,7 +12,7 @@ use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
     nonce::State as NonceState,
     pubkey::Pubkey,
-    signature::{keypair_from_seed, Keypair, Signer},
+    signature::{keypair_from_seed, Keypair, NullSigner, Signer},
 };
 use std::{fs::remove_dir_all, sync::mpsc::channel, thread::sleep, time::Duration};
 
@@ -41,6 +40,7 @@ fn test_transfer() {
     } = TestValidator::run_with_options(TestValidatorOptions {
         fees: 1,
         bootstrap_validator_lamports: 42_000,
+        ..TestValidatorOptions::default()
     });
 
     let (sender, receiver) = channel();
@@ -69,6 +69,7 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        no_wait: false,
         blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
@@ -96,20 +97,23 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: true,
+        no_wait: false,
         blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
     };
     let sign_only_reply = process_command(&offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sign_only_reply);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_only_reply);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::Transfer {
         lamports: 10,
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        no_wait: false,
         blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,
@@ -147,6 +151,7 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        no_wait: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
             nonce_hash,
@@ -187,23 +192,26 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: true,
+        no_wait: false,
         blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
     };
     let sign_only_reply = process_command(&offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sign_only_reply);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_only_reply);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::Transfer {
         lamports: 10,
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
+        no_wait: false,
         blockhash_query: BlockhashQuery::FeeCalculator(
             blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
-            blockhash,
+            sign_only.blockhash,
         ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
@@ -216,3 +224,118 @@ fn test_transfer() {
     server.close().unwrap();
     remove_dir_all(ledger_path).unwrap();
 }
+
+#[test]
+fn test_transfer_multisession_signing() {
+    let TestValidator {
+        server,
+        leader_data,
+        alice: mint_keypair,
+        ledger_path,
+        ..
+    } = TestValidator::run_with_options(TestValidatorOptions {
+        fees: 1,
+        bootstrap_validator_lamports: 42_000,
+        ..TestValidatorOptions::default()
+    });
+
+    let (sender, receiver) = channel();
+    run_local_faucet(mint_keypair, sender, None);
+    let faucet_addr = receiver.recv().unwrap();
+
+    let to_pubkey = Pubkey::new(&[1u8; 32]);
+    let offline_from_signer = keypair_from_seed(&[2u8; 32]).unwrap();
+    let offline_fee_payer_signer = keypair_from_seed(&[3u8; 32]).unwrap();
+    let from_null_signer = NullSigner::new(&offline_from_signer.pubkey());
+
+    // Setup accounts
+    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_from_signer.pubkey(), 43)
+        .unwrap();
+    request_and_confirm_airdrop(
+        &rpc_client,
+        &faucet_addr,
+        &offline_fee_payer_signer.pubkey(),
+        3,
+    )
+    .unwrap();
+    check_balance(43, &rpc_client, &offline_from_signer.pubkey());
+    check_balance(3, &rpc_client, &offline_fee_payer_signer.pubkey());
+    check_balance(0, &rpc_client, &to_pubkey);
+
+    let (blockhash, _) = rpc_client.get_recent_blockhash().unwrap();
+
+    // Offline fee-payer signs first
+    let mut fee_payer_config = CliConfig::default();
+    fee_payer_config.json_rpc_url = String::default();
+    fee_payer_config.signers = vec![&offline_fee_payer_signer, &from_null_signer];
+    // Verify we cannot contact the cluster
+    fee_payer_config.command = CliCommand::ClusterVersion;
+    process_command(&fee_payer_config).unwrap_err();
+    fee_payer_config.command = CliCommand::Transfer {
+        lamports: 42,
+        to: to_pubkey,
+        from: 1,
+        sign_only: true,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::None(blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        fee_payer: 0,
+    };
+    let sign_only_reply = process_command(&fee_payer_config).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_only_reply);
+    assert!(!sign_only.has_all_signers());
+    let fee_payer_presigner = sign_only
+        .presigner_of(&offline_fee_payer_signer.pubkey())
+        .unwrap();
+
+    // Now the offline fund source
+    let mut from_config = CliConfig::default();
+    from_config.json_rpc_url = String::default();
+    from_config.signers = vec![&fee_payer_presigner, &offline_from_signer];
+    // Verify we cannot contact the cluster
+    from_config.command = CliCommand::ClusterVersion;
+    process_command(&from_config).unwrap_err();
+    from_config.command = CliCommand::Transfer {
+        lamports: 42,
+        to: to_pubkey,
+        from: 1,
+        sign_only: true,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::None(blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        fee_payer: 0,
+    };
+    let sign_only_reply = process_command(&from_config).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_only_reply);
+    assert!(sign_only.has_all_signers());
+    let from_presigner = sign_only
+        .presigner_of(&offline_from_signer.pubkey())
+        .unwrap();
+
+    // Finally submit to the cluster
+    let mut config = CliConfig::default();
+    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.signers = vec![&fee_payer_presigner, &from_presigner];
+    config.command = CliCommand::Transfer {
+        lamports: 42,
+        to: to_pubkey,
+        from: 1,
+        sign_only: false,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        fee_payer: 0,
+    };
+    process_command(&config).unwrap();
+
+    check_balance(1, &rpc_client, &offline_from_signer.pubkey());
+    check_balance(1, &rpc_client, &offline_fee_payer_signer.pubkey());
+    check_balance(42, &rpc_client, &to_pubkey);
+
+    server.close().unwrap();
+    remove_dir_all(ledger_path).unwrap();
+}

cli/tests/vote.rs

@@ -0,0 +1,118 @@
+use solana_cli::cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig};
+use solana_client::rpc_client::RpcClient;
+use solana_core::validator::TestValidator;
+use solana_faucet::faucet::run_local_faucet;
+use solana_sdk::{
+    account_utils::StateMut,
+    pubkey::Pubkey,
+    signature::{Keypair, Signer},
+};
+use solana_vote_program::vote_state::{VoteAuthorize, VoteState, VoteStateVersions};
+use std::{fs::remove_dir_all, sync::mpsc::channel, thread::sleep, time::Duration};
+
+fn check_balance(expected_balance: u64, client: &RpcClient, pubkey: &Pubkey) {
+    (0..5).for_each(|tries| {
+        let balance = client.retry_get_balance(pubkey, 1).unwrap().unwrap();
+        if balance == expected_balance {
+            return;
+        }
+        if tries == 4 {
+            assert_eq!(balance, expected_balance);
+        }
+        sleep(Duration::from_millis(500));
+    });
+}
+
+#[test]
+fn test_vote_authorize_and_withdraw() {
+    let TestValidator {
+        server,
+        leader_data,
+        alice,
+        ledger_path,
+        ..
+    } = TestValidator::run();
+    let (sender, receiver) = channel();
+    run_local_faucet(alice, sender, None);
+    let faucet_addr = receiver.recv().unwrap();
+
+    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let default_signer = Keypair::new();
+
+    let mut config = CliConfig::default();
+    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.signers = vec![&default_signer];
+
+    request_and_confirm_airdrop(
+        &rpc_client,
+        &faucet_addr,
+        &config.signers[0].pubkey(),
+        100_000,
+    )
+    .unwrap();
+
+    // Create vote account
+    let vote_account_keypair = Keypair::new();
+    let vote_account_pubkey = vote_account_keypair.pubkey();
+    config.signers = vec![&default_signer, &vote_account_keypair];
+    config.command = CliCommand::CreateVoteAccount {
+        seed: None,
+        identity_account: 0,
+        authorized_voter: None,
+        authorized_withdrawer: Some(config.signers[0].pubkey()),
+        commission: 0,
+    };
+    process_command(&config).unwrap();
+    let vote_account = rpc_client
+        .get_account(&vote_account_keypair.pubkey())
+        .unwrap();
+    let vote_state: VoteStateVersions = vote_account.state().unwrap();
+    let authorized_withdrawer = vote_state.convert_to_current().authorized_withdrawer;
+    assert_eq!(authorized_withdrawer, config.signers[0].pubkey());
+    let expected_balance = rpc_client
+        .get_minimum_balance_for_rent_exemption(VoteState::size_of())
+        .unwrap()
+        .max(1);
+    check_balance(expected_balance, &rpc_client, &vote_account_pubkey);
+
+    // Authorize vote account withdrawal to another signer
+    let withdraw_authority = Keypair::new();
+    config.signers = vec![&default_signer];
+    config.command = CliCommand::VoteAuthorize {
+        vote_account_pubkey,
+        new_authorized_pubkey: withdraw_authority.pubkey(),
+        vote_authorize: VoteAuthorize::Withdrawer,
+    };
+    process_command(&config).unwrap();
+    let vote_account = rpc_client
+        .get_account(&vote_account_keypair.pubkey())
+        .unwrap();
+    let vote_state: VoteStateVersions = vote_account.state().unwrap();
+    let authorized_withdrawer = vote_state.convert_to_current().authorized_withdrawer;
+    assert_eq!(authorized_withdrawer, withdraw_authority.pubkey());
+
+    // Withdraw from vote account
+    let destination_account = Pubkey::new_rand(); // Send withdrawal to new account to make balance check easy
+    config.signers = vec![&default_signer, &withdraw_authority];
+    config.command = CliCommand::WithdrawFromVoteAccount {
+        vote_account_pubkey,
+        withdraw_authority: 1,
+        lamports: 100,
+        destination_account_pubkey: destination_account,
+    };
+    process_command(&config).unwrap();
+    check_balance(expected_balance - 100, &rpc_client, &vote_account_pubkey);
+    check_balance(100, &rpc_client, &destination_account);
+
+    // Re-assign validator identity
+    let new_identity_keypair = Keypair::new();
+    config.signers.push(&new_identity_keypair);
+    config.command = CliCommand::VoteUpdateValidator {
+        vote_account_pubkey,
+        new_identity_account: 2,
+    };
+    process_command(&config).unwrap();
+
+    server.close().unwrap();
+    remove_dir_all(ledger_path).unwrap();
+}

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.0.6"
+version = "1.1.5"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,15 +11,18 @@ edition = "2018"
 [dependencies]
 bincode = "1.2.1"
 bs58 = "0.3.0"
+indicatif = "0.14.0"
 jsonrpc-core = "14.0.5"
 log = "0.4.8"
-rayon = "1.2.0"
-reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
-serde = "1.0.104"
+rayon = "1.3.0"
+reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+serde = "1.0.105"
 serde_derive = "1.0.103"
-serde_json = "1.0.46"
-solana-net-utils = { path = "../net-utils", version = "1.0.6" }
-solana-sdk = { path = "../sdk", version = "1.0.6" }
+serde_json = "1.0.48"
+solana-transaction-status = { path = "../transaction-status", version = "1.1.5" }
+solana-net-utils = { path = "../net-utils", version = "1.1.5" }
+solana-sdk = { path = "../sdk", version = "1.1.5" }
+solana-vote-program = { path = "../programs/vote", version = "1.1.5" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -28,4 +31,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "14.0.5"
 jsonrpc-http-server = "14.0.6"
-solana-logger = { path = "../logger", version = "1.0.6" }
+solana-logger = { path = "../logger", version = "1.1.5" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

client/src/mock_rpc_client_request.rs

@@ -8,8 +8,10 @@ use serde_json::{Number, Value};
 use solana_sdk::{
     fee_calculator::{FeeCalculator, FeeRateGovernor},
     instruction::InstructionError,
-    transaction::{self, TransactionError},
+    signature::Signature,
+    transaction::{self, Transaction, TransactionError},
 };
+use solana_transaction_status::TransactionStatus;
 use std::{collections::HashMap, sync::RwLock};
 
 pub const PUBKEY: &str = "7RoSF9fUmdphVCpabEoefH81WwrW7orsWonXWqTXkKV8";
@@ -49,17 +51,6 @@ impl GenericRpcClientRequest for MockRpcClientRequest {
             return Ok(Value::Null);
         }
         let val = match request {
-            RpcRequest::ConfirmTransaction => {
-                if let Some(params_array) = params.as_array() {
-                    if let Value::String(param_string) = &params_array[0] {
-                        Value::Bool(param_string == SIGNATURE)
-                    } else {
-                        Value::Null
-                    }
-                } else {
-                    Value::Null
-                }
-            }
             RpcRequest::GetBalance => serde_json::to_value(Response {
                 context: RpcResponseContext { slot: 1 },
                 value: Value::Number(Number::from(50)),
@@ -86,24 +77,46 @@ impl GenericRpcClientRequest for MockRpcClientRequest {
                 context: RpcResponseContext { slot: 1 },
                 value: serde_json::to_value(FeeRateGovernor::default()).unwrap(),
             })?,
-            RpcRequest::GetSignatureStatus => {
-                let response: Option<transaction::Result<()>> = if self.url == "account_in_use" {
-                    Some(Err(TransactionError::AccountInUse))
+            RpcRequest::GetSignatureStatuses => {
+                let status: transaction::Result<()> = if self.url == "account_in_use" {
+                    Err(TransactionError::AccountInUse)
                 } else if self.url == "instruction_error" {
-                    Some(Err(TransactionError::InstructionError(
+                    Err(TransactionError::InstructionError(
                         0,
                         InstructionError::UninitializedAccount,
-                    )))
-                } else if self.url == "sig_not_found" {
+                    ))
+                } else {
+                    Ok(())
+                };
+                let status = if self.url == "sig_not_found" {
                     None
                 } else {
-                    Some(Ok(()))
+                    let err = status.clone().err();
+                    Some(TransactionStatus {
+                        status,
+                        slot: 1,
+                        confirmations: None,
+                        err,
+                    })
                 };
-                serde_json::to_value(response).unwrap()
+                serde_json::to_value(Response {
+                    context: RpcResponseContext { slot: 1 },
+                    value: vec![status],
+                })?
             }
             RpcRequest::GetTransactionCount => Value::Number(Number::from(1234)),
             RpcRequest::GetSlot => Value::Number(Number::from(0)),
-            RpcRequest::SendTransaction => Value::String(SIGNATURE.to_string()),
+            RpcRequest::SendTransaction => {
+                let signature = if self.url == "malicious" {
+                    Signature::new(&[8; 64]).to_string()
+                } else {
+                    let tx_str = params.as_array().unwrap()[0].as_str().unwrap().to_string();
+                    let data = bs58::decode(tx_str).into_vec().unwrap();
+                    let tx: Transaction = bincode::deserialize(&data).unwrap();
+                    tx.signatures[0].to_string()
+                };
+                Value::String(signature)
+            }
             RpcRequest::GetMinimumBalanceForRentExemption => Value::Number(Number::from(1234)),
             _ => Value::Null,
         };

client/src/rpc_client.rs

@@ -1,16 +1,13 @@
 use crate::{
-    client_error::{ClientError, Result as ClientResult},
+    client_error::{ClientError, ClientErrorKind, Result as ClientResult},
     generic_rpc_client_request::GenericRpcClientRequest,
     mock_rpc_client_request::{MockRpcClientRequest, Mocks},
     rpc_client_request::RpcClientRequest,
     rpc_request::{RpcError, RpcRequest},
-    rpc_response::{
-        Response, RpcAccount, RpcBlockhashFeeCalculator, RpcConfirmedBlock, RpcContactInfo,
-        RpcEpochInfo, RpcFeeCalculator, RpcFeeRateGovernor, RpcIdentity, RpcKeyedAccount,
-        RpcLeaderSchedule, RpcResult, RpcVersionInfo, RpcVoteAccountStatus,
-    },
+    rpc_response::*,
 };
 use bincode::serialize;
+use indicatif::{ProgressBar, ProgressStyle};
 use log::*;
 use serde_json::{json, Value};
 use solana_sdk::{
@@ -26,6 +23,10 @@ use solana_sdk::{
     signers::Signers,
     transaction::{self, Transaction, TransactionError},
 };
+use solana_transaction_status::{
+    ConfirmedBlock, ConfirmedTransaction, TransactionEncoding, TransactionStatus,
+};
+use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
 use std::{
     error,
     net::SocketAddr,
@@ -67,7 +68,7 @@ impl RpcClient {
         }
     }
 
-    pub fn confirm_transaction(&self, signature: &str) -> ClientResult<bool> {
+    pub fn confirm_transaction(&self, signature: &Signature) -> ClientResult<bool> {
         Ok(self
             .confirm_transaction_with_commitment(signature, CommitmentConfig::default())?
             .value)
@@ -75,54 +76,110 @@ impl RpcClient {
 
     pub fn confirm_transaction_with_commitment(
         &self,
-        signature: &str,
+        signature: &Signature,
         commitment_config: CommitmentConfig,
     ) -> RpcResult<bool> {
-        let response = self
-            .client
-            .send(
-                &RpcRequest::ConfirmTransaction,
-                json!([signature, commitment_config]),
-                0,
-            )
-            .map_err(|err| err.into_with_command("ConfirmTransaction"))?;
+        let Response { context, value } = self.get_signature_statuses(&[*signature])?;
 
-        serde_json::from_value::<Response<bool>>(response)
-            .map_err(|err| ClientError::new_with_command(err.into(), "ConfirmTransaction"))
+        Ok(Response {
+            context,
+            value: value[0]
+                .as_ref()
+                .filter(|result| result.satisfies_commitment(commitment_config))
+                .map(|result| result.status.is_ok())
+                .unwrap_or_default(),
+        })
     }
 
-    pub fn send_transaction(&self, transaction: &Transaction) -> ClientResult<String> {
+    pub fn send_transaction(&self, transaction: &Transaction) -> ClientResult<Signature> {
         let serialized_encoded = bs58::encode(serialize(transaction).unwrap()).into_string();
-        let signature =
+        let response =
             self.client
                 .send(&RpcRequest::SendTransaction, json!([serialized_encoded]), 5)?;
-        if signature.as_str().is_none() {
-            Err(RpcError::ForUser("Received result of an unexpected type".to_string()).into())
-        } else {
-            Ok(signature.as_str().unwrap().to_string())
+
+        match response.as_str() {
+            None => {
+                Err(RpcError::ForUser("Received result of an unexpected type".to_string()).into())
+            }
+            Some(signature_base58_str) => {
+                let signature = signature_base58_str.parse::<Signature>().map_err(|err| {
+                    Into::<ClientError>::into(RpcError::ParseError(err.to_string()))
+                })?;
+                // A mismatching RPC response signature indicates an issue with the RPC node, and
+                // should not be passed along to confirmation methods. The transaction may or may
+                // not have been submitted to the cluster, so callers should verify the success of
+                // the correct transaction signature independently.
+                if signature != transaction.signatures[0] {
+                    Err(RpcError::RpcRequestError(format!(
+                        "RPC node returned mismatched signature {:?}, expected {:?}",
+                        signature, transaction.signatures[0]
+                    ))
+                    .into())
+                } else {
+                    Ok(transaction.signatures[0])
+                }
+            }
         }
     }
 
     pub fn get_signature_status(
         &self,
-        signature: &str,
+        signature: &Signature,
     ) -> ClientResult<Option<transaction::Result<()>>> {
         self.get_signature_status_with_commitment(signature, CommitmentConfig::default())
     }
 
+    pub fn get_signature_statuses(
+        &self,
+        signatures: &[Signature],
+    ) -> RpcResult<Vec<Option<TransactionStatus>>> {
+        let signatures: Vec<_> = signatures.iter().map(|s| s.to_string()).collect();
+        let signature_status =
+            self.client
+                .send(&RpcRequest::GetSignatureStatuses, json!([signatures]), 5)?;
+        Ok(serde_json::from_value(signature_status)
+            .map_err(|err| ClientError::new_with_command(err.into(), "GetSignatureStatuses"))?)
+    }
+
     pub fn get_signature_status_with_commitment(
         &self,
-        signature: &str,
+        signature: &Signature,
         commitment_config: CommitmentConfig,
     ) -> ClientResult<Option<transaction::Result<()>>> {
         let signature_status = self.client.send(
-            &RpcRequest::GetSignatureStatus,
-            json!([signature.to_string(), commitment_config]),
+            &RpcRequest::GetSignatureStatuses,
+            json!([[signature.to_string()]]),
             5,
         )?;
-        let result: Option<transaction::Result<()>> =
-            serde_json::from_value(signature_status).unwrap();
-        Ok(result)
+        let result: Response<Vec<Option<TransactionStatus>>> =
+            serde_json::from_value(signature_status)
+                .map_err(|err| ClientError::new_with_command(err.into(), "GetSignatureStatuses"))?;
+        Ok(result.value[0]
+            .clone()
+            .filter(|result| result.satisfies_commitment(commitment_config))
+            .map(|status_meta| status_meta.status))
+    }
+
+    pub fn get_signature_status_with_commitment_and_history(
+        &self,
+        signature: &Signature,
+        commitment_config: CommitmentConfig,
+        search_transaction_history: bool,
+    ) -> ClientResult<Option<transaction::Result<()>>> {
+        let signature_status = self.client.send(
+            &RpcRequest::GetSignatureStatuses,
+            json!([[signature.to_string()], {
+                "searchTransactionHistory": search_transaction_history
+            }]),
+            5,
+        )?;
+        let result: Response<Vec<Option<TransactionStatus>>> =
+            serde_json::from_value(signature_status)
+                .map_err(|err| ClientError::new_with_command(err.into(), "GetSignatureStatuses"))?;
+        Ok(result.value[0]
+            .clone()
+            .filter(|result| result.satisfies_commitment(commitment_config))
+            .map(|status_meta| status_meta.status))
     }
 
     pub fn get_slot(&self) -> ClientResult<Slot> {
@@ -142,6 +199,23 @@ impl RpcClient {
             .map_err(|err| ClientError::new_with_command(err.into(), "GetSlot"))
     }
 
+    pub fn total_supply(&self) -> ClientResult<u64> {
+        self.total_supply_with_commitment(CommitmentConfig::default())
+    }
+
+    pub fn total_supply_with_commitment(
+        &self,
+        commitment_config: CommitmentConfig,
+    ) -> ClientResult<u64> {
+        let response = self
+            .client
+            .send(&RpcRequest::GetTotalSupply, json!([commitment_config]), 0)
+            .map_err(|err| err.into_with_command("GetTotalSupply"))?;
+
+        serde_json::from_value(response)
+            .map_err(|err| ClientError::new_with_command(err.into(), "GetTotalSupply"))
+    }
+
     pub fn get_vote_accounts(&self) -> ClientResult<RpcVoteAccountStatus> {
         self.get_vote_accounts_with_commitment(CommitmentConfig::default())
     }
@@ -169,10 +243,18 @@ impl RpcClient {
             .map_err(|err| ClientError::new_with_command(err.into(), "GetClusterNodes"))
     }
 
-    pub fn get_confirmed_block(&self, slot: Slot) -> ClientResult<RpcConfirmedBlock> {
+    pub fn get_confirmed_block(&self, slot: Slot) -> ClientResult<ConfirmedBlock> {
+        self.get_confirmed_block_with_encoding(slot, TransactionEncoding::Json)
+    }
+
+    pub fn get_confirmed_block_with_encoding(
+        &self,
+        slot: Slot,
+        encoding: TransactionEncoding,
+    ) -> ClientResult<ConfirmedBlock> {
         let response = self
             .client
-            .send(&RpcRequest::GetConfirmedBlock, json!([slot]), 0)
+            .send(&RpcRequest::GetConfirmedBlock, json!([slot, encoding]), 0)
             .map_err(|err| err.into_with_command("GetConfirmedBlock"))?;
 
         serde_json::from_value(response)
@@ -197,6 +279,44 @@ impl RpcClient {
             .map_err(|err| ClientError::new_with_command(err.into(), "GetConfirmedBlocks"))
     }
 
+    pub fn get_confirmed_signatures_for_address(
+        &self,
+        address: &Pubkey,
+        start_slot: Slot,
+        end_slot: Slot,
+    ) -> ClientResult<Vec<Signature>> {
+        let response = self
+            .client
+            .send(
+                &RpcRequest::GetConfirmedSignaturesForAddress,
+                json!([address, start_slot, end_slot]),
+                0,
+            )
+            .map_err(|err| err.into_with_command("GetConfirmedSignaturesForAddress"))?;
+
+        serde_json::from_value(response).map_err(|err| {
+            ClientError::new_with_command(err.into(), "GetConfirmedSignaturesForAddress")
+        })
+    }
+
+    pub fn get_confirmed_transaction(
+        &self,
+        signature: &Signature,
+        encoding: TransactionEncoding,
+    ) -> ClientResult<ConfirmedTransaction> {
+        let response = self
+            .client
+            .send(
+                &RpcRequest::GetConfirmedTransaction,
+                json!([signature.to_string(), encoding]),
+                0,
+            )
+            .map_err(|err| err.into_with_command("GetConfirmedTransaction"))?;
+
+        serde_json::from_value(response)
+            .map_err(|err| ClientError::new_with_command(err.into(), "GetConfirmedTransaction"))
+    }
+
     pub fn get_block_time(&self, slot: Slot) -> ClientResult<UnixTimestamp> {
         let response = self
             .client
@@ -319,13 +439,13 @@ impl RpcClient {
         &self,
         transaction: &mut Transaction,
         signer_keys: &T,
-    ) -> ClientResult<String> {
+    ) -> ClientResult<Signature> {
         let mut send_retries = 20;
         loop {
             let mut status_retries = 15;
-            let signature_str = self.send_transaction(transaction)?;
+            let signature = self.send_transaction(transaction)?;
             let status = loop {
-                let status = self.get_signature_status(&signature_str)?;
+                let status = self.get_signature_status(&signature)?;
                 if status.is_none() {
                     status_retries -= 1;
                     if status_retries == 0 {
@@ -341,7 +461,7 @@ impl RpcClient {
             };
             send_retries = if let Some(result) = status.clone() {
                 match result {
-                    Ok(_) => return Ok(signature_str),
+                    Ok(_) => return Ok(signature),
                     Err(TransactionError::AccountInUse) => {
                         // Fetch a new blockhash and re-sign the transaction before sending it again
                         self.resign_transaction(transaction, signer_keys)?;
@@ -803,10 +923,9 @@ impl RpcClient {
     ) -> ClientResult<()> {
         let now = Instant::now();
         loop {
-            if let Ok(Some(_)) = self.get_signature_status_with_commitment(
-                &signature.to_string(),
-                commitment_config.clone(),
-            ) {
+            if let Ok(Some(_)) =
+                self.get_signature_status_with_commitment(&signature, commitment_config.clone())
+            {
                 break;
             }
             if now.elapsed().as_secs() > 15 {
@@ -826,14 +945,13 @@ impl RpcClient {
         trace!("check_signature: {:?}", signature);
 
         for _ in 0..30 {
-            let response = self.client.send(
-                &RpcRequest::ConfirmTransaction,
-                json!([signature.to_string(), CommitmentConfig::recent()]),
-                0,
-            );
-
+            let response =
+                self.confirm_transaction_with_commitment(signature, CommitmentConfig::recent());
             match response {
-                Ok(Value::Bool(signature_status)) => {
+                Ok(Response {
+                    value: signature_status,
+                    ..
+                }) => {
                     if signature_status {
                         trace!("Response found signature");
                     } else {
@@ -842,12 +960,6 @@ impl RpcClient {
 
                     return signature_status;
                 }
-                Ok(other) => {
-                    debug!(
-                        "check_signature request failed, expected bool, got: {:?}",
-                        other
-                    );
-                }
                 Err(err) => {
                     debug!("check_signature request failed: {:?}", err);
                 }
@@ -919,14 +1031,112 @@ impl RpcClient {
         let response = self
             .client
             .send(
-                &RpcRequest::GetNumBlocksSinceSignatureConfirmation,
-                json!([signature.to_string(), CommitmentConfig::recent().ok()]),
-                1,
+                &RpcRequest::GetSignatureStatuses,
+                json!([[signature.to_string()]]),
+                5,
             )
-            .map_err(|err| err.into_with_command("GetNumBlocksSinceSignatureConfirmation"))?;
-        serde_json::from_value(response).map_err(|err| {
-            ClientError::new_with_command(err.into(), "GetNumBlocksSinceSignatureConfirmation")
-        })
+            .map_err(|err| err.into_with_command("GetSignatureStatuses"))?;
+        let result: Response<Vec<Option<TransactionStatus>>> = serde_json::from_value(response)
+            .map_err(|err| ClientError::new_with_command(err.into(), "GetSignatureStatuses"))?;
+
+        let confirmations = result.value[0]
+            .clone()
+            .ok_or_else(|| {
+                ClientError::new_with_command(
+                    ClientErrorKind::Custom("signature not found".to_string()),
+                    "GetSignatureStatuses",
+                )
+            })?
+            .confirmations
+            .unwrap_or(MAX_LOCKOUT_HISTORY + 1);
+        Ok(confirmations)
+    }
+
+    pub fn send_and_confirm_transaction_with_spinner<T: Signers>(
+        &self,
+        transaction: &mut Transaction,
+        signer_keys: &T,
+    ) -> ClientResult<Signature> {
+        let mut confirmations = 0;
+
+        let progress_bar = new_spinner_progress_bar();
+
+        let mut send_retries = 20;
+        let signature = loop {
+            progress_bar.set_message(&format!(
+                "[{}/{}] Finalizing transaction {}",
+                confirmations,
+                MAX_LOCKOUT_HISTORY + 1,
+                transaction.signatures[0],
+            ));
+            let mut status_retries = 15;
+            let (signature, status) = loop {
+                let signature = self.send_transaction(transaction)?;
+
+                // Get recent commitment in order to count confirmations for successful transactions
+                let status = self
+                    .get_signature_status_with_commitment(&signature, CommitmentConfig::recent())?;
+                if status.is_none() {
+                    status_retries -= 1;
+                    if status_retries == 0 {
+                        break (signature, status);
+                    }
+                } else {
+                    break (signature, status);
+                }
+
+                if cfg!(not(test)) {
+                    sleep(Duration::from_millis(500));
+                }
+            };
+            send_retries = if let Some(result) = status.clone() {
+                match result {
+                    Ok(_) => 0,
+                    Err(TransactionError::AccountInUse) => {
+                        // Fetch a new blockhash and re-sign the transaction before sending it again
+                        self.resign_transaction(transaction, signer_keys)?;
+                        send_retries - 1
+                    }
+                    // If transaction errors, return right away; no point in counting confirmations
+                    Err(_) => 0,
+                }
+            } else {
+                send_retries - 1
+            };
+            if send_retries == 0 {
+                if let Some(result) = status {
+                    match result {
+                        Ok(_) => {
+                            break signature;
+                        }
+                        Err(err) => {
+                            return Err(err.into());
+                        }
+                    }
+                } else {
+                    return Err(
+                        RpcError::ForUser("unable to confirm transaction. This can happen in situations such as transaction expiration and insufficient fee-payer funds".to_string()).into(),
+                    );
+                }
+            }
+        };
+        loop {
+            // Return when default (max) commitment is reached
+            // Failed transactions have already been eliminated, `is_some` check is sufficient
+            if self.get_signature_status(&signature)?.is_some() {
+                progress_bar.set_message("Transaction confirmed");
+                progress_bar.finish_and_clear();
+                return Ok(signature);
+            }
+            progress_bar.set_message(&format!(
+                "[{}/{}] Finalizing transaction {}",
+                confirmations + 1,
+                MAX_LOCKOUT_HISTORY + 1,
+                signature,
+            ));
+            sleep(Duration::from_millis(500));
+            confirmations = self.get_num_blocks_since_signature_confirmation(&signature)?;
+        }
     }
 
     pub fn validator_exit(&self) -> ClientResult<bool> {
@@ -944,6 +1154,14 @@ impl RpcClient {
     }
 }
 
+fn new_spinner_progress_bar() -> ProgressBar {
+    let progress_bar = ProgressBar::new(42);
+    progress_bar
+        .set_style(ProgressStyle::default_spinner().template("{spinner:.green} {wide_msg}"));
+    progress_bar.enable_steady_tick(100);
+    progress_bar
+}
+
 pub fn get_rpc_request_str(rpc_addr: SocketAddr, tls: bool) -> String {
     if tls {
         format!("https://{}", rpc_addr)
@@ -955,10 +1173,7 @@ pub fn get_rpc_request_str(rpc_addr: SocketAddr, tls: bool) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{
-        client_error::ClientErrorKind,
-        mock_rpc_client_request::{PUBKEY, SIGNATURE},
-    };
+    use crate::{client_error::ClientErrorKind, mock_rpc_client_request::PUBKEY};
     use assert_matches::assert_matches;
     use jsonrpc_core::{Error, IoHandler, Params};
     use jsonrpc_http_server::{AccessControlAllowOrigin, DomainsValidation, ServerBuilder};
@@ -1071,12 +1286,17 @@ mod tests {
         let tx = system_transaction::transfer(&key, &to, 50, blockhash);
 
         let signature = rpc_client.send_transaction(&tx);
-        assert_eq!(signature.unwrap(), SIGNATURE.to_string());
+        assert_eq!(signature.unwrap(), tx.signatures[0]);
 
         let rpc_client = RpcClient::new_mock("fails".to_string());
 
         let signature = rpc_client.send_transaction(&tx);
         assert!(signature.is_err());
+
+        // Test bad signature returned from rpc node
+        let rpc_client = RpcClient::new_mock("malicious".to_string());
+        let signature = rpc_client.send_transaction(&tx);
+        assert!(signature.is_err());
     }
     #[test]
     fn test_get_recent_blockhash() {
@@ -1094,18 +1314,17 @@ mod tests {
 
     #[test]
     fn test_get_signature_status() {
+        let signature = Signature::default();
+
         let rpc_client = RpcClient::new_mock("succeeds".to_string());
-        let signature = "good_signature";
         let status = rpc_client.get_signature_status(&signature).unwrap();
         assert_eq!(status, Some(Ok(())));
 
         let rpc_client = RpcClient::new_mock("sig_not_found".to_string());
-        let signature = "sig_not_found";
         let status = rpc_client.get_signature_status(&signature).unwrap();
         assert_eq!(status, None);
 
         let rpc_client = RpcClient::new_mock("account_in_use".to_string());
-        let signature = "account_in_use";
         let status = rpc_client.get_signature_status(&signature).unwrap();
         assert_eq!(status, Some(Err(TransactionError::AccountInUse)));
     }

client/src/rpc_request.rs

@@ -3,7 +3,6 @@ use thiserror::Error;
 
 #[derive(Debug, PartialEq, Eq, Hash)]
 pub enum RpcRequest {
-    ConfirmTransaction,
     DeregisterNode,
     ValidatorExit,
     GetAccountInfo,
@@ -12,24 +11,26 @@ pub enum RpcRequest {
     GetClusterNodes,
     GetConfirmedBlock,
     GetConfirmedBlocks,
+    GetConfirmedSignaturesForAddress,
+    GetConfirmedTransaction,
     GetEpochInfo,
     GetEpochSchedule,
     GetGenesisHash,
     GetIdentity,
     GetInflation,
     GetLeaderSchedule,
-    GetNumBlocksSinceSignatureConfirmation,
     GetProgramAccounts,
     GetRecentBlockhash,
     GetFeeCalculatorForBlockhash,
     GetFeeRateGovernor,
-    GetSignatureStatus,
+    GetSignatureStatuses,
     GetSlot,
     GetSlotLeader,
     GetStorageTurn,
     GetStorageTurnRate,
     GetSlotsPerSegment,
     GetStoragePubkeysForSlot,
+    GetTotalSupply,
     GetTransactionCount,
     GetVersion,
     GetVoteAccounts,
@@ -45,7 +46,6 @@ impl RpcRequest {
     pub(crate) fn build_request_json(&self, id: u64, params: Value) -> Value {
         let jsonrpc = "2.0";
         let method = match self {
-            RpcRequest::ConfirmTransaction => "confirmTransaction",
             RpcRequest::DeregisterNode => "deregisterNode",
             RpcRequest::ValidatorExit => "validatorExit",
             RpcRequest::GetAccountInfo => "getAccountInfo",
@@ -54,26 +54,26 @@ impl RpcRequest {
             RpcRequest::GetClusterNodes => "getClusterNodes",
             RpcRequest::GetConfirmedBlock => "getConfirmedBlock",
             RpcRequest::GetConfirmedBlocks => "getConfirmedBlocks",
+            RpcRequest::GetConfirmedSignaturesForAddress => "getConfirmedSignaturesForAddress",
+            RpcRequest::GetConfirmedTransaction => "getConfirmedTransaction",
             RpcRequest::GetEpochInfo => "getEpochInfo",
             RpcRequest::GetEpochSchedule => "getEpochSchedule",
             RpcRequest::GetGenesisHash => "getGenesisHash",
             RpcRequest::GetIdentity => "getIdentity",
             RpcRequest::GetInflation => "getInflation",
             RpcRequest::GetLeaderSchedule => "getLeaderSchedule",
-            RpcRequest::GetNumBlocksSinceSignatureConfirmation => {
-                "getNumBlocksSinceSignatureConfirmation"
-            }
             RpcRequest::GetProgramAccounts => "getProgramAccounts",
             RpcRequest::GetRecentBlockhash => "getRecentBlockhash",
             RpcRequest::GetFeeCalculatorForBlockhash => "getFeeCalculatorForBlockhash",
             RpcRequest::GetFeeRateGovernor => "getFeeRateGovernor",
-            RpcRequest::GetSignatureStatus => "getSignatureStatus",
+            RpcRequest::GetSignatureStatuses => "getSignatureStatuses",
             RpcRequest::GetSlot => "getSlot",
             RpcRequest::GetSlotLeader => "getSlotLeader",
             RpcRequest::GetStorageTurn => "getStorageTurn",
             RpcRequest::GetStorageTurnRate => "getStorageTurnRate",
             RpcRequest::GetSlotsPerSegment => "getSlotsPerSegment",
             RpcRequest::GetStoragePubkeysForSlot => "getStoragePubkeysForSlot",
+            RpcRequest::GetTotalSupply => "getTotalSupply",
             RpcRequest::GetTransactionCount => "getTransactionCount",
             RpcRequest::GetVersion => "getVersion",
             RpcRequest::GetVoteAccounts => "getVoteAccounts",
@@ -95,7 +95,7 @@ impl RpcRequest {
 
 #[derive(Debug, Error)]
 pub enum RpcError {
-    #[error("rpc reques error: {0}")]
+    #[error("rpc request error: {0}")]
     RpcRequestError(String),
     #[error("parse error: expected {0}")]
     ParseError(String), /* "expected" */

client/src/rpc_response.rs

@@ -1,12 +1,10 @@
 use crate::{client_error, rpc_request::RpcError};
-use bincode::serialize;
 use solana_sdk::{
     account::Account,
     clock::{Epoch, Slot},
     fee_calculator::{FeeCalculator, FeeRateGovernor},
-    message::MessageHeader,
     pubkey::Pubkey,
-    transaction::{Result, Transaction},
+    transaction::{Result, TransactionError},
 };
 use std::{collections::HashMap, net::SocketAddr, str::FromStr};
 
@@ -30,119 +28,6 @@ pub struct RpcBlockCommitment<T> {
     pub total_stake: u64,
 }
 
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-pub struct RpcReward {
-    pub pubkey: String,
-    pub lamports: i64,
-}
-
-pub type RpcRewards = Vec<RpcReward>;
-
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcConfirmedBlock {
-    pub previous_blockhash: String,
-    pub blockhash: String,
-    pub parent_slot: Slot,
-    pub transactions: Vec<RpcTransactionWithStatusMeta>,
-    pub rewards: RpcRewards,
-}
-
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcTransactionWithStatusMeta {
-    pub transaction: RpcEncodedTransaction,
-    pub meta: Option<RpcTransactionStatus>,
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-#[serde(rename_all = "camelCase")]
-pub enum RpcTransactionEncoding {
-    Binary,
-    Json,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase", untagged)]
-pub enum RpcEncodedTransaction {
-    Binary(String),
-    Json(RpcTransaction),
-}
-
-impl RpcEncodedTransaction {
-    pub fn encode(transaction: Transaction, encoding: RpcTransactionEncoding) -> Self {
-        if encoding == RpcTransactionEncoding::Json {
-            RpcEncodedTransaction::Json(RpcTransaction {
-                signatures: transaction
-                    .signatures
-                    .iter()
-                    .map(|sig| sig.to_string())
-                    .collect(),
-                message: RpcMessage {
-                    header: transaction.message.header,
-                    account_keys: transaction
-                        .message
-                        .account_keys
-                        .iter()
-                        .map(|pubkey| pubkey.to_string())
-                        .collect(),
-                    recent_blockhash: transaction.message.recent_blockhash.to_string(),
-                    instructions: transaction
-                        .message
-                        .instructions
-                        .iter()
-                        .map(|instruction| RpcCompiledInstruction {
-                            program_id_index: instruction.program_id_index,
-                            accounts: instruction.accounts.clone(),
-                            data: bs58::encode(instruction.data.clone()).into_string(),
-                        })
-                        .collect(),
-                },
-            })
-        } else {
-            RpcEncodedTransaction::Binary(
-                bs58::encode(serialize(&transaction).unwrap()).into_string(),
-            )
-        }
-    }
-}
-
-/// A duplicate representation of a Transaction for pretty JSON serialization
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcTransaction {
-    pub signatures: Vec<String>,
-    pub message: RpcMessage,
-}
-
-/// A duplicate representation of a Message for pretty JSON serialization
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcMessage {
-    pub header: MessageHeader,
-    pub account_keys: Vec<String>,
-    pub recent_blockhash: String,
-    pub instructions: Vec<RpcCompiledInstruction>,
-}
-
-/// A duplicate representation of a Message for pretty JSON serialization
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcCompiledInstruction {
-    pub program_id_index: u8,
-    pub accounts: Vec<u8>,
-    pub data: String,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcTransactionStatus {
-    pub status: Result<()>,
-    pub fee: u64,
-    pub pre_balances: Vec<u64>,
-    pub post_balances: Vec<u64>,
-}
-
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcBlockhashFeeCalculator {
@@ -169,6 +54,12 @@ pub struct RpcKeyedAccount {
     pub account: RpcAccount,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSignatureResult {
+    pub err: Option<TransactionError>,
+}
+
 /// A duplicate representation of a Message for pretty JSON serialization
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]

client/src/thin_client.rs

@@ -471,7 +471,7 @@ impl SyncClient for ThinClient {
     ) -> TransportResult<Option<transaction::Result<()>>> {
         let status = self
             .rpc_client()
-            .get_signature_status(&signature.to_string())
+            .get_signature_status(&signature)
             .map_err(|err| {
                 io::Error::new(
                     io::ErrorKind::Other,
@@ -488,7 +488,7 @@ impl SyncClient for ThinClient {
     ) -> TransportResult<Option<transaction::Result<()>>> {
         let status = self
             .rpc_client()
-            .get_signature_status_with_commitment(&signature.to_string(), commitment_config)
+            .get_signature_status_with_commitment(&signature, commitment_config)
             .map_err(|err| {
                 io::Error::new(
                     io::ErrorKind::Other,

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.6"
+version = "1.1.5"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -15,66 +15,67 @@ codecov = { repository = "solana-labs/solana", branch = "master", service = "git
 
 [dependencies]
 bincode = "1.2.1"
+bv = { version = "0.11.1", features = ["serde"] }
 bs58 = "0.3.0"
-byteorder = "1.3.2"
-chrono = { version = "0.4.10", features = ["serde"] }
-compression = "0.1.5"
+byteorder = "1.3.4"
+chrono = { version = "0.4.11", features = ["serde"] }
 core_affinity = "0.5.10"
-crossbeam-channel = "0.3"
+crossbeam-channel = "0.4"
 fs_extra = "1.1.0"
+flate2 = "1.0"
 indexmap = "1.3"
-itertools = "0.8.2"
+itertools = "0.9.0"
 jsonrpc-core = "14.0.5"
 jsonrpc-core-client = { version = "14.0.5", features = ["ws"] }
 jsonrpc-derive = "14.0.5"
 jsonrpc-http-server = "14.0.6"
 jsonrpc-pubsub = "14.0.6"
 jsonrpc-ws-server = "14.0.6"
-libc = "0.2.66"
 log = "0.4.8"
-nix = "0.17.0"
+num_cpus = "1.0.0"
 num-traits = "0.2"
 rand = "0.6.5"
 rand_chacha = "0.1.1"
-rayon = "1.2.0"
-regex = "1.3.4"
-serde = "1.0.104"
+rayon = "1.3.0"
+regex = "1.3.6"
+serde = "1.0.105"
 serde_derive = "1.0.103"
-serde_json = "1.0.46"
-solana-budget-program = { path = "../programs/budget", version = "1.0.6" }
-solana-clap-utils = { path = "../clap-utils", version = "1.0.6" }
-solana-client = { path = "../client", version = "1.0.6" }
-solana-faucet = { path = "../faucet", version = "1.0.6" }
+serde_json = "1.0.48"
+solana-budget-program = { path = "../programs/budget", version = "1.1.5" }
+solana-clap-utils = { path = "../clap-utils", version = "1.1.5" }
+solana-client = { path = "../client", version = "1.1.5" }
+solana-transaction-status = { path = "../transaction-status", version = "1.1.5" }
+solana-faucet = { path = "../faucet", version = "1.1.5" }
 ed25519-dalek = "=1.0.0-pre.1"
-solana-ledger = { path = "../ledger", version = "1.0.6" }
-solana-logger = { path = "../logger", version = "1.0.6" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.0.6" }
-solana-metrics = { path = "../metrics", version = "1.0.6" }
-solana-measure = { path = "../measure", version = "1.0.6" }
-solana-net-utils = { path = "../net-utils", version = "1.0.6" }
-solana-chacha-cuda = { path = "../chacha-cuda", version = "1.0.6" }
-solana-perf = { path = "../perf", version = "1.0.6" }
-solana-runtime = { path = "../runtime", version = "1.0.6" }
-solana-sdk = { path = "../sdk", version = "1.0.6" }
-solana-stake-program = { path = "../programs/stake", version = "1.0.6" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.6" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.6" }
-solana-vote-signer = { path = "../vote-signer", version = "1.0.6" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.0.6" }
-sys-info = "0.5.9"
+solana-ledger = { path = "../ledger", version = "1.1.5" }
+solana-logger = { path = "../logger", version = "1.1.5" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.1.5" }
+solana-metrics = { path = "../metrics", version = "1.1.5" }
+solana-measure = { path = "../measure", version = "1.1.5" }
+solana-net-utils = { path = "../net-utils", version = "1.1.5" }
+solana-chacha-cuda = { path = "../chacha-cuda", version = "1.1.5" }
+solana-perf = { path = "../perf", version = "1.1.5" }
+solana-runtime = { path = "../runtime", version = "1.1.5" }
+solana-sdk = { path = "../sdk", version = "1.1.5" }
+solana-stake-program = { path = "../programs/stake", version = "1.1.5" }
+solana-storage-program = { path = "../programs/storage", version = "1.1.5" }
+solana-streamer = { path = "../streamer", version = "1.1.5" }
+solana-vote-program = { path = "../programs/vote", version = "1.1.5" }
+solana-vote-signer = { path = "../vote-signer", version = "1.1.5" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.1.5" }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = "0.1"
 tokio-codec = "0.1"
 tokio-fs = "0.1"
 tokio-io = "0.1"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.0.6" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.1.5" }
 trees = "0.2.1"
 
 [dev-dependencies]
 matches = "0.1.6"
-reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
-serial_test = "0.3.2"
+reqwest = { version = "0.10.4", default-features = false, features = ["blocking", "rustls-tls", "json"] }
+serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
 systemstat = "0.1.5"
 
@@ -102,3 +103,6 @@ name = "cluster_info"
 [[bench]]
 name = "chacha"
 required-features = ["chacha"]
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

core/benches/banking_stage.rs

@@ -9,12 +9,12 @@ use rayon::prelude::*;
 use solana_core::banking_stage::{create_test_recorder, BankingStage};
 use solana_core::cluster_info::ClusterInfo;
 use solana_core::cluster_info::Node;
-use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-use solana_core::packet::to_packets_chunked;
 use solana_core::poh_recorder::WorkingBankEntry;
 use solana_ledger::blockstore_processor::process_entries;
 use solana_ledger::entry::{next_hash, Entry};
+use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
 use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
+use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_runtime::bank::Bank;
 use solana_sdk::genesis_config::GenesisConfig;

core/benches/cluster_info.rs

@@ -3,13 +3,18 @@
 extern crate test;
 
 use rand::{thread_rng, Rng};
+use solana_core::broadcast_stage::{broadcast_shreds, get_broadcast_peers};
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::contact_info::ContactInfo;
+use solana_ledger::shred::Shred;
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::timing::timestamp;
-use std::collections::HashMap;
-use std::net::UdpSocket;
-use std::sync::Arc;
+use std::sync::RwLock;
+use std::{
+    collections::HashMap,
+    net::UdpSocket,
+    sync::{atomic::AtomicU64, Arc},
+};
 use test::Bencher;
 
 #[bench]
@@ -20,10 +25,8 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
     let mut cluster_info = ClusterInfo::new_with_invalid_keypair(leader_info.info.clone());
     let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
 
-    const SHRED_SIZE: usize = 1024;
     const NUM_SHREDS: usize = 32;
-    let shreds = vec![vec![0; SHRED_SIZE]; NUM_SHREDS];
-    let seeds = vec![[0u8; 32]; NUM_SHREDS];
+    let shreds = vec![Shred::new_empty_data_shred(); NUM_SHREDS];
     let mut stakes = HashMap::new();
     const NUM_PEERS: usize = 200;
     for _ in 0..NUM_PEERS {
@@ -33,10 +36,20 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
         stakes.insert(id, thread_rng().gen_range(1, NUM_PEERS) as u64);
     }
     let stakes = Arc::new(stakes);
+    let cluster_info = Arc::new(RwLock::new(cluster_info));
+    let (peers, peers_and_stakes) = get_broadcast_peers(&cluster_info, Some(stakes.clone()));
+    let shreds = Arc::new(shreds);
+    let last_datapoint = Arc::new(AtomicU64::new(0));
     bencher.iter(move || {
         let shreds = shreds.clone();
-        cluster_info
-            .broadcast_shreds(&socket, shreds, &seeds, Some(stakes.clone()))
-            .unwrap();
+        broadcast_shreds(
+            &socket,
+            &shreds,
+            &peers_and_stakes,
+            &peers,
+            &last_datapoint,
+            &mut 0,
+        )
+        .unwrap();
     });
 }

core/benches/retransmit_stage.rs

@@ -6,12 +6,12 @@ extern crate test;
 use log::*;
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::contact_info::ContactInfo;
-use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-use solana_core::packet::to_packets_chunked;
 use solana_core::retransmit_stage::retransmitter;
 use solana_ledger::bank_forks::BankForks;
+use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
 use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
 use solana_measure::measure::Measure;
+use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_runtime::bank::Bank;
 use solana_sdk::pubkey::Pubkey;

core/benches/sigverify_stage.rs

@@ -6,9 +6,9 @@ extern crate test;
 use crossbeam_channel::unbounded;
 use log::*;
 use rand::{thread_rng, Rng};
-use solana_core::packet::to_packets_chunked;
 use solana_core::sigverify::TransactionSigVerifier;
 use solana_core::sigverify_stage::SigVerifyStage;
+use solana_perf::packet::to_packets_chunked;
 use solana_perf::test_tx::test_tx;
 use solana_sdk::hash::Hash;
 use solana_sdk::signature::{Keypair, Signer};

core/src/accounts_background_service.rs

@@ -0,0 +1,45 @@
+// Service to clean up dead slots in accounts_db
+//
+// This can be expensive since we have to walk the append vecs being cleaned up.
+
+use solana_ledger::bank_forks::BankForks;
+use std::sync::{
+    atomic::{AtomicBool, Ordering},
+    Arc, RwLock,
+};
+use std::thread::{self, sleep, Builder, JoinHandle};
+use std::time::Duration;
+
+pub struct AccountsBackgroundService {
+    t_background: JoinHandle<()>,
+}
+
+const INTERVAL_MS: u64 = 100;
+
+impl AccountsBackgroundService {
+    pub fn new(bank_forks: Arc<RwLock<BankForks>>, exit: &Arc<AtomicBool>) -> Self {
+        info!("AccountsBackgroundService active");
+        let exit = exit.clone();
+        let t_background = Builder::new()
+            .name("solana-accounts-background".to_string())
+            .spawn(move || loop {
+                if exit.load(Ordering::Relaxed) {
+                    break;
+                }
+                let bank = bank_forks.read().unwrap().working_bank();
+
+                bank.process_dead_slots();
+
+                // Currently, given INTERVAL_MS, we process 1 slot/100 ms
+                bank.process_stale_slot();
+
+                sleep(Duration::from_millis(INTERVAL_MS));
+            })
+            .unwrap();
+        Self { t_background }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.t_background.join()
+    }
+}

core/src/accounts_hash_verifier.rs

@@ -0,0 +1,263 @@
+// Service to verify accounts hashes with other trusted validator nodes.
+//
+// Each interval, publish the snapshat hash which is the full accounts state
+// hash on gossip. Monitor gossip for messages from validators in the --trusted-validators
+// set and halt the node if a mismatch is detected.
+
+use crate::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES};
+use solana_ledger::{
+    snapshot_package::SnapshotPackage, snapshot_package::SnapshotPackageReceiver,
+    snapshot_package::SnapshotPackageSender,
+};
+use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
+use std::collections::{HashMap, HashSet};
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        mpsc::RecvTimeoutError,
+        Arc, RwLock,
+    },
+    thread::{self, Builder, JoinHandle},
+    time::Duration,
+};
+
+pub struct AccountsHashVerifier {
+    t_accounts_hash_verifier: JoinHandle<()>,
+}
+
+impl AccountsHashVerifier {
+    pub fn new(
+        snapshot_package_receiver: SnapshotPackageReceiver,
+        snapshot_package_sender: Option<SnapshotPackageSender>,
+        exit: &Arc<AtomicBool>,
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        trusted_validators: Option<HashSet<Pubkey>>,
+        halt_on_trusted_validators_accounts_hash_mismatch: bool,
+        fault_injection_rate_slots: u64,
+    ) -> Self {
+        let exit = exit.clone();
+        let cluster_info = cluster_info.clone();
+        let t_accounts_hash_verifier = Builder::new()
+            .name("solana-accounts-hash".to_string())
+            .spawn(move || {
+                let mut hashes = vec![];
+                loop {
+                    if exit.load(Ordering::Relaxed) {
+                        break;
+                    }
+
+                    match snapshot_package_receiver.recv_timeout(Duration::from_secs(1)) {
+                        Ok(snapshot_package) => {
+                            Self::process_snapshot(
+                                snapshot_package,
+                                &cluster_info,
+                                &trusted_validators,
+                                halt_on_trusted_validators_accounts_hash_mismatch,
+                                &snapshot_package_sender,
+                                &mut hashes,
+                                &exit,
+                                fault_injection_rate_slots,
+                            );
+                        }
+                        Err(RecvTimeoutError::Disconnected) => break,
+                        Err(RecvTimeoutError::Timeout) => (),
+                    }
+                }
+            })
+            .unwrap();
+        Self {
+            t_accounts_hash_verifier,
+        }
+    }
+
+    fn process_snapshot(
+        snapshot_package: SnapshotPackage,
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        trusted_validators: &Option<HashSet<Pubkey>>,
+        halt_on_trusted_validator_accounts_hash_mismatch: bool,
+        snapshot_package_sender: &Option<SnapshotPackageSender>,
+        hashes: &mut Vec<(Slot, Hash)>,
+        exit: &Arc<AtomicBool>,
+        fault_injection_rate_slots: u64,
+    ) {
+        if fault_injection_rate_slots != 0
+            && snapshot_package.root % fault_injection_rate_slots == 0
+        {
+            // For testing, publish an invalid hash to gossip.
+            use rand::{thread_rng, Rng};
+            use solana_sdk::hash::extend_and_hash;
+            warn!("inserting fault at slot: {}", snapshot_package.root);
+            let rand = thread_rng().gen_range(0, 10);
+            let hash = extend_and_hash(&snapshot_package.hash, &[rand]);
+            hashes.push((snapshot_package.root, hash));
+        } else {
+            hashes.push((snapshot_package.root, snapshot_package.hash));
+        }
+
+        while hashes.len() > MAX_SNAPSHOT_HASHES {
+            hashes.remove(0);
+        }
+
+        if halt_on_trusted_validator_accounts_hash_mismatch {
+            let mut slot_to_hash = HashMap::new();
+            for (slot, hash) in hashes.iter() {
+                slot_to_hash.insert(*slot, *hash);
+            }
+            if Self::should_halt(&cluster_info, trusted_validators, &mut slot_to_hash) {
+                exit.store(true, Ordering::Relaxed);
+            }
+        }
+        if let Some(sender) = snapshot_package_sender.as_ref() {
+            if sender.send(snapshot_package).is_err() {}
+        }
+
+        cluster_info
+            .write()
+            .unwrap()
+            .push_accounts_hashes(hashes.clone());
+    }
+
+    fn should_halt(
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        trusted_validators: &Option<HashSet<Pubkey>>,
+        slot_to_hash: &mut HashMap<Slot, Hash>,
+    ) -> bool {
+        let mut verified_count = 0;
+        let mut highest_slot = 0;
+        if let Some(trusted_validators) = trusted_validators.as_ref() {
+            for trusted_validator in trusted_validators {
+                let cluster_info_r = cluster_info.read().unwrap();
+                if let Some(accounts_hashes) =
+                    cluster_info_r.get_accounts_hash_for_node(trusted_validator)
+                {
+                    for (slot, hash) in accounts_hashes {
+                        if let Some(reference_hash) = slot_to_hash.get(slot) {
+                            if *hash != *reference_hash {
+                                error!("Trusted validator {} produced conflicting hashes for slot: {} ({} != {})",
+                                    trusted_validator,
+                                    slot,
+                                    hash,
+                                    reference_hash,
+                                );
+
+                                return true;
+                            } else {
+                                verified_count += 1;
+                            }
+                        } else {
+                            highest_slot = std::cmp::max(*slot, highest_slot);
+                            slot_to_hash.insert(*slot, *hash);
+                        }
+                    }
+                }
+            }
+        }
+        inc_new_counter_info!("accounts_hash_verifier-hashes_verified", verified_count);
+        datapoint_info!(
+            "accounts_hash_verifier",
+            ("highest_slot_verified", highest_slot, i64),
+        );
+        false
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.t_accounts_hash_verifier.join()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::cluster_info::make_accounts_hashes_message;
+    use crate::contact_info::ContactInfo;
+    use solana_sdk::{
+        hash::hash,
+        signature::{Keypair, Signer},
+    };
+
+    #[test]
+    fn test_should_halt() {
+        let keypair = Keypair::new();
+
+        let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0);
+        let cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
+        let cluster_info = Arc::new(RwLock::new(cluster_info));
+
+        let mut trusted_validators = HashSet::new();
+        let mut slot_to_hash = HashMap::new();
+        assert!(!AccountsHashVerifier::should_halt(
+            &cluster_info,
+            &Some(trusted_validators.clone()),
+            &mut slot_to_hash,
+        ));
+
+        let validator1 = Keypair::new();
+        let hash1 = hash(&[1]);
+        let hash2 = hash(&[2]);
+        {
+            let message = make_accounts_hashes_message(&validator1, vec![(0, hash1)]).unwrap();
+            let mut cluster_info_w = cluster_info.write().unwrap();
+            cluster_info_w.push_message(message);
+        }
+        slot_to_hash.insert(0, hash2);
+        trusted_validators.insert(validator1.pubkey());
+        assert!(AccountsHashVerifier::should_halt(
+            &cluster_info,
+            &Some(trusted_validators.clone()),
+            &mut slot_to_hash,
+        ));
+    }
+
+    #[test]
+    fn test_max_hashes() {
+        solana_logger::setup();
+        use std::path::PathBuf;
+        use tempfile::TempDir;
+        let keypair = Keypair::new();
+
+        let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0);
+        let cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
+        let cluster_info = Arc::new(RwLock::new(cluster_info));
+
+        let trusted_validators = HashSet::new();
+        let exit = Arc::new(AtomicBool::new(false));
+        let mut hashes = vec![];
+        for i in 0..MAX_SNAPSHOT_HASHES + 1 {
+            let snapshot_links = TempDir::new().unwrap();
+            let snapshot_package = SnapshotPackage {
+                hash: hash(&[i as u8]),
+                root: 100 + i as u64,
+                slot_deltas: vec![],
+                snapshot_links,
+                tar_output_file: PathBuf::from("."),
+                storages: vec![],
+            };
+
+            AccountsHashVerifier::process_snapshot(
+                snapshot_package,
+                &cluster_info,
+                &Some(trusted_validators.clone()),
+                false,
+                &None,
+                &mut hashes,
+                &exit,
+                0,
+            );
+        }
+        let cluster_info_r = cluster_info.read().unwrap();
+        let cluster_hashes = cluster_info_r
+            .get_accounts_hash_for_node(&keypair.pubkey())
+            .unwrap();
+        info!("{:?}", cluster_hashes);
+        assert_eq!(hashes.len(), MAX_SNAPSHOT_HASHES);
+        assert_eq!(cluster_hashes.len(), MAX_SNAPSHOT_HASHES);
+        assert_eq!(cluster_hashes[0], (101, hash(&[1])));
+        assert_eq!(
+            cluster_hashes[MAX_SNAPSHOT_HASHES - 1],
+            (
+                100 + MAX_SNAPSHOT_HASHES as u64,
+                hash(&[MAX_SNAPSHOT_HASHES as u8])
+            )
+        );
+    }
+}

core/src/banking_stage.rs

@@ -3,7 +3,6 @@
 //! can do its processing in parallel with signature verification on the GPU.
 use crate::{
     cluster_info::ClusterInfo,
-    packet::{limited_deserialize, Packet, Packets, PACKETS_PER_BATCH},
     poh_recorder::{PohRecorder, PohRecorderError, WorkingBankEntry},
     poh_service::PohService,
 };
@@ -17,7 +16,11 @@ use solana_ledger::{
 };
 use solana_measure::{measure::Measure, thread_mem_usage};
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_info, inc_new_counter_warn};
-use solana_perf::{cuda_runtime::PinnedVec, perf_libs};
+use solana_perf::{
+    cuda_runtime::PinnedVec,
+    packet::{limited_deserialize, Packet, Packets, PACKETS_PER_BATCH},
+    perf_libs,
+};
 use solana_runtime::{
     accounts_db::ErrorCounters,
     bank::{Bank, TransactionBalancesSet, TransactionProcessResult},
@@ -1009,20 +1012,18 @@ pub fn create_test_recorder(
 mod tests {
     use super::*;
     use crate::{
-        cluster_info::Node,
-        genesis_utils::{create_genesis_config, GenesisConfigInfo},
-        packet::to_packets,
-        poh_recorder::WorkingBank,
+        cluster_info::Node, poh_recorder::WorkingBank,
         transaction_status_service::TransactionStatusService,
     };
     use crossbeam_channel::unbounded;
     use itertools::Itertools;
-    use solana_client::rpc_response::{RpcEncodedTransaction, RpcTransactionWithStatusMeta};
     use solana_ledger::{
         blockstore::entries_to_test_shreds,
         entry::{next_entry, Entry, EntrySlice},
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
         get_tmp_ledger_path,
     };
+    use solana_perf::packet::to_packets;
     use solana_runtime::bank::HashAgeKind;
     use solana_sdk::{
         instruction::InstructionError,
@@ -1030,6 +1031,7 @@ mod tests {
         system_transaction,
         transaction::TransactionError,
     };
+    use solana_transaction_status::{EncodedTransaction, TransactionWithStatusMeta};
     use std::{sync::atomic::Ordering, thread::sleep};
 
     #[test]
@@ -1975,15 +1977,25 @@ mod tests {
             let confirmed_block = blockstore.get_confirmed_block(bank.slot(), None).unwrap();
             assert_eq!(confirmed_block.transactions.len(), 3);
 
-            for RpcTransactionWithStatusMeta { transaction, meta } in
+            for TransactionWithStatusMeta { transaction, meta } in
                 confirmed_block.transactions.into_iter()
             {
-                if let RpcEncodedTransaction::Json(transaction) = transaction {
+                if let EncodedTransaction::Json(transaction) = transaction {
                     if transaction.signatures[0] == success_signature.to_string() {
-                        assert_eq!(meta.unwrap().status, Ok(()));
+                        let meta = meta.unwrap();
+                        assert_eq!(meta.err, None);
+                        assert_eq!(meta.status, Ok(()));
                     } else if transaction.signatures[0] == ix_error_signature.to_string() {
+                        let meta = meta.unwrap();
                         assert_eq!(
-                            meta.unwrap().status,
+                            meta.err,
+                            Some(TransactionError::InstructionError(
+                                0,
+                                InstructionError::CustomError(1)
+                            ))
+                        );
+                        assert_eq!(
+                            meta.status,
                             Err(TransactionError::InstructionError(
                                 0,
                                 InstructionError::CustomError(1)

core/src/blockstream.rs

@@ -1,283 +0,0 @@
-//! The `blockstream` module provides a method for streaming entries out via a
-//! local unix socket, to provide client services such as a block explorer with
-//! real-time access to entries.
-
-use bincode::serialize;
-use chrono::{SecondsFormat, Utc};
-use serde_json::json;
-use solana_ledger::entry::Entry;
-use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
-use std::cell::RefCell;
-use std::io::Result;
-use std::path::{Path, PathBuf};
-
-pub trait EntryWriter: std::fmt::Debug {
-    fn write(&self, payload: String) -> Result<()>;
-}
-
-#[derive(Debug, Default)]
-pub struct EntryVec {
-    values: RefCell<Vec<String>>,
-}
-
-impl EntryWriter for EntryVec {
-    fn write(&self, payload: String) -> Result<()> {
-        self.values.borrow_mut().push(payload);
-        Ok(())
-    }
-}
-
-impl EntryVec {
-    pub fn new() -> Self {
-        EntryVec {
-            values: RefCell::new(Vec::new()),
-        }
-    }
-
-    pub fn entries(&self) -> Vec<String> {
-        self.values.borrow().clone()
-    }
-}
-
-#[derive(Debug)]
-pub struct EntrySocket {
-    unix_socket: PathBuf,
-}
-
-impl EntryWriter for EntrySocket {
-    #[cfg(not(windows))]
-    fn write(&self, payload: String) -> Result<()> {
-        use std::io::prelude::*;
-        use std::net::Shutdown;
-        use std::os::unix::net::UnixStream;
-
-        const MESSAGE_TERMINATOR: &str = "\n";
-
-        let mut socket = UnixStream::connect(&self.unix_socket)?;
-        socket.write_all(payload.as_bytes())?;
-        socket.write_all(MESSAGE_TERMINATOR.as_bytes())?;
-        socket.shutdown(Shutdown::Write)?;
-        Ok(())
-    }
-    #[cfg(windows)]
-    fn write(&self, _payload: String) -> Result<()> {
-        Err(std::io::Error::new(
-            std::io::ErrorKind::Other,
-            "EntryWriter::write() not implemented for windows",
-        ))
-    }
-}
-
-pub trait BlockstreamEvents {
-    fn emit_entry_event(
-        &self,
-        slot: Slot,
-        tick_height: u64,
-        leader_pubkey: &Pubkey,
-        entries: &Entry,
-    ) -> Result<()>;
-    fn emit_block_event(
-        &self,
-        slot: Slot,
-        tick_height: u64,
-        leader_pubkey: &Pubkey,
-        blockhash: Hash,
-    ) -> Result<()>;
-}
-
-#[derive(Debug)]
-pub struct Blockstream<T: EntryWriter> {
-    pub output: T,
-}
-
-impl<T> BlockstreamEvents for Blockstream<T>
-where
-    T: EntryWriter,
-{
-    fn emit_entry_event(
-        &self,
-        slot: Slot,
-        tick_height: u64,
-        leader_pubkey: &Pubkey,
-        entry: &Entry,
-    ) -> Result<()> {
-        let transactions: Vec<Vec<u8>> = serialize_transactions(entry);
-        let stream_entry = json!({
-            "num_hashes": entry.num_hashes,
-            "hash": entry.hash,
-            "transactions": transactions
-        });
-        let json_entry = serde_json::to_string(&stream_entry)?;
-        let payload = format!(
-            r#"{{"dt":"{}","t":"entry","s":{},"h":{},"l":"{:?}","entry":{}}}"#,
-            Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
-            slot,
-            tick_height,
-            leader_pubkey,
-            json_entry,
-        );
-        self.output.write(payload)?;
-        Ok(())
-    }
-
-    fn emit_block_event(
-        &self,
-        slot: Slot,
-        tick_height: u64,
-        leader_pubkey: &Pubkey,
-        blockhash: Hash,
-    ) -> Result<()> {
-        let payload = format!(
-            r#"{{"dt":"{}","t":"block","s":{},"h":{},"l":"{:?}","hash":"{:?}"}}"#,
-            Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true),
-            slot,
-            tick_height,
-            leader_pubkey,
-            blockhash,
-        );
-        self.output.write(payload)?;
-        Ok(())
-    }
-}
-
-pub type SocketBlockstream = Blockstream<EntrySocket>;
-
-impl SocketBlockstream {
-    pub fn new(unix_socket: &Path) -> Self {
-        Blockstream {
-            output: EntrySocket {
-                unix_socket: unix_socket.to_path_buf(),
-            },
-        }
-    }
-}
-
-pub type MockBlockstream = Blockstream<EntryVec>;
-
-impl MockBlockstream {
-    pub fn new(_: &Path) -> Self {
-        Blockstream {
-            output: EntryVec::new(),
-        }
-    }
-
-    pub fn entries(&self) -> Vec<String> {
-        self.output.entries()
-    }
-}
-
-fn serialize_transactions(entry: &Entry) -> Vec<Vec<u8>> {
-    entry
-        .transactions
-        .iter()
-        .map(|tx| serialize(&tx).unwrap())
-        .collect()
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use chrono::{DateTime, FixedOffset};
-    use serde_json::Value;
-    use solana_sdk::hash::Hash;
-    use solana_sdk::signature::{Keypair, Signer};
-    use solana_sdk::system_transaction;
-    use std::collections::HashSet;
-    use std::path::PathBuf;
-
-    #[test]
-    fn test_serialize_transactions() {
-        let entry = Entry::new(&Hash::default(), 1, vec![]);
-        let empty_vec: Vec<Vec<u8>> = vec![];
-        assert_eq!(serialize_transactions(&entry), empty_vec);
-
-        let keypair0 = Keypair::new();
-        let keypair1 = Keypair::new();
-        let tx0 = system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default());
-        let tx1 = system_transaction::transfer(&keypair1, &keypair0.pubkey(), 2, Hash::default());
-        let serialized_tx0 = serialize(&tx0).unwrap();
-        let serialized_tx1 = serialize(&tx1).unwrap();
-        let entry = Entry::new(&Hash::default(), 1, vec![tx0, tx1]);
-        assert_eq!(
-            serialize_transactions(&entry),
-            vec![serialized_tx0, serialized_tx1]
-        );
-    }
-
-    #[test]
-    fn test_blockstream() -> () {
-        let blockstream = MockBlockstream::new(&PathBuf::from("test_stream"));
-        let ticks_per_slot = 5;
-
-        let mut blockhash = Hash::default();
-        let mut entries = Vec::new();
-        let mut expected_entries = Vec::new();
-
-        let tick_height_initial = 1;
-        let tick_height_final = tick_height_initial + ticks_per_slot + 2;
-        let mut curr_slot = 0;
-        let leader_pubkey = Pubkey::new_rand();
-
-        for tick_height in tick_height_initial..=tick_height_final {
-            if tick_height == 5 {
-                blockstream
-                    .emit_block_event(curr_slot, tick_height, &leader_pubkey, blockhash)
-                    .unwrap();
-                curr_slot += 1;
-            }
-            let entry = Entry::new(&mut blockhash, 1, vec![]); // just ticks
-            blockhash = entry.hash;
-            blockstream
-                .emit_entry_event(curr_slot, tick_height, &leader_pubkey, &entry)
-                .unwrap();
-            expected_entries.push(entry.clone());
-            entries.push(entry);
-        }
-
-        assert_eq!(
-            blockstream.entries().len() as u64,
-            // one entry per tick (1..=N+2) is +3, plus one block
-            ticks_per_slot + 3 + 1
-        );
-
-        let mut j = 0;
-        let mut matched_entries = 0;
-        let mut matched_slots = HashSet::new();
-        let mut matched_blocks = HashSet::new();
-
-        for item in blockstream.entries() {
-            let json: Value = serde_json::from_str(&item).unwrap();
-            let dt_str = json["dt"].as_str().unwrap();
-
-            // Ensure `ts` field parses as valid DateTime
-            let _dt: DateTime<FixedOffset> = DateTime::parse_from_rfc3339(dt_str).unwrap();
-
-            let item_type = json["t"].as_str().unwrap();
-            match item_type {
-                "block" => {
-                    let hash = json["hash"].to_string();
-                    matched_blocks.insert(hash);
-                }
-
-                "entry" => {
-                    let slot = json["s"].as_u64().unwrap();
-                    matched_slots.insert(slot);
-                    let entry_obj = json["entry"].clone();
-                    let entry: Entry = serde_json::from_value(entry_obj).unwrap();
-
-                    assert_eq!(entry, expected_entries[j]);
-                    matched_entries += 1;
-                    j += 1;
-                }
-
-                _ => {
-                    assert!(false, "unknown item type {}", item);
-                }
-            }
-        }
-
-        assert_eq!(matched_entries, expected_entries.len());
-        assert_eq!(matched_slots.len(), 2);
-        assert_eq!(matched_blocks.len(), 1);
-    }
-}

core/src/blockstream_service.rs

@@ -1,228 +0,0 @@
-//! The `blockstream_service` implements optional streaming of entries and block metadata
-//! using the `blockstream` module, providing client services such as a block explorer with
-//! real-time access to entries.
-
-use crate::blockstream::BlockstreamEvents;
-#[cfg(test)]
-use crate::blockstream::MockBlockstream as Blockstream;
-#[cfg(not(test))]
-use crate::blockstream::SocketBlockstream as Blockstream;
-use crate::result::{Error, Result};
-use solana_ledger::blockstore::Blockstore;
-use solana_sdk::pubkey::Pubkey;
-use std::path::Path;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::mpsc::{Receiver, RecvTimeoutError};
-use std::sync::Arc;
-use std::thread::{self, Builder, JoinHandle};
-use std::time::Duration;
-
-pub struct BlockstreamService {
-    t_blockstream: JoinHandle<()>,
-}
-
-impl BlockstreamService {
-    #[allow(clippy::new_ret_no_self)]
-    pub fn new(
-        slot_full_receiver: Receiver<(u64, Pubkey)>,
-        blockstore: Arc<Blockstore>,
-        unix_socket: &Path,
-        exit: &Arc<AtomicBool>,
-    ) -> Self {
-        let mut blockstream = Blockstream::new(unix_socket);
-        let exit = exit.clone();
-        let t_blockstream = Builder::new()
-            .name("solana-blockstream".to_string())
-            .spawn(move || loop {
-                if exit.load(Ordering::Relaxed) {
-                    break;
-                }
-                if let Err(e) =
-                    Self::process_entries(&slot_full_receiver, &blockstore, &mut blockstream)
-                {
-                    match e {
-                        Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
-                        Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
-                        _ => info!("Error from process_entries: {:?}", e),
-                    }
-                }
-            })
-            .unwrap();
-        Self { t_blockstream }
-    }
-    fn process_entries(
-        slot_full_receiver: &Receiver<(u64, Pubkey)>,
-        blockstore: &Arc<Blockstore>,
-        blockstream: &mut Blockstream,
-    ) -> Result<()> {
-        let timeout = Duration::new(1, 0);
-        let (slot, slot_leader) = slot_full_receiver.recv_timeout(timeout)?;
-
-        // Slot might not exist due to LedgerCleanupService, check first
-        let blockstore_meta = blockstore.meta(slot).unwrap();
-        if let Some(blockstore_meta) = blockstore_meta {
-            // Return error to main loop. Thread won't exit, will just log the error
-            let entries = blockstore.get_slot_entries(slot, 0, None)?;
-            let _parent_slot = if slot == 0 {
-                None
-            } else {
-                Some(blockstore_meta.parent_slot)
-            };
-            let ticks_per_slot = entries.iter().filter(|entry| entry.is_tick()).count() as u64;
-            let mut tick_height = ticks_per_slot * slot;
-
-            for (i, entry) in entries.iter().enumerate() {
-                if entry.is_tick() {
-                    tick_height += 1;
-                }
-                blockstream
-                    .emit_entry_event(slot, tick_height, &slot_leader, &entry)
-                    .unwrap_or_else(|e| {
-                        debug!("Blockstream error: {:?}, {:?}", e, blockstream.output);
-                    });
-                if i == entries.len() - 1 {
-                    blockstream
-                        .emit_block_event(slot, tick_height, &slot_leader, entry.hash)
-                        .unwrap_or_else(|e| {
-                            debug!("Blockstream error: {:?}, {:?}", e, blockstream.output);
-                        });
-                }
-            }
-        }
-        Ok(())
-    }
-
-    pub fn join(self) -> thread::Result<()> {
-        self.t_blockstream.join()
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-    use bincode::{deserialize, serialize};
-    use chrono::{DateTime, FixedOffset};
-    use serde_json::Value;
-    use solana_ledger::create_new_tmp_ledger;
-    use solana_ledger::entry::{create_ticks, Entry};
-    use solana_sdk::hash::Hash;
-    use solana_sdk::signature::{Keypair, Signer};
-    use solana_sdk::system_transaction;
-    use std::path::PathBuf;
-    use std::sync::mpsc::channel;
-
-    #[test]
-    fn test_blockstream_service_process_entries() {
-        let ticks_per_slot = 5;
-        let leader_pubkey = Pubkey::new_rand();
-
-        // Set up genesis config and blockstore
-        let GenesisConfigInfo {
-            mut genesis_config, ..
-        } = create_genesis_config(1000);
-        genesis_config.ticks_per_slot = ticks_per_slot;
-
-        let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
-        let blockstore = Blockstore::open(&ledger_path).unwrap();
-
-        // Set up blockstream
-        let mut blockstream = Blockstream::new(&PathBuf::from("test_stream"));
-
-        // Set up dummy channel to receive a full-slot notification
-        let (slot_full_sender, slot_full_receiver) = channel();
-
-        // Create entries - 4 ticks + 1 populated entry + 1 tick
-        let mut entries = create_ticks(4, 0, Hash::default());
-
-        let keypair = Keypair::new();
-        let mut blockhash = entries[3].hash;
-        let tx = system_transaction::transfer(&keypair, &keypair.pubkey(), 1, Hash::default());
-        let entry = Entry::new(&mut blockhash, 1, vec![tx]);
-        blockhash = entry.hash;
-        entries.push(entry);
-        let final_tick = create_ticks(1, 0, blockhash);
-        entries.extend_from_slice(&final_tick);
-
-        let expected_entries = entries.clone();
-        let expected_tick_heights = [6, 7, 8, 9, 9, 10];
-
-        blockstore
-            .write_entries(
-                1,
-                0,
-                0,
-                ticks_per_slot,
-                None,
-                true,
-                &Arc::new(Keypair::new()),
-                entries,
-                0,
-            )
-            .unwrap();
-
-        slot_full_sender.send((1, leader_pubkey)).unwrap();
-        BlockstreamService::process_entries(
-            &slot_full_receiver,
-            &Arc::new(blockstore),
-            &mut blockstream,
-        )
-        .unwrap();
-        assert_eq!(blockstream.entries().len(), 7);
-
-        let (entry_events, block_events): (Vec<Value>, Vec<Value>) = blockstream
-            .entries()
-            .iter()
-            .map(|item| {
-                let json: Value = serde_json::from_str(&item).unwrap();
-                let dt_str = json["dt"].as_str().unwrap();
-                // Ensure `ts` field parses as valid DateTime
-                let _dt: DateTime<FixedOffset> = DateTime::parse_from_rfc3339(dt_str).unwrap();
-                json
-            })
-            .partition(|json| {
-                let item_type = json["t"].as_str().unwrap();
-                item_type == "entry"
-            });
-        for (i, json) in entry_events.iter().enumerate() {
-            let height = json["h"].as_u64().unwrap();
-            assert_eq!(height, expected_tick_heights[i]);
-            let entry_obj = json["entry"].clone();
-            let tx = entry_obj["transactions"].as_array().unwrap();
-            let entry: Entry;
-            if tx.len() == 0 {
-                entry = serde_json::from_value(entry_obj).unwrap();
-            } else {
-                let entry_json = entry_obj.as_object().unwrap();
-                entry = Entry {
-                    num_hashes: entry_json.get("num_hashes").unwrap().as_u64().unwrap(),
-                    hash: serde_json::from_value(entry_json.get("hash").unwrap().clone()).unwrap(),
-                    transactions: entry_json
-                        .get("transactions")
-                        .unwrap()
-                        .as_array()
-                        .unwrap()
-                        .into_iter()
-                        .enumerate()
-                        .map(|(j, tx)| {
-                            let tx_vec: Vec<u8> = serde_json::from_value(tx.clone()).unwrap();
-                            // Check explicitly that transaction matches bincode-serialized format
-                            assert_eq!(
-                                tx_vec,
-                                serialize(&expected_entries[i].transactions[j]).unwrap()
-                            );
-                            deserialize(&tx_vec).unwrap()
-                        })
-                        .collect(),
-                };
-            }
-            assert_eq!(entry, expected_entries[i]);
-        }
-        for json in block_events {
-            let slot = json["s"].as_u64().unwrap();
-            assert_eq!(1, slot);
-            let height = json["h"].as_u64().unwrap();
-            assert_eq!(2 * ticks_per_slot, height);
-        }
-    }
-}

core/src/broadcast_stage.rs

@@ -1,30 +1,51 @@
 //! A stage to broadcast data from a leader node to validators
-use self::broadcast_fake_shreds_run::BroadcastFakeShredsRun;
-use self::fail_entry_verification_broadcast_run::FailEntryVerificationBroadcastRun;
-use self::standard_broadcast_run::StandardBroadcastRun;
-use crate::cluster_info::{ClusterInfo, ClusterInfoError};
-use crate::poh_recorder::WorkingBankEntry;
-use crate::result::{Error, Result};
-use solana_ledger::blockstore::Blockstore;
-use solana_ledger::shred::Shred;
-use solana_ledger::staking_utils;
+use self::{
+    broadcast_fake_shreds_run::BroadcastFakeShredsRun, broadcast_metrics::*,
+    fail_entry_verification_broadcast_run::FailEntryVerificationBroadcastRun,
+    standard_broadcast_run::StandardBroadcastRun,
+};
+use crate::contact_info::ContactInfo;
+use crate::crds_gossip_pull::CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS;
+use crate::weighted_shuffle::weighted_best;
+use crate::{
+    cluster_info::{ClusterInfo, ClusterInfoError},
+    poh_recorder::WorkingBankEntry,
+    result::{Error, Result},
+};
+use crossbeam_channel::{
+    Receiver as CrossbeamReceiver, RecvTimeoutError as CrossbeamRecvTimeoutError,
+    Sender as CrossbeamSender,
+};
+use solana_ledger::{blockstore::Blockstore, shred::Shred, staking_utils};
+use solana_measure::measure::Measure;
 use solana_metrics::{inc_new_counter_error, inc_new_counter_info};
-use solana_sdk::pubkey::Pubkey;
-use std::collections::HashMap;
-use std::net::UdpSocket;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender};
-use std::sync::{Arc, Mutex, RwLock};
-use std::thread::{self, Builder, JoinHandle};
-use std::time::Instant;
-
-pub const NUM_INSERT_THREADS: usize = 2;
+use solana_runtime::bank::Bank;
+use solana_sdk::timing::timestamp;
+use solana_sdk::{clock::Slot, pubkey::Pubkey};
+use solana_streamer::sendmmsg::send_mmsg;
+use std::sync::atomic::AtomicU64;
+use std::{
+    collections::HashMap,
+    net::UdpSocket,
+    sync::atomic::{AtomicBool, Ordering},
+    sync::mpsc::{channel, Receiver, RecvError, RecvTimeoutError, Sender},
+    sync::{Arc, Mutex, RwLock},
+    thread::{self, Builder, JoinHandle},
+    time::{Duration, Instant},
+};
 
 mod broadcast_fake_shreds_run;
+pub(crate) mod broadcast_metrics;
 pub(crate) mod broadcast_utils;
 mod fail_entry_verification_broadcast_run;
 mod standard_broadcast_run;
 
+pub(crate) const NUM_INSERT_THREADS: usize = 2;
+pub(crate) type RetransmitSlotsSender = CrossbeamSender<HashMap<Slot, Arc<Bank>>>;
+pub(crate) type RetransmitSlotsReceiver = CrossbeamReceiver<HashMap<Slot, Arc<Bank>>>;
+pub(crate) type RecordReceiver = Receiver<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>;
+pub(crate) type TransmitReceiver = Receiver<(TransmitShreds, Option<BroadcastShredBatchInfo>)>;
+
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub enum BroadcastStageReturnType {
     ChannelDisconnected,
@@ -43,6 +64,7 @@ impl BroadcastStageType {
         sock: Vec<UdpSocket>,
         cluster_info: Arc<RwLock<ClusterInfo>>,
         receiver: Receiver<WorkingBankEntry>,
+        retransmit_slots_receiver: RetransmitSlotsReceiver,
         exit_sender: &Arc<AtomicBool>,
         blockstore: &Arc<Blockstore>,
         shred_version: u16,
@@ -53,6 +75,7 @@ impl BroadcastStageType {
                 sock,
                 cluster_info,
                 receiver,
+                retransmit_slots_receiver,
                 exit_sender,
                 blockstore,
                 StandardBroadcastRun::new(keypair, shred_version),
@@ -62,6 +85,7 @@ impl BroadcastStageType {
                 sock,
                 cluster_info,
                 receiver,
+                retransmit_slots_receiver,
                 exit_sender,
                 blockstore,
                 FailEntryVerificationBroadcastRun::new(keypair, shred_version),
@@ -71,6 +95,7 @@ impl BroadcastStageType {
                 sock,
                 cluster_info,
                 receiver,
+                retransmit_slots_receiver,
                 exit_sender,
                 blockstore,
                 BroadcastFakeShredsRun::new(keypair, 0, shred_version),
@@ -79,24 +104,24 @@ impl BroadcastStageType {
     }
 }
 
-type TransmitShreds = (Option<Arc<HashMap<Pubkey, u64>>>, Arc<Vec<Shred>>);
+pub type TransmitShreds = (Option<Arc<HashMap<Pubkey, u64>>>, Arc<Vec<Shred>>);
 trait BroadcastRun {
     fn run(
         &mut self,
         blockstore: &Arc<Blockstore>,
         receiver: &Receiver<WorkingBankEntry>,
-        socket_sender: &Sender<TransmitShreds>,
-        blockstore_sender: &Sender<Arc<Vec<Shred>>>,
+        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
     ) -> Result<()>;
     fn transmit(
-        &self,
-        receiver: &Arc<Mutex<Receiver<TransmitShreds>>>,
+        &mut self,
+        receiver: &Arc<Mutex<TransmitReceiver>>,
         cluster_info: &Arc<RwLock<ClusterInfo>>,
         sock: &UdpSocket,
     ) -> Result<()>;
     fn record(
-        &self,
-        receiver: &Arc<Mutex<Receiver<Arc<Vec<Shred>>>>>,
+        &mut self,
+        receiver: &Arc<Mutex<RecordReceiver>>,
         blockstore: &Arc<Blockstore>,
     ) -> Result<()>;
 }
@@ -128,32 +153,34 @@ impl BroadcastStage {
     fn run(
         blockstore: &Arc<Blockstore>,
         receiver: &Receiver<WorkingBankEntry>,
-        socket_sender: &Sender<TransmitShreds>,
-        blockstore_sender: &Sender<Arc<Vec<Shred>>>,
+        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
         mut broadcast_stage_run: impl BroadcastRun,
     ) -> BroadcastStageReturnType {
         loop {
             let res =
                 broadcast_stage_run.run(blockstore, receiver, socket_sender, blockstore_sender);
-            let res = Self::handle_error(res);
+            let res = Self::handle_error(res, "run");
             if let Some(res) = res {
                 return res;
             }
         }
     }
-    fn handle_error(r: Result<()>) -> Option<BroadcastStageReturnType> {
+    fn handle_error(r: Result<()>, name: &str) -> Option<BroadcastStageReturnType> {
         if let Err(e) = r {
             match e {
                 Error::RecvTimeoutError(RecvTimeoutError::Disconnected)
                 | Error::SendError
-                | Error::RecvError(RecvError) => {
+                | Error::RecvError(RecvError)
+                | Error::CrossbeamRecvTimeoutError(CrossbeamRecvTimeoutError::Disconnected) => {
                     return Some(BroadcastStageReturnType::ChannelDisconnected);
                 }
-                Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
+                Error::RecvTimeoutError(RecvTimeoutError::Timeout)
+                | Error::CrossbeamRecvTimeoutError(CrossbeamRecvTimeoutError::Timeout) => (),
                 Error::ClusterInfoError(ClusterInfoError::NoPeers) => (), // TODO: Why are the unit-tests throwing hundreds of these?
                 _ => {
                     inc_new_counter_error!("streamer-broadcaster-error", 1, 1);
-                    error!("broadcaster error: {:?}", e);
+                    error!("{} broadcaster error: {:?}", name, e);
                 }
             }
         }
@@ -180,6 +207,7 @@ impl BroadcastStage {
         socks: Vec<UdpSocket>,
         cluster_info: Arc<RwLock<ClusterInfo>>,
         receiver: Receiver<WorkingBankEntry>,
+        retransmit_slots_receiver: RetransmitSlotsReceiver,
         exit_sender: &Arc<AtomicBool>,
         blockstore: &Arc<Blockstore>,
         broadcast_stage_run: impl BroadcastRun + Send + 'static + Clone,
@@ -189,6 +217,8 @@ impl BroadcastStage {
         let (socket_sender, socket_receiver) = channel();
         let (blockstore_sender, blockstore_receiver) = channel();
         let bs_run = broadcast_stage_run.clone();
+
+        let socket_sender_ = socket_sender.clone();
         let thread_hdl = Builder::new()
             .name("solana-broadcaster".to_string())
             .spawn(move || {
@@ -196,7 +226,7 @@ impl BroadcastStage {
                 Self::run(
                     &btree,
                     &receiver,
-                    &socket_sender,
+                    &socket_sender_,
                     &blockstore_sender,
                     bs_run,
                 )
@@ -206,13 +236,13 @@ impl BroadcastStage {
         let socket_receiver = Arc::new(Mutex::new(socket_receiver));
         for sock in socks.into_iter() {
             let socket_receiver = socket_receiver.clone();
-            let bs_transmit = broadcast_stage_run.clone();
+            let mut bs_transmit = broadcast_stage_run.clone();
             let cluster_info = cluster_info.clone();
             let t = Builder::new()
                 .name("solana-broadcaster-transmit".to_string())
                 .spawn(move || loop {
                     let res = bs_transmit.transmit(&socket_receiver, &cluster_info, &sock);
-                    let res = Self::handle_error(res);
+                    let res = Self::handle_error(res, "solana-broadcaster-transmit");
                     if let Some(res) = res {
                         return res;
                     }
@@ -223,13 +253,13 @@ impl BroadcastStage {
         let blockstore_receiver = Arc::new(Mutex::new(blockstore_receiver));
         for _ in 0..NUM_INSERT_THREADS {
             let blockstore_receiver = blockstore_receiver.clone();
-            let bs_record = broadcast_stage_run.clone();
+            let mut bs_record = broadcast_stage_run.clone();
             let btree = blockstore.clone();
             let t = Builder::new()
                 .name("solana-broadcaster-record".to_string())
                 .spawn(move || loop {
                     let res = bs_record.record(&blockstore_receiver, &btree);
-                    let res = Self::handle_error(res);
+                    let res = Self::handle_error(res, "solana-broadcaster-record");
                     if let Some(res) = res {
                         return res;
                     }
@@ -238,9 +268,68 @@ impl BroadcastStage {
             thread_hdls.push(t);
         }
 
+        let blockstore = blockstore.clone();
+        let retransmit_thread = Builder::new()
+            .name("solana-broadcaster-retransmit".to_string())
+            .spawn(move || loop {
+                if let Some(res) = Self::handle_error(
+                    Self::check_retransmit_signals(
+                        &blockstore,
+                        &retransmit_slots_receiver,
+                        &socket_sender,
+                    ),
+                    "solana-broadcaster-retransmit-check_retransmit_signals",
+                ) {
+                    return res;
+                }
+            })
+            .unwrap();
+
+        thread_hdls.push(retransmit_thread);
         Self { thread_hdls }
     }
 
+    fn check_retransmit_signals(
+        blockstore: &Blockstore,
+        retransmit_slots_receiver: &RetransmitSlotsReceiver,
+        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+    ) -> Result<()> {
+        let timer = Duration::from_millis(100);
+
+        // Check for a retransmit signal
+        let mut retransmit_slots = retransmit_slots_receiver.recv_timeout(timer)?;
+        while let Ok(new_retransmit_slots) = retransmit_slots_receiver.try_recv() {
+            retransmit_slots.extend(new_retransmit_slots);
+        }
+
+        for (_, bank) in retransmit_slots.iter() {
+            let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
+            let stakes = staking_utils::staked_nodes_at_epoch(&bank, bank_epoch);
+            let stakes = stakes.map(Arc::new);
+            let data_shreds = Arc::new(
+                blockstore
+                    .get_data_shreds_for_slot(bank.slot(), 0)
+                    .expect("My own shreds must be reconstructable"),
+            );
+
+            if !data_shreds.is_empty() {
+                socket_sender.send(((stakes.clone(), data_shreds), None))?;
+            }
+
+            let coding_shreds = Arc::new(
+                blockstore
+                    .get_coding_shreds_for_slot(bank.slot(), 0)
+                    .expect("My own shreds must be reconstructable"),
+            );
+
+            if !coding_shreds.is_empty() {
+                socket_sender.send(((stakes.clone(), coding_shreds), None))?;
+            }
+        }
+
+        Ok(())
+    }
+
     pub fn join(self) -> thread::Result<BroadcastStageReturnType> {
         for thread_hdl in self.thread_hdls.into_iter() {
             let _ = thread_hdl.join();
@@ -249,23 +338,223 @@ impl BroadcastStage {
     }
 }
 
+fn update_peer_stats(
+    num_live_peers: i64,
+    broadcast_len: i64,
+    last_datapoint_submit: &Arc<AtomicU64>,
+) {
+    let now = timestamp();
+    let last = last_datapoint_submit.load(Ordering::Relaxed);
+    if now - last > 1000
+        && last_datapoint_submit.compare_and_swap(last, now, Ordering::Relaxed) == last
+    {
+        datapoint_info!(
+            "cluster_info-num_nodes",
+            ("live_count", num_live_peers, i64),
+            ("broadcast_count", broadcast_len, i64)
+        );
+    }
+}
+
+pub fn get_broadcast_peers<S: std::hash::BuildHasher>(
+    cluster_info: &Arc<RwLock<ClusterInfo>>,
+    stakes: Option<Arc<HashMap<Pubkey, u64, S>>>,
+) -> (Vec<ContactInfo>, Vec<(u64, usize)>) {
+    use crate::cluster_info;
+    let mut peers = cluster_info.read().unwrap().tvu_peers();
+    let peers_and_stakes = cluster_info::stake_weight_peers(&mut peers, stakes);
+    (peers, peers_and_stakes)
+}
+
+/// broadcast messages from the leader to layer 1 nodes
+/// # Remarks
+pub fn broadcast_shreds(
+    s: &UdpSocket,
+    shreds: &Arc<Vec<Shred>>,
+    peers_and_stakes: &[(u64, usize)],
+    peers: &[ContactInfo],
+    last_datapoint_submit: &Arc<AtomicU64>,
+    send_mmsg_total: &mut u64,
+) -> Result<()> {
+    let broadcast_len = peers_and_stakes.len();
+    if broadcast_len == 0 {
+        update_peer_stats(1, 1, last_datapoint_submit);
+        return Ok(());
+    }
+    let packets: Vec<_> = shreds
+        .iter()
+        .map(|shred| {
+            let broadcast_index = weighted_best(&peers_and_stakes, shred.seed());
+
+            (&shred.payload, &peers[broadcast_index].tvu)
+        })
+        .collect();
+
+    let mut sent = 0;
+    let mut send_mmsg_time = Measure::start("send_mmsg");
+    while sent < packets.len() {
+        match send_mmsg(s, &packets[sent..]) {
+            Ok(n) => sent += n,
+            Err(e) => {
+                return Err(Error::IO(e));
+            }
+        }
+    }
+    send_mmsg_time.stop();
+    *send_mmsg_total += send_mmsg_time.as_us();
+
+    let num_live_peers = num_live_peers(&peers);
+    update_peer_stats(
+        num_live_peers,
+        broadcast_len as i64 + 1,
+        last_datapoint_submit,
+    );
+    Ok(())
+}
+
+fn num_live_peers(peers: &[ContactInfo]) -> i64 {
+    let mut num_live_peers = 1i64;
+    peers.iter().for_each(|p| {
+        // A peer is considered live if they generated their contact info recently
+        if timestamp() - p.wallclock <= CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS {
+            num_live_peers += 1;
+        }
+    });
+    num_live_peers
+}
+
 #[cfg(test)]
-mod test {
+pub mod test {
     use super::*;
     use crate::cluster_info::{ClusterInfo, Node};
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-    use solana_ledger::entry::create_ticks;
-    use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
+    use crossbeam_channel::unbounded;
+    use solana_ledger::{
+        blockstore::{make_slot_entries, Blockstore},
+        entry::create_ticks,
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        get_tmp_ledger_path,
+        shred::{max_ticks_per_n_shreds, Shredder, RECOMMENDED_FEC_RATE},
+    };
     use solana_runtime::bank::Bank;
-    use solana_sdk::hash::Hash;
-    use solana_sdk::pubkey::Pubkey;
-    use solana_sdk::signature::{Keypair, Signer};
-    use std::path::Path;
-    use std::sync::atomic::AtomicBool;
-    use std::sync::mpsc::channel;
-    use std::sync::{Arc, RwLock};
-    use std::thread::sleep;
-    use std::time::Duration;
+    use solana_sdk::{
+        hash::Hash,
+        pubkey::Pubkey,
+        signature::{Keypair, Signer},
+    };
+    use std::{
+        path::Path,
+        sync::atomic::AtomicBool,
+        sync::mpsc::channel,
+        sync::{Arc, RwLock},
+        thread::sleep,
+    };
+
+    pub fn make_transmit_shreds(
+        slot: Slot,
+        num: u64,
+        stakes: Option<Arc<HashMap<Pubkey, u64>>>,
+    ) -> (
+        Vec<Shred>,
+        Vec<Shred>,
+        Vec<TransmitShreds>,
+        Vec<TransmitShreds>,
+    ) {
+        let num_entries = max_ticks_per_n_shreds(num);
+        let (data_shreds, _) = make_slot_entries(slot, 0, num_entries);
+        let keypair = Arc::new(Keypair::new());
+        let shredder = Shredder::new(slot, 0, RECOMMENDED_FEC_RATE, keypair, 0, 0)
+            .expect("Expected to create a new shredder");
+
+        let coding_shreds = shredder.data_shreds_to_coding_shreds(&data_shreds[0..]);
+        (
+            data_shreds.clone(),
+            coding_shreds.clone(),
+            data_shreds
+                .into_iter()
+                .map(|s| (stakes.clone(), Arc::new(vec![s])))
+                .collect(),
+            coding_shreds
+                .into_iter()
+                .map(|s| (stakes.clone(), Arc::new(vec![s])))
+                .collect(),
+        )
+    }
+
+    fn check_all_shreds_received(
+        transmit_receiver: &TransmitReceiver,
+        mut data_index: u64,
+        mut coding_index: u64,
+        num_expected_data_shreds: u64,
+        num_expected_coding_shreds: u64,
+    ) {
+        while let Ok((new_retransmit_slots, _)) = transmit_receiver.try_recv() {
+            if new_retransmit_slots.1[0].is_data() {
+                for data_shred in new_retransmit_slots.1.iter() {
+                    assert_eq!(data_shred.index() as u64, data_index);
+                    data_index += 1;
+                }
+            } else {
+                assert_eq!(new_retransmit_slots.1[0].index() as u64, coding_index);
+                for coding_shred in new_retransmit_slots.1.iter() {
+                    assert_eq!(coding_shred.index() as u64, coding_index);
+                    coding_index += 1;
+                }
+            }
+        }
+
+        assert_eq!(num_expected_data_shreds, data_index);
+        assert_eq!(num_expected_coding_shreds, coding_index);
+    }
+
+    #[test]
+    fn test_duplicate_retransmit_signal() {
+        // Setup
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let (transmit_sender, transmit_receiver) = channel();
+        let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
+        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(100_000);
+        let bank0 = Arc::new(Bank::new(&genesis_config));
+
+        // Make some shreds
+        let updated_slot = 0;
+        let (all_data_shreds, all_coding_shreds, _, _all_coding_transmit_shreds) =
+            make_transmit_shreds(updated_slot, 10, None);
+        let num_data_shreds = all_data_shreds.len();
+        let num_coding_shreds = all_coding_shreds.len();
+        assert!(num_data_shreds >= 10);
+
+        // Insert all the shreds
+        blockstore
+            .insert_shreds(all_data_shreds, None, true)
+            .unwrap();
+        blockstore
+            .insert_shreds(all_coding_shreds, None, true)
+            .unwrap();
+
+        // Insert duplicate retransmit signal, blocks should
+        // only be retransmitted once
+        retransmit_slots_sender
+            .send(vec![(updated_slot, bank0.clone())].into_iter().collect())
+            .unwrap();
+        retransmit_slots_sender
+            .send(vec![(updated_slot, bank0.clone())].into_iter().collect())
+            .unwrap();
+        BroadcastStage::check_retransmit_signals(
+            &blockstore,
+            &retransmit_slots_receiver,
+            &transmit_sender,
+        )
+        .unwrap();
+        // Check all the data shreds were received only once
+        check_all_shreds_received(
+            &transmit_receiver,
+            0,
+            0,
+            num_data_shreds as u64,
+            num_coding_shreds as u64,
+        );
+    }
 
     struct MockBroadcastStage {
         blockstore: Arc<Blockstore>,
@@ -277,6 +566,7 @@ mod test {
         leader_pubkey: &Pubkey,
         ledger_path: &Path,
         entry_receiver: Receiver<WorkingBankEntry>,
+        retransmit_slots_receiver: RetransmitSlotsReceiver,
     ) -> MockBroadcastStage {
         // Make the database ledger
         let blockstore = Arc::new(Blockstore::open(ledger_path).unwrap());
@@ -304,6 +594,7 @@ mod test {
             leader_info.sockets.broadcast,
             cluster_info,
             entry_receiver,
+            retransmit_slots_receiver,
             &exit_sender,
             &blockstore,
             StandardBroadcastRun::new(leader_keypair, 0),
@@ -326,10 +617,12 @@ mod test {
             let leader_keypair = Keypair::new();
 
             let (entry_sender, entry_receiver) = channel();
+            let (retransmit_slots_sender, retransmit_slots_receiver) = unbounded();
             let broadcast_service = setup_dummy_broadcast_service(
                 &leader_keypair.pubkey(),
                 &ledger_path,
                 entry_receiver,
+                retransmit_slots_receiver,
             );
             let start_tick_height;
             let max_tick_height;
@@ -348,6 +641,7 @@ mod test {
                         .expect("Expect successful send to broadcast service");
                 }
             }
+
             sleep(Duration::from_millis(2000));
 
             trace!(
@@ -364,6 +658,7 @@ mod test {
             assert_eq!(entries.len(), max_tick_height as usize);
 
             drop(entry_sender);
+            drop(retransmit_slots_sender);
             broadcast_service
                 .broadcast_service
                 .join()

core/src/broadcast_stage/broadcast_fake_shreds_run.rs

@@ -28,8 +28,8 @@ impl BroadcastRun for BroadcastFakeShredsRun {
         &mut self,
         blockstore: &Arc<Blockstore>,
         receiver: &Receiver<WorkingBankEntry>,
-        socket_sender: &Sender<TransmitShreds>,
-        blockstore_sender: &Sender<Arc<Vec<Shred>>>,
+        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
     ) -> Result<()> {
         // 1) Pull entries from banking stage
         let receive_results = broadcast_utils::recv_slot_entries(receiver)?;
@@ -83,25 +83,31 @@ impl BroadcastRun for BroadcastFakeShredsRun {
         }
 
         let data_shreds = Arc::new(data_shreds);
-        blockstore_sender.send(data_shreds.clone())?;
+        blockstore_sender.send((data_shreds.clone(), None))?;
 
         // 3) Start broadcast step
         //some indicates fake shreds
-        socket_sender.send((Some(Arc::new(HashMap::new())), Arc::new(fake_data_shreds)))?;
-        socket_sender.send((Some(Arc::new(HashMap::new())), Arc::new(fake_coding_shreds)))?;
+        socket_sender.send((
+            (Some(Arc::new(HashMap::new())), Arc::new(fake_data_shreds)),
+            None,
+        ))?;
+        socket_sender.send((
+            (Some(Arc::new(HashMap::new())), Arc::new(fake_coding_shreds)),
+            None,
+        ))?;
         //none indicates real shreds
-        socket_sender.send((None, data_shreds))?;
-        socket_sender.send((None, Arc::new(coding_shreds)))?;
+        socket_sender.send(((None, data_shreds), None))?;
+        socket_sender.send(((None, Arc::new(coding_shreds)), None))?;
 
         Ok(())
     }
     fn transmit(
-        &self,
-        receiver: &Arc<Mutex<Receiver<TransmitShreds>>>,
+        &mut self,
+        receiver: &Arc<Mutex<TransmitReceiver>>,
         cluster_info: &Arc<RwLock<ClusterInfo>>,
         sock: &UdpSocket,
     ) -> Result<()> {
-        for (stakes, data_shreds) in receiver.lock().unwrap().iter() {
+        for ((stakes, data_shreds), _) in receiver.lock().unwrap().iter() {
             let peers = cluster_info.read().unwrap().tvu_peers();
             peers.iter().enumerate().for_each(|(i, peer)| {
                 if i <= self.partition && stakes.is_some() {
@@ -119,11 +125,11 @@ impl BroadcastRun for BroadcastFakeShredsRun {
         Ok(())
     }
     fn record(
-        &self,
-        receiver: &Arc<Mutex<Receiver<Arc<Vec<Shred>>>>>,
+        &mut self,
+        receiver: &Arc<Mutex<RecordReceiver>>,
         blockstore: &Arc<Blockstore>,
     ) -> Result<()> {
-        for data_shreds in receiver.lock().unwrap().iter() {
+        for (data_shreds, _) in receiver.lock().unwrap().iter() {
             blockstore.insert_shreds(data_shreds.to_vec(), None, true)?;
         }
         Ok(())

core/src/broadcast_stage/broadcast_metrics.rs

@@ -0,0 +1,288 @@
+use super::*;
+
+pub(crate) trait BroadcastStats {
+    fn update(&mut self, new_stats: &Self);
+    fn report_stats(&mut self, slot: Slot, slot_start: Instant);
+}
+
+#[derive(Clone)]
+pub(crate) struct BroadcastShredBatchInfo {
+    pub(crate) slot: Slot,
+    pub(crate) num_expected_batches: Option<usize>,
+    pub(crate) slot_start_ts: Instant,
+}
+
+#[derive(Default, Clone)]
+pub(crate) struct ProcessShredsStats {
+    // Per-slot elapsed time
+    pub(crate) shredding_elapsed: u64,
+    pub(crate) receive_elapsed: u64,
+}
+impl ProcessShredsStats {
+    pub(crate) fn update(&mut self, new_stats: &ProcessShredsStats) {
+        self.shredding_elapsed += new_stats.shredding_elapsed;
+        self.receive_elapsed += new_stats.receive_elapsed;
+    }
+    pub(crate) fn reset(&mut self) {
+        *self = Self::default();
+    }
+}
+
+#[derive(Default, Clone)]
+pub(crate) struct TransmitShredsStats {
+    pub(crate) transmit_elapsed: u64,
+    pub(crate) send_mmsg_elapsed: u64,
+    pub(crate) get_peers_elapsed: u64,
+    pub(crate) num_shreds: usize,
+}
+
+impl BroadcastStats for TransmitShredsStats {
+    fn update(&mut self, new_stats: &TransmitShredsStats) {
+        self.transmit_elapsed += new_stats.transmit_elapsed;
+        self.send_mmsg_elapsed += new_stats.send_mmsg_elapsed;
+        self.get_peers_elapsed += new_stats.get_peers_elapsed;
+        self.num_shreds += new_stats.num_shreds;
+    }
+    fn report_stats(&mut self, slot: Slot, slot_start: Instant) {
+        datapoint_info!(
+            "broadcast-transmit-shreds-stats",
+            ("slot", slot as i64, i64),
+            (
+                "end_to_end_elapsed",
+                // `slot_start` signals when the first batch of shreds was
+                // received, used to measure duration of broadcast
+                slot_start.elapsed().as_micros() as i64,
+                i64
+            ),
+            ("transmit_elapsed", self.transmit_elapsed as i64, i64),
+            ("send_mmsg_elapsed", self.send_mmsg_elapsed as i64, i64),
+            ("get_peers_elapsed", self.get_peers_elapsed as i64, i64),
+            ("num_shreds", self.num_shreds as i64, i64),
+        );
+    }
+}
+
+#[derive(Default, Clone)]
+pub(crate) struct InsertShredsStats {
+    pub(crate) insert_shreds_elapsed: u64,
+    pub(crate) num_shreds: usize,
+}
+impl BroadcastStats for InsertShredsStats {
+    fn update(&mut self, new_stats: &InsertShredsStats) {
+        self.insert_shreds_elapsed += new_stats.insert_shreds_elapsed;
+        self.num_shreds += new_stats.num_shreds;
+    }
+    fn report_stats(&mut self, slot: Slot, slot_start: Instant) {
+        datapoint_info!(
+            "broadcast-insert-shreds-stats",
+            ("slot", slot as i64, i64),
+            (
+                "end_to_end_elapsed",
+                // `slot_start` signals when the first batch of shreds was
+                // received, used to measure duration of broadcast
+                slot_start.elapsed().as_micros() as i64,
+                i64
+            ),
+            (
+                "insert_shreds_elapsed",
+                self.insert_shreds_elapsed as i64,
+                i64
+            ),
+            ("num_shreds", self.num_shreds as i64, i64),
+        );
+    }
+}
+
+// Tracks metrics of type `T` acrosss multiple threads
+#[derive(Default)]
+pub(crate) struct BatchCounter<T: BroadcastStats + Default> {
+    // The number of batches processed across all threads so far
+    num_batches: usize,
+    // Filled in when the last batch of shreds is received,
+    // signals how many batches of shreds to expect
+    num_expected_batches: Option<usize>,
+    broadcast_shred_stats: T,
+}
+
+impl<T: BroadcastStats + Default> BatchCounter<T> {
+    #[cfg(test)]
+    pub(crate) fn num_batches(&self) -> usize {
+        self.num_batches
+    }
+}
+
+#[derive(Default)]
+pub(crate) struct SlotBroadcastStats<T: BroadcastStats + Default>(HashMap<Slot, BatchCounter<T>>);
+
+impl<T: BroadcastStats + Default> SlotBroadcastStats<T> {
+    #[cfg(test)]
+    pub(crate) fn get(&self, slot: Slot) -> Option<&BatchCounter<T>> {
+        self.0.get(&slot)
+    }
+    pub(crate) fn update(&mut self, new_stats: &T, batch_info: &Option<BroadcastShredBatchInfo>) {
+        if let Some(batch_info) = batch_info {
+            let mut should_delete = false;
+            {
+                let slot_batch_counter = self.0.entry(batch_info.slot).or_default();
+                slot_batch_counter.broadcast_shred_stats.update(new_stats);
+                // Only count the ones where `broadcast_shred_batch_info`.is_some(), because
+                // there could potentially be other `retransmit` slots inserted into the
+                // transmit pipeline (signaled by ReplayStage) that are not created by the
+                // main shredding/broadcast pipeline
+                slot_batch_counter.num_batches += 1;
+                if let Some(num_expected_batches) = batch_info.num_expected_batches {
+                    slot_batch_counter.num_expected_batches = Some(num_expected_batches);
+                }
+                if let Some(num_expected_batches) = slot_batch_counter.num_expected_batches {
+                    if slot_batch_counter.num_batches == num_expected_batches {
+                        slot_batch_counter
+                            .broadcast_shred_stats
+                            .report_stats(batch_info.slot, batch_info.slot_start_ts);
+                        should_delete = true;
+                    }
+                }
+            }
+            if should_delete {
+                self.0
+                    .remove(&batch_info.slot)
+                    .expect("delete should be successful");
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[derive(Default)]
+    struct TestStats {
+        sender: Option<Sender<(usize, Slot, Instant)>>,
+        count: usize,
+    }
+
+    impl BroadcastStats for TestStats {
+        fn update(&mut self, new_stats: &TestStats) {
+            self.count += new_stats.count;
+            self.sender = new_stats.sender.clone();
+        }
+        fn report_stats(&mut self, slot: Slot, slot_start: Instant) {
+            self.sender
+                .as_ref()
+                .unwrap()
+                .send((self.count, slot, slot_start))
+                .unwrap()
+        }
+    }
+
+    #[test]
+    fn test_update() {
+        let start = Instant::now();
+        let mut slot_broadcast_stats = SlotBroadcastStats::default();
+        slot_broadcast_stats.update(
+            &TransmitShredsStats {
+                transmit_elapsed: 1,
+                get_peers_elapsed: 1,
+                send_mmsg_elapsed: 1,
+                num_shreds: 1,
+            },
+            &Some(BroadcastShredBatchInfo {
+                slot: 0,
+                num_expected_batches: Some(2),
+                slot_start_ts: start.clone(),
+            }),
+        );
+
+        // Singular update
+        let slot_0_stats = slot_broadcast_stats.0.get(&0).unwrap();
+        assert_eq!(slot_0_stats.num_batches, 1);
+        assert_eq!(slot_0_stats.num_expected_batches.unwrap(), 2);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.transmit_elapsed, 1);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 1);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 1);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 1);
+
+        slot_broadcast_stats.update(
+            &TransmitShredsStats {
+                transmit_elapsed: 1,
+                get_peers_elapsed: 1,
+                send_mmsg_elapsed: 1,
+                num_shreds: 1,
+            },
+            &None,
+        );
+
+        // If BroadcastShredBatchInfo == None, then update should be ignored
+        let slot_0_stats = slot_broadcast_stats.0.get(&0).unwrap();
+        assert_eq!(slot_0_stats.num_batches, 1);
+        assert_eq!(slot_0_stats.num_expected_batches.unwrap(), 2);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.transmit_elapsed, 1);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.get_peers_elapsed, 1);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.send_mmsg_elapsed, 1);
+        assert_eq!(slot_0_stats.broadcast_shred_stats.num_shreds, 1);
+
+        // If another batch is given, then total number of batches == num_expected_batches == 2,
+        // so the batch should be purged from the HashMap
+        slot_broadcast_stats.update(
+            &TransmitShredsStats {
+                transmit_elapsed: 1,
+                get_peers_elapsed: 1,
+                send_mmsg_elapsed: 1,
+                num_shreds: 1,
+            },
+            &Some(BroadcastShredBatchInfo {
+                slot: 0,
+                num_expected_batches: None,
+                slot_start_ts: start.clone(),
+            }),
+        );
+
+        assert!(slot_broadcast_stats.0.get(&0).is_none());
+    }
+
+    #[test]
+    fn test_update_multi_threaded() {
+        for round in 0..50 {
+            let start = Instant::now();
+            let slot_broadcast_stats = Arc::new(Mutex::new(SlotBroadcastStats::default()));
+            let num_threads = 5;
+            let slot = 0;
+            let (sender, receiver) = channel();
+            let thread_handles: Vec<_> = (0..num_threads)
+                .into_iter()
+                .map(|i| {
+                    let slot_broadcast_stats = slot_broadcast_stats.clone();
+                    let sender = Some(sender.clone());
+                    let test_stats = TestStats { sender, count: 1 };
+                    let mut broadcast_batch_info = BroadcastShredBatchInfo {
+                        slot,
+                        num_expected_batches: None,
+                        slot_start_ts: start.clone(),
+                    };
+                    if i == round % num_threads {
+                        broadcast_batch_info.num_expected_batches = Some(num_threads);
+                    }
+                    Builder::new()
+                        .name("test_update_multi_threaded".to_string())
+                        .spawn(move || {
+                            slot_broadcast_stats
+                                .lock()
+                                .unwrap()
+                                .update(&test_stats, &Some(broadcast_batch_info))
+                        })
+                        .unwrap()
+                })
+                .collect();
+
+            for t in thread_handles {
+                t.join().unwrap();
+            }
+
+            assert!(slot_broadcast_stats.lock().unwrap().0.get(&slot).is_none());
+            let (returned_count, returned_slot, returned_instant) = receiver.recv().unwrap();
+            assert_eq!(returned_count, num_threads);
+            assert_eq!(returned_slot, slot);
+            assert_eq!(returned_instant, returned_instant);
+        }
+    }
+}

core/src/broadcast_stage/broadcast_utils.rs

@@ -76,7 +76,7 @@ pub(super) fn recv_slot_entries(receiver: &Receiver<WorkingBankEntry>) -> Result
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_sdk::genesis_config::GenesisConfig;
     use solana_sdk::pubkey::Pubkey;
     use solana_sdk::system_transaction;

core/src/broadcast_stage/fail_entry_verification_broadcast_run.rs

@@ -23,8 +23,8 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
         &mut self,
         blockstore: &Arc<Blockstore>,
         receiver: &Receiver<WorkingBankEntry>,
-        socket_sender: &Sender<TransmitShreds>,
-        blockstore_sender: &Sender<Arc<Vec<Shred>>>,
+        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
     ) -> Result<()> {
         // 1) Pull entries from banking stage
         let mut receive_results = broadcast_utils::recv_slot_entries(receiver)?;
@@ -61,38 +61,44 @@ impl BroadcastRun for FailEntryVerificationBroadcastRun {
         );
 
         let data_shreds = Arc::new(data_shreds);
-        blockstore_sender.send(data_shreds.clone())?;
+        blockstore_sender.send((data_shreds.clone(), None))?;
         // 3) Start broadcast step
         let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
         let stakes = staking_utils::staked_nodes_at_epoch(&bank, bank_epoch);
 
         let stakes = stakes.map(Arc::new);
-        socket_sender.send((stakes.clone(), data_shreds))?;
-        socket_sender.send((stakes, Arc::new(coding_shreds)))?;
+        socket_sender.send(((stakes.clone(), data_shreds), None))?;
+        socket_sender.send(((stakes, Arc::new(coding_shreds)), None))?;
         Ok(())
     }
     fn transmit(
-        &self,
-        receiver: &Arc<Mutex<Receiver<TransmitShreds>>>,
+        &mut self,
+        receiver: &Arc<Mutex<TransmitReceiver>>,
         cluster_info: &Arc<RwLock<ClusterInfo>>,
         sock: &UdpSocket,
     ) -> Result<()> {
-        let (stakes, shreds) = receiver.lock().unwrap().recv()?;
-        let all_seeds: Vec<[u8; 32]> = shreds.iter().map(|s| s.seed()).collect();
+        let ((stakes, shreds), _) = receiver.lock().unwrap().recv()?;
         // Broadcast data
-        let all_shred_bufs: Vec<Vec<u8>> = shreds.to_vec().into_iter().map(|s| s.payload).collect();
-        cluster_info
-            .write()
-            .unwrap()
-            .broadcast_shreds(sock, all_shred_bufs, &all_seeds, stakes)?;
+        let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
+
+        let mut send_mmsg_total = 0;
+        broadcast_shreds(
+            sock,
+            &shreds,
+            &peers_and_stakes,
+            &peers,
+            &Arc::new(AtomicU64::new(0)),
+            &mut send_mmsg_total,
+        )?;
+
         Ok(())
     }
     fn record(
-        &self,
-        receiver: &Arc<Mutex<Receiver<Arc<Vec<Shred>>>>>,
+        &mut self,
+        receiver: &Arc<Mutex<RecordReceiver>>,
         blockstore: &Arc<Blockstore>,
     ) -> Result<()> {
-        let all_shreds = receiver.lock().unwrap().recv()?;
+        let (all_shreds, _) = receiver.lock().unwrap().recv()?;
         blockstore
             .insert_shreds(all_shreds.to_vec(), None, true)
             .expect("Failed to insert shreds in blockstore");

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -1,53 +1,43 @@
-use super::broadcast_utils::{self, ReceiveResults};
-use super::*;
+use super::{
+    broadcast_utils::{self, ReceiveResults},
+    *,
+};
 use crate::broadcast_stage::broadcast_utils::UnfinishedSlotInfo;
-use solana_ledger::entry::Entry;
-use solana_ledger::shred::{Shred, Shredder, RECOMMENDED_FEC_RATE, SHRED_TICK_REFERENCE_MASK};
-use solana_sdk::pubkey::Pubkey;
-use solana_sdk::signature::Keypair;
-use solana_sdk::timing::duration_as_us;
+use solana_ledger::{
+    entry::Entry,
+    shred::{Shred, Shredder, RECOMMENDED_FEC_RATE, SHRED_TICK_REFERENCE_MASK},
+};
+use solana_sdk::{pubkey::Pubkey, signature::Keypair, timing::duration_as_us};
 use std::collections::HashMap;
 use std::time::Duration;
 
-#[derive(Default)]
-struct BroadcastStats {
-    // Per-slot elapsed time
-    shredding_elapsed: u64,
-    insert_shreds_elapsed: u64,
-    broadcast_elapsed: u64,
-    receive_elapsed: u64,
-    seed_elapsed: u64,
-}
-
-impl BroadcastStats {
-    fn reset(&mut self) {
-        self.insert_shreds_elapsed = 0;
-        self.shredding_elapsed = 0;
-        self.broadcast_elapsed = 0;
-        self.receive_elapsed = 0;
-        self.seed_elapsed = 0;
-    }
-}
-
 #[derive(Clone)]
-pub(super) struct StandardBroadcastRun {
-    stats: Arc<RwLock<BroadcastStats>>,
+pub struct StandardBroadcastRun {
+    process_shreds_stats: ProcessShredsStats,
+    transmit_shreds_stats: Arc<Mutex<SlotBroadcastStats<TransmitShredsStats>>>,
+    insert_shreds_stats: Arc<Mutex<SlotBroadcastStats<InsertShredsStats>>>,
     unfinished_slot: Option<UnfinishedSlotInfo>,
     current_slot_and_parent: Option<(u64, u64)>,
     slot_broadcast_start: Option<Instant>,
     keypair: Arc<Keypair>,
     shred_version: u16,
+    last_datapoint_submit: Arc<AtomicU64>,
+    num_batches: usize,
 }
 
 impl StandardBroadcastRun {
     pub(super) fn new(keypair: Arc<Keypair>, shred_version: u16) -> Self {
         Self {
-            stats: Arc::new(RwLock::new(BroadcastStats::default())),
+            process_shreds_stats: ProcessShredsStats::default(),
+            transmit_shreds_stats: Arc::new(Mutex::new(SlotBroadcastStats::default())),
+            insert_shreds_stats: Arc::new(Mutex::new(SlotBroadcastStats::default())),
             unfinished_slot: None,
             current_slot_and_parent: None,
             slot_broadcast_start: None,
             keypair,
             shred_version,
+            last_datapoint_submit: Arc::new(AtomicU64::new(0)),
+            num_batches: 0,
         }
     }
 
@@ -142,6 +132,7 @@ impl StandardBroadcastRun {
         let brecv = Arc::new(Mutex::new(brecv));
         //data
         let _ = self.transmit(&srecv, cluster_info, sock);
+        let _ = self.record(&brecv, blockstore);
         //coding
         let _ = self.transmit(&srecv, cluster_info, sock);
         let _ = self.record(&brecv, blockstore);
@@ -151,8 +142,8 @@ impl StandardBroadcastRun {
     fn process_receive_results(
         &mut self,
         blockstore: &Arc<Blockstore>,
-        socket_sender: &Sender<TransmitShreds>,
-        blockstore_sender: &Sender<Arc<Vec<Shred>>>,
+        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
         receive_results: ReceiveResults,
     ) -> Result<()> {
         let mut receive_elapsed = receive_results.time_elapsed;
@@ -160,11 +151,13 @@ impl StandardBroadcastRun {
         let bank = receive_results.bank.clone();
         let last_tick_height = receive_results.last_tick_height;
         inc_new_counter_info!("broadcast_service-entries_received", num_entries);
-
+        let old_broadcast_start = self.slot_broadcast_start;
+        let old_num_batches = self.num_batches;
         if self.current_slot_and_parent.is_none()
             || bank.slot() != self.current_slot_and_parent.unwrap().0
         {
             self.slot_broadcast_start = Some(Instant::now());
+            self.num_batches = 0;
             let slot = bank.slot();
             let parent_slot = bank.parent_slot();
 
@@ -179,19 +172,19 @@ impl StandardBroadcastRun {
             self.check_for_interrupted_slot(bank.ticks_per_slot() as u8);
 
         // 2) Convert entries to shreds and coding shreds
-
         let (shredder, next_shred_index) = self.init_shredder(
             blockstore,
             (bank.tick_height() % bank.ticks_per_slot()) as u8,
         );
-        let mut data_shreds = self.entries_to_data_shreds(
+        let is_last_in_slot = last_tick_height == bank.max_tick_height();
+        let data_shreds = self.entries_to_data_shreds(
             &shredder,
             next_shred_index,
             &receive_results.entries,
-            last_tick_height == bank.max_tick_height(),
+            is_last_in_slot,
         );
-        //Insert the first shred so blockstore stores that the leader started this block
-        //This must be done before the blocks are sent out over the wire.
+        // Insert the first shred so blockstore stores that the leader started this block
+        // This must be done before the blocks are sent out over the wire.
         if !data_shreds.is_empty() && data_shreds[0].index() == 0 {
             let first = vec![data_shreds[0].clone()];
             blockstore
@@ -199,26 +192,56 @@ impl StandardBroadcastRun {
                 .expect("Failed to insert shreds in blockstore");
         }
         let last_data_shred = data_shreds.len();
-        if let Some(last_shred) = last_unfinished_slot_shred {
-            data_shreds.push(last_shred);
-        }
         let to_shreds_elapsed = to_shreds_start.elapsed();
 
         let bank_epoch = bank.get_leader_schedule_epoch(bank.slot());
         let stakes = staking_utils::staked_nodes_at_epoch(&bank, bank_epoch);
         let stakes = stakes.map(Arc::new);
-        let data_shreds = Arc::new(data_shreds);
-        socket_sender.send((stakes.clone(), data_shreds.clone()))?;
-        blockstore_sender.send(data_shreds.clone())?;
-        let coding_shreds = shredder.data_shreds_to_coding_shreds(&data_shreds[0..last_data_shred]);
-        let coding_shreds = Arc::new(coding_shreds);
-        socket_sender.send((stakes, coding_shreds))?;
-        self.update_broadcast_stats(BroadcastStats {
-            shredding_elapsed: duration_as_us(&to_shreds_elapsed),
-            receive_elapsed: duration_as_us(&receive_elapsed),
-            ..BroadcastStats::default()
+
+        // Broadcast the last shred of the interrupted slot if necessary
+        if let Some(last_shred) = last_unfinished_slot_shred {
+            let batch_info = Some(BroadcastShredBatchInfo {
+                slot: last_shred.slot(),
+                num_expected_batches: Some(old_num_batches + 1),
+                slot_start_ts: old_broadcast_start.expect(
+                    "Old broadcast start time for previous slot must exist if the previous slot
+                 was interrupted",
+                ),
+            });
+            let last_shred = Arc::new(vec![last_shred]);
+            socket_sender.send(((stakes.clone(), last_shred.clone()), batch_info.clone()))?;
+            blockstore_sender.send((last_shred, batch_info))?;
+        }
+
+        // Increment by two batches, one for the data batch, one for the coding batch.
+        self.num_batches += 2;
+        let num_expected_batches = {
+            if is_last_in_slot {
+                Some(self.num_batches)
+            } else {
+                None
+            }
+        };
+        let batch_info = Some(BroadcastShredBatchInfo {
+            slot: bank.slot(),
+            num_expected_batches,
+            slot_start_ts: self
+                .slot_broadcast_start
+                .clone()
+                .expect("Start timestamp must exist for a slot if we're broadcasting the slot"),
         });
 
+        let data_shreds = Arc::new(data_shreds);
+        socket_sender.send(((stakes.clone(), data_shreds.clone()), batch_info.clone()))?;
+        blockstore_sender.send((data_shreds.clone(), batch_info.clone()))?;
+        let coding_shreds = shredder.data_shreds_to_coding_shreds(&data_shreds[0..last_data_shred]);
+        let coding_shreds = Arc::new(coding_shreds);
+        socket_sender.send(((stakes, coding_shreds.clone()), batch_info.clone()))?;
+        blockstore_sender.send((coding_shreds, batch_info))?;
+        self.process_shreds_stats.update(&ProcessShredsStats {
+            shredding_elapsed: duration_as_us(&to_shreds_elapsed),
+            receive_elapsed: duration_as_us(&receive_elapsed),
+        });
         if last_tick_height == bank.max_tick_height() {
             self.report_and_reset_stats();
             self.unfinished_slot = None;
@@ -227,10 +250,15 @@ impl StandardBroadcastRun {
         Ok(())
     }
 
-    fn insert(&self, blockstore: &Arc<Blockstore>, shreds: Arc<Vec<Shred>>) -> Result<()> {
+    fn insert(
+        &mut self,
+        blockstore: &Arc<Blockstore>,
+        shreds: Arc<Vec<Shred>>,
+        broadcast_shred_batch_info: Option<BroadcastShredBatchInfo>,
+    ) -> Result<()> {
         // Insert shreds into blockstore
         let insert_shreds_start = Instant::now();
-        //The first shred is inserted synchronously
+        // The first shred is inserted synchronously
         let data_shreds = if !shreds.is_empty() && shreds[0].index() == 0 {
             shreds[1..].to_vec()
         } else {
@@ -240,66 +268,80 @@ impl StandardBroadcastRun {
             .insert_shreds(data_shreds, None, true)
             .expect("Failed to insert shreds in blockstore");
         let insert_shreds_elapsed = insert_shreds_start.elapsed();
-        self.update_broadcast_stats(BroadcastStats {
+        let new_insert_shreds_stats = InsertShredsStats {
             insert_shreds_elapsed: duration_as_us(&insert_shreds_elapsed),
-            ..BroadcastStats::default()
-        });
+            num_shreds: shreds.len(),
+        };
+        self.update_insertion_metrics(&new_insert_shreds_stats, &broadcast_shred_batch_info);
         Ok(())
     }
 
+    fn update_insertion_metrics(
+        &mut self,
+        new_insertion_shreds_stats: &InsertShredsStats,
+        broadcast_shred_batch_info: &Option<BroadcastShredBatchInfo>,
+    ) {
+        let mut insert_shreds_stats = self.insert_shreds_stats.lock().unwrap();
+        insert_shreds_stats.update(new_insertion_shreds_stats, broadcast_shred_batch_info);
+    }
+
     fn broadcast(
-        &self,
+        &mut self,
         sock: &UdpSocket,
         cluster_info: &Arc<RwLock<ClusterInfo>>,
         stakes: Option<Arc<HashMap<Pubkey, u64>>>,
         shreds: Arc<Vec<Shred>>,
+        broadcast_shred_batch_info: Option<BroadcastShredBatchInfo>,
     ) -> Result<()> {
-        let seed_start = Instant::now();
-        let seeds: Vec<[u8; 32]> = shreds.iter().map(|s| s.seed()).collect();
-        let seed_elapsed = seed_start.elapsed();
+        trace!("Broadcasting {:?} shreds", shreds.len());
+        // Get the list of peers to broadcast to
+        let get_peers_start = Instant::now();
+        let (peers, peers_and_stakes) = get_broadcast_peers(cluster_info, stakes);
+        let get_peers_elapsed = get_peers_start.elapsed();
 
         // Broadcast the shreds
-        let broadcast_start = Instant::now();
-        let shred_bufs: Vec<Vec<u8>> = shreds.to_vec().into_iter().map(|s| s.payload).collect();
-        trace!("Broadcasting {:?} shreds", shred_bufs.len());
+        let transmit_start = Instant::now();
+        let mut send_mmsg_total = 0;
+        broadcast_shreds(
+            sock,
+            &shreds,
+            &peers_and_stakes,
+            &peers,
+            &self.last_datapoint_submit,
+            &mut send_mmsg_total,
+        )?;
+        let transmit_elapsed = transmit_start.elapsed();
+        let new_transmit_shreds_stats = TransmitShredsStats {
+            transmit_elapsed: duration_as_us(&transmit_elapsed),
+            get_peers_elapsed: duration_as_us(&get_peers_elapsed),
+            send_mmsg_elapsed: send_mmsg_total,
+            num_shreds: shreds.len(),
+        };
 
-        cluster_info
-            .write()
-            .unwrap()
-            .broadcast_shreds(sock, shred_bufs, &seeds, stakes)?;
-
-        let broadcast_elapsed = broadcast_start.elapsed();
-
-        self.update_broadcast_stats(BroadcastStats {
-            broadcast_elapsed: duration_as_us(&broadcast_elapsed),
-            seed_elapsed: duration_as_us(&seed_elapsed),
-            ..BroadcastStats::default()
-        });
+        // Process metrics
+        self.update_transmit_metrics(&new_transmit_shreds_stats, &broadcast_shred_batch_info);
         Ok(())
     }
 
-    fn update_broadcast_stats(&self, stats: BroadcastStats) {
-        let mut wstats = self.stats.write().unwrap();
-        wstats.receive_elapsed += stats.receive_elapsed;
-        wstats.shredding_elapsed += stats.shredding_elapsed;
-        wstats.insert_shreds_elapsed += stats.insert_shreds_elapsed;
-        wstats.broadcast_elapsed += stats.broadcast_elapsed;
-        wstats.seed_elapsed += stats.seed_elapsed;
+    fn update_transmit_metrics(
+        &mut self,
+        new_transmit_shreds_stats: &TransmitShredsStats,
+        broadcast_shred_batch_info: &Option<BroadcastShredBatchInfo>,
+    ) {
+        let mut transmit_shreds_stats = self.transmit_shreds_stats.lock().unwrap();
+        transmit_shreds_stats.update(new_transmit_shreds_stats, broadcast_shred_batch_info);
     }
 
     fn report_and_reset_stats(&mut self) {
-        let stats = self.stats.read().unwrap();
+        let stats = &self.process_shreds_stats;
         assert!(self.unfinished_slot.is_some());
         datapoint_info!(
-            "broadcast-bank-stats",
+            "broadcast-process-shreds-stats",
             ("slot", self.unfinished_slot.unwrap().slot as i64, i64),
             ("shredding_time", stats.shredding_elapsed as i64, i64),
-            ("insertion_time", stats.insert_shreds_elapsed as i64, i64),
-            ("broadcast_time", stats.broadcast_elapsed as i64, i64),
             ("receive_time", stats.receive_elapsed as i64, i64),
-            ("seed", stats.seed_elapsed as i64, i64),
             (
-                "num_shreds",
+                "num_data_shreds",
                 i64::from(self.unfinished_slot.unwrap().next_shred_index),
                 i64
             ),
@@ -309,8 +351,7 @@ impl StandardBroadcastRun {
                 i64
             ),
         );
-        drop(stats);
-        self.stats.write().unwrap().reset();
+        self.process_shreds_stats.reset();
     }
 }
 
@@ -319,8 +360,8 @@ impl BroadcastRun for StandardBroadcastRun {
         &mut self,
         blockstore: &Arc<Blockstore>,
         receiver: &Receiver<WorkingBankEntry>,
-        socket_sender: &Sender<TransmitShreds>,
-        blockstore_sender: &Sender<Arc<Vec<Shred>>>,
+        socket_sender: &Sender<(TransmitShreds, Option<BroadcastShredBatchInfo>)>,
+        blockstore_sender: &Sender<(Arc<Vec<Shred>>, Option<BroadcastShredBatchInfo>)>,
     ) -> Result<()> {
         let receive_results = broadcast_utils::recv_slot_entries(receiver)?;
         self.process_receive_results(
@@ -331,21 +372,21 @@ impl BroadcastRun for StandardBroadcastRun {
         )
     }
     fn transmit(
-        &self,
-        receiver: &Arc<Mutex<Receiver<TransmitShreds>>>,
+        &mut self,
+        receiver: &Arc<Mutex<TransmitReceiver>>,
         cluster_info: &Arc<RwLock<ClusterInfo>>,
         sock: &UdpSocket,
     ) -> Result<()> {
-        let (stakes, shreds) = receiver.lock().unwrap().recv()?;
-        self.broadcast(sock, cluster_info, stakes, shreds)
+        let ((stakes, shreds), slot_start_ts) = receiver.lock().unwrap().recv()?;
+        self.broadcast(sock, cluster_info, stakes, shreds, slot_start_ts)
     }
     fn record(
-        &self,
-        receiver: &Arc<Mutex<Receiver<Arc<Vec<Shred>>>>>,
+        &mut self,
+        receiver: &Arc<Mutex<RecordReceiver>>,
         blockstore: &Arc<Blockstore>,
     ) -> Result<()> {
-        let shreds = receiver.lock().unwrap().recv()?;
-        self.insert(blockstore, shreds)
+        let (shreds, slot_start_ts) = receiver.lock().unwrap().recv()?;
+        self.insert(blockstore, shreds, slot_start_ts)
     }
 }
 
@@ -353,14 +394,13 @@ impl BroadcastRun for StandardBroadcastRun {
 mod test {
     use super::*;
     use crate::cluster_info::{ClusterInfo, Node};
-    use crate::genesis_utils::create_genesis_config;
+    use solana_ledger::genesis_utils::create_genesis_config;
     use solana_ledger::{
         blockstore::Blockstore, entry::create_ticks, get_tmp_ledger_path,
         shred::max_ticks_per_n_shreds,
     };
     use solana_runtime::bank::Bank;
     use solana_sdk::{
-        clock::Slot,
         genesis_config::GenesisConfig,
         signature::{Keypair, Signer},
     };
@@ -462,25 +502,40 @@ mod test {
         // Make sure the slot is not complete
         assert!(!blockstore.is_full(0));
         // Modify the stats, should reset later
-        standard_broadcast_run
-            .stats
-            .write()
-            .unwrap()
-            .receive_elapsed = 10;
-
-        // Try to fetch ticks from blockstore, nothing should break
-        assert_eq!(blockstore.get_slot_entries(0, 0, None).unwrap(), ticks0);
+        standard_broadcast_run.process_shreds_stats.receive_elapsed = 10;
+        // Broadcast stats should exist, and 2 batches should have been sent,
+        // one for data, one for coding
         assert_eq!(
-            blockstore
-                .get_slot_entries(0, num_shreds_per_slot, None)
-                .unwrap(),
+            standard_broadcast_run
+                .transmit_shreds_stats
+                .lock()
+                .unwrap()
+                .get(unfinished_slot.slot)
+                .unwrap()
+                .num_batches(),
+            2
+        );
+        assert_eq!(
+            standard_broadcast_run
+                .insert_shreds_stats
+                .lock()
+                .unwrap()
+                .get(unfinished_slot.slot)
+                .unwrap()
+                .num_batches(),
+            2
+        );
+        // Try to fetch ticks from blockstore, nothing should break
+        assert_eq!(blockstore.get_slot_entries(0, 0).unwrap(), ticks0);
+        assert_eq!(
+            blockstore.get_slot_entries(0, num_shreds_per_slot).unwrap(),
             vec![],
         );
 
         // Step 2: Make a transmission for another bank that interrupts the transmission for
         // slot 0
         let bank2 = Arc::new(Bank::new_from_parent(&bank0, &leader_keypair.pubkey(), 2));
-
+        let interrupted_slot = unfinished_slot.slot;
         // Interrupting the slot should cause the unfinished_slot and stats to reset
         let num_shreds = 1;
         assert!(num_shreds < num_shreds_per_slot);
@@ -504,16 +559,28 @@ mod test {
 
         // Check that the stats were reset as well
         assert_eq!(
-            standard_broadcast_run.stats.read().unwrap().receive_elapsed,
+            standard_broadcast_run.process_shreds_stats.receive_elapsed,
             0
         );
 
+        // Broadcast stats for interrupted slot should be cleared
+        assert!(standard_broadcast_run
+            .transmit_shreds_stats
+            .lock()
+            .unwrap()
+            .get(interrupted_slot)
+            .is_none());
+        assert!(standard_broadcast_run
+            .insert_shreds_stats
+            .lock()
+            .unwrap()
+            .get(interrupted_slot)
+            .is_none());
+
         // Try to fetch the incomplete ticks from blockstore, should succeed
-        assert_eq!(blockstore.get_slot_entries(0, 0, None).unwrap(), ticks0);
+        assert_eq!(blockstore.get_slot_entries(0, 0).unwrap(), ticks0);
         assert_eq!(
-            blockstore
-                .get_slot_entries(0, num_shreds_per_slot, None)
-                .unwrap(),
+            blockstore.get_slot_entries(0, num_shreds_per_slot).unwrap(),
             vec![],
         );
     }

core/src/cluster_slots.rs

@@ -0,0 +1,348 @@
+use crate::{
+    cluster_info::ClusterInfo, contact_info::ContactInfo, epoch_slots::EpochSlots,
+    serve_repair::RepairType,
+};
+use solana_ledger::bank_forks::BankForks;
+use solana_runtime::epoch_stakes::NodeIdToVoteAccounts;
+use solana_sdk::{clock::Slot, pubkey::Pubkey};
+use std::{
+    collections::{HashMap, HashSet},
+    sync::{Arc, RwLock},
+};
+
+pub type SlotPubkeys = HashMap<Arc<Pubkey>, u64>;
+pub type ClusterSlotsMap = RwLock<HashMap<Slot, Arc<RwLock<SlotPubkeys>>>>;
+
+#[derive(Default)]
+pub struct ClusterSlots {
+    cluster_slots: ClusterSlotsMap,
+    keys: RwLock<HashSet<Arc<Pubkey>>>,
+    since: RwLock<Option<u64>>,
+    validator_stakes: RwLock<Arc<NodeIdToVoteAccounts>>,
+    epoch: RwLock<Option<u64>>,
+    self_id: RwLock<Pubkey>,
+}
+
+impl ClusterSlots {
+    pub fn lookup(&self, slot: Slot) -> Option<Arc<RwLock<SlotPubkeys>>> {
+        self.cluster_slots.read().unwrap().get(&slot).cloned()
+    }
+    pub fn update(
+        &self,
+        root: Slot,
+        cluster_info: &RwLock<ClusterInfo>,
+        bank_forks: &RwLock<BankForks>,
+    ) {
+        self.update_peers(cluster_info, bank_forks);
+        let since = *self.since.read().unwrap();
+        let epoch_slots = cluster_info.read().unwrap().get_epoch_slots_since(since);
+        self.update_internal(root, epoch_slots);
+    }
+    fn update_internal(&self, root: Slot, epoch_slots: (Vec<EpochSlots>, Option<u64>)) {
+        let (epoch_slots_list, since) = epoch_slots;
+        for epoch_slots in epoch_slots_list {
+            let slots = epoch_slots.to_slots(root);
+            for slot in &slots {
+                if *slot <= root {
+                    continue;
+                }
+                let pubkey = Arc::new(epoch_slots.from);
+                let exists = self.keys.read().unwrap().get(&pubkey).is_some();
+                if !exists {
+                    self.keys.write().unwrap().insert(pubkey.clone());
+                }
+                let from = self.keys.read().unwrap().get(&pubkey).unwrap().clone();
+                let balance = self
+                    .validator_stakes
+                    .read()
+                    .unwrap()
+                    .get(&from)
+                    .map(|v| v.total_stake)
+                    .unwrap_or(0);
+
+                let mut slot_pubkeys = self.cluster_slots.read().unwrap().get(slot).cloned();
+                if slot_pubkeys.is_none() {
+                    let new_slot_pubkeys = Arc::new(RwLock::new(HashMap::default()));
+                    self.cluster_slots
+                        .write()
+                        .unwrap()
+                        .insert(*slot, new_slot_pubkeys.clone());
+                    slot_pubkeys = Some(new_slot_pubkeys);
+                }
+
+                slot_pubkeys
+                    .unwrap()
+                    .write()
+                    .unwrap()
+                    .insert(from.clone(), balance);
+            }
+        }
+        self.cluster_slots.write().unwrap().retain(|x, _| *x > root);
+        self.keys
+            .write()
+            .unwrap()
+            .retain(|x| Arc::strong_count(x) > 1);
+        *self.since.write().unwrap() = since;
+    }
+    pub fn collect(&self, id: &Pubkey) -> HashSet<Slot> {
+        self.cluster_slots
+            .read()
+            .unwrap()
+            .iter()
+            .filter(|(_, keys)| keys.read().unwrap().get(id).is_some())
+            .map(|(slot, _)| slot)
+            .cloned()
+            .collect()
+    }
+
+    fn update_peers(&self, cluster_info: &RwLock<ClusterInfo>, bank_forks: &RwLock<BankForks>) {
+        let root_bank = bank_forks.read().unwrap().root_bank().clone();
+        let root_epoch = root_bank.epoch();
+        let my_epoch = *self.epoch.read().unwrap();
+
+        if Some(root_epoch) != my_epoch {
+            let validator_stakes = root_bank
+                .epoch_stakes(root_epoch)
+                .expect(
+                    "Bank must have epoch stakes
+                for its own epoch",
+                )
+                .node_id_to_vote_accounts()
+                .clone();
+
+            *self.validator_stakes.write().unwrap() = validator_stakes;
+            let id = cluster_info.read().unwrap().id();
+            *self.self_id.write().unwrap() = id;
+            *self.epoch.write().unwrap() = Some(root_epoch);
+        }
+    }
+
+    pub fn compute_weights(&self, slot: Slot, repair_peers: &[ContactInfo]) -> Vec<(u64, usize)> {
+        let slot_peers = self.lookup(slot);
+        repair_peers
+            .iter()
+            .enumerate()
+            .map(|(i, x)| {
+                let peer_stake = slot_peers
+                    .as_ref()
+                    .and_then(|v| v.read().unwrap().get(&x.id).cloned())
+                    .unwrap_or(0);
+                (
+                    1 + peer_stake
+                        + self
+                            .validator_stakes
+                            .read()
+                            .unwrap()
+                            .get(&x.id)
+                            .map(|v| v.total_stake)
+                            .unwrap_or(0),
+                    i,
+                )
+            })
+            .collect()
+    }
+
+    pub fn generate_repairs_for_missing_slots(
+        &self,
+        self_id: &Pubkey,
+        root: Slot,
+    ) -> Vec<RepairType> {
+        let my_slots = self.collect(self_id);
+        self.cluster_slots
+            .read()
+            .unwrap()
+            .keys()
+            .filter(|x| **x > root)
+            .filter(|x| !my_slots.contains(*x))
+            .map(|x| RepairType::HighestShred(*x, 0))
+            .collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_runtime::epoch_stakes::NodeVoteAccounts;
+
+    #[test]
+    fn test_default() {
+        let cs = ClusterSlots::default();
+        assert!(cs.cluster_slots.read().unwrap().is_empty());
+        assert!(cs.since.read().unwrap().is_none());
+    }
+
+    #[test]
+    fn test_update_noop() {
+        let cs = ClusterSlots::default();
+        cs.update_internal(0, (vec![], None));
+        assert!(cs.cluster_slots.read().unwrap().is_empty());
+        assert!(cs.since.read().unwrap().is_none());
+    }
+
+    #[test]
+    fn test_update_empty() {
+        let cs = ClusterSlots::default();
+        let epoch_slot = EpochSlots::default();
+        cs.update_internal(0, (vec![epoch_slot], Some(0)));
+        assert_eq!(*cs.since.read().unwrap(), Some(0));
+        assert!(cs.lookup(0).is_none());
+    }
+
+    #[test]
+    fn test_update_rooted() {
+        //root is 0, so it should clear out the slot
+        let cs = ClusterSlots::default();
+        let mut epoch_slot = EpochSlots::default();
+        epoch_slot.fill(&[0], 0);
+        cs.update_internal(0, (vec![epoch_slot], Some(0)));
+        assert_eq!(*cs.since.read().unwrap(), Some(0));
+        assert!(cs.lookup(0).is_none());
+    }
+
+    #[test]
+    fn test_update_new_slot() {
+        let cs = ClusterSlots::default();
+        let mut epoch_slot = EpochSlots::default();
+        epoch_slot.fill(&[1], 0);
+        cs.update_internal(0, (vec![epoch_slot], Some(0)));
+        assert_eq!(*cs.since.read().unwrap(), Some(0));
+        assert!(cs.lookup(0).is_none());
+        assert!(cs.lookup(1).is_some());
+        assert_eq!(
+            cs.lookup(1)
+                .unwrap()
+                .read()
+                .unwrap()
+                .get(&Pubkey::default()),
+            Some(&0)
+        );
+    }
+
+    #[test]
+    fn test_compute_weights() {
+        let cs = ClusterSlots::default();
+        let ci = ContactInfo::default();
+        assert_eq!(cs.compute_weights(0, &[ci]), vec![(1, 0)]);
+    }
+
+    #[test]
+    fn test_best_peer_2() {
+        let cs = ClusterSlots::default();
+        let mut c1 = ContactInfo::default();
+        let mut c2 = ContactInfo::default();
+        let mut map = HashMap::new();
+        let k1 = Pubkey::new_rand();
+        let k2 = Pubkey::new_rand();
+        map.insert(Arc::new(k1.clone()), std::u64::MAX / 2);
+        map.insert(Arc::new(k2.clone()), 0);
+        cs.cluster_slots
+            .write()
+            .unwrap()
+            .insert(0, Arc::new(RwLock::new(map)));
+        c1.id = k1;
+        c2.id = k2;
+        assert_eq!(
+            cs.compute_weights(0, &[c1, c2]),
+            vec![(std::u64::MAX / 2 + 1, 0), (1, 1)]
+        );
+    }
+
+    #[test]
+    fn test_best_peer_3() {
+        let cs = ClusterSlots::default();
+        let mut c1 = ContactInfo::default();
+        let mut c2 = ContactInfo::default();
+        let mut map = HashMap::new();
+        let k1 = Pubkey::new_rand();
+        let k2 = Pubkey::new_rand();
+        map.insert(Arc::new(k2.clone()), 0);
+        cs.cluster_slots
+            .write()
+            .unwrap()
+            .insert(0, Arc::new(RwLock::new(map)));
+        //make sure default weights are used as well
+        let validator_stakes: HashMap<_, _> = vec![(
+            *Arc::new(k1.clone()),
+            NodeVoteAccounts {
+                total_stake: std::u64::MAX / 2,
+                vote_accounts: vec![Pubkey::default()],
+            },
+        )]
+        .into_iter()
+        .collect();
+        *cs.validator_stakes.write().unwrap() = Arc::new(validator_stakes);
+        c1.id = k1;
+        c2.id = k2;
+        assert_eq!(
+            cs.compute_weights(0, &[c1, c2]),
+            vec![(std::u64::MAX / 2 + 1, 0), (1, 1)]
+        );
+    }
+
+    #[test]
+    fn test_update_new_staked_slot() {
+        let cs = ClusterSlots::default();
+        let mut epoch_slot = EpochSlots::default();
+        epoch_slot.fill(&[1], 0);
+
+        let map = Arc::new(
+            vec![(
+                Pubkey::default(),
+                NodeVoteAccounts {
+                    total_stake: 1,
+                    vote_accounts: vec![Pubkey::default()],
+                },
+            )]
+            .into_iter()
+            .collect(),
+        );
+
+        *cs.validator_stakes.write().unwrap() = map;
+        cs.update_internal(0, (vec![epoch_slot], None));
+        assert!(cs.lookup(1).is_some());
+        assert_eq!(
+            cs.lookup(1)
+                .unwrap()
+                .read()
+                .unwrap()
+                .get(&Pubkey::default()),
+            Some(&1)
+        );
+    }
+
+    #[test]
+    fn test_generate_repairs() {
+        let cs = ClusterSlots::default();
+        let mut epoch_slot = EpochSlots::default();
+        epoch_slot.fill(&[1], 0);
+        cs.update_internal(0, (vec![epoch_slot], None));
+        let self_id = Pubkey::new_rand();
+        assert_eq!(
+            cs.generate_repairs_for_missing_slots(&self_id, 0),
+            vec![RepairType::HighestShred(1, 0)]
+        )
+    }
+
+    #[test]
+    fn test_collect_my_slots() {
+        let cs = ClusterSlots::default();
+        let mut epoch_slot = EpochSlots::default();
+        epoch_slot.fill(&[1], 0);
+        let self_id = epoch_slot.from;
+        cs.update_internal(0, (vec![epoch_slot], None));
+        let slots: Vec<Slot> = cs.collect(&self_id).into_iter().collect();
+        assert_eq!(slots, vec![1]);
+    }
+
+    #[test]
+    fn test_generate_repairs_existing() {
+        let cs = ClusterSlots::default();
+        let mut epoch_slot = EpochSlots::default();
+        epoch_slot.fill(&[1], 0);
+        let self_id = epoch_slot.from;
+        cs.update_internal(0, (vec![epoch_slot], None));
+        assert!(cs
+            .generate_repairs_for_missing_slots(&self_id, 0)
+            .is_empty());
+    }
+}

core/src/commitment.rs

@@ -1,3 +1,6 @@
+use crate::consensus::VOTE_THRESHOLD_SIZE;
+use solana_measure::measure::Measure;
+use solana_metrics::inc_new_counter_info;
 use solana_runtime::bank::Bank;
 use solana_sdk::clock::Slot;
 use solana_vote_program::{vote_state::VoteState, vote_state::MAX_LOCKOUT_HISTORY};
@@ -31,17 +34,40 @@ impl BlockCommitment {
     }
 }
 
-#[derive(Debug, Default)]
+#[derive(Default)]
 pub struct BlockCommitmentCache {
     block_commitment: HashMap<Slot, BlockCommitment>,
     total_stake: u64,
+    bank: Arc<Bank>,
+    root: Slot,
+}
+
+impl std::fmt::Debug for BlockCommitmentCache {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("BlockCommitmentCache")
+            .field("block_commitment", &self.block_commitment)
+            .field("total_stake", &self.total_stake)
+            .field(
+                "bank",
+                &format_args!("Bank({{current_slot: {:?}}})", self.bank.slot()),
+            )
+            .field("root", &self.root)
+            .finish()
+    }
 }
 
 impl BlockCommitmentCache {
-    pub fn new(block_commitment: HashMap<Slot, BlockCommitment>, total_stake: u64) -> Self {
+    pub fn new(
+        block_commitment: HashMap<Slot, BlockCommitment>,
+        total_stake: u64,
+        bank: Arc<Bank>,
+        root: Slot,
+    ) -> Self {
         Self {
             block_commitment,
             total_stake,
+            bank,
+            root,
         }
     }
 
@@ -53,38 +79,62 @@ impl BlockCommitmentCache {
         self.total_stake
     }
 
-    pub fn get_block_with_depth_commitment(
-        &self,
-        minimum_depth: usize,
-        minimum_stake_percentage: f64,
-    ) -> Option<Slot> {
-        self.block_commitment
-            .iter()
-            .filter(|&(_, block_commitment)| {
-                let fork_stake_minimum_depth: u64 = block_commitment.commitment[minimum_depth..]
-                    .iter()
-                    .cloned()
-                    .sum();
-                fork_stake_minimum_depth as f64 / self.total_stake as f64
-                    >= minimum_stake_percentage
-            })
-            .map(|(slot, _)| *slot)
-            .max()
+    pub fn bank(&self) -> Arc<Bank> {
+        self.bank.clone()
     }
 
-    pub fn get_rooted_block_with_commitment(&self, minimum_stake_percentage: f64) -> Option<u64> {
-        self.get_block_with_depth_commitment(MAX_LOCKOUT_HISTORY - 1, minimum_stake_percentage)
+    pub fn slot(&self) -> Slot {
+        self.bank.slot()
+    }
+
+    pub fn root(&self) -> Slot {
+        self.root
+    }
+
+    pub fn get_confirmation_count(&self, slot: Slot) -> Option<usize> {
+        self.get_lockout_count(slot, VOTE_THRESHOLD_SIZE)
+    }
+
+    // Returns the lowest level at which at least `minimum_stake_percentage` of the total epoch
+    // stake is locked out
+    fn get_lockout_count(&self, slot: Slot, minimum_stake_percentage: f64) -> Option<usize> {
+        self.get_block_commitment(slot).map(|block_commitment| {
+            let iterator = block_commitment.commitment.iter().enumerate().rev();
+            let mut sum = 0;
+            for (i, stake) in iterator {
+                sum += stake;
+                if (sum as f64 / self.total_stake as f64) > minimum_stake_percentage {
+                    return i + 1;
+                }
+            }
+            0
+        })
+    }
+    #[cfg(test)]
+    pub fn new_for_tests() -> Self {
+        let mut block_commitment: HashMap<Slot, BlockCommitment> = HashMap::new();
+        block_commitment.insert(0, BlockCommitment::default());
+        Self {
+            block_commitment,
+            total_stake: 42,
+            ..Self::default()
+        }
     }
 }
 
 pub struct CommitmentAggregationData {
     bank: Arc<Bank>,
+    root: Slot,
     total_staked: u64,
 }
 
 impl CommitmentAggregationData {
-    pub fn new(bank: Arc<Bank>, total_staked: u64) -> Self {
-        Self { bank, total_staked }
+    pub fn new(bank: Arc<Bank>, root: Slot, total_staked: u64) -> Self {
+        Self {
+            bank,
+            root,
+            total_staked,
+        }
     }
 }
 
@@ -144,14 +194,24 @@ impl AggregateCommitmentService {
                 continue;
             }
 
+            let mut aggregate_commitment_time = Measure::start("aggregate-commitment-ms");
             let block_commitment = Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
 
-            let mut new_block_commitment =
-                BlockCommitmentCache::new(block_commitment, aggregation_data.total_staked);
+            let mut new_block_commitment = BlockCommitmentCache::new(
+                block_commitment,
+                aggregation_data.total_staked,
+                aggregation_data.bank,
+                aggregation_data.root,
+            );
 
             let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
 
             std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
+            aggregate_commitment_time.stop();
+            inc_new_counter_info!(
+                "aggregate-commitment-ms",
+                aggregate_commitment_time.as_ms() as usize
+            );
         }
     }
 
@@ -230,7 +290,7 @@ impl AggregateCommitmentService {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use solana_sdk::pubkey::Pubkey;
     use solana_stake_program::stake_state;
     use solana_vote_program::vote_state::{self, VoteStateVersions};
@@ -246,84 +306,31 @@ mod tests {
     }
 
     #[test]
-    fn test_get_block_with_depth_commitment() {
+    fn test_get_confirmations() {
+        let bank = Arc::new(Bank::default());
         // Build BlockCommitmentCache with votes at depths 0 and 1 for 2 slots
         let mut cache0 = BlockCommitment::default();
-        cache0.increase_confirmation_stake(1, 15);
-        cache0.increase_confirmation_stake(2, 25);
+        cache0.increase_confirmation_stake(1, 5);
+        cache0.increase_confirmation_stake(2, 40);
 
         let mut cache1 = BlockCommitment::default();
-        cache1.increase_confirmation_stake(1, 10);
-        cache1.increase_confirmation_stake(2, 20);
+        cache1.increase_confirmation_stake(1, 40);
+        cache1.increase_confirmation_stake(2, 5);
+
+        let mut cache2 = BlockCommitment::default();
+        cache2.increase_confirmation_stake(1, 20);
+        cache2.increase_confirmation_stake(2, 5);
 
         let mut block_commitment = HashMap::new();
         block_commitment.entry(0).or_insert(cache0.clone());
         block_commitment.entry(1).or_insert(cache1.clone());
-        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 50);
+        block_commitment.entry(2).or_insert(cache2.clone());
+        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 50, bank, 0);
 
-        // Neither slot has rooted votes
-        assert_eq!(
-            block_commitment_cache.get_rooted_block_with_commitment(0.1),
-            None
-        );
-        // Neither slot meets the minimum level of commitment 0.6 at depth 1
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(1, 0.6),
-            None
-        );
-        // Only slot 0 meets the minimum level of commitment 0.5 at depth 1
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(1, 0.5),
-            Some(0)
-        );
-        // If multiple slots meet the minimum level of commitment, method should return the most recent
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(1, 0.4),
-            Some(1)
-        );
-        // If multiple slots meet the minimum level of commitment, method should return the most recent
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(0, 0.6),
-            Some(1)
-        );
-        // Neither slot meets the minimum level of commitment 0.9 at depth 0
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(0, 0.9),
-            None
-        );
-    }
-
-    #[test]
-    fn test_get_rooted_block_with_commitment() {
-        // Build BlockCommitmentCache with rooted votes
-        let mut cache0 = BlockCommitment::new([0; MAX_LOCKOUT_HISTORY]);
-        cache0.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, 40);
-        cache0.increase_confirmation_stake(MAX_LOCKOUT_HISTORY - 1, 10);
-        let mut cache1 = BlockCommitment::new([0; MAX_LOCKOUT_HISTORY]);
-        cache1.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, 30);
-        cache1.increase_confirmation_stake(MAX_LOCKOUT_HISTORY - 1, 10);
-        cache1.increase_confirmation_stake(MAX_LOCKOUT_HISTORY - 2, 10);
-
-        let mut block_commitment = HashMap::new();
-        block_commitment.entry(0).or_insert(cache0.clone());
-        block_commitment.entry(1).or_insert(cache1.clone());
-        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 50);
-
-        // Only slot 0 meets the minimum level of commitment 0.66 at root
-        assert_eq!(
-            block_commitment_cache.get_rooted_block_with_commitment(0.66),
-            Some(0)
-        );
-        // If multiple slots meet the minimum level of commitment, method should return the most recent
-        assert_eq!(
-            block_commitment_cache.get_rooted_block_with_commitment(0.6),
-            Some(1)
-        );
-        // Neither slot meets the minimum level of commitment 0.9 at root
-        assert_eq!(
-            block_commitment_cache.get_rooted_block_with_commitment(0.9),
-            None
-        );
+        assert_eq!(block_commitment_cache.get_confirmation_count(0), Some(2));
+        assert_eq!(block_commitment_cache.get_confirmation_count(1), Some(1));
+        assert_eq!(block_commitment_cache.get_confirmation_count(2), Some(0),);
+        assert_eq!(block_commitment_cache.get_confirmation_count(3), None,);
     }
 
     #[test]

core/src/consensus.rs

@@ -1,3 +1,4 @@
+use crate::progress_map::ProgressMap;
 use chrono::prelude::*;
 use solana_ledger::bank_forks::BankForks;
 use solana_runtime::bank::Bank;
@@ -320,7 +321,7 @@ impl Tower {
     }
     pub fn check_vote_stake_threshold(
         &self,
-        slot: u64,
+        slot: Slot,
         stake_lockouts: &HashMap<u64, StakeLockout>,
         total_staked: u64,
     ) -> bool {
@@ -331,11 +332,8 @@ impl Tower {
             if let Some(fork_stake) = stake_lockouts.get(&vote.slot) {
                 let lockout = fork_stake.stake as f64 / total_staked as f64;
                 trace!(
-                    "fork_stake slot: {} lockout: {} fork_stake: {} total_stake: {}",
-                    slot,
-                    lockout,
-                    fork_stake.stake,
-                    total_staked
+                    "fork_stake slot: {}, vote slot: {}, lockout: {} fork_stake: {} total_stake: {}",
+                    slot, vote.slot, lockout, fork_stake.stake, total_staked
                 );
                 if vote.confirmation_count as usize > self.threshold_depth {
                     for old_vote in &self.lockouts.votes {
@@ -355,6 +353,18 @@ impl Tower {
         }
     }
 
+    pub(crate) fn check_switch_threshold(
+        &self,
+        _slot: Slot,
+        _ancestors: &HashMap<Slot, HashSet<u64>>,
+        _descendants: &HashMap<Slot, HashSet<u64>>,
+        _progress: &ProgressMap,
+        _total_epoch_stake: u64,
+        _epoch_vote_accounts: &HashMap<Pubkey, (u64, Account)>,
+    ) -> bool {
+        true
+    }
+
     /// Update lockouts for all the ancestors
     fn update_ancestor_lockouts(
         stake_lockouts: &mut HashMap<Slot, StakeLockout>,
@@ -468,7 +478,12 @@ impl Tower {
 #[cfg(test)]
 pub mod test {
     use super::*;
-    use crate::replay_stage::{ForkProgress, ReplayStage};
+    use crate::{
+        cluster_info_vote_listener::VoteTracker,
+        cluster_slots::ClusterSlots,
+        progress_map::ForkProgress,
+        replay_stage::{HeaviestForkFailures, ReplayStage},
+    };
     use solana_ledger::bank_forks::BankForks;
     use solana_runtime::{
         bank::Bank,
@@ -481,107 +496,96 @@ pub mod test {
         hash::Hash,
         pubkey::Pubkey,
         signature::{Keypair, Signer},
-        transaction::Transaction,
     };
     use solana_vote_program::{
-        vote_instruction,
         vote_state::{Vote, VoteStateVersions},
+        vote_transaction,
     };
-    use std::collections::{HashMap, VecDeque};
+    use std::collections::HashMap;
     use std::sync::RwLock;
     use std::{thread::sleep, time::Duration};
-    use trees::{tr, Node, Tree};
+    use trees::{tr, Tree, TreeWalk};
 
-    pub(crate) struct VoteSimulator<'a> {
-        searchable_nodes: HashMap<u64, &'a Node<u64>>,
+    pub(crate) struct VoteSimulator {
+        pub validator_keypairs: HashMap<Pubkey, ValidatorVoteKeypairs>,
+        pub node_pubkeys: Vec<Pubkey>,
+        pub vote_pubkeys: Vec<Pubkey>,
+        pub bank_forks: RwLock<BankForks>,
+        pub progress: ProgressMap,
     }
 
-    impl<'a> VoteSimulator<'a> {
-        pub(crate) fn new(forks: &'a Tree<u64>) -> Self {
-            let mut searchable_nodes = HashMap::new();
-            let root = forks.root();
-            searchable_nodes.insert(root.data, root);
-            Self { searchable_nodes }
+    impl VoteSimulator {
+        pub(crate) fn new(num_keypairs: usize) -> Self {
+            let (validator_keypairs, node_pubkeys, vote_pubkeys, bank_forks, progress) =
+                Self::init_state(num_keypairs);
+            Self {
+                validator_keypairs,
+                node_pubkeys,
+                vote_pubkeys,
+                bank_forks: RwLock::new(bank_forks),
+                progress,
+            }
+        }
+        pub(crate) fn fill_bank_forks(
+            &mut self,
+            forks: Tree<u64>,
+            cluster_votes: &HashMap<Pubkey, Vec<u64>>,
+        ) {
+            let root = forks.root().data;
+            assert!(self.bank_forks.read().unwrap().get(root).is_some());
+
+            let mut walk = TreeWalk::from(forks);
+            loop {
+                if let Some(visit) = walk.get() {
+                    let slot = visit.node().data;
+                    self.progress
+                        .entry(slot)
+                        .or_insert_with(|| ForkProgress::new(Hash::default(), None, None, 0, 0));
+                    if self.bank_forks.read().unwrap().get(slot).is_some() {
+                        walk.forward();
+                        continue;
+                    }
+                    let parent = walk.get_parent().unwrap().data;
+                    let parent_bank = self.bank_forks.read().unwrap().get(parent).unwrap().clone();
+                    let new_bank = Bank::new_from_parent(&parent_bank, &Pubkey::default(), slot);
+                    for (pubkey, vote) in cluster_votes.iter() {
+                        if vote.contains(&parent) {
+                            let keypairs = self.validator_keypairs.get(pubkey).unwrap();
+                            let last_blockhash = parent_bank.last_blockhash();
+                            let vote_tx = vote_transaction::new_vote_transaction(
+                                // Must vote > root to be processed
+                                vec![parent],
+                                parent_bank.hash(),
+                                last_blockhash,
+                                &keypairs.node_keypair,
+                                &keypairs.vote_keypair,
+                                &keypairs.vote_keypair,
+                            );
+                            info!("voting {} {}", parent_bank.slot(), parent_bank.hash());
+                            new_bank.process_transaction(&vote_tx).unwrap();
+                        }
+                    }
+                    new_bank.freeze();
+                    self.bank_forks.write().unwrap().insert(new_bank);
+                    walk.forward();
+                } else {
+                    break;
+                }
+            }
         }
 
         pub(crate) fn simulate_vote(
             &mut self,
             vote_slot: Slot,
-            bank_forks: &RwLock<BankForks>,
-            cluster_votes: &mut HashMap<Pubkey, Vec<u64>>,
-            validator_keypairs: &HashMap<Pubkey, ValidatorVoteKeypairs>,
-            my_keypairs: &ValidatorVoteKeypairs,
-            progress: &mut HashMap<u64, ForkProgress>,
+            my_pubkey: &Pubkey,
             tower: &mut Tower,
-        ) -> Vec<VoteFailures> {
-            let node = self
-                .find_node_and_update_simulation(vote_slot)
-                .expect("Vote to simulate must be for a slot in the tree");
-
-            let mut missing_nodes = VecDeque::new();
-            let mut current = node;
-            loop {
-                let current_slot = current.data;
-                if bank_forks.read().unwrap().get(current_slot).is_some()
-                    || tower.root().map(|r| current_slot < r).unwrap_or(false)
-                {
-                    break;
-                } else {
-                    missing_nodes.push_front(current);
-                }
-
-                if let Some(parent) = current.parent() {
-                    current = parent;
-                } else {
-                    break;
-                }
-            }
-
-            // Create any missing banks along the path
-            for missing_node in missing_nodes {
-                let missing_slot = missing_node.data;
-                let parent = missing_node.parent().unwrap().data;
-                let parent_bank = bank_forks
-                    .read()
-                    .unwrap()
-                    .get(parent)
-                    .expect("parent bank must exist")
-                    .clone();
-                info!("parent of {} is {}", missing_slot, parent_bank.slot(),);
-                progress
-                    .entry(missing_slot)
-                    .or_insert_with(|| ForkProgress::new(parent_bank.last_blockhash()));
-
-                // Create the missing bank
-                let new_bank =
-                    Bank::new_from_parent(&parent_bank, &Pubkey::default(), missing_slot);
-
-                // Simulate ingesting the cluster's votes for the parent into this bank
-                for (pubkey, vote) in cluster_votes.iter() {
-                    if vote.contains(&parent_bank.slot()) {
-                        let keypairs = validator_keypairs.get(pubkey).unwrap();
-                        let node_pubkey = keypairs.node_keypair.pubkey();
-                        let vote_pubkey = keypairs.vote_keypair.pubkey();
-                        let last_blockhash = parent_bank.last_blockhash();
-                        let votes = Vote::new(vec![parent_bank.slot()], parent_bank.hash());
-                        info!("voting {} {}", parent_bank.slot(), parent_bank.hash());
-                        let vote_ix = vote_instruction::vote(&vote_pubkey, &vote_pubkey, votes);
-                        let mut vote_tx =
-                            Transaction::new_with_payer(vec![vote_ix], Some(&node_pubkey));
-                        vote_tx.partial_sign(&[&keypairs.node_keypair], last_blockhash);
-                        vote_tx.partial_sign(&[&keypairs.vote_keypair], last_blockhash);
-                        new_bank.process_transaction(&vote_tx).unwrap();
-                    }
-                }
-                new_bank.freeze();
-                bank_forks.write().unwrap().insert(new_bank);
-            }
-
-            // Now try to simulate the vote
-            let my_pubkey = my_keypairs.node_keypair.pubkey();
+        ) -> Vec<HeaviestForkFailures> {
+            // Try to simulate the vote
+            let my_keypairs = self.validator_keypairs.get(&my_pubkey).unwrap();
             let my_vote_pubkey = my_keypairs.vote_keypair.pubkey();
-            let ancestors = bank_forks.read().unwrap().ancestors();
-            let mut frozen_banks: Vec<_> = bank_forks
+            let ancestors = self.bank_forks.read().unwrap().ancestors();
+            let mut frozen_banks: Vec<_> = self
+                .bank_forks
                 .read()
                 .unwrap()
                 .frozen_banks()
@@ -594,99 +598,132 @@ pub mod test {
                 &ancestors,
                 &mut frozen_banks,
                 tower,
-                progress,
+                &mut self.progress,
+                &VoteTracker::default(),
+                &ClusterSlots::default(),
+                &self.bank_forks,
+                &mut HashSet::new(),
             );
 
-            let bank = bank_forks
+            let vote_bank = self
+                .bank_forks
                 .read()
                 .unwrap()
                 .get(vote_slot)
                 .expect("Bank must have been created before vote simulation")
                 .clone();
+
+            // Try to vote on the given slot
+            let descendants = self.bank_forks.read().unwrap().descendants();
+            let (_, _, failure_reasons) = ReplayStage::select_vote_and_reset_forks(
+                &Some(vote_bank.clone()),
+                &None,
+                &ancestors,
+                &descendants,
+                &self.progress,
+                &tower,
+            );
+
             // Make sure this slot isn't locked out or failing threshold
-            let fork_progress = progress
-                .get(&vote_slot)
-                .expect("Slot for vote must exist in progress map");
-            info!("Checking vote: {}", vote_slot);
-            info!("lockouts: {:?}", fork_progress.fork_stats.stake_lockouts);
-            let mut failures = vec![];
-            if fork_progress.fork_stats.is_locked_out {
-                failures.push(VoteFailures::LockedOut(vote_slot));
+            info!("Checking vote: {}", vote_bank.slot());
+            if !failure_reasons.is_empty() {
+                return failure_reasons;
             }
-            if !fork_progress.fork_stats.vote_threshold {
-                failures.push(VoteFailures::FailedThreshold(vote_slot));
-            }
-            if !failures.is_empty() {
-                return failures;
-            }
-            let vote = tower.new_vote_from_bank(&bank, &my_vote_pubkey).0;
+            let vote = tower.new_vote_from_bank(&vote_bank, &my_vote_pubkey).0;
             if let Some(new_root) = tower.record_bank_vote(vote) {
-                ReplayStage::handle_new_root(new_root, bank_forks, progress, &None);
+                ReplayStage::handle_new_root(
+                    new_root,
+                    &self.bank_forks,
+                    &mut self.progress,
+                    &None,
+                    &mut HashSet::new(),
+                );
             }
 
-            // Mark the vote for this bank under this node's pubkey so it will be
-            // integrated into any future child banks
-            cluster_votes.entry(my_pubkey).or_default().push(vote_slot);
             vec![]
         }
 
-        // Find a node representing the given slot
-        fn find_node_and_update_simulation(&mut self, slot: u64) -> Option<&'a Node<u64>> {
-            let mut successful_search_node: Option<&'a Node<u64>> = None;
-            let mut found_node = None;
-            for search_node in self.searchable_nodes.values() {
-                if let Some((target, new_searchable_nodes)) = Self::find_node(search_node, slot) {
-                    successful_search_node = Some(search_node);
-                    found_node = Some(target);
-                    for node in new_searchable_nodes {
-                        self.searchable_nodes.insert(node.data, node);
-                    }
-                    break;
+        fn can_progress_on_fork(
+            &mut self,
+            my_pubkey: &Pubkey,
+            tower: &mut Tower,
+            start_slot: u64,
+            num_slots: u64,
+            cluster_votes: &mut HashMap<Pubkey, Vec<u64>>,
+        ) -> bool {
+            // Check that within some reasonable time, validator can make a new
+            // root on this fork
+            let old_root = tower.root();
+
+            for i in 1..num_slots {
+                // The parent of the tip of the fork
+                let mut fork_tip_parent = tr(start_slot + i - 1);
+                // The tip of the fork
+                fork_tip_parent.push_front(tr(start_slot + i));
+                self.fill_bank_forks(fork_tip_parent, cluster_votes);
+                if self
+                    .simulate_vote(i + start_slot, &my_pubkey, tower)
+                    .is_empty()
+                {
+                    cluster_votes
+                        .entry(*my_pubkey)
+                        .or_default()
+                        .push(start_slot + i);
+                }
+                if old_root != tower.root() {
+                    return true;
                 }
             }
-            successful_search_node.map(|node| {
-                self.searchable_nodes.remove(&node.data);
-            });
-            found_node
+
+            false
         }
 
-        fn find_node(
-            node: &'a Node<u64>,
-            slot: u64,
-        ) -> Option<(&'a Node<u64>, Vec<&'a Node<u64>>)> {
-            if node.data == slot {
-                Some((node, node.iter().collect()))
-            } else {
-                let mut search_result: Option<(&'a Node<u64>, Vec<&'a Node<u64>>)> = None;
-                for child in node.iter() {
-                    if let Some((_, ref mut new_searchable_nodes)) = search_result {
-                        new_searchable_nodes.push(child);
-                        continue;
-                    }
-                    search_result = Self::find_node(child, slot);
-                }
+        fn init_state(
+            num_keypairs: usize,
+        ) -> (
+            HashMap<Pubkey, ValidatorVoteKeypairs>,
+            Vec<Pubkey>,
+            Vec<Pubkey>,
+            BankForks,
+            ProgressMap,
+        ) {
+            let keypairs: HashMap<_, _> = std::iter::repeat_with(|| {
+                let node_keypair = Keypair::new();
+                let vote_keypair = Keypair::new();
+                let stake_keypair = Keypair::new();
+                let node_pubkey = node_keypair.pubkey();
+                (
+                    node_pubkey,
+                    ValidatorVoteKeypairs::new(node_keypair, vote_keypair, stake_keypair),
+                )
+            })
+            .take(num_keypairs)
+            .collect();
+            let node_pubkeys: Vec<_> = keypairs
+                .values()
+                .map(|keys| keys.node_keypair.pubkey())
+                .collect();
+            let vote_pubkeys: Vec<_> = keypairs
+                .values()
+                .map(|keys| keys.vote_keypair.pubkey())
+                .collect();
 
-                search_result
-            }
+            let (bank_forks, progress) = initialize_state(&keypairs, 10_000);
+            (keypairs, node_pubkeys, vote_pubkeys, bank_forks, progress)
         }
     }
 
-    #[derive(PartialEq, Debug)]
-    pub(crate) enum VoteFailures {
-        LockedOut(u64),
-        FailedThreshold(u64),
-    }
-
     // Setup BankForks with bank 0 and all the validator accounts
     pub(crate) fn initialize_state(
         validator_keypairs_map: &HashMap<Pubkey, ValidatorVoteKeypairs>,
-    ) -> (BankForks, HashMap<u64, ForkProgress>) {
+        stake: u64,
+    ) -> (BankForks, ProgressMap) {
         let validator_keypairs: Vec<_> = validator_keypairs_map.values().collect();
         let GenesisConfigInfo {
             genesis_config,
             mint_keypair,
             voting_keypair: _,
-        } = create_genesis_config_with_vote_accounts(1_000_000_000, &validator_keypairs);
+        } = create_genesis_config_with_vote_accounts(1_000_000_000, &validator_keypairs, stake);
 
         let bank0 = Bank::new(&genesis_config);
 
@@ -695,8 +732,11 @@ pub mod test {
         }
 
         bank0.freeze();
-        let mut progress = HashMap::new();
-        progress.insert(0, ForkProgress::new(bank0.last_blockhash()));
+        let mut progress = ProgressMap::default();
+        progress.insert(
+            0,
+            ForkProgress::new(bank0.last_blockhash(), None, None, 0, 0),
+        );
         (BankForks::new(0, bank0), progress)
     }
 
@@ -720,84 +760,26 @@ pub mod test {
         stakes
     }
 
-    fn can_progress_on_fork(
-        my_pubkey: &Pubkey,
-        tower: &mut Tower,
-        start_slot: u64,
-        num_slots: u64,
-        bank_forks: &RwLock<BankForks>,
-        cluster_votes: &mut HashMap<Pubkey, Vec<u64>>,
-        keypairs: &HashMap<Pubkey, ValidatorVoteKeypairs>,
-        progress: &mut HashMap<u64, ForkProgress>,
-    ) -> bool {
-        // Check that within some reasonable time, validator can make a new
-        // root on this fork
-        let old_root = tower.root();
-        let mut main_fork = tr(start_slot);
-        let mut tip = main_fork.root_mut();
-
-        for i in 1..num_slots {
-            tip.push_front(tr(start_slot + i));
-            tip = tip.first_mut().unwrap();
-        }
-        let mut voting_simulator = VoteSimulator::new(&main_fork);
-        for i in 1..num_slots {
-            voting_simulator.simulate_vote(
-                i + start_slot,
-                &bank_forks,
-                cluster_votes,
-                &keypairs,
-                keypairs.get(&my_pubkey).unwrap(),
-                progress,
-                tower,
-            );
-            if old_root != tower.root() {
-                return true;
-            }
-        }
-
-        false
-    }
-
     #[test]
     fn test_simple_votes() {
-        let node_keypair = Keypair::new();
-        let vote_keypair = Keypair::new();
-        let stake_keypair = Keypair::new();
-        let node_pubkey = node_keypair.pubkey();
-
-        let mut keypairs = HashMap::new();
-        keypairs.insert(
-            node_pubkey,
-            ValidatorVoteKeypairs::new(node_keypair, vote_keypair, stake_keypair),
-        );
-
-        // Initialize BankForks
-        let (bank_forks, mut progress) = initialize_state(&keypairs);
-        let bank_forks = RwLock::new(bank_forks);
+        // Init state
+        let mut vote_simulator = VoteSimulator::new(1);
+        let node_pubkey = vote_simulator.node_pubkeys[0];
+        let mut tower = Tower::new_with_key(&node_pubkey);
 
         // Create the tree of banks
         let forks = tr(0) / (tr(1) / (tr(2) / (tr(3) / (tr(4) / tr(5)))));
 
         // Set the voting behavior
-        let mut voting_simulator = VoteSimulator::new(&forks);
+        let mut cluster_votes = HashMap::new();
         let votes = vec![0, 1, 2, 3, 4, 5];
+        cluster_votes.insert(node_pubkey, votes.clone());
+        vote_simulator.fill_bank_forks(forks, &cluster_votes);
 
         // Simulate the votes
-        let mut tower = Tower::new_with_key(&node_pubkey);
-
-        let mut cluster_votes = HashMap::new();
         for vote in votes {
-            assert!(voting_simulator
-                .simulate_vote(
-                    vote,
-                    &bank_forks,
-                    &mut cluster_votes,
-                    &keypairs,
-                    keypairs.get(&node_pubkey).unwrap(),
-                    &mut progress,
-                    &mut tower,
-                )
+            assert!(vote_simulator
+                .simulate_vote(vote, &node_pubkey, &mut tower,)
                 .is_empty());
         }
 
@@ -809,21 +791,14 @@ pub mod test {
 
     #[test]
     fn test_double_partition() {
-        solana_logger::setup();
-        let node_keypair = Keypair::new();
-        let vote_keypair = Keypair::new();
-        let stake_keypair = Keypair::new();
-        let node_pubkey = node_keypair.pubkey();
-        let vote_pubkey = vote_keypair.pubkey();
+        // Init state
+        let mut vote_simulator = VoteSimulator::new(2);
+        let node_pubkey = vote_simulator.node_pubkeys[0];
+        let vote_pubkey = vote_simulator.vote_pubkeys[0];
+        let mut tower = Tower::new_with_key(&node_pubkey);
 
-        let mut keypairs = HashMap::new();
-        info!("my_pubkey: {}", node_pubkey);
-        keypairs.insert(
-            node_pubkey,
-            ValidatorVoteKeypairs::new(node_keypair, vote_keypair, stake_keypair),
-        );
-
-        // Create the tree of banks in a BankForks object
+        let num_slots_to_try = 200;
+        // Create the tree of banks
         let forks = tr(0)
             / (tr(1)
                 / (tr(2)
@@ -840,56 +815,37 @@ pub mod test {
                                                     / (tr(44)
                                                         // Minor fork 2
                                                         / (tr(45) / (tr(46) / (tr(47) / (tr(48) / (tr(49) / (tr(50)))))))
-                                                        / (tr(110)))))))))))));
+                                                        / (tr(110) / (tr(110 + 2 * num_slots_to_try))))))))))))));
 
-        // Set the voting behavior
-        let mut voting_simulator = VoteSimulator::new(&forks);
-        let mut votes: Vec<Slot> = vec![];
+        // Set the successful voting behavior
+        let mut cluster_votes = HashMap::new();
+        let mut my_votes: Vec<Slot> = vec![];
+        let next_unlocked_slot = 110;
         // Vote on the first minor fork
-        votes.extend((0..=14).into_iter());
+        my_votes.extend((0..=14).into_iter());
         // Come back to the main fork
-        votes.extend((43..=44).into_iter());
+        my_votes.extend((43..=44).into_iter());
         // Vote on the second minor fork
-        votes.extend((45..=50).into_iter());
+        my_votes.extend((45..=50).into_iter());
+        // Vote to come back to main fork
+        my_votes.push(next_unlocked_slot);
+        cluster_votes.insert(node_pubkey, my_votes.clone());
+        // Make the other validator vote fork to pass the threshold checks
+        let other_votes = my_votes.clone();
+        cluster_votes.insert(vote_simulator.node_pubkeys[1], other_votes);
+        vote_simulator.fill_bank_forks(forks, &cluster_votes);
 
-        let mut cluster_votes: HashMap<Pubkey, Vec<Slot>> = HashMap::new();
-        let (bank_forks, mut progress) = initialize_state(&keypairs);
-        let bank_forks = RwLock::new(bank_forks);
-
-        // Simulate the votes. Should fail on trying to come back to the main fork
-        // at 106 exclusively due to threshold failure
-        let mut tower = Tower::new_with_key(&node_pubkey);
-        for vote in &votes {
+        // Simulate the votes.
+        for vote in &my_votes {
             // All these votes should be ok
-            assert!(voting_simulator
-                .simulate_vote(
-                    *vote,
-                    &bank_forks,
-                    &mut cluster_votes,
-                    &keypairs,
-                    keypairs.get(&node_pubkey).unwrap(),
-                    &mut progress,
-                    &mut tower,
-                )
+            assert!(vote_simulator
+                .simulate_vote(*vote, &node_pubkey, &mut tower,)
                 .is_empty());
         }
 
-        // Try to come back to main fork
-        let next_unlocked_slot = 110;
-        assert!(voting_simulator
-            .simulate_vote(
-                next_unlocked_slot,
-                &bank_forks,
-                &mut cluster_votes,
-                &keypairs,
-                keypairs.get(&node_pubkey).unwrap(),
-                &mut progress,
-                &mut tower,
-            )
-            .is_empty());
-
         info!("local tower: {:#?}", tower.lockouts.votes);
-        let vote_accounts = bank_forks
+        let vote_accounts = vote_simulator
+            .bank_forks
             .read()
             .unwrap()
             .get(next_unlocked_slot)
@@ -899,15 +855,17 @@ pub mod test {
         let state = VoteState::from(&observed.1).unwrap();
         info!("observed tower: {:#?}", state.votes);
 
-        assert!(can_progress_on_fork(
+        let num_slots_to_try = 200;
+        cluster_votes
+            .get_mut(&vote_simulator.node_pubkeys[1])
+            .unwrap()
+            .extend(next_unlocked_slot + 1..next_unlocked_slot + num_slots_to_try);
+        assert!(vote_simulator.can_progress_on_fork(
             &node_pubkey,
             &mut tower,
             next_unlocked_slot,
-            200,
-            &bank_forks,
+            num_slots_to_try,
             &mut cluster_votes,
-            &keypairs,
-            &mut progress
         ));
     }
 

core/src/crds_gossip_pull.rs

@@ -278,7 +278,7 @@ impl CrdsGossipPull {
         failed
     }
     // build a set of filters of the current crds table
-    // num_filters - used to increase the likely hood of a value in crds being added to some filter
+    // num_filters - used to increase the likelyhood of a value in crds being added to some filter
     pub fn build_crds_filters(&self, crds: &Crds, bloom_size: usize) -> Vec<CrdsFilter> {
         let num = cmp::max(
             CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS,

core/src/crds_value.rs

@@ -1,5 +1,8 @@
 use crate::contact_info::ContactInfo;
+use crate::deprecated;
+use crate::epoch_slots::EpochSlots;
 use bincode::{serialize, serialized_size};
+use solana_sdk::timing::timestamp;
 use solana_sdk::{
     clock::Slot,
     hash::Hash,
@@ -16,7 +19,8 @@ use std::{
 pub type VoteIndex = u8;
 pub const MAX_VOTES: VoteIndex = 32;
 
-pub type EpochSlotIndex = u8;
+pub type EpochSlotsIndex = u8;
+pub const MAX_EPOCH_SLOTS: EpochSlotsIndex = 255;
 
 /// CrdsValue that is replicated across the cluster
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -48,6 +52,7 @@ impl Signable for CrdsValue {
             .verify(&self.pubkey().as_ref(), self.signable_data().borrow());
         let data_check = match &self.data {
             CrdsData::Vote(ix, _) => *ix < MAX_VOTES,
+            CrdsData::EpochSlots(ix, _) => *ix < MAX_EPOCH_SLOTS,
             _ => true,
         };
         sig_check && data_check
@@ -56,33 +61,16 @@ impl Signable for CrdsValue {
 
 /// CrdsData that defines the different types of items CrdsValues can hold
 /// * Merge Strategy - Latest wallclock is picked
+/// * LowestSlot index is deprecated
 #[allow(clippy::large_enum_variant)]
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub enum CrdsData {
     ContactInfo(ContactInfo),
     Vote(VoteIndex, Vote),
-    EpochSlots(EpochSlotIndex, EpochSlots),
-    SnapshotHash(SnapshotHash),
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-pub enum CompressionType {
-    Uncompressed,
-    GZip,
-    BZip2,
-}
-
-impl Default for CompressionType {
-    fn default() -> Self {
-        Self::Uncompressed
-    }
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
-pub struct EpochIncompleteSlots {
-    pub first: Slot,
-    pub compression: CompressionType,
-    pub compressed_list: Vec<u8>,
+    LowestSlot(u8, LowestSlot),
+    SnapshotHashes(SnapshotHash),
+    AccountsHashes(SnapshotHash),
+    EpochSlots(EpochSlotsIndex, EpochSlots),
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -93,40 +81,32 @@ pub struct SnapshotHash {
 }
 
 impl SnapshotHash {
-    pub fn new(from: Pubkey, hashes: Vec<(Slot, Hash)>, wallclock: u64) -> Self {
+    pub fn new(from: Pubkey, hashes: Vec<(Slot, Hash)>) -> Self {
         Self {
             from,
             hashes,
-            wallclock,
+            wallclock: timestamp(),
         }
     }
 }
-
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-pub struct EpochSlots {
+pub struct LowestSlot {
     pub from: Pubkey,
-    pub root: Slot,
+    root: Slot, //deprecated
     pub lowest: Slot,
-    pub slots: BTreeSet<Slot>,
-    pub stash: Vec<EpochIncompleteSlots>,
+    slots: BTreeSet<Slot>,                        //deprecated
+    stash: Vec<deprecated::EpochIncompleteSlots>, //deprecated
     pub wallclock: u64,
 }
 
-impl EpochSlots {
-    pub fn new(
-        from: Pubkey,
-        root: Slot,
-        lowest: Slot,
-        slots: BTreeSet<Slot>,
-        stash: Vec<EpochIncompleteSlots>,
-        wallclock: u64,
-    ) -> Self {
+impl LowestSlot {
+    pub fn new(from: Pubkey, lowest: Slot, wallclock: u64) -> Self {
         Self {
             from,
-            root,
+            root: 0,
             lowest,
-            slots,
-            stash,
+            slots: BTreeSet::new(),
+            stash: vec![],
             wallclock,
         }
     }
@@ -155,8 +135,10 @@ impl Vote {
 pub enum CrdsValueLabel {
     ContactInfo(Pubkey),
     Vote(VoteIndex, Pubkey),
-    EpochSlots(Pubkey),
-    SnapshotHash(Pubkey),
+    LowestSlot(Pubkey),
+    SnapshotHashes(Pubkey),
+    EpochSlots(EpochSlotsIndex, Pubkey),
+    AccountsHashes(Pubkey),
 }
 
 impl fmt::Display for CrdsValueLabel {
@@ -164,8 +146,10 @@ impl fmt::Display for CrdsValueLabel {
         match self {
             CrdsValueLabel::ContactInfo(_) => write!(f, "ContactInfo({})", self.pubkey()),
             CrdsValueLabel::Vote(ix, _) => write!(f, "Vote({}, {})", ix, self.pubkey()),
-            CrdsValueLabel::EpochSlots(_) => write!(f, "EpochSlots({})", self.pubkey()),
-            CrdsValueLabel::SnapshotHash(_) => write!(f, "SnapshotHash({})", self.pubkey()),
+            CrdsValueLabel::LowestSlot(_) => write!(f, "LowestSlot({})", self.pubkey()),
+            CrdsValueLabel::SnapshotHashes(_) => write!(f, "SnapshotHash({})", self.pubkey()),
+            CrdsValueLabel::EpochSlots(ix, _) => write!(f, "EpochSlots({}, {})", ix, self.pubkey()),
+            CrdsValueLabel::AccountsHashes(_) => write!(f, "AccountsHashes({})", self.pubkey()),
         }
     }
 }
@@ -175,8 +159,10 @@ impl CrdsValueLabel {
         match self {
             CrdsValueLabel::ContactInfo(p) => *p,
             CrdsValueLabel::Vote(_, p) => *p,
-            CrdsValueLabel::EpochSlots(p) => *p,
-            CrdsValueLabel::SnapshotHash(p) => *p,
+            CrdsValueLabel::LowestSlot(p) => *p,
+            CrdsValueLabel::SnapshotHashes(p) => *p,
+            CrdsValueLabel::EpochSlots(_, p) => *p,
+            CrdsValueLabel::AccountsHashes(p) => *p,
         }
     }
 }
@@ -201,24 +187,30 @@ impl CrdsValue {
         match &self.data {
             CrdsData::ContactInfo(contact_info) => contact_info.wallclock,
             CrdsData::Vote(_, vote) => vote.wallclock,
-            CrdsData::EpochSlots(_, vote) => vote.wallclock,
-            CrdsData::SnapshotHash(hash) => hash.wallclock,
+            CrdsData::LowestSlot(_, obj) => obj.wallclock,
+            CrdsData::SnapshotHashes(hash) => hash.wallclock,
+            CrdsData::AccountsHashes(hash) => hash.wallclock,
+            CrdsData::EpochSlots(_, p) => p.wallclock,
         }
     }
     pub fn pubkey(&self) -> Pubkey {
         match &self.data {
             CrdsData::ContactInfo(contact_info) => contact_info.id,
             CrdsData::Vote(_, vote) => vote.from,
-            CrdsData::EpochSlots(_, slots) => slots.from,
-            CrdsData::SnapshotHash(hash) => hash.from,
+            CrdsData::LowestSlot(_, slots) => slots.from,
+            CrdsData::SnapshotHashes(hash) => hash.from,
+            CrdsData::AccountsHashes(hash) => hash.from,
+            CrdsData::EpochSlots(_, p) => p.from,
         }
     }
     pub fn label(&self) -> CrdsValueLabel {
         match &self.data {
             CrdsData::ContactInfo(_) => CrdsValueLabel::ContactInfo(self.pubkey()),
             CrdsData::Vote(ix, _) => CrdsValueLabel::Vote(*ix, self.pubkey()),
-            CrdsData::EpochSlots(_, _) => CrdsValueLabel::EpochSlots(self.pubkey()),
-            CrdsData::SnapshotHash(_) => CrdsValueLabel::SnapshotHash(self.pubkey()),
+            CrdsData::LowestSlot(_, _) => CrdsValueLabel::LowestSlot(self.pubkey()),
+            CrdsData::SnapshotHashes(_) => CrdsValueLabel::SnapshotHashes(self.pubkey()),
+            CrdsData::AccountsHashes(_) => CrdsValueLabel::AccountsHashes(self.pubkey()),
+            CrdsData::EpochSlots(ix, _) => CrdsValueLabel::EpochSlots(*ix, self.pubkey()),
         }
     }
     pub fn contact_info(&self) -> Option<&ContactInfo> {
@@ -241,16 +233,30 @@ impl CrdsValue {
         }
     }
 
-    pub fn epoch_slots(&self) -> Option<&EpochSlots> {
+    pub fn lowest_slot(&self) -> Option<&LowestSlot> {
         match &self.data {
-            CrdsData::EpochSlots(_, slots) => Some(slots),
+            CrdsData::LowestSlot(_, slots) => Some(slots),
             _ => None,
         }
     }
 
     pub fn snapshot_hash(&self) -> Option<&SnapshotHash> {
         match &self.data {
-            CrdsData::SnapshotHash(slots) => Some(slots),
+            CrdsData::SnapshotHashes(slots) => Some(slots),
+            _ => None,
+        }
+    }
+
+    pub fn accounts_hash(&self) -> Option<&SnapshotHash> {
+        match &self.data {
+            CrdsData::AccountsHashes(slots) => Some(slots),
+            _ => None,
+        }
+    }
+
+    pub fn epoch_slots(&self) -> Option<&EpochSlots> {
+        match &self.data {
+            CrdsData::EpochSlots(_, slots) => Some(slots),
             _ => None,
         }
     }
@@ -259,10 +265,12 @@ impl CrdsValue {
     pub fn record_labels(key: &Pubkey) -> Vec<CrdsValueLabel> {
         let mut labels = vec![
             CrdsValueLabel::ContactInfo(*key),
-            CrdsValueLabel::EpochSlots(*key),
-            CrdsValueLabel::SnapshotHash(*key),
+            CrdsValueLabel::LowestSlot(*key),
+            CrdsValueLabel::SnapshotHashes(*key),
+            CrdsValueLabel::AccountsHashes(*key),
         ];
         labels.extend((0..MAX_VOTES).map(|ix| CrdsValueLabel::Vote(ix, *key)));
+        labels.extend((0..MAX_EPOCH_SLOTS).map(|ix| CrdsValueLabel::EpochSlots(ix, *key)));
         labels
     }
 
@@ -310,14 +318,18 @@ mod test {
 
     #[test]
     fn test_labels() {
-        let mut hits = [false; 3 + MAX_VOTES as usize];
+        let mut hits = [false; 4 + MAX_VOTES as usize + MAX_EPOCH_SLOTS as usize];
         // this method should cover all the possible labels
         for v in &CrdsValue::record_labels(&Pubkey::default()) {
             match v {
                 CrdsValueLabel::ContactInfo(_) => hits[0] = true,
-                CrdsValueLabel::EpochSlots(_) => hits[1] = true,
-                CrdsValueLabel::SnapshotHash(_) => hits[2] = true,
-                CrdsValueLabel::Vote(ix, _) => hits[*ix as usize + 3] = true,
+                CrdsValueLabel::LowestSlot(_) => hits[1] = true,
+                CrdsValueLabel::SnapshotHashes(_) => hits[2] = true,
+                CrdsValueLabel::AccountsHashes(_) => hits[3] = true,
+                CrdsValueLabel::Vote(ix, _) => hits[*ix as usize + 4] = true,
+                CrdsValueLabel::EpochSlots(ix, _) => {
+                    hits[*ix as usize + MAX_VOTES as usize + 4] = true
+                }
             }
         }
         assert!(hits.iter().all(|x| *x));
@@ -337,13 +349,13 @@ mod test {
         let key = v.clone().vote().unwrap().from;
         assert_eq!(v.label(), CrdsValueLabel::Vote(0, key));
 
-        let v = CrdsValue::new_unsigned(CrdsData::EpochSlots(
+        let v = CrdsValue::new_unsigned(CrdsData::LowestSlot(
             0,
-            EpochSlots::new(Pubkey::default(), 0, 0, BTreeSet::new(), vec![], 0),
+            LowestSlot::new(Pubkey::default(), 0, 0),
         ));
         assert_eq!(v.wallclock(), 0);
-        let key = v.clone().epoch_slots().unwrap().from;
-        assert_eq!(v.label(), CrdsValueLabel::EpochSlots(key));
+        let key = v.clone().lowest_slot().unwrap().from;
+        assert_eq!(v.label(), CrdsValueLabel::LowestSlot(key));
     }
 
     #[test]
@@ -360,10 +372,9 @@ mod test {
             Vote::new(&keypair.pubkey(), test_tx(), timestamp()),
         ));
         verify_signatures(&mut v, &keypair, &wrong_keypair);
-        let btreeset: BTreeSet<Slot> = vec![1, 2, 3, 6, 8].into_iter().collect();
-        v = CrdsValue::new_unsigned(CrdsData::EpochSlots(
+        v = CrdsValue::new_unsigned(CrdsData::LowestSlot(
             0,
-            EpochSlots::new(keypair.pubkey(), 0, 0, btreeset, vec![], timestamp()),
+            LowestSlot::new(keypair.pubkey(), 0, timestamp()),
         ));
         verify_signatures(&mut v, &keypair, &wrong_keypair);
     }
@@ -381,6 +392,18 @@ mod test {
         assert!(!vote.verify());
     }
 
+    #[test]
+    fn test_max_epoch_slots_index() {
+        let keypair = Keypair::new();
+        let item = CrdsValue::new_signed(
+            CrdsData::EpochSlots(
+                MAX_EPOCH_SLOTS,
+                EpochSlots::new(keypair.pubkey(), timestamp()),
+            ),
+            &keypair,
+        );
+        assert!(!item.verify());
+    }
     #[test]
     fn test_compute_vote_index_empty() {
         for i in 0..MAX_VOTES {

core/src/deprecated.rs

@@ -0,0 +1,21 @@
+use solana_sdk::clock::Slot;
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+enum CompressionType {
+    Uncompressed,
+    GZip,
+    BZip2,
+}
+
+impl Default for CompressionType {
+    fn default() -> Self {
+        Self::Uncompressed
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
+pub(crate) struct EpochIncompleteSlots {
+    first: Slot,
+    compression: CompressionType,
+    compressed_list: Vec<u8>,
+}

core/src/epoch_slots.rs

@@ -0,0 +1,401 @@
+use crate::cluster_info::MAX_CRDS_OBJECT_SIZE;
+use bincode::serialized_size;
+use bv::BitVec;
+use flate2::{Compress, Compression, Decompress, FlushCompress, FlushDecompress};
+use solana_sdk::clock::Slot;
+use solana_sdk::pubkey::Pubkey;
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+pub struct Uncompressed {
+    pub first_slot: Slot,
+    pub num: usize,
+    pub slots: BitVec<u8>,
+}
+
+#[derive(Deserialize, Serialize, Clone, Debug, PartialEq)]
+pub struct Flate2 {
+    pub first_slot: Slot,
+    pub num: usize,
+    pub compressed: Vec<u8>,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Error {
+    CompressError,
+    DecompressError,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl std::convert::From<flate2::CompressError> for Error {
+    fn from(_e: flate2::CompressError) -> Error {
+        Error::CompressError
+    }
+}
+impl std::convert::From<flate2::DecompressError> for Error {
+    fn from(_e: flate2::DecompressError) -> Error {
+        Error::DecompressError
+    }
+}
+
+impl Flate2 {
+    fn deflate(mut unc: Uncompressed) -> Result<Self> {
+        let mut compressed = Vec::with_capacity(unc.slots.block_capacity());
+        let mut compressor = Compress::new(Compression::best(), false);
+        let first_slot = unc.first_slot;
+        let num = unc.num;
+        unc.slots.shrink_to_fit();
+        let bits = unc.slots.into_boxed_slice();
+        compressor.compress_vec(&bits, &mut compressed, FlushCompress::Finish)?;
+        let rv = Self {
+            first_slot,
+            num,
+            compressed,
+        };
+        let _ = rv.inflate()?;
+        Ok(rv)
+    }
+    pub fn inflate(&self) -> Result<Uncompressed> {
+        //add some head room for the decompressor which might spill more bits
+        let mut uncompressed = Vec::with_capacity(32 + (self.num + 4) / 8);
+        let mut decompress = Decompress::new(false);
+        decompress.decompress_vec(&self.compressed, &mut uncompressed, FlushDecompress::Finish)?;
+        Ok(Uncompressed {
+            first_slot: self.first_slot,
+            num: self.num,
+            slots: BitVec::from_bits(&uncompressed),
+        })
+    }
+}
+
+impl Uncompressed {
+    pub fn new(max_size: usize) -> Self {
+        Self {
+            num: 0,
+            first_slot: 0,
+            slots: BitVec::new_fill(false, 8 * max_size as u64),
+        }
+    }
+    pub fn to_slots(&self, min_slot: Slot) -> Vec<Slot> {
+        let mut rv = vec![];
+        let start = if min_slot < self.first_slot {
+            0 as usize
+        } else {
+            (min_slot - self.first_slot) as usize
+        };
+        for i in start..self.num {
+            if self.slots.get(i as u64) {
+                rv.push(self.first_slot + i as Slot);
+            }
+        }
+        rv
+    }
+    pub fn add(&mut self, slots: &[Slot]) -> usize {
+        for (i, s) in slots.iter().enumerate() {
+            if self.num == 0 {
+                self.first_slot = *s;
+            }
+            if *s < self.first_slot {
+                return i;
+            }
+            if *s - self.first_slot >= self.slots.capacity() {
+                return i;
+            }
+            self.slots.set(*s - self.first_slot, true);
+            self.num = std::cmp::max(self.num, 1 + (*s - self.first_slot) as usize);
+        }
+        slots.len()
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+pub enum CompressedSlots {
+    Flate2(Flate2),
+    Uncompressed(Uncompressed),
+}
+
+impl Default for CompressedSlots {
+    fn default() -> Self {
+        CompressedSlots::new(0)
+    }
+}
+
+impl CompressedSlots {
+    fn new(max_size: usize) -> Self {
+        CompressedSlots::Uncompressed(Uncompressed::new(max_size))
+    }
+
+    pub fn first_slot(&self) -> Slot {
+        match self {
+            CompressedSlots::Uncompressed(a) => a.first_slot,
+            CompressedSlots::Flate2(b) => b.first_slot,
+        }
+    }
+
+    pub fn num_slots(&self) -> usize {
+        match self {
+            CompressedSlots::Uncompressed(a) => a.num,
+            CompressedSlots::Flate2(b) => b.num,
+        }
+    }
+
+    pub fn add(&mut self, slots: &[Slot]) -> usize {
+        match self {
+            CompressedSlots::Uncompressed(vals) => vals.add(slots),
+            CompressedSlots::Flate2(_) => 0,
+        }
+    }
+    pub fn to_slots(&self, min_slot: Slot) -> Result<Vec<Slot>> {
+        match self {
+            CompressedSlots::Uncompressed(vals) => Ok(vals.to_slots(min_slot)),
+            CompressedSlots::Flate2(vals) => {
+                let unc = vals.inflate()?;
+                Ok(unc.to_slots(min_slot))
+            }
+        }
+    }
+    pub fn deflate(&mut self) -> Result<()> {
+        match self {
+            CompressedSlots::Uncompressed(vals) => {
+                let unc = vals.clone();
+                let compressed = Flate2::deflate(unc)?;
+                let mut new = CompressedSlots::Flate2(compressed);
+                std::mem::swap(self, &mut new);
+                Ok(())
+            }
+            CompressedSlots::Flate2(_) => Ok(()),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
+pub struct EpochSlots {
+    pub from: Pubkey,
+    pub slots: Vec<CompressedSlots>,
+    pub wallclock: u64,
+}
+
+impl EpochSlots {
+    pub fn new(from: Pubkey, now: u64) -> Self {
+        Self {
+            from,
+            wallclock: now,
+            slots: vec![],
+        }
+    }
+    pub fn fill(&mut self, slots: &[Slot], now: u64) -> usize {
+        let mut num = 0;
+        self.wallclock = std::cmp::max(now, self.wallclock + 1);
+        while num < slots.len() {
+            num += self.add(&slots[num..]);
+            if num < slots.len() {
+                if self.deflate().is_err() {
+                    return num;
+                }
+                let space = self.max_compressed_slot_size();
+                if space > 0 {
+                    let cslot = CompressedSlots::new(space as usize);
+                    self.slots.push(cslot);
+                } else {
+                    return num;
+                }
+            }
+        }
+        num
+    }
+    pub fn add(&mut self, slots: &[Slot]) -> usize {
+        let mut num = 0;
+        for s in &mut self.slots {
+            num += s.add(&slots[num..]);
+            if num >= slots.len() {
+                break;
+            }
+        }
+        num
+    }
+    pub fn deflate(&mut self) -> Result<()> {
+        for s in self.slots.iter_mut() {
+            s.deflate()?;
+        }
+        Ok(())
+    }
+    pub fn max_compressed_slot_size(&self) -> isize {
+        let len_header = serialized_size(self).unwrap();
+        let len_slot = serialized_size(&CompressedSlots::default()).unwrap();
+        MAX_CRDS_OBJECT_SIZE as isize - (len_header + len_slot) as isize
+    }
+
+    pub fn first_slot(&self) -> Option<Slot> {
+        self.slots.iter().map(|s| s.first_slot()).min()
+    }
+
+    pub fn to_slots(&self, min_slot: Slot) -> Vec<Slot> {
+        self.slots
+            .iter()
+            .filter(|s| min_slot < s.first_slot() + s.num_slots() as u64)
+            .filter_map(|s| s.to_slots(min_slot).ok())
+            .flatten()
+            .collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn test_epoch_slots_max_size() {
+        let epoch_slots = EpochSlots::default();
+        assert!(epoch_slots.max_compressed_slot_size() > 0);
+    }
+
+    #[test]
+    fn test_epoch_slots_uncompressed_add_1() {
+        let mut slots = Uncompressed::new(1);
+        assert_eq!(slots.slots.capacity(), 8);
+        assert_eq!(slots.add(&[1]), 1);
+        assert_eq!(slots.to_slots(1), vec![1]);
+        assert!(slots.to_slots(2).is_empty());
+    }
+    #[test]
+    fn test_epoch_slots_uncompressed_add_2() {
+        let mut slots = Uncompressed::new(1);
+        assert_eq!(slots.add(&[1, 2]), 2);
+        assert_eq!(slots.to_slots(1), vec![1, 2]);
+    }
+    #[test]
+    fn test_epoch_slots_uncompressed_add_3a() {
+        let mut slots = Uncompressed::new(1);
+        assert_eq!(slots.add(&[1, 3, 2]), 3);
+        assert_eq!(slots.to_slots(1), vec![1, 2, 3]);
+    }
+
+    #[test]
+    fn test_epoch_slots_uncompressed_add_3b() {
+        let mut slots = Uncompressed::new(1);
+        assert_eq!(slots.add(&[1, 10, 2]), 1);
+        assert_eq!(slots.to_slots(1), vec![1]);
+    }
+
+    #[test]
+    fn test_epoch_slots_uncompressed_add_3c() {
+        let mut slots = Uncompressed::new(2);
+        assert_eq!(slots.add(&[1, 10, 2]), 3);
+        assert_eq!(slots.to_slots(1), vec![1, 2, 10]);
+        assert_eq!(slots.to_slots(2), vec![2, 10]);
+        assert_eq!(slots.to_slots(3), vec![10]);
+        assert_eq!(slots.to_slots(11).is_empty(), true);
+    }
+    #[test]
+    fn test_epoch_slots_compressed() {
+        let mut slots = Uncompressed::new(100);
+        slots.add(&[1, 701, 2]);
+        assert_eq!(slots.num, 701);
+        let compressed = Flate2::deflate(slots).unwrap();
+        assert_eq!(compressed.first_slot, 1);
+        assert_eq!(compressed.num, 701);
+        assert!(compressed.compressed.len() < 32);
+        let slots = compressed.inflate().unwrap();
+        assert_eq!(slots.first_slot, 1);
+        assert_eq!(slots.num, 701);
+        assert_eq!(slots.to_slots(1), vec![1, 2, 701]);
+    }
+    #[test]
+    fn test_epoch_slots_fill_range() {
+        let range: Vec<Slot> = (0..5000).into_iter().collect();
+        let mut slots = EpochSlots::default();
+        assert_eq!(slots.fill(&range, 1), 5000);
+        assert_eq!(slots.wallclock, 1);
+        assert_eq!(slots.to_slots(0), range);
+        assert_eq!(slots.to_slots(4999), vec![4999]);
+        assert_eq!(slots.to_slots(5000).is_empty(), true);
+    }
+    #[test]
+    fn test_epoch_slots_fill_sparce_range() {
+        let range: Vec<Slot> = (0..5000).into_iter().map(|x| x * 3).collect();
+        let mut slots = EpochSlots::default();
+        assert_eq!(slots.fill(&range, 2), 5000);
+        assert_eq!(slots.wallclock, 2);
+        assert_eq!(slots.slots.len(), 3);
+        assert_eq!(slots.slots[0].first_slot(), 0);
+        assert_ne!(slots.slots[0].num_slots(), 0);
+        let next = slots.slots[0].num_slots() as u64 + slots.slots[0].first_slot();
+        assert!(slots.slots[1].first_slot() >= next);
+        assert_ne!(slots.slots[1].num_slots(), 0);
+        assert_ne!(slots.slots[2].num_slots(), 0);
+        assert_eq!(slots.to_slots(0), range);
+        assert_eq!(slots.to_slots(4999 * 3), vec![4999 * 3]);
+    }
+
+    #[test]
+    fn test_epoch_slots_fill_large_sparce_range() {
+        let range: Vec<Slot> = (0..5000).into_iter().map(|x| x * 7).collect();
+        let mut slots = EpochSlots::default();
+        assert_eq!(slots.fill(&range, 2), 5000);
+        assert_eq!(slots.to_slots(0), range);
+    }
+
+    #[test]
+    fn test_epoch_slots_fill_uncompressed_random_range() {
+        use rand::Rng;
+        for _ in 0..10 {
+            let mut range: Vec<Slot> = vec![];
+            for _ in 0..5000 {
+                let last = *range.last().unwrap_or(&0);
+                range.push(last + rand::thread_rng().gen_range(1, 5));
+            }
+            let sz = EpochSlots::default().max_compressed_slot_size();
+            let mut slots = Uncompressed::new(sz as usize);
+            let sz = slots.add(&range);
+            let slots = slots.to_slots(0);
+            assert_eq!(slots.len(), sz);
+            assert_eq!(slots[..], range[..sz]);
+        }
+    }
+
+    #[test]
+    fn test_epoch_slots_fill_compressed_random_range() {
+        use rand::Rng;
+        for _ in 0..10 {
+            let mut range: Vec<Slot> = vec![];
+            for _ in 0..5000 {
+                let last = *range.last().unwrap_or(&0);
+                range.push(last + rand::thread_rng().gen_range(1, 5));
+            }
+            let sz = EpochSlots::default().max_compressed_slot_size();
+            let mut slots = Uncompressed::new(sz as usize);
+            let sz = slots.add(&range);
+            let mut slots = CompressedSlots::Uncompressed(slots);
+            slots.deflate().unwrap();
+            let slots = slots.to_slots(0).unwrap();
+            assert_eq!(slots.len(), sz);
+            assert_eq!(slots[..], range[..sz]);
+        }
+    }
+
+    #[test]
+    fn test_epoch_slots_fill_random_range() {
+        use rand::Rng;
+        for _ in 0..10 {
+            let mut range: Vec<Slot> = vec![];
+            for _ in 0..5000 {
+                let last = *range.last().unwrap_or(&0);
+                range.push(last + rand::thread_rng().gen_range(1, 5));
+            }
+            let mut slots = EpochSlots::default();
+            let sz = slots.fill(&range, 1);
+            let last = range[sz - 1];
+            assert_eq!(
+                last,
+                slots.slots.last().unwrap().first_slot()
+                    + slots.slots.last().unwrap().num_slots() as u64
+                    - 1
+            );
+            for s in &slots.slots {
+                assert!(s.to_slots(0).is_ok());
+            }
+            let slots = slots.to_slots(0);
+            assert_eq!(slots[..], range[..slots.len()]);
+            assert_eq!(sz, slots.len())
+        }
+    }
+}

core/src/fetch_stage.rs

@@ -1,14 +1,14 @@
 //! The `fetch_stage` batches input from a UDP socket and sends it to a channel.
 
 use crate::banking_stage::FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET;
-use crate::packet::PacketsRecycler;
 use crate::poh_recorder::PohRecorder;
 use crate::result::{Error, Result};
-use crate::streamer::{self, PacketReceiver, PacketSender};
 use solana_measure::thread_mem_usage;
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_info};
+use solana_perf::packet::PacketsRecycler;
 use solana_perf::recycler::Recycler;
 use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;
+use solana_streamer::streamer::{self, PacketReceiver, PacketSender};
 use std::net::UdpSocket;
 use std::sync::atomic::AtomicBool;
 use std::sync::mpsc::{channel, RecvTimeoutError};

core/src/genesis_utils.rs

@@ -1 +0,0 @@
-pub use solana_ledger::genesis_utils::*;

core/src/gossip_service.rs

@@ -2,13 +2,13 @@
 
 use crate::cluster_info::{ClusterInfo, VALIDATOR_PORT_RANGE};
 use crate::contact_info::ContactInfo;
-use crate::streamer;
 use rand::{thread_rng, Rng};
 use solana_client::thin_client::{create_client, ThinClient};
 use solana_ledger::bank_forks::BankForks;
 use solana_perf::recycler::Recycler;
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::signature::{Keypair, Signer};
+use solana_streamer::streamer;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, UdpSocket};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::mpsc::channel;

core/src/ledger_cleanup_service.rs

@@ -1,6 +1,8 @@
 //! The `ledger_cleanup_service` drops older ledger data to limit disk space usage
 
 use solana_ledger::blockstore::Blockstore;
+use solana_ledger::blockstore_db::Result as BlockstoreResult;
+use solana_measure::measure::Measure;
 use solana_metrics::datapoint_debug;
 use solana_sdk::clock::Slot;
 use std::string::ToString;
@@ -11,13 +13,25 @@ use std::thread;
 use std::thread::{Builder, JoinHandle};
 use std::time::Duration;
 
+// - To try and keep the RocksDB size under 400GB:
+//   Seeing about 1600b/shred, using 2000b/shred for margin, so 200m shreds can be stored in 400gb.
+//   at 5k shreds/slot at 50k tps, this is 500k slots (~5 hours).
+//   At idle, 60 shreds/slot this is about 4m slots (18 days)
 // This is chosen to allow enough time for
-// - To try and keep the RocksDB size under 512GB at 50k tps (100 slots take ~2GB).
 // - A validator to download a snapshot from a peer and boot from it
 // - To make sure that if a validator needs to reboot from its own snapshot, it has enough slots locally
 //   to catch back up to where it was when it stopped
-pub const DEFAULT_MAX_LEDGER_SLOTS: u64 = 270_000;
-// Remove a fixed number of slots at a time, it's more efficient than doing it one-by-one
+pub const DEFAULT_MAX_LEDGER_SHREDS: u64 = 200_000_000;
+
+// Allow down to 50m, or 3.5 days at idle, 1hr at 50k load, around ~100GB
+pub const DEFAULT_MIN_MAX_LEDGER_SHREDS: u64 = 50_000_000;
+
+// Check for removing slots at this interval so we don't purge too often
+// and starve other blockstore users.
+pub const DEFAULT_PURGE_SLOT_INTERVAL: u64 = 512;
+
+// Remove a limited number of slots at a time, so the operation
+// does not take too long and block other blockstore users.
 pub const DEFAULT_PURGE_BATCH_SIZE: u64 = 256;
 
 pub struct LedgerCleanupService {
@@ -36,7 +50,7 @@ impl LedgerCleanupService {
             max_ledger_slots
         );
         let exit = exit.clone();
-        let mut next_purge_batch = max_ledger_slots;
+        let mut last_purge_slot = 0;
         let t_cleanup = Builder::new()
             .name("solana-ledger-cleanup".to_string())
             .spawn(move || loop {
@@ -47,7 +61,8 @@ impl LedgerCleanupService {
                     &new_root_receiver,
                     &blockstore,
                     max_ledger_slots,
-                    &mut next_purge_batch,
+                    &mut last_purge_slot,
+                    DEFAULT_PURGE_SLOT_INTERVAL,
                 ) {
                     match e {
                         RecvTimeoutError::Disconnected => break,
@@ -59,45 +74,123 @@ impl LedgerCleanupService {
         Self { t_cleanup }
     }
 
+    fn find_slots_to_clean(
+        blockstore: &Arc<Blockstore>,
+        root: Slot,
+        max_ledger_shreds: u64,
+    ) -> (u64, Slot, Slot) {
+        let mut shreds = Vec::new();
+        let mut iterate_time = Measure::start("iterate_time");
+        let mut total_shreds = 0;
+        let mut first_slot = 0;
+        for (i, (slot, meta)) in blockstore.slot_meta_iterator(0).unwrap().enumerate() {
+            if i == 0 {
+                first_slot = slot;
+                debug!("purge: searching from slot: {}", slot);
+            }
+            // Not exact since non-full slots will have holes
+            total_shreds += meta.received;
+            shreds.push((slot, meta.received));
+            if slot > root {
+                break;
+            }
+        }
+        iterate_time.stop();
+        info!(
+            "checking for ledger purge: max_shreds: {} slots: {} total_shreds: {} {}",
+            max_ledger_shreds,
+            shreds.len(),
+            total_shreds,
+            iterate_time
+        );
+        if (total_shreds as u64) < max_ledger_shreds {
+            return (0, 0, 0);
+        }
+        let mut cur_shreds = 0;
+        let mut lowest_slot_to_clean = shreds[0].0;
+        for (slot, num_shreds) in shreds.iter().rev() {
+            cur_shreds += *num_shreds as u64;
+            if cur_shreds > max_ledger_shreds {
+                lowest_slot_to_clean = *slot;
+                break;
+            }
+        }
+
+        (cur_shreds, lowest_slot_to_clean, first_slot)
+    }
+
     fn cleanup_ledger(
         new_root_receiver: &Receiver<Slot>,
         blockstore: &Arc<Blockstore>,
-        max_ledger_slots: u64,
-        next_purge_batch: &mut u64,
+        max_ledger_shreds: u64,
+        last_purge_slot: &mut u64,
+        purge_interval: u64,
     ) -> Result<(), RecvTimeoutError> {
-        let disk_utilization_pre = blockstore.storage_size();
-
-        let root = new_root_receiver.recv_timeout(Duration::from_secs(1))?;
-
-        // Notify blockstore of impending purge
-        if root > *next_purge_batch {
-            //cleanup
-            let lowest_slot = root - max_ledger_slots;
-            *blockstore.lowest_cleanup_slot.write().unwrap() = lowest_slot;
-            blockstore.purge_slots(0, Some(lowest_slot));
-            *next_purge_batch += DEFAULT_PURGE_BATCH_SIZE;
+        let mut root = new_root_receiver.recv_timeout(Duration::from_secs(1))?;
+        // Get the newest root
+        while let Ok(new_root) = new_root_receiver.try_recv() {
+            root = new_root;
         }
 
-        let disk_utilization_post = blockstore.storage_size();
-
-        if let (Ok(disk_utilization_pre), Ok(disk_utilization_post)) =
-            (disk_utilization_pre, disk_utilization_post)
-        {
-            datapoint_debug!(
-                "ledger_disk_utilization",
-                ("disk_utilization_pre", disk_utilization_pre as i64, i64),
-                ("disk_utilization_post", disk_utilization_post as i64, i64),
-                (
-                    "disk_utilization_delta",
-                    (disk_utilization_pre as i64 - disk_utilization_post as i64),
-                    i64
-                )
+        if root - *last_purge_slot > purge_interval {
+            let disk_utilization_pre = blockstore.storage_size();
+            info!(
+                "purge: new root: {} last_purge: {} purge_interval: {} disk: {:?}",
+                root, last_purge_slot, purge_interval, disk_utilization_pre
             );
+            *last_purge_slot = root;
+
+            let (num_shreds_to_clean, lowest_slot_to_clean, mut first_slot) =
+                Self::find_slots_to_clean(blockstore, root, max_ledger_shreds);
+
+            if num_shreds_to_clean > 0 {
+                debug!(
+                    "cleaning up to: {} shreds: {} first: {}",
+                    lowest_slot_to_clean, num_shreds_to_clean, first_slot
+                );
+                loop {
+                    let current_lowest =
+                        std::cmp::min(lowest_slot_to_clean, first_slot + DEFAULT_PURGE_BATCH_SIZE);
+
+                    let mut slot_update_time = Measure::start("slot_update");
+                    *blockstore.lowest_cleanup_slot.write().unwrap() = current_lowest;
+                    slot_update_time.stop();
+
+                    let mut clean_time = Measure::start("ledger_clean");
+                    blockstore.purge_slots(first_slot, Some(current_lowest));
+                    clean_time.stop();
+
+                    debug!(
+                        "ledger purge {} -> {}: {} {}",
+                        first_slot, current_lowest, slot_update_time, clean_time
+                    );
+                    first_slot += DEFAULT_PURGE_BATCH_SIZE;
+                    if current_lowest == lowest_slot_to_clean {
+                        break;
+                    }
+                    thread::sleep(Duration::from_millis(500));
+                }
+            }
+
+            let disk_utilization_post = blockstore.storage_size();
+
+            Self::report_disk_metrics(disk_utilization_pre, disk_utilization_post);
         }
 
         Ok(())
     }
 
+    fn report_disk_metrics(pre: BlockstoreResult<u64>, post: BlockstoreResult<u64>) {
+        if let (Ok(pre), Ok(post)) = (pre, post) {
+            datapoint_debug!(
+                "ledger_disk_utilization",
+                ("disk_utilization_pre", pre as i64, i64),
+                ("disk_utilization_post", post as i64, i64),
+                ("disk_utilization_delta", (pre as i64 - post as i64), i64)
+            );
+        }
+    }
+
     pub fn join(self) -> thread::Result<()> {
         self.t_cleanup.join()
     }
@@ -111,6 +204,7 @@ mod tests {
 
     #[test]
     fn test_cleanup() {
+        solana_logger::setup();
         let blockstore_path = get_tmp_ledger_path!();
         let blockstore = Blockstore::open(&blockstore_path).unwrap();
         let (shreds, _) = make_many_slot_entries(0, 50, 5);
@@ -118,10 +212,10 @@ mod tests {
         let blockstore = Arc::new(blockstore);
         let (sender, receiver) = channel();
 
-        //send a signal to kill slots 0-40
-        let mut next_purge_slot = 0;
+        //send a signal to kill all but 5 shreds, which will be in the newest slots
+        let mut last_purge_slot = 0;
         sender.send(50).unwrap();
-        LedgerCleanupService::cleanup_ledger(&receiver, &blockstore, 10, &mut next_purge_slot)
+        LedgerCleanupService::cleanup_ledger(&receiver, &blockstore, 5, &mut last_purge_slot, 10)
             .unwrap();
 
         //check that 0-40 don't exist
@@ -134,6 +228,62 @@ mod tests {
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
 
+    #[test]
+    fn test_cleanup_speed() {
+        solana_logger::setup();
+        let blockstore_path = get_tmp_ledger_path!();
+        let mut blockstore = Blockstore::open(&blockstore_path).unwrap();
+        blockstore.set_no_compaction(true);
+        let blockstore = Arc::new(blockstore);
+        let (sender, receiver) = channel();
+
+        let mut first_insert = Measure::start("first_insert");
+        let initial_slots = 50;
+        let initial_entries = 5;
+        let (shreds, _) = make_many_slot_entries(0, initial_slots, initial_entries);
+        blockstore.insert_shreds(shreds, None, false).unwrap();
+        first_insert.stop();
+        info!("{}", first_insert);
+
+        let mut last_purge_slot = 0;
+        let mut slot = initial_slots;
+        let mut num_slots = 6;
+        for _ in 0..5 {
+            let mut insert_time = Measure::start("insert time");
+            let batch_size = 2;
+            let batches = num_slots / batch_size;
+            for i in 0..batches {
+                let (shreds, _) = make_many_slot_entries(slot + i * batch_size, batch_size, 5);
+                blockstore.insert_shreds(shreds, None, false).unwrap();
+                if i % 100 == 0 {
+                    info!("inserting..{} of {}", i, batches);
+                }
+            }
+            insert_time.stop();
+
+            let mut time = Measure::start("purge time");
+            sender.send(slot + num_slots).unwrap();
+            LedgerCleanupService::cleanup_ledger(
+                &receiver,
+                &blockstore,
+                initial_slots,
+                &mut last_purge_slot,
+                10,
+            )
+            .unwrap();
+            time.stop();
+            info!(
+                "slot: {} size: {} {} {}",
+                slot, num_slots, insert_time, time
+            );
+            slot += num_slots;
+            num_slots *= 2;
+        }
+
+        drop(blockstore);
+        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
+    }
+
     #[test]
     fn test_compaction() {
         let blockstore_path = get_tmp_ledger_path!();
@@ -142,7 +292,7 @@ mod tests {
         let n = 10_000;
         let batch_size = 100;
         let batches = n / batch_size;
-        let max_ledger_slots = 100;
+        let max_ledger_shreds = 100;
 
         for i in 0..batches {
             let (shreds, _) = make_many_slot_entries(i * batch_size, batch_size, 1);
@@ -158,8 +308,9 @@ mod tests {
         LedgerCleanupService::cleanup_ledger(
             &receiver,
             &blockstore,
-            max_ledger_slots,
+            max_ledger_shreds,
             &mut next_purge_batch,
+            10,
         )
         .unwrap();
 
@@ -170,7 +321,7 @@ mod tests {
         assert!(u2 < u1, "insufficient compaction! pre={},post={}", u1, u2,);
 
         // check that early slots don't exist
-        let max_slot = n - max_ledger_slots;
+        let max_slot = n - max_ledger_shreds - 1;
         blockstore
             .slot_meta_iterator(0)
             .unwrap()

core/src/lib.rs

@@ -5,16 +5,18 @@
 //! command-line tools to spin up validators and a Rust library
 //!
 
+pub mod accounts_background_service;
+pub mod accounts_hash_verifier;
 pub mod banking_stage;
 pub mod broadcast_stage;
 pub mod cluster_info_vote_listener;
 pub mod commitment;
+mod deprecated;
 pub mod shred_fetch_stage;
 #[macro_use]
 pub mod contact_info;
-pub mod blockstream;
-pub mod blockstream_service;
 pub mod cluster_info;
+pub mod cluster_slots;
 pub mod consensus;
 pub mod crds;
 pub mod crds_gossip;
@@ -22,16 +24,15 @@ pub mod crds_gossip_error;
 pub mod crds_gossip_pull;
 pub mod crds_gossip_push;
 pub mod crds_value;
+pub mod epoch_slots;
 pub mod fetch_stage;
 pub mod gen_keys;
-pub mod genesis_utils;
 pub mod gossip_service;
 pub mod ledger_cleanup_service;
 pub mod local_vote_signer_service;
-pub mod packet;
 pub mod poh_recorder;
 pub mod poh_service;
-pub mod recvmmsg;
+pub mod progress_map;
 pub mod repair_service;
 pub mod replay_stage;
 mod result;
@@ -42,7 +43,6 @@ pub mod rpc_pubsub;
 pub mod rpc_pubsub_service;
 pub mod rpc_service;
 pub mod rpc_subscriptions;
-pub mod sendmmsg;
 pub mod serve_repair;
 pub mod serve_repair_service;
 pub mod sigverify;
@@ -50,11 +50,11 @@ pub mod sigverify_shreds;
 pub mod sigverify_stage;
 pub mod snapshot_packager_service;
 pub mod storage_stage;
-pub mod streamer;
 pub mod tpu;
 pub mod transaction_status_service;
 pub mod tvu;
 pub mod validator;
+pub mod verified_vote_packets;
 pub mod weighted_shuffle;
 pub mod window_service;