Rpc: Add getCirculatingSupply endpoint, redux (#9953) (#9954)

automerge

.buildkite/env/secrets.ejson

@@ -7,9 +7,6 @@
       "GITHUB_TOKEN": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:Vq2dkGTOzfEpRht0BAGHFp/hDogMvXJe:tFXHg1epVt2mq9hkuc5sRHe+KAnVREi/p8S+IZu67XRyzdiA/nGak1k860FXYuuzuaE0QWekaEc=]",
       "INFLUX_DATABASE": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:5KI9WBkXx3R/W4m256mU5MJOE7N8aAT9:Cb8QFELZ9I60t5zhJ9h55Kcs]",
       "INFLUX_PASSWORD": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:hQRMpLCrav+OYkNphkeM4hagdVoZv5Iw:AUO76rr6+gF1OLJA8ZLSG8wHKXgYCPNk6gRCV8rBhZBJ4KwDaxpvOhMl7bxxXG6jol7v4aRa/Lk=]",
-      "INFLUX_USERNAME": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:R7BNmQjfeqoGDAFTJu9bYTGHol2NgnYN:Q2tOT/EBcFvhFk+DKLKmVU7tLCpVC3Ui]",
-      "SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:Egc2dMrHDU0NcZ71LwGv/V66shUhwYUE:04VoIb8CKy7KYhQ5W4cEW9SDKZltxWBL5Hob106lMBbUOD/yUvKYcG3Ep8JfTMwO3K8zowW5HpU/IdGoilX0XWLiJJ6t+p05WWK0TA16nOEtwrEG+UK8wm3sN+xCO20i4jDhpNpgg3FYFHT5rKTHW8+zaBTNUX/SFxkN67Lm+92IM28CXYE43SU1WV6H99hGFFVpTK5JVM3JuYU1ex/dHRE+xCzTr4MYUB/F+nGoNFW8HUDV/y0e1jxT9to3x0SmnytEEuk+5RUzFuEt9cKNFeNml3fOCi4qL+sfj/Y5pjH9xDiUxsvH/8NL35jbLP244aFHgWcp]",
-      "SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_apple_darwin": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:NeOxSoWCvXB9AL4H6OK26l/7bmsKd/oz:Ijfoxtvk2CHlN1ZXHup3Gg/914kbbAkEGWJfvozA8UIe+aUzUObMyTrKkVOeNAH8Q8YH9tNzk7RRnrTcpnzeCCBLlWcVEeruMxHox3mPRzmSeDLxtbzCl9VePlRO3T7jg90K5hW+ZAkd5J/WJNzpAcmr93ts/of3MbvGHSujId/efCTzJEcP6JInnBb8Vrj7TlgKbzUlnqpq1+NjYPSXN3maKa9pKeo2JWxZlGBMoy6QWUUY5GbYEylw9smwh1LJcHZjlaZNMuOl4gNKtaSr38IXQkAXaRUJDPAmPras00YObKzXU8RkTrP4EoP/jx5LPR7f]",
-      "SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_pc_windows_msvc": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:7t+56twjW+jR7fpFNNeRFLPd7E4lbmyN:JuviDpkQrfVcNUGRGsa2e/UhvH6tTYyk1s4cHHE5xZH1NByL7Kpqx36VG/+o1AUGEeSQdsBnKgzYdMoFYbO8o50DoRPc86QIEVXCupD6J9avxLFtQgOWgJp+/mCdUVXlqXiFs/vQgS/L4psrcKdF6WHd77BeUr6ll8DjH+9m5FC9Rcai2pXno6VbPpunHQ0oUdYzhFR64+LiRacBaefQ9igZ+nSEWDLqbaZSyfm9viWkijoVFTq8gAgdXXEh7g0QdxVE5T6bPristJhT6jWBhWunPUCDNFFErWIsbRGctepl4pbCWqh2hNTw9btSgVfeY6uGCOsdy9E=]"
+      "INFLUX_USERNAME": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:R7BNmQjfeqoGDAFTJu9bYTGHol2NgnYN:Q2tOT/EBcFvhFk+DKLKmVU7tLCpVC3Ui]"
     }
 }

.travis.yml

@@ -1,5 +1,6 @@
 os:
   - osx
+  - windows
 
 language: rust
 rust:

CONTRIBUTING.md

@@ -45,7 +45,7 @@ $ git pull --rebase upstream master
 
 If there are no functional changes, PRs can be very large and that's no
 problem. If, however, your changes are making meaningful changes or additions,
-then about 1.0.5 lines of changes is about the most you should ask a Solana
+then about 1000 lines of changes is about the most you should ask a Solana
 maintainer to review.
 
 ### Should I send small PRs as I develop large, new components?

Cargo.toml

@@ -48,7 +48,9 @@ members = [
     "sdk",
     "sdk-c",
     "scripts",
+    "stake-monitor",
     "sys-tuner",
+    "transaction-status",
     "upload-perf",
     "net-utils",
     "vote-signer",

README.md

@@ -9,20 +9,6 @@ Blockchain Rebuilt for Scale
 Solana™ is a new blockchain architecture built from the ground up for scale. The architecture supports
 up to 710 thousand transactions per second on a gigabit network.
 
-Disclaimer
-===
-
-All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort.  It is up to the reader to check and validate their accuracy and truthfulness.  Furthermore nothing in this project constitutes a solicitation for investment.
-
-Introduction
-===
-
-It's possible for a centralized database to process 710,000 transactions per second on a standard gigabit network if the transactions are, on average, no more than 176 bytes. A centralized database can also replicate itself and maintain high availability without significantly compromising that transaction rate using the distributed system technique known as Optimistic Concurrency Control [\[H.T.Kung, J.T.Robinson (1981)\]](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.65.4735). At Solana, we're demonstrating that these same theoretical limits apply just as well to blockchain on an adversarial network. The key ingredient? Finding a way to share time when nodes can't trust one-another. Once nodes can trust time, suddenly ~40 years of distributed systems research becomes applicable to blockchain!
-
-> Perhaps the most striking difference between algorithms obtained by our method and ones based upon timeout is that using timeout produces a traditional distributed algorithm in which the processes operate asynchronously, while our method produces a globally synchronous one in which every process does the same thing at (approximately) the same time. Our method seems to contradict the whole purpose of distributed processing, which is to permit different processes to operate independently and perform different functions. However, if a distributed system is really a single system, then the processes must be synchronized in some way. Conceptually, the easiest way to synchronize processes is to get them all to do the same thing at the same time. Therefore, our method is used to implement a kernel that performs the necessary synchronization--for example, making sure that two different processes do not try to modify a file at the same time. Processes might spend only a small fraction of their time executing the synchronizing kernel; the rest of the time, they can operate independently--e.g., accessing different files. This is an approach we have advocated even when fault-tolerance is not required. The method's basic simplicity makes it easier to understand the precise properties of a system, which is crucial if one is to know just how fault-tolerant the system is. [\[L.Lamport (1984)\]](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.71.1078)
-
-Furthermore, and much to our surprise, it can be implemented using a mechanism that has existed in Bitcoin since day one. The Bitcoin feature is called nLocktime and it can be used to postdate transactions using block height instead of a timestamp. As a Bitcoin client, you'd use block height instead of a timestamp if you don't trust the network. Block height turns out to be an instance of what's being called a Verifiable Delay Function in cryptography circles. It's a cryptographically secure way to say time has passed. In Solana, we use a far more granular verifiable delay function, a SHA 256 hash chain, to checkpoint the ledger and coordinate consensus. With it, we implement Optimistic Concurrency Control and are now well en route towards that theoretical limit of 710,000 transactions per second.
-
 Documentation
 ===
 
@@ -238,3 +224,8 @@ problem is solved by this code?" On the other hand, if a test does fail and you
 better way to solve the same problem, a Pull Request with your solution would most certainly be
 welcome! Likewise, if rewriting a test can better communicate what code it's protecting, please
 send us that patch!
+
+Disclaimer
+===
+
+All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort.  It is up to the reader to check and validate their accuracy and truthfulness.  Furthermore nothing in this project constitutes a solicitation for investment.

archiver-lib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-archiver-lib"
-version = "1.0.5"
+version = "1.0.22"
 description = "Solana Archiver Library"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,29 +11,32 @@ edition = "2018"
 [dependencies]
 bincode = "1.2.1"
 crossbeam-channel = "0.3"
-ed25519-dalek = "=1.0.0-pre.1"
+ed25519-dalek = "=1.0.0-pre.3"
 log = "0.4.8"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
-solana-client = { path = "../client", version = "1.0.5" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.5" }
+rand = "0.7.0"
+rand_chacha = "0.2.2"
+solana-client = { path = "../client", version = "1.0.22" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.22" }
 thiserror = "1.0"
 serde = "1.0.104"
 serde_json = "1.0.46"
 serde_derive = "1.0.103"
-solana-net-utils = { path = "../net-utils", version = "1.0.5" }
-solana-chacha = { path = "../chacha", version = "1.0.5" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.5" }
-solana-ledger = { path = "../ledger", version = "1.0.5" }
-solana-logger = { path = "../logger", version = "1.0.5" }
-solana-perf = { path = "../perf", version = "1.0.5" }
-solana-sdk = { path = "../sdk", version = "1.0.5" }
-solana-core = { path = "../core", version = "1.0.5" }
-solana-archiver-utils = { path = "../archiver-utils", version = "1.0.5" }
-solana-metrics = { path = "../metrics", version = "1.0.5" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-chacha = { path = "../chacha", version = "1.0.22" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.0.22" }
+solana-ledger = { path = "../ledger", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-perf = { path = "../perf", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
+solana-core = { path = "../core", version = "1.0.22" }
+solana-archiver-utils = { path = "../archiver-utils", version = "1.0.22" }
+solana-metrics = { path = "../metrics", version = "1.0.22" }
 
 [dev-dependencies]
 hex = "0.4.0"
 
 [lib]
 name = "solana_archiver_lib"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

archiver-lib/src/archiver.rs

@@ -1,7 +1,7 @@
 use crate::result::ArchiverError;
 use crossbeam_channel::unbounded;
-use rand::{thread_rng, Rng, SeedableRng};
-use rand_chacha::ChaChaRng;
+use rand::{thread_rng, Rng};
+use rand_chacha::{rand_core::SeedableRng, ChaChaRng};
 use solana_archiver_utils::sample_file;
 use solana_chacha::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE};
 use solana_client::{
@@ -199,7 +199,7 @@ impl Archiver {
 
         info!("Connecting to the cluster via {:?}", cluster_entrypoint);
         let (nodes, _) =
-            match solana_core::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 1) {
+            match solana_core::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 2) {
                 Ok(nodes_and_archivers) => nodes_and_archivers,
                 Err(e) => {
                     //shutdown services before exiting
@@ -235,6 +235,7 @@ impl Archiver {
             shred_forward_sockets,
             repair_socket.clone(),
             &shred_fetch_sender,
+            None,
             &exit,
         );
         let (slot_sender, slot_receiver) = channel();
@@ -379,8 +380,7 @@ impl Archiver {
                         &archiver_keypair.pubkey(),
                         &storage_keypair.pubkey(),
                     );
-                    let message =
-                        Message::new_with_payer(vec![ix], Some(&archiver_keypair.pubkey()));
+                    let message = Message::new_with_payer(&[ix], Some(&archiver_keypair.pubkey()));
                     if let Err(e) = client.send_message(&[archiver_keypair.as_ref()], message) {
                         error!("unable to redeem reward, tx failed: {:?}", e);
                     } else {
@@ -613,6 +613,7 @@ impl Archiver {
                         ErrorKind::Other,
                         "setup_mining_account: signature not found",
                     ),
+                    TransportError::Custom(e) => io::Error::new(ErrorKind::Other, e),
                 })?;
         }
         Ok(())
@@ -655,7 +656,7 @@ impl Archiver {
             Signature::new(&meta.signature.as_ref()),
             meta.blockhash,
         );
-        let message = Message::new_with_payer(vec![instruction], Some(&archiver_keypair.pubkey()));
+        let message = Message::new_with_payer(&[instruction], Some(&archiver_keypair.pubkey()));
         let mut transaction = Transaction::new(
             &[archiver_keypair.as_ref(), storage_keypair.as_ref()],
             message,

archiver-lib/src/result.rs

@@ -1,4 +1,3 @@
-use serde_json;
 use solana_client::client_error;
 use solana_ledger::blockstore;
 use solana_sdk::transport;

archiver-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-archiver-utils"
-version = "1.0.5"
+version = "1.0.22"
 description = "Solana Archiver Utils"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,16 +10,19 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-rand = "0.6.5"
-solana-chacha = { path = "../chacha", version = "1.0.5" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.5" }
-solana-ledger = { path = "../ledger", version = "1.0.5" }
-solana-logger = { path = "../logger", version = "1.0.5" }
-solana-perf = { path = "../perf", version = "1.0.5" }
-solana-sdk = { path = "../sdk", version = "1.0.5" }
+rand = "0.7.0"
+solana-chacha = { path = "../chacha", version = "1.0.22" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.0.22" }
+solana-ledger = { path = "../ledger", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-perf = { path = "../perf", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
 
 [dev-dependencies]
 hex = "0.4.0"
 
 [lib]
 name = "solana_archiver_utils"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

archiver/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-archiver"
-version = "1.0.5"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,14 @@ homepage = "https://solana.com/"
 [dependencies]
 clap = "2.33.0"
 console = "0.9.2"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.5" }
-solana-core = { path = "../core", version = "1.0.5" }
-solana-logger = { path = "../logger", version = "1.0.5" }
-solana-metrics = { path = "../metrics", version = "1.0.5" }
-solana-archiver-lib = { path = "../archiver-lib", version = "1.0.5" }
-solana-net-utils = { path = "../net-utils", version = "1.0.5" }
-solana-sdk = { path = "../sdk", version = "1.0.5" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.22" }
+solana-core = { path = "../core", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-metrics = { path = "../metrics", version = "1.0.22" }
+solana-archiver-lib = { path = "../archiver-lib", version = "1.0.22" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
 
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

archiver/src/main.rs

@@ -28,7 +28,7 @@ fn main() {
         .arg(
             Arg::with_name("identity_keypair")
                 .short("i")
-                .long("identity-keypair")
+                .long("identity")
                 .value_name("PATH")
                 .takes_value(true)
                 .validator(is_keypair_or_ask_keyword)

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.0.5"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,14 @@ homepage = "https://solana.com/"
 [dependencies]
 log = "0.4.6"
 rayon = "1.2.0"
-solana-core = { path = "../core", version = "1.0.5" }
-solana-ledger = { path = "../ledger", version = "1.0.5" }
-solana-logger = { path = "../logger", version = "1.0.5" }
-solana-runtime = { path = "../runtime", version = "1.0.5" }
-solana-measure = { path = "../measure", version = "1.0.5" }
-solana-sdk = { path = "../sdk", version = "1.0.5" }
-rand = "0.6.5"
+solana-core = { path = "../core", version = "1.0.22" }
+solana-ledger = { path = "../ledger", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-runtime = { path = "../runtime", version = "1.0.22" }
+solana-measure = { path = "../measure", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
+rand = "0.7.0"
 crossbeam-channel = "0.3"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -246,7 +246,7 @@ fn main() {
                 poh_recorder.lock().unwrap().set_bank(&bank);
                 assert!(poh_recorder.lock().unwrap().bank().is_some());
                 if bank.slot() > 32 {
-                    bank_forks.set_root(root, &None);
+                    bank_forks.set_root(root, &None, None);
                     root += 1;
                 }
                 debug!(

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.0.5"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,21 +14,24 @@ itertools = "0.8.2"
 log = "0.4.8"
 num-derive = "0.3"
 num-traits = "0.2"
-rand = "0.6.5"
+rand = "0.7.0"
 rayon = "1.2.0"
 serde_json = "1.0.46"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.5" }
-solana-core = { path = "../core", version = "1.0.5" }
-solana-genesis = { path = "../genesis", version = "1.0.5" }
-solana-client = { path = "../client", version = "1.0.5" }
-solana-faucet = { path = "../faucet", version = "1.0.5" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.0.5" }
-solana-logger = { path = "../logger", version = "1.0.5" }
-solana-metrics = { path = "../metrics", version = "1.0.5" }
-solana-net-utils = { path = "../net-utils", version = "1.0.5" }
-solana-runtime = { path = "../runtime", version = "1.0.5" }
-solana-sdk = { path = "../sdk", version = "1.0.5" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.22" }
+solana-core = { path = "../core", version = "1.0.22" }
+solana-genesis = { path = "../genesis", version = "1.0.22" }
+solana-client = { path = "../client", version = "1.0.22" }
+solana-faucet = { path = "../faucet", version = "1.0.22" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-metrics = { path = "../metrics", version = "1.0.22" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-runtime = { path = "../runtime", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.0.5" }
+solana-local-cluster = { path = "../local-cluster", version = "1.0.22" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/Cargo.toml

@@ -2,14 +2,17 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.0.5"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.5" }
-solana-core = { path = "../core", version = "1.0.5" }
-solana-logger = { path = "../logger", version = "1.0.5" }
-solana-net-utils = { path = "../net-utils", version = "1.0.5" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.22" }
+solana-core = { path = "../core", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.0.5"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,24 +14,27 @@ log = "0.4.8"
 rayon = "1.2.0"
 serde_json = "1.0.46"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.5" }
-solana-core = { path = "../core", version = "1.0.5" }
-solana-genesis = { path = "../genesis", version = "1.0.5" }
-solana-client = { path = "../client", version = "1.0.5" }
-solana-faucet = { path = "../faucet", version = "1.0.5" }
-solana-librapay = { path = "../programs/librapay", version = "1.0.5", optional = true }
-solana-logger = { path = "../logger", version = "1.0.5" }
-solana-metrics = { path = "../metrics", version = "1.0.5" }
-solana-measure = { path = "../measure", version = "1.0.5" }
-solana-net-utils = { path = "../net-utils", version = "1.0.5" }
-solana-runtime = { path = "../runtime", version = "1.0.5" }
-solana-sdk = { path = "../sdk", version = "1.0.5" }
-solana-move-loader-program = { path = "../programs/move_loader", version = "1.0.5", optional = true }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.22" }
+solana-core = { path = "../core", version = "1.0.22" }
+solana-genesis = { path = "../genesis", version = "1.0.22" }
+solana-client = { path = "../client", version = "1.0.22" }
+solana-faucet = { path = "../faucet", version = "1.0.22" }
+#solana-librapay = { path = "../programs/librapay", version = "1.0.20", optional = true }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-metrics = { path = "../metrics", version = "1.0.22" }
+solana-measure = { path = "../measure", version = "1.0.22" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-runtime = { path = "../runtime", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
+#solana-move-loader-program = { path = "../programs/move_loader", version = "1.0.20", optional = true }
 
 [dev-dependencies]
 serial_test = "0.3.2"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.0.5" }
+solana-local-cluster = { path = "../local-cluster", version = "1.0.22" }
 
-[features]
-move = ["solana-librapay", "solana-move-loader-program"]
+#[features]
+#move = ["solana-librapay", "solana-move-loader-program"]
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

chacha-cuda/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha-cuda"
-version = "1.0.5"
+version = "1.0.22"
 description = "Solana Chacha Cuda APIs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,15 +10,18 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-solana-archiver-utils = { path = "../archiver-utils", version = "1.0.5" }
-solana-chacha = { path = "../chacha", version = "1.0.5" }
-solana-ledger = { path = "../ledger", version = "1.0.5" }
-solana-logger = { path = "../logger", version = "1.0.5" }
-solana-perf = { path = "../perf", version = "1.0.5" }
-solana-sdk = { path = "../sdk", version = "1.0.5" }
+solana-archiver-utils = { path = "../archiver-utils", version = "1.0.22" }
+solana-chacha = { path = "../chacha", version = "1.0.22" }
+solana-ledger = { path = "../ledger", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-perf = { path = "../perf", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
 
 [dev-dependencies]
 hex-literal = "0.2.1"
 
 [lib]
 name = "solana_chacha_cuda"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

chacha-sys/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha-sys"
-version = "1.0.5"
+version = "1.0.22"
 description = "Solana chacha-sys"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,3 +10,6 @@ edition = "2018"
 
 [build-dependencies]
 cc = "1.0.49"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

chacha/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha"
-version = "1.0.5"
+version = "1.0.22"
 description = "Solana Chacha APIs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,16 +10,19 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.5" }
-solana-ledger = { path = "../ledger", version = "1.0.5" }
-solana-logger = { path = "../logger", version = "1.0.5" }
-solana-perf = { path = "../perf", version = "1.0.5" }
-solana-sdk = { path = "../sdk", version = "1.0.5" }
+rand = "0.7.0"
+rand_chacha = "0.2.2"
+solana-chacha-sys = { path = "../chacha-sys", version = "1.0.22" }
+solana-ledger = { path = "../ledger", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-perf = { path = "../perf", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
 
 [dev-dependencies]
 hex-literal = "0.2.1"
 
 [lib]
 name = "solana_chacha"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

ci/_

@@ -5,7 +5,13 @@
 # |source| me
 #
 
+base_dir=$(realpath --strip "$(dirname "$0")/..")
+
 _() {
-  echo "--- $*"
+  if [[ $(pwd) = $base_dir ]]; then
+    echo "--- $*"
+  else
+    echo "--- $* (wd: $(pwd))"
+  fi
   "$@"
 }

ci/buildkite-secondary.yml

@@ -2,6 +2,16 @@
 # Build steps that run after the primary pipeline on pushes and tags.
 # Pull requests to not run these steps.
 steps:
+  - command: "ci/publish-tarball.sh"
+    timeout_in_minutes: 60
+    name: "publish tarball"
+  - command: "ci/publish-docs.sh"
+    timeout_in_minutes: 15
+    name: "publish docs"
+  - command: "ci/publish-bpf-sdk.sh"
+    timeout_in_minutes: 5
+    name: "publish bpf sdk"
+  - wait
   - command: "sdk/docker-solana/build.sh"
     timeout_in_minutes: 60
     name: "publish docker"
@@ -9,12 +19,6 @@ steps:
     timeout_in_minutes: 240
     name: "publish crate"
     branches: "!master"
-  - command: "ci/publish-bpf-sdk.sh"
-    timeout_in_minutes: 5
-    name: "publish bpf sdk"
-  - command: "ci/publish-tarball.sh"
-    timeout_in_minutes: 60
-    name: "publish tarball"
-  - command: "ci/publish-docs.sh"
-    timeout_in_minutes: 15
-    name: "publish docs"
+    #  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
+    #    name: "move"
+    #    timeout_in_minutes: 20

ci/buildkite-tests.yml

@@ -0,0 +1,26 @@
+# These steps are conditionally triggered by ci/buildkite.yml when files
+# other than those in docs/ are modified
+
+steps:
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh"
+    name: "coverage"
+    timeout_in_minutes: 30
+  - wait
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh"
+    name: "stable"
+    timeout_in_minutes: 60
+    artifact_paths: "log-*.txt"
+  - wait
+  - command: "ci/test-stable-perf.sh"
+    name: "stable-perf"
+    timeout_in_minutes: 40
+    artifact_paths: "log-*.txt"
+    agents:
+      - "queue=cuda"
+  - command: "ci/test-bench.sh"
+    name: "bench"
+    timeout_in_minutes: 30
+  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-local-cluster.sh"
+    name: "local-cluster"
+    timeout_in_minutes: 45
+    artifact_paths: "log-*.txt"

ci/buildkite.yml

@@ -1,38 +1,25 @@
 # Build steps that run on pushes and pull requests.
+# If files other than those in docs/ were modified, this will be followed up by
+# ci/buildkite-tests.yml
 #
 # Release tags use buildkite-release.yml instead
+
 steps:
-  - command: "ci/shellcheck.sh"
-    name: "shellcheck"
-    timeout_in_minutes: 5
   - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-checks.sh"
     name: "checks"
     timeout_in_minutes: 20
+  - command: "ci/shellcheck.sh"
+    name: "shellcheck"
+    timeout_in_minutes: 5
+
   - wait
-  - command: "ci/test-stable-perf.sh"
-    name: "stable-perf"
-    timeout_in_minutes: 40
-    artifact_paths: "log-*.txt"
-    agents:
-      - "queue=cuda"
-  - command: "ci/test-bench.sh"
-    name: "bench"
-    timeout_in_minutes: 30
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh"
-    name: "stable"
-    timeout_in_minutes: 60
-    artifact_paths: "log-*.txt"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
-    name: "move"
-    timeout_in_minutes: 20
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-local-cluster.sh"
-    name: "local-cluster"
-    timeout_in_minutes: 30
-    artifact_paths: "log-*.txt"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh"
-    name: "coverage"
-    timeout_in_minutes: 30
+
+  - command: "ci/maybe-trigger-tests.sh"
+    name: "maybe-trigger-tests"
+    timeout_in_minutes: 2
+
   - wait
+
   - trigger: "solana-secondary"
     branches: "!pull/*"
     async: true

ci/docker-run.sh

@@ -49,7 +49,7 @@ else
   # ~/.cargo
   ARGS+=(--volume "$PWD:/home")
 fi
-ARGS+=(--env "CARGO_HOME=/home/.cargo")
+ARGS+=(--env "HOME=/home" --env "CARGO_HOME=/home/.cargo")
 
 # kcov tries to set the personality of the binary which docker
 # doesn't allow by default.
@@ -67,6 +67,7 @@ ARGS+=(
   --env BUILDKITE_JOB_ID
   --env CI
   --env CI_BRANCH
+  --env CI_TAG
   --env CI_BUILD_ID
   --env CI_COMMIT
   --env CI_JOB_ID

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.41.1
+FROM solanalabs/rust:1.43.0
 ARG date
 
 RUN set -x \

ci/docker-rust-nightly/README.md

@@ -15,6 +15,8 @@ To update the pinned version:
 1. Run `ci/docker-rust-nightly/build.sh` to rebuild the nightly image locally,
    or potentially `ci/docker-rust-nightly/build.sh YYYY-MM-DD` if there's a
    specific YYYY-MM-DD that is desired (default is today's build).
+   Check https://rust-lang.github.io/rustup-components-history/ for build
+   status
 1. Update `ci/rust-version.sh` to reflect the new nightly `YYY-MM-DD`
 1. Run `SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-coverage.sh`
    to confirm the new nightly image builds.  Fix any issues as needed

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.41.1
+FROM rust:1.43.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0

ci/localnet-sanity.sh

@@ -178,7 +178,7 @@ startNodes() {
 
       (
         set -x
-        $solana_cli --keypair config/bootstrap-validator/identity-keypair.json \
+        $solana_cli --keypair config/bootstrap-validator/identity.json \
           --url http://127.0.0.1:8899 genesis-hash
       ) | tee genesis-hash.log
       maybeExpectedGenesisHash="--expected-genesis-hash $(tail -n1 genesis-hash.log)"

ci/maybe-trigger-tests.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -e
+cd "$(dirname "$0")/.."
+
+annotate() {
+  ${BUILDKITE:-false} && {
+    buildkite-agent annotate "$@"
+  }
+}
+
+# Skip if only the docs have been modified
+ci/affects-files.sh \
+  \!^docs/ \
+|| {
+  annotate --style info \
+    "Skipping all further tests as only docs/ files were modified"
+  exit 0
+}
+
+annotate --style info "Triggering tests"
+buildkite-agent pipeline upload ci/buildkite-tests.yml

ci/publish-docs.sh

@@ -13,9 +13,8 @@ if [[ -n $CI_BRANCH ]]; then
       . ci/rust-version.sh stable
       ci/docker-run.sh "$rust_stable_docker_image" make -C docs
     )
-    # make a local commit for the svgs
-    git add -A -f docs/src/.gitbook/assets/.
-    git add -f docs/src/cli/usage.md
+    # make a local commit for the svgs and generated/updated markdown
+    git add -f docs/src
     if ! git diff-index --quiet HEAD; then
       git config user.email maintainers@solana.com
       git config user.name "$me"

ci/publish-tarball.sh

@@ -45,7 +45,7 @@ linux)
   TARGET=x86_64-unknown-linux-gnu
   ;;
 windows)
-  TARGET=x86_64-pc-windows-msvc
+  TARGET=x86_64-pc-windows-gnu
   ;;
 *)
   echo CI_OS_NAME unset
@@ -71,16 +71,7 @@ echo --- Creating release tarball
   export CHANNEL
 
   source ci/rust-version.sh stable
-  scripts/cargo-install-all.sh +"$rust_stable" --use-move solana-release
-
-  # Reduce the Windows archive size until
-  # https://github.com/appveyor/ci/issues/2997 is fixed
-  if [[ -n $APPVEYOR ]]; then
-    rm -f \
-      solana-release/bin/solana-validator.exe \
-      solana-release/bin/solana-bench-exchange.exe \
-
-  fi
+  scripts/cargo-install-all.sh +"$rust_stable" solana-release
 
   tar cvf solana-release-$TARGET.tar solana-release
   bzip2 solana-release-$TARGET.tar
@@ -104,9 +95,8 @@ fi
 source ci/upload-ci-artifact.sh
 
 for file in solana-release-$TARGET.tar.bz2 solana-release-$TARGET.yml solana-install-init-"$TARGET"* $MAYBE_TARBALLS; do
-  upload-ci-artifact "$file"
-
   if [[ -n $DO_NOT_PUBLISH_TAR ]]; then
+    upload-ci-artifact "$file"
     echo "Skipped $file due to DO_NOT_PUBLISH_TAR"
     continue
   fi

ci/rust-version.sh

@@ -1,28 +1,30 @@
 #
 # This file maintains the rust versions for use by CI.
 #
-# Build with stable rust, updating the stable toolchain if necessary:
-#   $ source ci/rust-version.sh stable
-#   $ cargo +"$rust_stable" build
-#
-# Build with nightly rust, updating the nightly toolchain if necessary:
-#   $ source ci/rust-version.sh nightly
-#   $ cargo +"$rust_nightly" build
-#
 # Obtain the environment variables without any automatic toolchain updating:
 #   $ source ci/rust-version.sh
 #
+# Obtain the environment variables updating both stable and nightly, only stable, or
+# only nightly:
+#   $ source ci/rust-version.sh all
+#   $ source ci/rust-version.sh stable
+#   $ source ci/rust-version.sh nightly
+
+# Then to build with either stable or nightly:
+#   $ cargo +"$rust_stable" build
+#   $ cargo +"$rust_nightly" build
+#
 
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.41.1
+  stable_version=1.43.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2020-02-27
+  nightly_version=2020-04-23
 fi
 
 
@@ -51,6 +53,10 @@ export rust_nightly_docker_image=solanalabs/rust-nightly:"$nightly_version"
   nightly)
      rustup_install "$rust_nightly"
     ;;
+  all)
+     rustup_install "$rust_stable"
+     rustup_install "$rust_nightly"
+    ;;
   *)
     echo "Note: ignoring unknown argument: $1"
     ;;

ci/test-bench.sh

@@ -25,7 +25,7 @@ source ci/_
 source ci/upload-ci-artifact.sh
 
 eval "$(ci/channel-info.sh)"
-source ci/rust-version.sh nightly
+source ci/rust-version.sh all
 
 set -o pipefail
 export RUST_BACKTRACE=1

ci/test-checks.sh

@@ -22,7 +22,7 @@ _ cargo +"$rust_stable" clippy --all --exclude solana-sdk-c -- --deny=warnings
 _ cargo +"$rust_stable" clippy --manifest-path sdk-c/Cargo.toml -- --deny=warnings
 
 _ cargo +"$rust_stable" audit --version
-_ cargo +"$rust_stable" audit --ignore RUSTSEC-2020-0002
+_ cargo +"$rust_stable" audit --ignore RUSTSEC-2020-0002 --ignore RUSTSEC-2020-0008
 _ ci/nits.sh
 _ ci/order-crates-for-publishing.py
 _ docs/build.sh

ci/test-stable.sh

@@ -38,11 +38,16 @@ test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
 # Clear the BPF sysroot files, they are not automatically rebuilt
 rm -rf target/xargo # Issue #3105
 
+# Limit compiler jobs to reduce memory usage
+# on machines with 1gb/thread of memory
+NPROC=$(nproc)
+NPROC=$((NPROC>16 ? 16 : NPROC))
+
 echo "Executing $testName"
 case $testName in
 test-stable)
-  _ cargo +"$rust_stable" test --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
-  _ cargo +"$rust_stable" test --manifest-path bench-tps/Cargo.toml --features=move ${V:+--verbose} test_bench_tps_local_cluster_move -- --nocapture
+  _ cargo +"$rust_stable" test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
+  #_ cargo +"$rust_stable" test --manifest-path bench-tps/Cargo.toml --features=move ${V:+--verbose} test_bench_tps_local_cluster_move -- --nocapture
   ;;
 test-stable-perf)
   ci/affects-files.sh \

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.0.5"
+version = "1.0.22"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,11 +11,15 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.0.5" }
-solana-sdk = { path = "../sdk", version = "1.0.5" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
+thiserror = "1.0.11"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"
 chrono = "0.4"
 
 [lib]
 name = "solana_clap_utils"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

clap-utils/src/commitment.rs

@@ -0,0 +1,18 @@
+use crate::ArgConstant;
+use clap::Arg;
+
+pub const COMMITMENT_ARG: ArgConstant<'static> = ArgConstant {
+    name: "commitment",
+    long: "commitment",
+    help: "Return information at the selected commitment level",
+};
+
+pub fn commitment_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(COMMITMENT_ARG.name)
+        .long(COMMITMENT_ARG.long)
+        .takes_value(true)
+        .possible_values(&["recent", "root", "max"])
+        .default_value("recent")
+        .value_name("COMMITMENT_LEVEL")
+        .help(COMMITMENT_ARG.help)
+}

clap-utils/src/input_parsers.rs

@@ -1,12 +1,13 @@
 use crate::keypair::{
-    keypair_from_seed_phrase, pubkey_from_path, signer_from_path, ASK_KEYWORD,
-    SKIP_SEED_PHRASE_VALIDATION_ARG,
+    keypair_from_seed_phrase, pubkey_from_path, resolve_signer_from_path, signer_from_path,
+    ASK_KEYWORD, SKIP_SEED_PHRASE_VALIDATION_ARG,
 };
 use chrono::DateTime;
 use clap::ArgMatches;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     clock::UnixTimestamp,
+    commitment_config::CommitmentConfig,
     native_token::sol_to_lamports,
     pubkey::Pubkey,
     signature::{read_keypair_file, Keypair, Signature, Signer},
@@ -129,10 +130,48 @@ pub fn pubkey_of_signer(
     }
 }
 
+pub fn pubkeys_of_multiple_signers(
+    matches: &ArgMatches<'_>,
+    name: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<Option<Vec<Pubkey>>, Box<dyn std::error::Error>> {
+    if let Some(pubkey_matches) = matches.values_of(name) {
+        let mut pubkeys: Vec<Pubkey> = vec![];
+        for signer in pubkey_matches {
+            pubkeys.push(pubkey_from_path(matches, signer, name, wallet_manager)?);
+        }
+        Ok(Some(pubkeys))
+    } else {
+        Ok(None)
+    }
+}
+
+pub fn resolve_signer(
+    matches: &ArgMatches<'_>,
+    name: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<Option<String>, Box<dyn std::error::Error>> {
+    Ok(resolve_signer_from_path(
+        matches,
+        matches.value_of(name).unwrap(),
+        name,
+        wallet_manager,
+    )?)
+}
+
 pub fn lamports_of_sol(matches: &ArgMatches<'_>, name: &str) -> Option<u64> {
     value_of(matches, name).map(sol_to_lamports)
 }
 
+pub fn commitment_of(matches: &ArgMatches<'_>, name: &str) -> Option<CommitmentConfig> {
+    matches.value_of(name).map(|value| match value {
+        "max" => CommitmentConfig::max(),
+        "recent" => CommitmentConfig::recent(),
+        "root" => CommitmentConfig::root(),
+        _ => CommitmentConfig::default(),
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

clap-utils/src/input_validators.rs

@@ -12,7 +12,7 @@ use std::str::FromStr;
 pub fn is_pubkey(string: String) -> Result<(), String> {
     match string.parse::<Pubkey>() {
         Ok(_) => Ok(()),
-        Err(err) => Err(format!("{:?}", err)),
+        Err(err) => Err(format!("{}", err)),
     }
 }
 
@@ -20,7 +20,7 @@ pub fn is_pubkey(string: String) -> Result<(), String> {
 pub fn is_hash(string: String) -> Result<(), String> {
     match string.parse::<Hash>() {
         Ok(_) => Ok(()),
-        Err(err) => Err(format!("{:?}", err)),
+        Err(err) => Err(format!("{}", err)),
     }
 }
 
@@ -28,7 +28,7 @@ pub fn is_hash(string: String) -> Result<(), String> {
 pub fn is_keypair(string: String) -> Result<(), String> {
     read_keypair_file(&string)
         .map(|_| ())
-        .map_err(|err| format!("{:?}", err))
+        .map_err(|err| format!("{}", err))
 }
 
 // Return an error if a keypair file cannot be parsed
@@ -38,7 +38,7 @@ pub fn is_keypair_or_ask_keyword(string: String) -> Result<(), String> {
     }
     read_keypair_file(&string)
         .map(|_| ())
-        .map_err(|err| format!("{:?}", err))
+        .map_err(|err| format!("{}", err))
 }
 
 // Return an error if string cannot be parsed as pubkey string or keypair file location
@@ -46,18 +46,27 @@ pub fn is_pubkey_or_keypair(string: String) -> Result<(), String> {
     is_pubkey(string.clone()).or_else(|_| is_keypair(string))
 }
 
-// Return an error if string cannot be parsed as pubkey or keypair file or keypair ask keyword
-pub fn is_pubkey_or_keypair_or_ask_keyword(string: String) -> Result<(), String> {
-    is_pubkey(string.clone()).or_else(|_| is_keypair_or_ask_keyword(string))
-}
-
-pub fn is_valid_signer(string: String) -> Result<(), String> {
+// Return an error if string cannot be parsed as a pubkey string, or a valid Signer that can
+// produce a pubkey()
+pub fn is_valid_pubkey(string: String) -> Result<(), String> {
     match parse_keypair_path(&string) {
         KeypairUrl::Filepath(path) => is_keypair(path),
         _ => Ok(()),
     }
 }
 
+// Return an error if string cannot be parsed as a valid Signer. This is an alias of
+// `is_valid_pubkey`, and does accept pubkey strings, even though a Pubkey is not by itself
+// sufficient to sign a transaction.
+//
+// In the current offline-signing implementation, a pubkey is the valid input for a signer field
+// when paired with an offline `--signer` argument to provide a Presigner (pubkey + signature).
+// Clap validators can't check multiple fields at once, so the verification that a `--signer` is
+// also provided and correct happens in parsing, not in validation.
+pub fn is_valid_signer(string: String) -> Result<(), String> {
+    is_valid_pubkey(string)
+}
+
 // Return an error if string cannot be parsed as pubkey=signature string
 pub fn is_pubkey_sig(string: String) -> Result<(), String> {
     let mut signer = string.split('=');
@@ -73,10 +82,10 @@ pub fn is_pubkey_sig(string: String) -> Result<(), String> {
                     .ok_or_else(|| "Malformed signer string".to_string())?,
             ) {
                 Ok(_) => Ok(()),
-                Err(err) => Err(format!("{:?}", err)),
+                Err(err) => Err(format!("{}", err)),
             }
         }
-        Err(err) => Err(format!("{:?}", err)),
+        Err(err) => Err(format!("{}", err)),
     }
 }
 
@@ -90,20 +99,20 @@ pub fn is_url(string: String) -> Result<(), String> {
                 Err("no host provided".to_string())
             }
         }
-        Err(err) => Err(format!("{:?}", err)),
+        Err(err) => Err(format!("{}", err)),
     }
 }
 
 pub fn is_slot(slot: String) -> Result<(), String> {
     slot.parse::<Slot>()
         .map(|_| ())
-        .map_err(|e| format!("{:?}", e))
+        .map_err(|e| format!("{}", e))
 }
 
 pub fn is_port(port: String) -> Result<(), String> {
     port.parse::<u16>()
         .map(|_| ())
-        .map_err(|e| format!("{:?}", e))
+        .map_err(|e| format!("{}", e))
 }
 
 pub fn is_valid_percentage(percentage: String) -> Result<(), String> {
@@ -111,7 +120,7 @@ pub fn is_valid_percentage(percentage: String) -> Result<(), String> {
         .parse::<u8>()
         .map_err(|e| {
             format!(
-                "Unable to parse input percentage, provided: {}, err: {:?}",
+                "Unable to parse input percentage, provided: {}, err: {}",
                 percentage, e
             )
         })
@@ -141,7 +150,7 @@ pub fn is_amount(amount: String) -> Result<(), String> {
 pub fn is_rfc3339_datetime(value: String) -> Result<(), String> {
     DateTime::parse_from_rfc3339(&value)
         .map(|_| ())
-        .map_err(|e| format!("{:?}", e))
+        .map_err(|e| format!("{}", e))
 }
 
 pub fn is_derivation(value: String) -> Result<(), String> {
@@ -152,7 +161,7 @@ pub fn is_derivation(value: String) -> Result<(), String> {
         .parse::<u32>()
         .map_err(|e| {
             format!(
-                "Unable to parse derivation, provided: {}, err: {:?}",
+                "Unable to parse derivation, provided: {}, err: {}",
                 account, e
             )
         })
@@ -160,7 +169,7 @@ pub fn is_derivation(value: String) -> Result<(), String> {
             if let Some(change) = parts.next() {
                 change.parse::<u32>().map_err(|e| {
                     format!(
-                        "Unable to parse derivation, provided: {}, err: {:?}",
+                        "Unable to parse derivation, provided: {}, err: {}",
                         change, e
                     )
                 })

clap-utils/src/keypair.rs

@@ -1,6 +1,10 @@
-use crate::{input_parsers::pubkeys_sigs_of, offline::SIGNER_ARG, ArgConstant};
+use crate::{
+    input_parsers::pubkeys_sigs_of,
+    offline::{SIGNER_ARG, SIGN_ONLY_ARG},
+    ArgConstant,
+};
 use bip39::{Language, Mnemonic, Seed};
-use clap::{ArgMatches, Error, ErrorKind};
+use clap::ArgMatches;
 use rpassword::prompt_password_stderr;
 use solana_remote_wallet::{
     remote_keypair::generate_remote_keypair,
@@ -10,7 +14,7 @@ use solana_sdk::{
     pubkey::Pubkey,
     signature::{
         keypair_from_seed, keypair_from_seed_phrase_and_passphrase, read_keypair,
-        read_keypair_file, Keypair, Presigner, Signature, Signer,
+        read_keypair_file, Keypair, NullSigner, Presigner, Signature, Signer,
     },
 };
 use std::{
@@ -71,7 +75,14 @@ pub fn signer_from_path(
                 false,
             )?))
         }
-        KeypairUrl::Filepath(path) => Ok(Box::new(read_keypair_file(&path)?)),
+        KeypairUrl::Filepath(path) => match read_keypair_file(&path) {
+            Err(e) => Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                format!("could not find keypair file: {} error: {}", path, e),
+            )
+            .into()),
+            Ok(file) => Ok(Box::new(file)),
+        },
         KeypairUrl::Stdin => {
             let mut stdin = std::io::stdin();
             Ok(Box::new(read_keypair(&mut stdin)?))
@@ -94,10 +105,12 @@ pub fn signer_from_path(
                 .and_then(|presigners| presigner_from_pubkey_sigs(&pubkey, presigners));
             if let Some(presigner) = presigner {
                 Ok(Box::new(presigner))
+            } else if matches.is_present(SIGN_ONLY_ARG.name) {
+                Ok(Box::new(NullSigner::new(&pubkey)))
             } else {
-                Err(Error::with_description(
-                    "Missing signature for supplied pubkey",
-                    ErrorKind::MissingRequiredArgument,
+                Err(std::io::Error::new(
+                    std::io::ErrorKind::Other,
+                    format!("missing signature for supplied pubkey: {}", pubkey),
                 )
                 .into())
             }
@@ -117,6 +130,51 @@ pub fn pubkey_from_path(
     }
 }
 
+pub fn resolve_signer_from_path(
+    matches: &ArgMatches,
+    path: &str,
+    keypair_name: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<Option<String>, Box<dyn error::Error>> {
+    match parse_keypair_path(path) {
+        KeypairUrl::Ask => {
+            let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
+            // This method validates the seed phrase, but returns `None` because there is no path
+            // on disk or to a device
+            keypair_from_seed_phrase(keypair_name, skip_validation, false).map(|_| None)
+        }
+        KeypairUrl::Filepath(path) => match read_keypair_file(&path) {
+            Err(e) => Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                format!("could not find keypair file: {} error: {}", path, e),
+            )
+            .into()),
+            Ok(_) => Ok(Some(path.to_string())),
+        },
+        KeypairUrl::Stdin => {
+            let mut stdin = std::io::stdin();
+            // This method validates the keypair from stdin, but returns `None` because there is no
+            // path on disk or to a device
+            read_keypair(&mut stdin).map(|_| None)
+        }
+        KeypairUrl::Usb(path) => {
+            if let Some(wallet_manager) = wallet_manager {
+                let path = generate_remote_keypair(
+                    path,
+                    wallet_manager,
+                    matches.is_present("confirm_key"),
+                    keypair_name,
+                )
+                .map(|keypair| keypair.path)?;
+                Ok(Some(path))
+            } else {
+                Err(RemoteWalletError::NoDeviceFound.into())
+            }
+        }
+        _ => Ok(Some(path.to_string())),
+    }
+}
+
 // Keyword used to indicate that the user should be asked for a keypair seed phrase
 pub const ASK_KEYWORD: &str = "ASK";
 

clap-utils/src/lib.rs

@@ -1,3 +1,5 @@
+use thiserror::Error;
+
 #[macro_export]
 macro_rules! version {
     () => {
@@ -23,6 +25,24 @@ pub struct ArgConstant<'a> {
     pub help: &'a str,
 }
 
+/// Error type for forwarding Errors out of `main()` of a `clap` app
+/// and still using the `Display` formatter
+#[derive(Error)]
+#[error("{0}")]
+pub struct DisplayError(Box<dyn std::error::Error>);
+impl DisplayError {
+    pub fn new_as_boxed(inner: Box<dyn std::error::Error>) -> Box<Self> {
+        DisplayError(inner).into()
+    }
+}
+
+impl std::fmt::Debug for DisplayError {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(fmt, "{}", self.0)
+    }
+}
+
+pub mod commitment;
 pub mod input_parsers;
 pub mod input_validators;
 pub mod keypair;

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.5"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,3 +15,6 @@ serde = "1.0.104"
 serde_derive = "1.0.103"
 serde_yaml = "0.8.11"
 url = "2.1.1"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

cli-config/src/config.rs

@@ -1,10 +1,6 @@
 // Wallet settings that can be configured for long-term use
 use serde_derive::{Deserialize, Serialize};
-use std::{
-    fs::{create_dir_all, File},
-    io::{self, Write},
-    path::Path,
-};
+use std::io;
 use url::Url;
 
 lazy_static! {
@@ -31,7 +27,10 @@ impl Default for Config {
             keypair_path.to_str().unwrap().to_string()
         };
         let json_rpc_url = "http://127.0.0.1:8899".to_string();
-        let websocket_url = Self::compute_websocket_url(&json_rpc_url);
+
+        // Empty websocket_url string indicates the client should
+        // `Config::compute_websocket_url(&json_rpc_url)`
+        let websocket_url = "".to_string();
 
         Self {
             json_rpc_url,
@@ -43,23 +42,11 @@ impl Default for Config {
 
 impl Config {
     pub fn load(config_file: &str) -> Result<Self, io::Error> {
-        let file = File::open(config_file.to_string())?;
-        let config = serde_yaml::from_reader(file)
-            .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
-        Ok(config)
+        crate::load_config_file(config_file)
     }
 
     pub fn save(&self, config_file: &str) -> Result<(), io::Error> {
-        let serialized = serde_yaml::to_string(self)
-            .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
-
-        if let Some(outdir) = Path::new(&config_file).parent() {
-            create_dir_all(outdir)?;
-        }
-        let mut file = File::create(config_file)?;
-        file.write_all(&serialized.into_bytes())?;
-
-        Ok(())
+        crate::save_config_file(self, config_file)
     }
 
     pub fn compute_websocket_url(json_rpc_url: &str) -> String {
@@ -73,17 +60,38 @@ impl Config {
         ws_url
             .set_scheme(if is_secure { "wss" } else { "ws" })
             .expect("unable to set scheme");
-        let ws_port = match json_rpc_url.port() {
-            Some(port) => port + 1,
-            None => {
-                if is_secure {
-                    8901
-                } else {
-                    8900
-                }
-            }
-        };
-        ws_url.set_port(Some(ws_port)).expect("unable to set port");
+        if let Some(port) = json_rpc_url.port() {
+            ws_url.set_port(Some(port + 1)).expect("unable to set port");
+        }
         ws_url.to_string()
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn compute_websocket_url() {
+        assert_eq!(
+            Config::compute_websocket_url(&"http://devnet.solana.com"),
+            "ws://devnet.solana.com/".to_string()
+        );
+
+        assert_eq!(
+            Config::compute_websocket_url(&"https://devnet.solana.com"),
+            "wss://devnet.solana.com/".to_string()
+        );
+
+        assert_eq!(
+            Config::compute_websocket_url(&"http://example.com:8899"),
+            "ws://example.com:8900/".to_string()
+        );
+        assert_eq!(
+            Config::compute_websocket_url(&"https://example.com:1234"),
+            "wss://example.com:1235/".to_string()
+        );
+
+        assert_eq!(Config::compute_websocket_url(&"garbage"), String::new());
+    }
+}

cli-config/src/lib.rs

@@ -3,3 +3,37 @@ extern crate lazy_static;
 
 mod config;
 pub use config::{Config, CONFIG_FILE};
+
+use std::{
+    fs::{create_dir_all, File},
+    io::{self, Write},
+    path::Path,
+};
+
+pub fn load_config_file<T, P>(config_file: P) -> Result<T, io::Error>
+where
+    T: serde::de::DeserializeOwned,
+    P: AsRef<Path>,
+{
+    let file = File::open(config_file)?;
+    let config = serde_yaml::from_reader(file)
+        .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
+    Ok(config)
+}
+
+pub fn save_config_file<T, P>(config: &T, config_file: P) -> Result<(), io::Error>
+where
+    T: serde::ser::Serialize,
+    P: AsRef<Path>,
+{
+    let serialized = serde_yaml::to_string(config)
+        .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
+
+    if let Some(outdir) = config_file.as_ref().parent() {
+        create_dir_all(outdir)?;
+    }
+    let mut file = File::create(config_file)?;
+    file.write_all(&serialized.into_bytes())?;
+
+    Ok(())
+}

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.5"
+version = "1.0.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -26,29 +26,34 @@ reqwest = { version = "0.10.1", default-features = false, features = ["blocking"
 serde = "1.0.104"
 serde_derive = "1.0.103"
 serde_json = "1.0.46"
-solana-budget-program = { path = "../programs/budget", version = "1.0.5" }
-solana-clap-utils = { path = "../clap-utils", version = "1.0.5" }
-solana-cli-config = { path = "../cli-config", version = "1.0.5" }
-solana-client = { path = "../client", version = "1.0.5" }
-solana-config-program = { path = "../programs/config", version = "1.0.5" }
-solana-faucet = { path = "../faucet", version = "1.0.5" }
-solana-logger = { path = "../logger", version = "1.0.5" }
-solana-net-utils = { path = "../net-utils", version = "1.0.5" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.0.5" }
-solana-runtime = { path = "../runtime", version = "1.0.5" }
-solana-sdk = { path = "../sdk", version = "1.0.5" }
-solana-stake-program = { path = "../programs/stake", version = "1.0.5" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.5" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.5" }
-solana-vote-signer = { path = "../vote-signer", version = "1.0.5" }
+solana-budget-program = { path = "../programs/budget", version = "1.0.22" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.22" }
+solana-cli-config = { path = "../cli-config", version = "1.0.22" }
+solana-client = { path = "../client", version = "1.0.22" }
+solana-config-program = { path = "../programs/config", version = "1.0.22" }
+solana-faucet = { path = "../faucet", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.0.22" }
+solana-runtime = { path = "../runtime", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
+solana-stake-program = { path = "../programs/stake", version = "1.0.22" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.22" }
+solana-transaction-status = { path = "../transaction-status", version = "1.0.22" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.22" }
+solana-vote-signer = { path = "../vote-signer", version = "1.0.22" }
 titlecase = "1.1.0"
+thiserror = "1.0.11"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.0.5" }
-solana-budget-program = { path = "../programs/budget", version = "1.0.5" }
+solana-core = { path = "../core", version = "1.0.22" }
+solana-budget-program = { path = "../programs/budget", version = "1.0.22" }
 tempfile = "3.1.0"
 
 [[bin]]
 name = "solana"
 path = "src/main.rs"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

cli/src/cluster_query.rs

@@ -8,7 +8,12 @@ use crate::{
 use clap::{value_t, value_t_or_exit, App, Arg, ArgMatches, SubCommand};
 use console::{style, Emoji};
 use indicatif::{ProgressBar, ProgressStyle};
-use solana_clap_utils::{input_parsers::*, input_validators::*, keypair::signer_from_path};
+use solana_clap_utils::{
+    commitment::{commitment_arg, COMMITMENT_ARG},
+    input_parsers::*,
+    input_validators::*,
+    keypair::signer_from_path,
+};
 use solana_client::{
     pubsub_client::{PubsubClient, SlotInfoMessage},
     rpc_client::RpcClient,
@@ -55,8 +60,8 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                     Arg::with_name("node_pubkey")
                         .index(1)
                         .takes_value(true)
-                        .value_name("PUBKEY")
-                        .validator(is_pubkey_or_keypair)
+                        .value_name("VALIDATOR_PUBKEY")
+                        .validator(is_valid_pubkey)
                         .required(true)
                         .help("Identity pubkey of the validator"),
                 )
@@ -91,14 +96,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
             SubCommand::with_name("epoch-info")
             .about("Get information about the current epoch")
             .alias("get-epoch-info")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return information at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("genesis-hash")
@@ -108,26 +106,16 @@ impl ClusterQuerySubCommands for App<'_, '_> {
         .subcommand(
             SubCommand::with_name("slot").about("Get current slot")
             .alias("get-slot")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return slot at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
+        )
+        .subcommand(
+            SubCommand::with_name("epoch").about("Get current epoch")
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("transaction-count").about("Get current transaction count")
             .alias("get-transaction-count")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return count at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("ping")
@@ -167,14 +155,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .default_value("15")
                         .help("Wait up to timeout seconds for transaction confirmation"),
                 )
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Wait until the transaction is confirmed at maximum-lockout commitment level",
-                        ),
-                ),
+                .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("live-slots")
@@ -208,10 +189,10 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("vote_account_pubkeys")
                         .index(1)
-                        .value_name("VOTE ACCOUNT PUBKEYS")
+                        .value_name("VOTE_ACCOUNT_PUBKEYS")
                         .takes_value(true)
                         .multiple(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Only show stake accounts delegated to the provided vote accounts"),
                 )
                 .arg(
@@ -225,26 +206,22 @@ impl ClusterQuerySubCommands for App<'_, '_> {
             SubCommand::with_name("validators")
                 .about("Show summary information about the current validators")
                 .alias("show-validators")
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Return information at maximum-lockout commitment level",
-                        ),
-                )
                 .arg(
                     Arg::with_name("lamports")
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
-                ),
+                )
+                .arg(commitment_arg()),
         )
     }
 }
 
-pub fn parse_catchup(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let node_pubkey = pubkey_of(matches, "node_pubkey").unwrap();
+pub fn parse_catchup(
+    matches: &ArgMatches<'_>,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let node_pubkey = pubkey_of_signer(matches, "node_pubkey", wallet_manager)?.unwrap();
     let node_json_rpc_url = value_t!(matches, "node_json_rpc_url", String).ok();
     Ok(CliCommandInfo {
         command: CliCommand::Catchup {
@@ -268,11 +245,7 @@ pub fn parse_cluster_ping(
         None
     };
     let timeout = Duration::from_secs(value_t_or_exit!(matches, "timeout", u64));
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::Ping {
             lamports,
@@ -299,11 +272,7 @@ pub fn parse_get_block_time(matches: &ArgMatches<'_>) -> Result<CliCommandInfo,
 }
 
 pub fn parse_get_epoch_info(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetEpochInfo { commitment_config },
         signers: vec![],
@@ -311,32 +280,36 @@ pub fn parse_get_epoch_info(matches: &ArgMatches<'_>) -> Result<CliCommandInfo,
 }
 
 pub fn parse_get_slot(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetSlot { commitment_config },
         signers: vec![],
     })
 }
 
+pub fn parse_get_epoch(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
+    Ok(CliCommandInfo {
+        command: CliCommand::GetEpoch { commitment_config },
+        signers: vec![],
+    })
+}
+
 pub fn parse_get_transaction_count(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetTransactionCount { commitment_config },
         signers: vec![],
     })
 }
 
-pub fn parse_show_stakes(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+pub fn parse_show_stakes(
+    matches: &ArgMatches<'_>,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
     let use_lamports_unit = matches.is_present("lamports");
-    let vote_account_pubkeys = pubkeys_of(matches, "vote_account_pubkeys");
+    let vote_account_pubkeys =
+        pubkeys_of_multiple_signers(matches, "vote_account_pubkeys", wallet_manager)?;
 
     Ok(CliCommandInfo {
         command: CliCommand::ShowStakes {
@@ -349,11 +322,7 @@ pub fn parse_show_stakes(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, Cli
 
 pub fn parse_show_validators(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let use_lamports_unit = matches.is_present("lamports");
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
 
     Ok(CliCommandInfo {
         command: CliCommand::ShowValidators {
@@ -424,25 +393,35 @@ pub fn process_catchup(
         }
 
         let slot_distance = rpc_slot as i64 - node_slot as i64;
+        let slots_per_second =
+            (previous_slot_distance - slot_distance) as f64 / f64::from(sleep_interval);
+        let time_remaining = (slot_distance as f64 / slots_per_second).round();
+        let time_remaining = if !time_remaining.is_normal() || time_remaining <= 0.0 {
+            "".to_string()
+        } else {
+            format!(
+                ". Time remaining: {}",
+                humantime::format_duration(Duration::from_secs_f64(time_remaining))
+            )
+        };
+
         progress_bar.set_message(&format!(
-            "Validator is {} slots away (us:{} them:{}){}",
+            "{} slots behind (us:{} them:{}){}",
             slot_distance,
             node_slot,
             rpc_slot,
             if previous_rpc_slot == std::u64::MAX {
                 "".to_string()
             } else {
-                let slots_per_second =
-                    (previous_slot_distance - slot_distance) as f64 / f64::from(sleep_interval);
-
                 format!(
-                    " and {} at {:.1} slots/second",
+                    ", {} at {:.1} slots/second{}",
                     if slots_per_second < 0.0 {
                         "falling behind"
                     } else {
                         "gaining"
                     },
                     slots_per_second,
+                    time_remaining
                 )
             }
         ));
@@ -568,6 +547,14 @@ pub fn process_get_slot(
     Ok(slot.to_string())
 }
 
+pub fn process_get_epoch(
+    rpc_client: &RpcClient,
+    commitment_config: CommitmentConfig,
+) -> ProcessResult {
+    let epoch_info = rpc_client.get_epoch_info_with_commitment(commitment_config.clone())?;
+    Ok(epoch_info.epoch.to_string())
+}
+
 pub fn parse_show_block_production(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let epoch = value_t!(matches, "epoch", Epoch).ok();
     let slot_limit = value_t!(matches, "slot_limit", u64).ok();
@@ -789,7 +776,7 @@ pub fn process_ping(
         last_blockhash = recent_blockhash;
 
         let ix = system_instruction::transfer(&config.signers[0].pubkey(), &to, lamports);
-        let message = Message::new(vec![ix]);
+        let message = Message::new(&[ix]);
         let mut transaction = Transaction::new_unsigned(message);
         transaction.try_sign(&config.signers, recent_blockhash)?;
         check_account_for_fee(
@@ -972,7 +959,7 @@ pub fn process_live_slots(url: &str) -> ProcessResult {
                 current = Some(new_info);
             }
             Err(err) => {
-                eprintln!("disconnected: {:?}", err);
+                eprintln!("disconnected: {}", err);
                 break;
             }
         }
@@ -1273,6 +1260,19 @@ mod tests {
             }
         );
 
+        let test_get_epoch = test_commands
+            .clone()
+            .get_matches_from(vec!["test", "epoch"]);
+        assert_eq!(
+            parse_command(&test_get_epoch, &default_keypair_file, None).unwrap(),
+            CliCommandInfo {
+                command: CliCommand::GetEpoch {
+                    commitment_config: CommitmentConfig::recent(),
+                },
+                signers: vec![],
+            }
+        );
+
         let test_transaction_count = test_commands
             .clone()
             .get_matches_from(vec!["test", "transaction-count"]);
@@ -1295,7 +1295,8 @@ mod tests {
             "2",
             "-t",
             "3",
-            "--confirmed",
+            "--commitment",
+            "max",
         ]);
         assert_eq!(
             parse_command(&test_ping, &default_keypair_file, None).unwrap(),
@@ -1305,7 +1306,7 @@ mod tests {
                     interval: Duration::from_secs(1),
                     count: Some(2),
                     timeout: Duration::from_secs(3),
-                    commitment_config: CommitmentConfig::default(),
+                    commitment_config: CommitmentConfig::max(),
                 },
                 signers: vec![default_keypair.into()],
             }

cli/src/display.rs

@@ -1,6 +1,11 @@
 use crate::cli::SettingType;
 use console::style;
-use solana_sdk::transaction::Transaction;
+use solana_sdk::{
+    hash::Hash, native_token::lamports_to_sol, program_utils::limited_deserialize,
+    transaction::Transaction,
+};
+use solana_transaction_status::RpcTransactionStatusMeta;
+use std::io;
 
 // Pretty print a "name value"
 pub fn println_name_value(name: &str, value: &str) {
@@ -27,13 +32,162 @@ pub fn println_name_value_or(name: &str, value: &str, setting_type: SettingType)
     );
 }
 
-pub fn println_signers(tx: &Transaction) {
+pub fn println_signers(
+    blockhash: &Hash,
+    signers: &[String],
+    absent: &[String],
+    bad_sig: &[String],
+) {
     println!();
-    println!("Blockhash: {}", tx.message.recent_blockhash);
-    println!("Signers (Pubkey=Signature):");
-    tx.signatures
-        .iter()
-        .zip(tx.message.account_keys.clone())
-        .for_each(|(signature, pubkey)| println!("  {:?}={:?}", pubkey, signature));
+    println!("Blockhash: {}", blockhash);
+    if !signers.is_empty() {
+        println!("Signers (Pubkey=Signature):");
+        signers.iter().for_each(|signer| println!("  {}", signer))
+    }
+    if !absent.is_empty() {
+        println!("Absent Signers (Pubkey):");
+        absent.iter().for_each(|pubkey| println!("  {}", pubkey))
+    }
+    if !bad_sig.is_empty() {
+        println!("Bad Signatures (Pubkey):");
+        bad_sig.iter().for_each(|pubkey| println!("  {}", pubkey))
+    }
     println!();
 }
+
+pub fn write_transaction<W: io::Write>(
+    w: &mut W,
+    transaction: &Transaction,
+    transaction_status: &Option<RpcTransactionStatusMeta>,
+    prefix: &str,
+) -> io::Result<()> {
+    let message = &transaction.message;
+    writeln!(
+        w,
+        "{}Recent Blockhash: {:?}",
+        prefix, message.recent_blockhash
+    )?;
+    for (signature_index, signature) in transaction.signatures.iter().enumerate() {
+        writeln!(
+            w,
+            "{}Signature {}: {:?}",
+            prefix, signature_index, signature
+        )?;
+    }
+    writeln!(w, "{}{:?}", prefix, message.header)?;
+    for (account_index, account) in message.account_keys.iter().enumerate() {
+        writeln!(w, "{}Account {}: {:?}", prefix, account_index, account)?;
+    }
+    for (instruction_index, instruction) in message.instructions.iter().enumerate() {
+        let program_pubkey = message.account_keys[instruction.program_id_index as usize];
+        writeln!(w, "{}Instruction {}", prefix, instruction_index)?;
+        writeln!(
+            w,
+            "{}  Program: {} ({})",
+            prefix, program_pubkey, instruction.program_id_index
+        )?;
+        for (account_index, account) in instruction.accounts.iter().enumerate() {
+            let account_pubkey = message.account_keys[*account as usize];
+            writeln!(
+                w,
+                "{}  Account {}: {} ({})",
+                prefix, account_index, account_pubkey, account
+            )?;
+        }
+
+        let mut raw = true;
+        if program_pubkey == solana_vote_program::id() {
+            if let Ok(vote_instruction) = limited_deserialize::<
+                solana_vote_program::vote_instruction::VoteInstruction,
+            >(&instruction.data)
+            {
+                writeln!(w, "{}  {:?}", prefix, vote_instruction)?;
+                raw = false;
+            }
+        } else if program_pubkey == solana_stake_program::id() {
+            if let Ok(stake_instruction) = limited_deserialize::<
+                solana_stake_program::stake_instruction::StakeInstruction,
+            >(&instruction.data)
+            {
+                writeln!(w, "{}  {:?}", prefix, stake_instruction)?;
+                raw = false;
+            }
+        } else if program_pubkey == solana_sdk::system_program::id() {
+            if let Ok(system_instruction) = limited_deserialize::<
+                solana_sdk::system_instruction::SystemInstruction,
+            >(&instruction.data)
+            {
+                writeln!(w, "{}  {:?}", prefix, system_instruction)?;
+                raw = false;
+            }
+        }
+
+        if raw {
+            writeln!(w, "{}  Data: {:?}", prefix, instruction.data)?;
+        }
+    }
+
+    if let Some(transaction_status) = transaction_status {
+        writeln!(
+            w,
+            "{}Status: {}",
+            prefix,
+            match &transaction_status.status {
+                Ok(_) => "Ok".into(),
+                Err(err) => err.to_string(),
+            }
+        )?;
+        writeln!(
+            w,
+            "{}  Fee: {} SOL",
+            prefix,
+            lamports_to_sol(transaction_status.fee)
+        )?;
+        assert_eq!(
+            transaction_status.pre_balances.len(),
+            transaction_status.post_balances.len()
+        );
+        for (i, (pre, post)) in transaction_status
+            .pre_balances
+            .iter()
+            .zip(transaction_status.post_balances.iter())
+            .enumerate()
+        {
+            if pre == post {
+                writeln!(
+                    w,
+                    "{}  Account {} balance: {} SOL",
+                    prefix,
+                    i,
+                    lamports_to_sol(*pre)
+                )?;
+            } else {
+                writeln!(
+                    w,
+                    "{}  Account {} balance: {} SOL -> {} SOL",
+                    prefix,
+                    i,
+                    lamports_to_sol(*pre),
+                    lamports_to_sol(*post)
+                )?;
+            }
+        }
+    } else {
+        writeln!(w, "{}Status: Unavailable", prefix)?;
+    }
+
+    Ok(())
+}
+
+pub fn println_transaction(
+    transaction: &Transaction,
+    transaction_status: &Option<RpcTransactionStatusMeta>,
+    prefix: &str,
+) {
+    let mut w = Vec::new();
+    if write_transaction(&mut w, transaction, transaction_status, prefix).is_ok() {
+        if let Ok(s) = String::from_utf8(w) {
+            print!("{}", s);
+        }
+    }
+}

cli/src/main.rs

@@ -1,7 +1,10 @@
 use clap::{crate_description, crate_name, AppSettings, Arg, ArgGroup, ArgMatches, SubCommand};
 use console::style;
 
-use solana_clap_utils::{input_validators::is_url, keypair::SKIP_SEED_PHRASE_VALIDATION_ARG};
+use solana_clap_utils::{
+    input_validators::is_url, keypair::SKIP_SEED_PHRASE_VALIDATION_ARG, offline::SIGN_ONLY_ARG,
+    DisplayError,
+};
 use solana_cli::{
     cli::{app, parse_command, process_command, CliCommandInfo, CliConfig, CliSigners},
     display::{println_name_value, println_name_value_or},
@@ -31,7 +34,7 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
                     if let Some(field) = subcommand_matches.value_of("specific_setting") {
                         let (field_name, value, setting_type) = match field {
                             "json_rpc_url" => ("RPC URL", json_rpc_url, url_setting_type),
-                            "websocket_url" => ("WS URL", websocket_url, ws_setting_type),
+                            "websocket_url" => ("WebSocket URL", websocket_url, ws_setting_type),
                             "keypair" => ("Key Path", keypair_path, keypair_setting_type),
                             _ => unreachable!(),
                         };
@@ -39,7 +42,7 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
                     } else {
                         println_name_value("Config File:", config_file);
                         println_name_value_or("RPC URL:", &json_rpc_url, url_setting_type);
-                        println_name_value_or("WS URL:", &websocket_url, ws_setting_type);
+                        println_name_value_or("WebSocket URL:", &websocket_url, ws_setting_type);
                         println_name_value_or("Keypair Path:", &keypair_path, keypair_setting_type);
                     }
                 } else {
@@ -55,6 +58,9 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
                     let mut config = Config::load(config_file).unwrap_or_default();
                     if let Some(url) = subcommand_matches.value_of("json_rpc_url") {
                         config.json_rpc_url = url.to_string();
+                        // Revert to a computed `websocket_url` value when `json_rpc_url` is
+                        // changed
+                        config.websocket_url = "".to_string();
                     }
                     if let Some(url) = subcommand_matches.value_of("websocket_url") {
                         config.websocket_url = url.to_string();
@@ -77,7 +83,7 @@ fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error
 
                     println_name_value("Config File:", config_file);
                     println_name_value_or("RPC URL:", &json_rpc_url, url_setting_type);
-                    println_name_value_or("WS URL:", &websocket_url, ws_setting_type);
+                    println_name_value_or("WebSocket URL:", &websocket_url, ws_setting_type);
                     println_name_value_or("Keypair Path:", &keypair_path, keypair_setting_type);
                 } else {
                     println!(
@@ -146,7 +152,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         let arg = Arg::with_name("config_file")
             .short("C")
             .long("config")
-            .value_name("PATH")
+            .value_name("FILEPATH")
             .takes_value(true)
             .global(true)
             .help("Configuration file to use");
@@ -179,17 +185,17 @@ fn main() -> Result<(), Box<dyn error::Error>> {
         Arg::with_name("keypair")
             .short("k")
             .long("keypair")
-            .value_name("PATH")
+            .value_name("KEYPAIR")
             .global(true)
             .takes_value(true)
-            .help("/path/to/id.json or usb://remote/wallet/path"),
+            .help("Filepath or URL to a keypair"),
     )
     .arg(
         Arg::with_name("verbose")
             .long("verbose")
             .short("v")
             .global(true)
-            .help("Show extra information header"),
+            .help("Show additional information"),
     )
     .arg(
         Arg::with_name(SKIP_SEED_PHRASE_VALIDATION_ARG.name)
@@ -227,13 +233,23 @@ fn main() -> Result<(), Box<dyn error::Error>> {
     )
     .get_matches();
 
+    do_main(&matches).map_err(|err| DisplayError::new_as_boxed(err).into())
+}
+
+fn do_main(matches: &ArgMatches<'_>) -> Result<(), Box<dyn error::Error>> {
     if parse_settings(&matches)? {
         let wallet_manager = maybe_wallet_manager()?;
 
         let (mut config, signers) = parse_args(&matches, wallet_manager)?;
         config.signers = signers.iter().map(|s| s.as_ref()).collect();
         let result = process_command(&config)?;
-        println!("{}", result);
-    }
+        let (_, submatches) = matches.subcommand();
+        let sign_only = submatches
+            .map(|m| m.is_present(SIGN_ONLY_ARG.name))
+            .unwrap_or(false);
+        if !sign_only {
+            println!("{}", result);
+        }
+    };
     Ok(())
 }

cli/src/nonce.rs

@@ -14,7 +14,11 @@ use solana_sdk::{
     account_utils::StateMut,
     hash::Hash,
     message::Message,
-    nonce::{self, state::Versions, State},
+    nonce::{
+        self,
+        state::{Data, Versions},
+        State,
+    },
     pubkey::Pubkey,
     system_instruction::{
         advance_nonce_account, authorize_nonce_account, create_address_with_seed,
@@ -25,14 +29,24 @@ use solana_sdk::{
     transaction::Transaction,
 };
 use std::sync::Arc;
+use thiserror::Error;
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Error, PartialEq)]
 pub enum CliNonceError {
+    #[error("invalid account owner")]
     InvalidAccountOwner,
+    #[error("invalid account data")]
     InvalidAccountData,
+    #[error("unexpected account data size")]
+    UnexpectedDataSize,
+    #[error("query hash does not match stored hash")]
     InvalidHash,
+    #[error("query authority does not match account authority")]
     InvalidAuthority,
-    InvalidState,
+    #[error("invalid state for requested operation")]
+    InvalidStateForOperation,
+    #[error("client error: {0}")]
+    Client(String),
 }
 
 pub const NONCE_ARG: ArgConstant<'static> = ArgConstant {
@@ -60,7 +74,7 @@ pub fn nonce_arg<'a, 'b>() -> Arg<'a, 'b> {
         .takes_value(true)
         .value_name("PUBKEY")
         .requires(BLOCKHASH_ARG.name)
-        .validator(is_pubkey)
+        .validator(is_valid_pubkey)
         .help(NONCE_ARG.help)
 }
 
@@ -68,7 +82,7 @@ pub fn nonce_authority_arg<'a, 'b>() -> Arg<'a, 'b> {
     Arg::with_name(NONCE_AUTHORITY_ARG.name)
         .long(NONCE_AUTHORITY_ARG.long)
         .takes_value(true)
-        .value_name("KEYPAIR or PUBKEY or REMOTE WALLET PATH")
+        .value_name("KEYPAIR")
         .validator(is_valid_signer)
         .help(NONCE_AUTHORITY_ARG.help)
 }
@@ -79,30 +93,23 @@ impl NonceSubCommands for App<'_, '_> {
             SubCommand::with_name("authorize-nonce-account")
                 .about("Assign account authority to a new entity")
                 .arg(
-                    Arg::with_name("nonce_account_keypair")
+                    Arg::with_name("nonce_account_pubkey")
                         .index(1)
-                        .value_name("NONCE_ACCOUNT")
+                        .value_name("NONCE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Address of the nonce account"),
                 )
                 .arg(
                     Arg::with_name("new_authority")
                         .index(2)
-                        .value_name("NEW_AUTHORITY_PUBKEY")
+                        .value_name("AUTHORITY_PUBKEY")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Account to be granted authority of the nonce account"),
                 )
-                .arg(
-                    Arg::with_name("seed")
-                        .long("seed")
-                        .value_name("SEED STRING")
-                        .takes_value(true)
-                        .help("Seed for address generation; if specified, the resulting account will be at a derived address of the NONCE_ACCOUNT pubkey")
-                )
                 .arg(nonce_authority_arg()),
         )
         .subcommand(
@@ -111,10 +118,10 @@ impl NonceSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("nonce_account_keypair")
                         .index(1)
-                        .value_name("NONCE ACCOUNT")
+                        .value_name("ACCOUNT_KEYPAIR")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_signer)
                         .help("Keypair of the nonce account to fund"),
                 )
                 .arg(
@@ -130,9 +137,16 @@ impl NonceSubCommands for App<'_, '_> {
                     Arg::with_name(NONCE_AUTHORITY_ARG.name)
                         .long(NONCE_AUTHORITY_ARG.long)
                         .takes_value(true)
-                        .value_name("BASE58_PUBKEY")
-                        .validator(is_pubkey_or_keypair)
+                        .value_name("PUBKEY")
+                        .validator(is_valid_pubkey)
                         .help("Assign noncing authority to another entity"),
+                )
+                .arg(
+                    Arg::with_name("seed")
+                        .long("seed")
+                        .value_name("STRING")
+                        .takes_value(true)
+                        .help("Seed for address generation; if specified, the resulting account will be at a derived address of the NONCE_ACCOUNT pubkey")
                 ),
         )
         .subcommand(
@@ -142,10 +156,10 @@ impl NonceSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("nonce_account_pubkey")
                         .index(1)
-                        .value_name("NONCE ACCOUNT")
+                        .value_name("NONCE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Address of the nonce account to display"),
                 ),
         )
@@ -153,12 +167,12 @@ impl NonceSubCommands for App<'_, '_> {
             SubCommand::with_name("new-nonce")
                 .about("Generate a new nonce, rendering the existing nonce useless")
                 .arg(
-                    Arg::with_name("nonce_account_keypair")
+                    Arg::with_name("nonce_account_pubkey")
                         .index(1)
-                        .value_name("NONCE ACCOUNT")
+                        .value_name("NONCE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Address of the nonce account"),
                 )
                 .arg(nonce_authority_arg()),
@@ -170,10 +184,10 @@ impl NonceSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("nonce_account_pubkey")
                         .index(1)
-                        .value_name("NONCE ACCOUNT")
+                        .value_name("NONCE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Address of the nonce account to display"),
                 )
                 .arg(
@@ -185,24 +199,24 @@ impl NonceSubCommands for App<'_, '_> {
         )
         .subcommand(
             SubCommand::with_name("withdraw-from-nonce-account")
-                .about("Withdraw lamports from the nonce account")
+                .about("Withdraw SOL from the nonce account")
                 .arg(
-                    Arg::with_name("nonce_account_keypair")
+                    Arg::with_name("nonce_account_pubkey")
                         .index(1)
-                        .value_name("NONCE ACCOUNT")
+                        .value_name("NONCE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_keypair_or_ask_keyword)
-                        .help("Nonce account from to withdraw from"),
+                        .validator(is_valid_pubkey)
+                        .help("Nonce account to withdraw from"),
                 )
                 .arg(
                     Arg::with_name("destination_account_pubkey")
                         .index(2)
-                        .value_name("DESTINATION ACCOUNT")
+                        .value_name("RECIPIENT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("The account to which the lamports should be transferred"),
+                        .validator(is_valid_pubkey)
+                        .help("The account to which the SOL should be transferred"),
                 )
                 .arg(
                     Arg::with_name("amount")
@@ -218,13 +232,55 @@ impl NonceSubCommands for App<'_, '_> {
     }
 }
 
+pub fn get_account(
+    rpc_client: &RpcClient,
+    nonce_pubkey: &Pubkey,
+) -> Result<Account, CliNonceError> {
+    rpc_client
+        .get_account(nonce_pubkey)
+        .map_err(|e| CliNonceError::Client(format!("{}", e)))
+        .and_then(|a| match account_identity_ok(&a) {
+            Ok(()) => Ok(a),
+            Err(e) => Err(e),
+        })
+}
+
+pub fn account_identity_ok(account: &Account) -> Result<(), CliNonceError> {
+    if account.owner != system_program::id() {
+        Err(CliNonceError::InvalidAccountOwner)
+    } else if account.data.is_empty() {
+        Err(CliNonceError::UnexpectedDataSize)
+    } else {
+        Ok(())
+    }
+}
+
+pub fn state_from_account(account: &Account) -> Result<State, CliNonceError> {
+    account_identity_ok(account)?;
+    StateMut::<Versions>::state(account)
+        .map_err(|_| CliNonceError::InvalidAccountData)
+        .map(|v| v.convert_to_current())
+}
+
+pub fn data_from_account(account: &Account) -> Result<Data, CliNonceError> {
+    account_identity_ok(account)?;
+    state_from_account(account).and_then(|ref s| data_from_state(s).map(|d| d.clone()))
+}
+
+pub fn data_from_state(state: &State) -> Result<&Data, CliNonceError> {
+    match state {
+        State::Uninitialized => Err(CliNonceError::InvalidStateForOperation),
+        State::Initialized(data) => Ok(data),
+    }
+}
+
 pub fn parse_authorize_nonce_account(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
     wallet_manager: Option<&Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let nonce_account = pubkey_of(matches, "nonce_account_keypair").unwrap();
-    let new_authority = pubkey_of(matches, "new_authority").unwrap();
+    let nonce_account = pubkey_of_signer(matches, "nonce_account_pubkey", wallet_manager)?.unwrap();
+    let new_authority = pubkey_of_signer(matches, "new_authority", wallet_manager)?.unwrap();
     let (nonce_authority, nonce_authority_pubkey) =
         signer_of(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
 
@@ -255,7 +311,7 @@ pub fn parse_nonce_create_account(
         signer_of(matches, "nonce_account_keypair", wallet_manager)?;
     let seed = matches.value_of("seed").map(|s| s.to_string());
     let lamports = lamports_of_sol(matches, "amount").unwrap();
-    let nonce_authority = pubkey_of(matches, NONCE_AUTHORITY_ARG.name);
+    let nonce_authority = pubkey_of_signer(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
 
     let payer_provided = None;
     let signer_info = generate_unique_signers(
@@ -276,8 +332,12 @@ pub fn parse_nonce_create_account(
     })
 }
 
-pub fn parse_get_nonce(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let nonce_account_pubkey = pubkey_of(matches, "nonce_account_pubkey").unwrap();
+pub fn parse_get_nonce(
+    matches: &ArgMatches<'_>,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let nonce_account_pubkey =
+        pubkey_of_signer(matches, "nonce_account_pubkey", wallet_manager)?.unwrap();
 
     Ok(CliCommandInfo {
         command: CliCommand::GetNonce(nonce_account_pubkey),
@@ -290,7 +350,7 @@ pub fn parse_new_nonce(
     default_signer_path: &str,
     wallet_manager: Option<&Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let nonce_account = pubkey_of(matches, "nonce_account_keypair").unwrap();
+    let nonce_account = pubkey_of_signer(matches, "nonce_account_pubkey", wallet_manager)?.unwrap();
     let (nonce_authority, nonce_authority_pubkey) =
         signer_of(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
 
@@ -311,8 +371,12 @@ pub fn parse_new_nonce(
     })
 }
 
-pub fn parse_show_nonce_account(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let nonce_account_pubkey = pubkey_of(matches, "nonce_account_pubkey").unwrap();
+pub fn parse_show_nonce_account(
+    matches: &ArgMatches<'_>,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let nonce_account_pubkey =
+        pubkey_of_signer(matches, "nonce_account_pubkey", wallet_manager)?.unwrap();
     let use_lamports_unit = matches.is_present("lamports");
 
     Ok(CliCommandInfo {
@@ -329,8 +393,9 @@ pub fn parse_withdraw_from_nonce_account(
     default_signer_path: &str,
     wallet_manager: Option<&Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let nonce_account = pubkey_of(matches, "nonce_account_keypair").unwrap();
-    let destination_account_pubkey = pubkey_of(matches, "destination_account_pubkey").unwrap();
+    let nonce_account = pubkey_of_signer(matches, "nonce_account_pubkey", wallet_manager)?.unwrap();
+    let destination_account_pubkey =
+        pubkey_of_signer(matches, "destination_account_pubkey", wallet_manager)?.unwrap();
     let lamports = lamports_of_sol(matches, "amount").unwrap();
     let (nonce_authority, nonce_authority_pubkey) =
         signer_of(matches, NONCE_AUTHORITY_ARG.name, wallet_manager)?;
@@ -359,24 +424,18 @@ pub fn check_nonce_account(
     nonce_account: &Account,
     nonce_authority: &Pubkey,
     nonce_hash: &Hash,
-) -> Result<(), Box<CliError>> {
-    if nonce_account.owner != system_program::ID {
-        return Err(CliError::InvalidNonce(CliNonceError::InvalidAccountOwner).into());
-    }
-    let nonce_state = StateMut::<Versions>::state(nonce_account)
-        .map(|v| v.convert_to_current())
-        .map_err(|_| Box::new(CliError::InvalidNonce(CliNonceError::InvalidAccountData)))?;
-    match nonce_state {
+) -> Result<(), CliError> {
+    match state_from_account(nonce_account)? {
         State::Initialized(ref data) => {
             if &data.blockhash != nonce_hash {
-                Err(CliError::InvalidNonce(CliNonceError::InvalidHash).into())
+                Err(CliNonceError::InvalidHash.into())
             } else if nonce_authority != &data.authority {
-                Err(CliError::InvalidNonce(CliNonceError::InvalidAuthority).into())
+                Err(CliNonceError::InvalidAuthority.into())
             } else {
                 Ok(())
             }
         }
-        State::Uninitialized => Err(CliError::InvalidNonce(CliNonceError::InvalidState).into()),
+        State::Uninitialized => Err(CliNonceError::InvalidStateForOperation.into()),
     }
 }
 
@@ -391,7 +450,7 @@ pub fn process_authorize_nonce_account(
 
     let nonce_authority = config.signers[nonce_authority];
     let ix = authorize_nonce_account(nonce_account, &nonce_authority.pubkey(), new_authority);
-    let message = Message::new_with_payer(vec![ix], Some(&config.signers[0].pubkey()));
+    let message = Message::new_with_payer(&[ix], Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
 
@@ -401,7 +460,7 @@ pub fn process_authorize_nonce_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
     log_instruction_custom_error::<NonceError>(result)
 }
 
@@ -425,10 +484,8 @@ pub fn process_create_nonce_account(
         (&nonce_account_address, "nonce_account".to_string()),
     )?;
 
-    if let Ok(nonce_account) = rpc_client.get_account(&nonce_account_address) {
-        let err_msg = if nonce_account.owner == system_program::id()
-            && StateMut::<Versions>::state(&nonce_account).is_ok()
-        {
+    if let Ok(nonce_account) = get_account(rpc_client, &nonce_account_address) {
+        let err_msg = if state_from_account(&nonce_account).is_ok() {
             format!("Nonce account {} already exists", nonce_account_address)
         } else {
             format!(
@@ -470,7 +527,7 @@ pub fn process_create_nonce_account(
 
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
 
-    let message = Message::new_with_payer(ixs, Some(&config.signers[0].pubkey()));
+    let message = Message::new_with_payer(&ixs, Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
 
@@ -480,28 +537,14 @@ pub fn process_create_nonce_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
     log_instruction_custom_error::<SystemError>(result)
 }
 
 pub fn process_get_nonce(rpc_client: &RpcClient, nonce_account_pubkey: &Pubkey) -> ProcessResult {
-    let nonce_account = rpc_client.get_account(nonce_account_pubkey)?;
-    if nonce_account.owner != system_program::id() {
-        return Err(CliError::RpcRequestError(format!(
-            "{:?} is not a nonce account",
-            nonce_account_pubkey
-        ))
-        .into());
-    }
-    let nonce_state = StateMut::<Versions>::state(&nonce_account).map(|v| v.convert_to_current());
-    match nonce_state {
-        Ok(State::Uninitialized) => Ok("Nonce account is uninitialized".to_string()),
-        Ok(State::Initialized(ref data)) => Ok(format!("{:?}", data.blockhash)),
-        Err(err) => Err(CliError::RpcRequestError(format!(
-            "Account data could not be deserialized to nonce state: {:?}",
-            err
-        ))
-        .into()),
+    match get_account(rpc_client, nonce_account_pubkey).and_then(|ref a| state_from_account(a))? {
+        State::Uninitialized => Ok("Nonce account is uninitialized".to_string()),
+        State::Initialized(ref data) => Ok(format!("{:?}", data.blockhash)),
     }
 }
 
@@ -526,7 +569,7 @@ pub fn process_new_nonce(
     let nonce_authority = config.signers[nonce_authority];
     let ix = advance_nonce_account(&nonce_account, &nonce_authority.pubkey());
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
-    let message = Message::new_with_payer(vec![ix], Some(&config.signers[0].pubkey()));
+    let message = Message::new_with_payer(&[ix], Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
@@ -535,8 +578,8 @@ pub fn process_new_nonce(
         &fee_calculator,
         &tx.message,
     )?;
-    let result =
-        rpc_client.send_and_confirm_transaction(&mut tx, &[config.signers[0], nonce_authority]);
+    let result = rpc_client
+        .send_and_confirm_transaction_with_spinner(&mut tx, &[config.signers[0], nonce_authority]);
     log_instruction_custom_error::<SystemError>(result)
 }
 
@@ -545,14 +588,7 @@ pub fn process_show_nonce_account(
     nonce_account_pubkey: &Pubkey,
     use_lamports_unit: bool,
 ) -> ProcessResult {
-    let nonce_account = rpc_client.get_account(nonce_account_pubkey)?;
-    if nonce_account.owner != system_program::id() {
-        return Err(CliError::RpcRequestError(format!(
-            "{:?} is not a nonce account",
-            nonce_account_pubkey
-        ))
-        .into());
-    }
+    let nonce_account = get_account(rpc_client, nonce_account_pubkey)?;
     let print_account = |data: Option<&nonce::state::Data>| {
         println!(
             "Balance: {}",
@@ -583,15 +619,9 @@ pub fn process_show_nonce_account(
         }
         Ok("".to_string())
     };
-    let nonce_state = StateMut::<Versions>::state(&nonce_account).map(|v| v.convert_to_current());
-    match nonce_state {
-        Ok(State::Uninitialized) => print_account(None),
-        Ok(State::Initialized(ref data)) => print_account(Some(data)),
-        Err(err) => Err(CliError::RpcRequestError(format!(
-            "Account data could not be deserialized to nonce state: {:?}",
-            err
-        ))
-        .into()),
+    match state_from_account(&nonce_account)? {
+        State::Uninitialized => print_account(None),
+        State::Initialized(ref data) => print_account(Some(data)),
     }
 }
 
@@ -612,7 +642,7 @@ pub fn process_withdraw_from_nonce_account(
         destination_account_pubkey,
         lamports,
     );
-    let message = Message::new_with_payer(vec![ix], Some(&config.signers[0].pubkey()));
+    let message = Message::new_with_payer(&[ix], Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
@@ -621,7 +651,7 @@ pub fn process_withdraw_from_nonce_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
     log_instruction_custom_error::<NonceError>(result)
 }
 
@@ -848,31 +878,6 @@ mod tests {
             }
         );
 
-        let test_withdraw_from_nonce_account = test_commands.clone().get_matches_from(vec![
-            "test",
-            "withdraw-from-nonce-account",
-            &keypair_file,
-            &nonce_account_string,
-            "42",
-        ]);
-        assert_eq!(
-            parse_command(
-                &test_withdraw_from_nonce_account,
-                &default_keypair_file,
-                None
-            )
-            .unwrap(),
-            CliCommandInfo {
-                command: CliCommand::WithdrawFromNonceAccount {
-                    nonce_account: read_keypair_file(&keypair_file).unwrap().pubkey(),
-                    nonce_authority: 0,
-                    destination_account_pubkey: nonce_account_pubkey,
-                    lamports: 42000000000
-                },
-                signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
-            }
-        );
-
         // Test WithdrawFromNonceAccount Subcommand with authority
         let test_withdraw_from_nonce_account = test_commands.clone().get_matches_from(vec![
             "test",
@@ -920,17 +925,13 @@ mod tests {
         let invalid_owner = Account::new_data(1, &data, &Pubkey::new(&[1u8; 32]));
         assert_eq!(
             check_nonce_account(&invalid_owner.unwrap(), &nonce_pubkey, &blockhash),
-            Err(Box::new(CliError::InvalidNonce(
-                CliNonceError::InvalidAccountOwner
-            ))),
+            Err(CliNonceError::InvalidAccountOwner.into()),
         );
 
         let invalid_data = Account::new_data(1, &"invalid", &system_program::ID);
         assert_eq!(
             check_nonce_account(&invalid_data.unwrap(), &nonce_pubkey, &blockhash),
-            Err(Box::new(CliError::InvalidNonce(
-                CliNonceError::InvalidAccountData
-            ))),
+            Err(CliNonceError::InvalidAccountData.into()),
         );
 
         let data = Versions::new_current(State::Initialized(nonce::state::Data {
@@ -941,7 +942,7 @@ mod tests {
         let invalid_hash = Account::new_data(1, &data, &system_program::ID);
         assert_eq!(
             check_nonce_account(&invalid_hash.unwrap(), &nonce_pubkey, &blockhash),
-            Err(Box::new(CliError::InvalidNonce(CliNonceError::InvalidHash))),
+            Err(CliNonceError::InvalidHash.into()),
         );
 
         let data = Versions::new_current(State::Initialized(nonce::state::Data {
@@ -952,18 +953,84 @@ mod tests {
         let invalid_authority = Account::new_data(1, &data, &system_program::ID);
         assert_eq!(
             check_nonce_account(&invalid_authority.unwrap(), &nonce_pubkey, &blockhash),
-            Err(Box::new(CliError::InvalidNonce(
-                CliNonceError::InvalidAuthority
-            ))),
+            Err(CliNonceError::InvalidAuthority.into()),
         );
 
         let data = Versions::new_current(State::Uninitialized);
         let invalid_state = Account::new_data(1, &data, &system_program::ID);
         assert_eq!(
             check_nonce_account(&invalid_state.unwrap(), &nonce_pubkey, &blockhash),
-            Err(Box::new(CliError::InvalidNonce(
-                CliNonceError::InvalidState
-            ))),
+            Err(CliNonceError::InvalidStateForOperation.into()),
         );
     }
+
+    #[test]
+    fn test_account_identity_ok() {
+        let nonce_account = nonce::create_account(1).into_inner();
+        assert_eq!(account_identity_ok(&nonce_account), Ok(()));
+
+        let system_account = Account::new(1, 0, &system_program::id());
+        assert_eq!(
+            account_identity_ok(&system_account),
+            Err(CliNonceError::UnexpectedDataSize),
+        );
+
+        let other_program = Pubkey::new(&[1u8; 32]);
+        let other_account_no_data = Account::new(1, 0, &other_program);
+        assert_eq!(
+            account_identity_ok(&other_account_no_data),
+            Err(CliNonceError::InvalidAccountOwner),
+        );
+    }
+
+    #[test]
+    fn test_state_from_account() {
+        let mut nonce_account = nonce::create_account(1).into_inner();
+        assert_eq!(state_from_account(&nonce_account), Ok(State::Uninitialized));
+
+        let data = nonce::state::Data {
+            authority: Pubkey::new(&[1u8; 32]),
+            blockhash: Hash::new(&[42u8; 32]),
+            fee_calculator: FeeCalculator::new(42),
+        };
+        nonce_account
+            .set_state(&Versions::new_current(State::Initialized(data.clone())))
+            .unwrap();
+        assert_eq!(
+            state_from_account(&nonce_account),
+            Ok(State::Initialized(data))
+        );
+
+        let wrong_data_size_account = Account::new(1, 1, &system_program::id());
+        assert_eq!(
+            state_from_account(&wrong_data_size_account),
+            Err(CliNonceError::InvalidAccountData),
+        );
+    }
+
+    #[test]
+    fn test_data_from_helpers() {
+        let mut nonce_account = nonce::create_account(1).into_inner();
+        let state = state_from_account(&nonce_account).unwrap();
+        assert_eq!(
+            data_from_state(&state),
+            Err(CliNonceError::InvalidStateForOperation)
+        );
+        assert_eq!(
+            data_from_account(&nonce_account),
+            Err(CliNonceError::InvalidStateForOperation)
+        );
+
+        let data = nonce::state::Data {
+            authority: Pubkey::new(&[1u8; 32]),
+            blockhash: Hash::new(&[42u8; 32]),
+            fee_calculator: FeeCalculator::new(42),
+        };
+        nonce_account
+            .set_state(&Versions::new_current(State::Initialized(data.clone())))
+            .unwrap();
+        let state = state_from_account(&nonce_account).unwrap();
+        assert_eq!(data_from_state(&state), Ok(&data));
+        assert_eq!(data_from_account(&nonce_account), Ok(data));
+    }
 }

cli/src/offline.rs

@@ -1,253 +0,0 @@
-use clap::{App, Arg, ArgMatches};
-use serde_json::Value;
-use solana_clap_utils::{
-    input_parsers::value_of,
-    input_validators::{is_hash, is_pubkey_sig},
-    offline::{BLOCKHASH_ARG, SIGNER_ARG, SIGN_ONLY_ARG},
-};
-use solana_client::rpc_client::RpcClient;
-use solana_sdk::{fee_calculator::FeeCalculator, hash::Hash, pubkey::Pubkey, signature::Signature};
-use std::str::FromStr;
-
-#[derive(Clone, Debug, PartialEq)]
-pub enum BlockhashQuery {
-    None(Hash, FeeCalculator),
-    FeeCalculator(Hash),
-    All,
-}
-
-impl BlockhashQuery {
-    pub fn new(blockhash: Option<Hash>, sign_only: bool) -> Self {
-        match blockhash {
-            Some(hash) if sign_only => Self::None(hash, FeeCalculator::default()),
-            Some(hash) if !sign_only => Self::FeeCalculator(hash),
-            None if !sign_only => Self::All,
-            _ => panic!("Cannot resolve blockhash"),
-        }
-    }
-
-    pub fn new_from_matches(matches: &ArgMatches<'_>) -> Self {
-        let blockhash = value_of(matches, BLOCKHASH_ARG.name);
-        let sign_only = matches.is_present(SIGN_ONLY_ARG.name);
-        BlockhashQuery::new(blockhash, sign_only)
-    }
-
-    pub fn get_blockhash_fee_calculator(
-        &self,
-        rpc_client: &RpcClient,
-    ) -> Result<(Hash, FeeCalculator), Box<dyn std::error::Error>> {
-        let (hash, fee_calc) = match self {
-            BlockhashQuery::None(hash, fee_calc) => (Some(hash), Some(fee_calc)),
-            BlockhashQuery::FeeCalculator(hash) => (Some(hash), None),
-            BlockhashQuery::All => (None, None),
-        };
-        if None == fee_calc {
-            let (cluster_hash, fee_calc) = rpc_client.get_recent_blockhash()?;
-            Ok((*hash.unwrap_or(&cluster_hash), fee_calc))
-        } else {
-            Ok((*hash.unwrap(), fee_calc.unwrap().clone()))
-        }
-    }
-}
-
-impl Default for BlockhashQuery {
-    fn default() -> Self {
-        BlockhashQuery::All
-    }
-}
-
-fn blockhash_arg<'a, 'b>() -> Arg<'a, 'b> {
-    Arg::with_name(BLOCKHASH_ARG.name)
-        .long(BLOCKHASH_ARG.long)
-        .takes_value(true)
-        .value_name("BLOCKHASH")
-        .validator(is_hash)
-        .help(BLOCKHASH_ARG.help)
-}
-
-fn sign_only_arg<'a, 'b>() -> Arg<'a, 'b> {
-    Arg::with_name(SIGN_ONLY_ARG.name)
-        .long(SIGN_ONLY_ARG.long)
-        .takes_value(false)
-        .requires(BLOCKHASH_ARG.name)
-        .help(SIGN_ONLY_ARG.help)
-}
-
-fn signer_arg<'a, 'b>() -> Arg<'a, 'b> {
-    Arg::with_name(SIGNER_ARG.name)
-        .long(SIGNER_ARG.long)
-        .takes_value(true)
-        .value_name("BASE58_PUBKEY=BASE58_SIG")
-        .validator(is_pubkey_sig)
-        .requires(BLOCKHASH_ARG.name)
-        .multiple(true)
-        .help(SIGNER_ARG.help)
-}
-
-pub trait OfflineArgs {
-    fn offline_args(self) -> Self;
-}
-
-impl OfflineArgs for App<'_, '_> {
-    fn offline_args(self) -> Self {
-        self.arg(blockhash_arg())
-            .arg(sign_only_arg())
-            .arg(signer_arg())
-    }
-}
-
-pub fn parse_sign_only_reply_string(reply: &str) -> (Hash, Vec<(Pubkey, Signature)>) {
-    let object: Value = serde_json::from_str(&reply).unwrap();
-    let blockhash_str = object.get("blockhash").unwrap().as_str().unwrap();
-    let blockhash = blockhash_str.parse::<Hash>().unwrap();
-    let signer_strings = object.get("signers").unwrap().as_array().unwrap();
-    let signers = signer_strings
-        .iter()
-        .map(|signer_string| {
-            let mut signer = signer_string.as_str().unwrap().split('=');
-            let key = Pubkey::from_str(signer.next().unwrap()).unwrap();
-            let sig = Signature::from_str(signer.next().unwrap()).unwrap();
-            (key, sig)
-        })
-        .collect();
-    (blockhash, signers)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use clap::App;
-    use serde_json::{self, json, Value};
-    use solana_client::{
-        rpc_request::RpcRequest,
-        rpc_response::{Response, RpcResponseContext},
-    };
-    use solana_sdk::{fee_calculator::FeeCalculator, hash::hash};
-    use std::collections::HashMap;
-
-    #[test]
-    fn test_blockhashspec_new_ok() {
-        let blockhash = hash(&[1u8]);
-
-        assert_eq!(
-            BlockhashQuery::new(Some(blockhash), true),
-            BlockhashQuery::None(blockhash, FeeCalculator::default()),
-        );
-        assert_eq!(
-            BlockhashQuery::new(Some(blockhash), false),
-            BlockhashQuery::FeeCalculator(blockhash),
-        );
-        assert_eq!(BlockhashQuery::new(None, false), BlockhashQuery::All,);
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_blockhashspec_new_fail() {
-        BlockhashQuery::new(None, true);
-    }
-
-    #[test]
-    fn test_blockhashspec_new_from_matches_ok() {
-        let test_commands = App::new("blockhashspec_test").offline_args();
-        let blockhash = hash(&[1u8]);
-        let blockhash_string = blockhash.to_string();
-
-        let matches = test_commands.clone().get_matches_from(vec![
-            "blockhashspec_test",
-            "--blockhash",
-            &blockhash_string,
-            "--sign-only",
-        ]);
-        assert_eq!(
-            BlockhashQuery::new_from_matches(&matches),
-            BlockhashQuery::None(blockhash, FeeCalculator::default()),
-        );
-
-        let matches = test_commands.clone().get_matches_from(vec![
-            "blockhashspec_test",
-            "--blockhash",
-            &blockhash_string,
-        ]);
-        assert_eq!(
-            BlockhashQuery::new_from_matches(&matches),
-            BlockhashQuery::FeeCalculator(blockhash),
-        );
-
-        let matches = test_commands
-            .clone()
-            .get_matches_from(vec!["blockhashspec_test"]);
-        assert_eq!(
-            BlockhashQuery::new_from_matches(&matches),
-            BlockhashQuery::All,
-        );
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_blockhashspec_new_from_matches_fail() {
-        let test_commands = App::new("blockhashspec_test")
-            .arg(blockhash_arg())
-            // We can really only hit this case unless the arg requirements
-            // are broken, so unset the requires() to recreate that condition
-            .arg(sign_only_arg().requires(""));
-
-        let matches = test_commands
-            .clone()
-            .get_matches_from(vec!["blockhashspec_test", "--sign-only"]);
-        BlockhashQuery::new_from_matches(&matches);
-    }
-
-    #[test]
-    fn test_blockhashspec_get_blockhash_fee_calc() {
-        let test_blockhash = hash(&[0u8]);
-        let rpc_blockhash = hash(&[1u8]);
-        let rpc_fee_calc = FeeCalculator::new(42);
-        let get_recent_blockhash_response = json!(Response {
-            context: RpcResponseContext { slot: 1 },
-            value: json!((
-                Value::String(rpc_blockhash.to_string()),
-                serde_json::to_value(rpc_fee_calc.clone()).unwrap()
-            )),
-        });
-        let mut mocks = HashMap::new();
-        mocks.insert(
-            RpcRequest::GetRecentBlockhash,
-            get_recent_blockhash_response.clone(),
-        );
-        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
-        assert_eq!(
-            BlockhashQuery::All
-                .get_blockhash_fee_calculator(&rpc_client)
-                .unwrap(),
-            (rpc_blockhash, rpc_fee_calc.clone()),
-        );
-        let mut mocks = HashMap::new();
-        mocks.insert(
-            RpcRequest::GetRecentBlockhash,
-            get_recent_blockhash_response.clone(),
-        );
-        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
-        assert_eq!(
-            BlockhashQuery::FeeCalculator(test_blockhash)
-                .get_blockhash_fee_calculator(&rpc_client)
-                .unwrap(),
-            (test_blockhash, rpc_fee_calc.clone()),
-        );
-        let mut mocks = HashMap::new();
-        mocks.insert(
-            RpcRequest::GetRecentBlockhash,
-            get_recent_blockhash_response.clone(),
-        );
-        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
-        assert_eq!(
-            BlockhashQuery::None(test_blockhash, FeeCalculator::default())
-                .get_blockhash_fee_calculator(&rpc_client)
-                .unwrap(),
-            (test_blockhash, FeeCalculator::default()),
-        );
-        let rpc_client = RpcClient::new_mock("fails".to_string());
-        assert!(BlockhashQuery::All
-            .get_blockhash_fee_calculator(&rpc_client)
-            .is_err());
-    }
-}

cli/src/offline/blockhash_query.rs

@@ -0,0 +1,394 @@
+use super::*;
+
+#[derive(Debug, PartialEq)]
+pub enum Source {
+    Cluster,
+    NonceAccount(Pubkey),
+}
+
+impl Source {
+    pub fn get_blockhash_and_fee_calculator(
+        &self,
+        rpc_client: &RpcClient,
+    ) -> Result<(Hash, FeeCalculator), Box<dyn std::error::Error>> {
+        match self {
+            Self::Cluster => {
+                let res = rpc_client.get_recent_blockhash()?;
+                Ok(res)
+            }
+            Self::NonceAccount(ref pubkey) => {
+                let data = nonce::get_account(rpc_client, pubkey)
+                    .and_then(|ref a| nonce::data_from_account(a))?;
+                Ok((data.blockhash, data.fee_calculator))
+            }
+        }
+    }
+
+    pub fn get_fee_calculator(
+        &self,
+        rpc_client: &RpcClient,
+        blockhash: &Hash,
+    ) -> Result<Option<FeeCalculator>, Box<dyn std::error::Error>> {
+        match self {
+            Self::Cluster => {
+                let res = rpc_client.get_fee_calculator_for_blockhash(blockhash)?;
+                Ok(res)
+            }
+            Self::NonceAccount(ref pubkey) => {
+                let res = nonce::get_account(rpc_client, pubkey)
+                    .and_then(|ref a| nonce::data_from_account(a))
+                    .and_then(|d| {
+                        if d.blockhash == *blockhash {
+                            Ok(Some(d.fee_calculator))
+                        } else {
+                            Ok(None)
+                        }
+                    })?;
+                Ok(res)
+            }
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum BlockhashQuery {
+    None(Hash),
+    FeeCalculator(Source, Hash),
+    All(Source),
+}
+
+impl BlockhashQuery {
+    pub fn new(blockhash: Option<Hash>, sign_only: bool, nonce_account: Option<Pubkey>) -> Self {
+        let source = nonce_account
+            .map(Source::NonceAccount)
+            .unwrap_or(Source::Cluster);
+        match blockhash {
+            Some(hash) if sign_only => Self::None(hash),
+            Some(hash) if !sign_only => Self::FeeCalculator(source, hash),
+            None if !sign_only => Self::All(source),
+            _ => panic!("Cannot resolve blockhash"),
+        }
+    }
+
+    pub fn new_from_matches(matches: &ArgMatches<'_>) -> Self {
+        let blockhash = value_of(matches, BLOCKHASH_ARG.name);
+        let sign_only = matches.is_present(SIGN_ONLY_ARG.name);
+        let nonce_account = pubkey_of(matches, nonce::NONCE_ARG.name);
+        BlockhashQuery::new(blockhash, sign_only, nonce_account)
+    }
+
+    pub fn get_blockhash_and_fee_calculator(
+        &self,
+        rpc_client: &RpcClient,
+    ) -> Result<(Hash, FeeCalculator), Box<dyn std::error::Error>> {
+        match self {
+            BlockhashQuery::None(hash) => Ok((*hash, FeeCalculator::default())),
+            BlockhashQuery::FeeCalculator(source, hash) => {
+                let fee_calculator = source
+                    .get_fee_calculator(rpc_client, hash)?
+                    .ok_or(format!("Hash has expired {:?}", hash))?;
+                Ok((*hash, fee_calculator))
+            }
+            BlockhashQuery::All(source) => source.get_blockhash_and_fee_calculator(rpc_client),
+        }
+    }
+}
+
+impl Default for BlockhashQuery {
+    fn default() -> Self {
+        BlockhashQuery::All(Source::Cluster)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{nonce::nonce_arg, offline::blockhash_query::BlockhashQuery};
+    use clap::App;
+    use serde_json::{self, json, Value};
+    use solana_client::{
+        rpc_request::RpcRequest,
+        rpc_response::{Response, RpcAccount, RpcFeeCalculator, RpcResponseContext},
+    };
+    use solana_sdk::{
+        account::Account, fee_calculator::FeeCalculator, hash::hash, nonce, system_program,
+    };
+    use std::collections::HashMap;
+
+    #[test]
+    fn test_blockhash_query_new_ok() {
+        let blockhash = hash(&[1u8]);
+        let nonce_pubkey = Pubkey::new(&[1u8; 32]);
+
+        assert_eq!(
+            BlockhashQuery::new(Some(blockhash), true, None),
+            BlockhashQuery::None(blockhash),
+        );
+        assert_eq!(
+            BlockhashQuery::new(Some(blockhash), false, None),
+            BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
+        );
+        assert_eq!(
+            BlockhashQuery::new(None, false, None),
+            BlockhashQuery::All(blockhash_query::Source::Cluster)
+        );
+
+        assert_eq!(
+            BlockhashQuery::new(Some(blockhash), true, Some(nonce_pubkey)),
+            BlockhashQuery::None(blockhash),
+        );
+        assert_eq!(
+            BlockhashQuery::new(Some(blockhash), false, Some(nonce_pubkey)),
+            BlockhashQuery::FeeCalculator(
+                blockhash_query::Source::NonceAccount(nonce_pubkey),
+                blockhash
+            ),
+        );
+        assert_eq!(
+            BlockhashQuery::new(None, false, Some(nonce_pubkey)),
+            BlockhashQuery::All(blockhash_query::Source::NonceAccount(nonce_pubkey)),
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_blockhash_query_new_no_nonce_fail() {
+        BlockhashQuery::new(None, true, None);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_blockhash_query_new_nonce_fail() {
+        let nonce_pubkey = Pubkey::new(&[1u8; 32]);
+        BlockhashQuery::new(None, true, Some(nonce_pubkey));
+    }
+
+    #[test]
+    fn test_blockhash_query_new_from_matches_ok() {
+        let test_commands = App::new("blockhash_query_test")
+            .arg(nonce_arg())
+            .offline_args();
+        let blockhash = hash(&[1u8]);
+        let blockhash_string = blockhash.to_string();
+
+        let matches = test_commands.clone().get_matches_from(vec![
+            "blockhash_query_test",
+            "--blockhash",
+            &blockhash_string,
+            "--sign-only",
+        ]);
+        assert_eq!(
+            BlockhashQuery::new_from_matches(&matches),
+            BlockhashQuery::None(blockhash),
+        );
+
+        let matches = test_commands.clone().get_matches_from(vec![
+            "blockhash_query_test",
+            "--blockhash",
+            &blockhash_string,
+        ]);
+        assert_eq!(
+            BlockhashQuery::new_from_matches(&matches),
+            BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
+        );
+
+        let matches = test_commands
+            .clone()
+            .get_matches_from(vec!["blockhash_query_test"]);
+        assert_eq!(
+            BlockhashQuery::new_from_matches(&matches),
+            BlockhashQuery::All(blockhash_query::Source::Cluster),
+        );
+
+        let nonce_pubkey = Pubkey::new(&[1u8; 32]);
+        let nonce_string = nonce_pubkey.to_string();
+        let matches = test_commands.clone().get_matches_from(vec![
+            "blockhash_query_test",
+            "--blockhash",
+            &blockhash_string,
+            "--sign-only",
+            "--nonce",
+            &nonce_string,
+        ]);
+        assert_eq!(
+            BlockhashQuery::new_from_matches(&matches),
+            BlockhashQuery::None(blockhash),
+        );
+
+        let matches = test_commands.clone().get_matches_from(vec![
+            "blockhash_query_test",
+            "--blockhash",
+            &blockhash_string,
+            "--nonce",
+            &nonce_string,
+        ]);
+        assert_eq!(
+            BlockhashQuery::new_from_matches(&matches),
+            BlockhashQuery::FeeCalculator(
+                blockhash_query::Source::NonceAccount(nonce_pubkey),
+                blockhash
+            ),
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_blockhash_query_new_from_matches_without_nonce_fail() {
+        let test_commands = App::new("blockhash_query_test")
+            .arg(blockhash_arg())
+            // We can really only hit this case if the arg requirements
+            // are broken, so unset the requires() to recreate that condition
+            .arg(sign_only_arg().requires(""));
+
+        let matches = test_commands
+            .clone()
+            .get_matches_from(vec!["blockhash_query_test", "--sign-only"]);
+        BlockhashQuery::new_from_matches(&matches);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_blockhash_query_new_from_matches_with_nonce_fail() {
+        let test_commands = App::new("blockhash_query_test")
+            .arg(blockhash_arg())
+            // We can really only hit this case if the arg requirements
+            // are broken, so unset the requires() to recreate that condition
+            .arg(sign_only_arg().requires(""));
+        let nonce_pubkey = Pubkey::new(&[1u8; 32]);
+        let nonce_string = nonce_pubkey.to_string();
+
+        let matches = test_commands.clone().get_matches_from(vec![
+            "blockhash_query_test",
+            "--sign-only",
+            "--nonce",
+            &nonce_string,
+        ]);
+        BlockhashQuery::new_from_matches(&matches);
+    }
+
+    #[test]
+    fn test_blockhash_query_get_blockhash_fee_calc() {
+        let test_blockhash = hash(&[0u8]);
+        let rpc_blockhash = hash(&[1u8]);
+        let rpc_fee_calc = FeeCalculator::new(42);
+        let get_recent_blockhash_response = json!(Response {
+            context: RpcResponseContext { slot: 1 },
+            value: json!((
+                Value::String(rpc_blockhash.to_string()),
+                serde_json::to_value(rpc_fee_calc.clone()).unwrap()
+            )),
+        });
+        let get_fee_calculator_for_blockhash_response = json!(Response {
+            context: RpcResponseContext { slot: 1 },
+            value: json!(RpcFeeCalculator {
+                fee_calculator: rpc_fee_calc.clone()
+            }),
+        });
+        let mut mocks = HashMap::new();
+        mocks.insert(
+            RpcRequest::GetRecentBlockhash,
+            get_recent_blockhash_response.clone(),
+        );
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert_eq!(
+            BlockhashQuery::default()
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .unwrap(),
+            (rpc_blockhash, rpc_fee_calc.clone()),
+        );
+        let mut mocks = HashMap::new();
+        mocks.insert(
+            RpcRequest::GetRecentBlockhash,
+            get_recent_blockhash_response.clone(),
+        );
+        mocks.insert(
+            RpcRequest::GetFeeCalculatorForBlockhash,
+            get_fee_calculator_for_blockhash_response.clone(),
+        );
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert_eq!(
+            BlockhashQuery::FeeCalculator(Source::Cluster, test_blockhash)
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .unwrap(),
+            (test_blockhash, rpc_fee_calc.clone()),
+        );
+        let mut mocks = HashMap::new();
+        mocks.insert(
+            RpcRequest::GetRecentBlockhash,
+            get_recent_blockhash_response.clone(),
+        );
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert_eq!(
+            BlockhashQuery::None(test_blockhash)
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .unwrap(),
+            (test_blockhash, FeeCalculator::default()),
+        );
+        let rpc_client = RpcClient::new_mock("fails".to_string());
+        assert!(BlockhashQuery::default()
+            .get_blockhash_and_fee_calculator(&rpc_client)
+            .is_err());
+
+        let nonce_blockhash = Hash::new(&[2u8; 32]);
+        let nonce_fee_calc = FeeCalculator::new(4242);
+        let data = nonce::state::Data {
+            authority: Pubkey::new(&[3u8; 32]),
+            blockhash: nonce_blockhash,
+            fee_calculator: nonce_fee_calc.clone(),
+        };
+        let nonce_account = Account::new_data_with_space(
+            42,
+            &nonce::state::Versions::new_current(nonce::State::Initialized(data)),
+            nonce::State::size(),
+            &system_program::id(),
+        )
+        .unwrap();
+        let nonce_pubkey = Pubkey::new(&[4u8; 32]);
+        let rpc_nonce_account = RpcAccount::encode(nonce_account);
+        let get_account_response = json!(Response {
+            context: RpcResponseContext { slot: 1 },
+            value: json!(Some(rpc_nonce_account.clone())),
+        });
+
+        let mut mocks = HashMap::new();
+        mocks.insert(RpcRequest::GetAccountInfo, get_account_response.clone());
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert_eq!(
+            BlockhashQuery::All(Source::NonceAccount(nonce_pubkey))
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .unwrap(),
+            (nonce_blockhash, nonce_fee_calc.clone()),
+        );
+        let mut mocks = HashMap::new();
+        mocks.insert(RpcRequest::GetAccountInfo, get_account_response.clone());
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert_eq!(
+            BlockhashQuery::FeeCalculator(Source::NonceAccount(nonce_pubkey), nonce_blockhash)
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .unwrap(),
+            (nonce_blockhash, nonce_fee_calc.clone()),
+        );
+        let mut mocks = HashMap::new();
+        mocks.insert(RpcRequest::GetAccountInfo, get_account_response.clone());
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert!(
+            BlockhashQuery::FeeCalculator(Source::NonceAccount(nonce_pubkey), test_blockhash)
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .is_err()
+        );
+        let mut mocks = HashMap::new();
+        mocks.insert(RpcRequest::GetAccountInfo, get_account_response.clone());
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert_eq!(
+            BlockhashQuery::None(nonce_blockhash)
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .unwrap(),
+            (nonce_blockhash, FeeCalculator::default()),
+        );
+
+        let rpc_client = RpcClient::new_mock("fails".to_string());
+        assert!(BlockhashQuery::All(Source::NonceAccount(nonce_pubkey))
+            .get_blockhash_and_fee_calculator(&rpc_client)
+            .is_err());
+    }
+}

cli/src/offline/mod.rs

@@ -0,0 +1,114 @@
+pub mod blockhash_query;
+
+use crate::nonce;
+use clap::{App, Arg, ArgMatches};
+use serde_json::Value;
+use solana_clap_utils::{
+    input_parsers::{pubkey_of, value_of},
+    input_validators::{is_hash, is_pubkey_sig},
+    keypair::presigner_from_pubkey_sigs,
+    offline::{BLOCKHASH_ARG, SIGNER_ARG, SIGN_ONLY_ARG},
+};
+use solana_client::rpc_client::RpcClient;
+use solana_sdk::{
+    fee_calculator::FeeCalculator,
+    hash::Hash,
+    pubkey::Pubkey,
+    signature::{Presigner, Signature},
+};
+use std::str::FromStr;
+
+fn blockhash_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(BLOCKHASH_ARG.name)
+        .long(BLOCKHASH_ARG.long)
+        .takes_value(true)
+        .value_name("BLOCKHASH")
+        .validator(is_hash)
+        .help(BLOCKHASH_ARG.help)
+}
+
+fn sign_only_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(SIGN_ONLY_ARG.name)
+        .long(SIGN_ONLY_ARG.long)
+        .takes_value(false)
+        .requires(BLOCKHASH_ARG.name)
+        .help(SIGN_ONLY_ARG.help)
+}
+
+fn signer_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(SIGNER_ARG.name)
+        .long(SIGNER_ARG.long)
+        .takes_value(true)
+        .value_name("PUBKEY=SIGNATURE")
+        .validator(is_pubkey_sig)
+        .requires(BLOCKHASH_ARG.name)
+        .multiple(true)
+        .help(SIGNER_ARG.help)
+}
+
+pub trait OfflineArgs {
+    fn offline_args(self) -> Self;
+}
+
+impl OfflineArgs for App<'_, '_> {
+    fn offline_args(self) -> Self {
+        self.arg(blockhash_arg())
+            .arg(sign_only_arg())
+            .arg(signer_arg())
+    }
+}
+
+pub struct SignOnly {
+    pub blockhash: Hash,
+    pub present_signers: Vec<(Pubkey, Signature)>,
+    pub absent_signers: Vec<Pubkey>,
+    pub bad_signers: Vec<Pubkey>,
+}
+
+impl SignOnly {
+    pub fn has_all_signers(&self) -> bool {
+        self.absent_signers.is_empty() && self.bad_signers.is_empty()
+    }
+
+    pub fn presigner_of(&self, pubkey: &Pubkey) -> Option<Presigner> {
+        presigner_from_pubkey_sigs(pubkey, &self.present_signers)
+    }
+}
+
+pub fn parse_sign_only_reply_string(reply: &str) -> SignOnly {
+    let object: Value = serde_json::from_str(&reply).unwrap();
+    let blockhash_str = object.get("blockhash").unwrap().as_str().unwrap();
+    let blockhash = blockhash_str.parse::<Hash>().unwrap();
+    let signer_strings = object.get("signers").unwrap().as_array().unwrap();
+    let present_signers = signer_strings
+        .iter()
+        .map(|signer_string| {
+            let mut signer = signer_string.as_str().unwrap().split('=');
+            let key = Pubkey::from_str(signer.next().unwrap()).unwrap();
+            let sig = Signature::from_str(signer.next().unwrap()).unwrap();
+            (key, sig)
+        })
+        .collect();
+    let signer_strings = object.get("absent").unwrap().as_array().unwrap();
+    let absent_signers = signer_strings
+        .iter()
+        .map(|val| {
+            let s = val.as_str().unwrap();
+            Pubkey::from_str(s).unwrap()
+        })
+        .collect();
+    let signer_strings = object.get("badSig").unwrap().as_array().unwrap();
+    let bad_signers = signer_strings
+        .iter()
+        .map(|val| {
+            let s = val.as_str().unwrap();
+            Pubkey::from_str(s).unwrap()
+        })
+        .collect();
+    SignOnly {
+        blockhash,
+        present_signers,
+        absent_signers,
+        bad_signers,
+    }
+}

cli/src/storage.rs

@@ -1,6 +1,7 @@
 use crate::cli::{
-    check_account_for_fee, check_unique_pubkeys, log_instruction_custom_error, CliCommand,
-    CliCommandInfo, CliConfig, CliError, ProcessResult, SignerIndex,
+    check_account_for_fee, check_unique_pubkeys, generate_unique_signers,
+    log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult,
+    SignerIndex,
 };
 use clap::{App, Arg, ArgMatches, SubCommand};
 use solana_clap_utils::{input_parsers::*, input_validators::*, keypair::signer_from_path};
@@ -25,18 +26,18 @@ impl StorageSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("storage_account_owner")
                         .index(1)
-                        .value_name("STORAGE ACCOUNT OWNER PUBKEY")
+                        .value_name("AUTHORITY_PUBKEY")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair),
+                        .validator(is_valid_pubkey),
                 )
                 .arg(
                     Arg::with_name("storage_account")
                         .index(2)
-                        .value_name("STORAGE ACCOUNT")
+                        .value_name("ACCOUNT_KEYPAIR")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_keypair_or_ask_keyword),
+                        .validator(is_valid_signer),
                 ),
         )
         .subcommand(
@@ -45,18 +46,18 @@ impl StorageSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("storage_account_owner")
                         .index(1)
-                        .value_name("STORAGE ACCOUNT OWNER PUBKEY")
+                        .value_name("AUTHORITY_PUBKEY")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair),
+                        .validator(is_valid_pubkey),
                 )
                 .arg(
                     Arg::with_name("storage_account")
                         .index(2)
-                        .value_name("STORAGE ACCOUNT")
+                        .value_name("ACCOUNT_KEYPAIR")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_keypair_or_ask_keyword),
+                        .validator(is_valid_signer),
                 ),
         )
         .subcommand(
@@ -65,19 +66,19 @@ impl StorageSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("node_account_pubkey")
                         .index(1)
-                        .value_name("NODE PUBKEY")
+                        .value_name("NODE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("The node account to credit the rewards to"),
                 )
                 .arg(
                     Arg::with_name("storage_account_pubkey")
                         .index(2)
-                        .value_name("STORAGE ACCOUNT PUBKEY")
+                        .value_name("STORAGE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Storage account address to redeem credits for"),
                 ),
         )
@@ -88,11 +89,11 @@ impl StorageSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("storage_account_pubkey")
                         .index(1)
-                        .value_name("STORAGE ACCOUNT PUBKEY")
+                        .value_name("STORAGE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("Storage account pubkey"),
+                        .validator(is_valid_pubkey)
+                        .help("Storage account address"),
                 ),
         )
     }
@@ -103,18 +104,26 @@ pub fn parse_storage_create_archiver_account(
     default_signer_path: &str,
     wallet_manager: Option<&Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let account_owner = pubkey_of(matches, "storage_account_owner").unwrap();
-    let storage_account = keypair_of(matches, "storage_account").unwrap();
+    let account_owner =
+        pubkey_of_signer(matches, "storage_account_owner", wallet_manager)?.unwrap();
+    let (storage_account, storage_account_pubkey) =
+        signer_of(matches, "storage_account", wallet_manager)?;
+
+    let payer_provided = None;
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, storage_account],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
+
     Ok(CliCommandInfo {
         command: CliCommand::CreateStorageAccount {
             account_owner,
-            storage_account: 1,
+            storage_account: signer_info.index_of(storage_account_pubkey).unwrap(),
             account_type: StorageAccountType::Archiver,
         },
-        signers: vec![
-            signer_from_path(matches, default_signer_path, "keypair", wallet_manager)?,
-            storage_account.into(),
-        ],
+        signers: signer_info.signers,
     })
 }
 
@@ -123,18 +132,26 @@ pub fn parse_storage_create_validator_account(
     default_signer_path: &str,
     wallet_manager: Option<&Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let account_owner = pubkey_of(matches, "storage_account_owner").unwrap();
-    let storage_account = keypair_of(matches, "storage_account").unwrap();
+    let account_owner =
+        pubkey_of_signer(matches, "storage_account_owner", wallet_manager)?.unwrap();
+    let (storage_account, storage_account_pubkey) =
+        signer_of(matches, "storage_account", wallet_manager)?;
+
+    let payer_provided = None;
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, storage_account],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
+
     Ok(CliCommandInfo {
         command: CliCommand::CreateStorageAccount {
             account_owner,
-            storage_account: 1,
+            storage_account: signer_info.index_of(storage_account_pubkey).unwrap(),
             account_type: StorageAccountType::Validator,
         },
-        signers: vec![
-            signer_from_path(matches, default_signer_path, "keypair", wallet_manager)?,
-            storage_account.into(),
-        ],
+        signers: signer_info.signers,
     })
 }
 
@@ -143,8 +160,10 @@ pub fn parse_storage_claim_reward(
     default_signer_path: &str,
     wallet_manager: Option<&Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let node_account_pubkey = pubkey_of(matches, "node_account_pubkey").unwrap();
-    let storage_account_pubkey = pubkey_of(matches, "storage_account_pubkey").unwrap();
+    let node_account_pubkey =
+        pubkey_of_signer(matches, "node_account_pubkey", wallet_manager)?.unwrap();
+    let storage_account_pubkey =
+        pubkey_of_signer(matches, "storage_account_pubkey", wallet_manager)?.unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::ClaimStorageReward {
             node_account_pubkey,
@@ -161,8 +180,10 @@ pub fn parse_storage_claim_reward(
 
 pub fn parse_storage_get_account_command(
     matches: &ArgMatches<'_>,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let storage_account_pubkey = pubkey_of(matches, "storage_account_pubkey").unwrap();
+    let storage_account_pubkey =
+        pubkey_of_signer(matches, "storage_account_pubkey", wallet_manager)?.unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::ShowStorageAccount(storage_account_pubkey),
         signers: vec![],
@@ -212,7 +233,7 @@ pub fn process_create_storage_account(
     );
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
 
-    let message = Message::new(ixs);
+    let message = Message::new(&ixs);
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
@@ -221,7 +242,7 @@ pub fn process_create_storage_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
     log_instruction_custom_error::<SystemError>(result)
 }
 
@@ -236,7 +257,7 @@ pub fn process_claim_storage_reward(
     let instruction =
         storage_instruction::claim_reward(node_account_pubkey, storage_account_pubkey);
     let signers = [config.signers[0]];
-    let message = Message::new_with_payer(vec![instruction], Some(&signers[0].pubkey()));
+    let message = Message::new_with_payer(&[instruction], Some(&signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&signers, recent_blockhash)?;
     check_account_for_fee(
@@ -245,8 +266,8 @@ pub fn process_claim_storage_reward(
         &fee_calculator,
         &tx.message,
     )?;
-    let signature_str = rpc_client.send_and_confirm_transaction(&mut tx, &signers)?;
-    Ok(signature_str)
+    let signature = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &signers)?;
+    Ok(signature.to_string())
 }
 
 pub fn process_show_storage_account(
@@ -266,7 +287,7 @@ pub fn process_show_storage_account(
 
     use solana_storage_program::storage_contract::StorageContract;
     let storage_contract: StorageContract = account.state().map_err(|err| {
-        CliError::RpcRequestError(format!("Unable to deserialize storage account: {:?}", err))
+        CliError::RpcRequestError(format!("Unable to deserialize storage account: {}", err))
     })?;
     println!("{:#?}", storage_contract);
     println!("Account Lamports: {}", account.lamports);

cli/src/validator_info.rs

@@ -274,7 +274,7 @@ pub fn process_set_validator_info(
                 println!("--force supplied, ignoring: {:?}", result);
             } else {
                 result.map_err(|err| {
-                    CliError::BadParameter(format!("Invalid validator keybase username: {:?}", err))
+                    CliError::BadParameter(format!("Invalid validator keybase username: {}", err))
                 })?;
             }
         }
@@ -339,7 +339,7 @@ pub fn process_set_validator_info(
             &validator_info,
         )]);
         let signers = vec![config.signers[0], &info_keypair];
-        let message = Message::new(instructions);
+        let message = Message::new(&instructions);
         (message, signers)
     } else {
         println!(
@@ -353,7 +353,7 @@ pub fn process_set_validator_info(
             keys,
             &validator_info,
         )];
-        let message = Message::new_with_payer(instructions, Some(&config.signers[0].pubkey()));
+        let message = Message::new_with_payer(&instructions, Some(&config.signers[0].pubkey()));
         let signers = vec![config.signers[0]];
         (message, signers)
     };
@@ -368,7 +368,7 @@ pub fn process_set_validator_info(
         &fee_calculator,
         &tx.message,
     )?;
-    let signature_str = rpc_client.send_and_confirm_transaction(&mut tx, &signers)?;
+    let signature_str = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &signers)?;
 
     println!("Success! Validator info published at: {:?}", info_pubkey);
     println!("{}", signature_str);

cli/src/vote.rs

@@ -1,10 +1,14 @@
 use crate::cli::{
     build_balance_message, check_account_for_fee, check_unique_pubkeys, generate_unique_signers,
     log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError, CliSignerInfo,
-    ProcessResult,
+    ProcessResult, SignerIndex,
 };
 use clap::{value_t_or_exit, App, Arg, ArgMatches, SubCommand};
-use solana_clap_utils::{input_parsers::*, input_validators::*};
+use solana_clap_utils::{
+    commitment::{commitment_arg, COMMITMENT_ARG},
+    input_parsers::*,
+    input_validators::*,
+};
 use solana_client::rpc_client::RpcClient;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
@@ -16,7 +20,7 @@ use solana_sdk::{
     transaction::Transaction,
 };
 use solana_vote_program::{
-    vote_instruction::{self, VoteError},
+    vote_instruction::{self, withdraw, VoteError},
     vote_state::{VoteAuthorize, VoteInit, VoteState},
 };
 use std::sync::Arc;
@@ -33,25 +37,25 @@ impl VoteSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("vote_account")
                         .index(1)
-                        .value_name("VOTE ACCOUNT KEYPAIR")
+                        .value_name("ACCOUNT_KEYPAIR")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_keypair_or_ask_keyword)
-                        .help("Vote account keypair to fund"),
+                        .validator(is_valid_signer)
+                        .help("Vote account keypair to create"),
                 )
                 .arg(
-                    Arg::with_name("identity_pubkey")
+                    Arg::with_name("identity_account")
                         .index(2)
-                        .value_name("VALIDATOR IDENTITY PUBKEY")
+                        .value_name("IDENTITY_KEYPAIR")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("Validator that will vote with this account"),
+                        .validator(is_valid_signer)
+                        .help("Keypair of validator that will vote with this account"),
                 )
                 .arg(
                     Arg::with_name("commission")
                         .long("commission")
-                        .value_name("NUM")
+                        .value_name("PERCENTAGE")
                         .takes_value(true)
                         .default_value("100")
                         .help("The commission taken on reward redemption (0-100)"),
@@ -59,78 +63,47 @@ impl VoteSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("authorized_voter")
                         .long("authorized-voter")
-                        .value_name("PUBKEY")
+                        .value_name("VOTER_PUBKEY")
                         .takes_value(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("Public key of the authorized voter (defaults to vote account)"),
+                        .validator(is_valid_pubkey)
+                        .help("Public key of the authorized voter [default: validator identity pubkey]"),
                 )
                 .arg(
                     Arg::with_name("authorized_withdrawer")
                         .long("authorized-withdrawer")
-                        .value_name("PUBKEY")
+                        .value_name("WITHDRAWER_PUBKEY")
                         .takes_value(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("Public key of the authorized withdrawer (defaults to cli config pubkey)"),
+                        .validator(is_valid_pubkey)
+                        .help("Public key of the authorized withdrawer [default: validator identity pubkey]"),
                 )
                 .arg(
                     Arg::with_name("seed")
                         .long("seed")
-                        .value_name("SEED STRING")
+                        .value_name("STRING")
                         .takes_value(true)
                         .help("Seed for address generation; if specified, the resulting account will be at a derived address of the VOTE ACCOUNT pubkey")
                 ),
         )
-        .subcommand(
-            SubCommand::with_name("vote-update-validator")
-                .about("Update the vote account's validator identity")
-                .arg(
-                    Arg::with_name("vote_account_pubkey")
-                        .index(1)
-                        .value_name("VOTE ACCOUNT PUBKEY")
-                        .takes_value(true)
-                        .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("Vote account to update"),
-                )
-                .arg(
-                    Arg::with_name("new_identity_pubkey")
-                        .index(2)
-                        .value_name("NEW VALIDATOR IDENTITY PUBKEY")
-                        .takes_value(true)
-                        .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("New validator that will vote with this account"),
-                )
-                .arg(
-                    Arg::with_name("authorized_voter")
-                        .index(3)
-                        .value_name("AUTHORIZED VOTER KEYPAIR")
-                        .takes_value(true)
-                        .required(true)
-                        .validator(is_keypair)
-                        .help("Authorized voter keypair"),
-                )
-        )
         .subcommand(
             SubCommand::with_name("vote-authorize-voter")
                 .about("Authorize a new vote signing keypair for the given vote account")
                 .arg(
                     Arg::with_name("vote_account_pubkey")
                         .index(1)
-                        .value_name("VOTE ACCOUNT PUBKEY")
+                        .value_name("VOTE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Vote account in which to set the authorized voter"),
                 )
                 .arg(
                     Arg::with_name("new_authorized_pubkey")
                         .index(2)
-                        .value_name("NEW VOTER PUBKEY")
+                        .value_name("AUTHORIZED_PUBKEY")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("New vote signer to authorize"),
+                        .validator(is_valid_pubkey)
+                        .help("New authorized vote signer"),
                 ),
         )
         .subcommand(
@@ -139,41 +112,64 @@ impl VoteSubCommands for App<'_, '_> {
                 .arg(
                     Arg::with_name("vote_account_pubkey")
                         .index(1)
-                        .value_name("VOTE ACCOUNT PUBKEY")
+                        .value_name("VOTE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Vote account in which to set the authorized withdrawer"),
                 )
                 .arg(
                     Arg::with_name("new_authorized_pubkey")
                         .index(2)
-                        .value_name("NEW WITHDRAWER PUBKEY")
+                        .value_name("AUTHORIZED_PUBKEY")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
-                        .help("New withdrawer to authorize"),
+                        .validator(is_valid_pubkey)
+                        .help("New authorized withdrawer"),
                 ),
         )
+        .subcommand(
+            SubCommand::with_name("vote-update-validator")
+                .about("Update the vote account's validator identity")
+                .arg(
+                    Arg::with_name("vote_account_pubkey")
+                        .index(1)
+                        .value_name("VOTE_ACCOUNT_ADDRESS")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_pubkey)
+                        .help("Vote account to update"),
+                )
+                .arg(
+                    Arg::with_name("new_identity_account")
+                        .index(2)
+                        .value_name("IDENTITY_KEYPAIR")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_signer)
+                        .help("Keypair of new validator that will vote with this account"),
+                )
+                .arg(
+                    Arg::with_name("authorized_withdrawer")
+                        .index(3)
+                        .value_name("AUTHORIZED_KEYPAIR")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_signer)
+                        .help("Authorized withdrawer keypair"),
+                )
+        )
         .subcommand(
             SubCommand::with_name("vote-account")
                 .about("Show the contents of a vote account")
                 .alias("show-vote-account")
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Return information at maximum-lockout commitment level",
-                        ),
-                )
                 .arg(
                     Arg::with_name("vote_account_pubkey")
                         .index(1)
-                        .value_name("VOTE ACCOUNT PUBKEY")
+                        .value_name("VOTE_ACCOUNT_ADDRESS")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_pubkey_or_keypair)
+                        .validator(is_valid_pubkey)
                         .help("Vote account pubkey"),
                 )
                 .arg(
@@ -181,26 +177,67 @@ impl VoteSubCommands for App<'_, '_> {
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
-                ),
+                )
+                .arg(commitment_arg()),
+        )
+        .subcommand(
+            SubCommand::with_name("withdraw-from-vote-account")
+                .about("Withdraw lamports from a vote account into a specified account")
+                .arg(
+                    Arg::with_name("vote_account_pubkey")
+                        .index(1)
+                        .value_name("VOTE_ACCOUNT_ADDRESS")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_pubkey)
+                        .help("Vote account from which to withdraw"),
+                )
+                .arg(
+                    Arg::with_name("destination_account_pubkey")
+                        .index(2)
+                        .value_name("RECIPIENT_ADDRESS")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_valid_pubkey)
+                        .help("The recipient of withdrawn SOL"),
+                )
+                .arg(
+                    Arg::with_name("amount")
+                        .index(3)
+                        .value_name("AMOUNT")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_amount)
+                        .help("The amount to withdraw, in SOL"),
+                )
+                .arg(
+                    Arg::with_name("authorized_withdrawer")
+                        .long("authorized-withdrawer")
+                        .value_name("AUTHORIZED_KEYPAIR")
+                        .takes_value(true)
+                        .validator(is_valid_signer)
+                        .help("Authorized withdrawer [default: cli config keypair]"),
+                )
         )
     }
 }
 
-pub fn parse_vote_create_account(
+pub fn parse_create_vote_account(
     matches: &ArgMatches<'_>,
     default_signer_path: &str,
     wallet_manager: Option<&Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
     let (vote_account, _) = signer_of(matches, "vote_account", wallet_manager)?;
     let seed = matches.value_of("seed").map(|s| s.to_string());
-    let identity_pubkey = pubkey_of(matches, "identity_pubkey").unwrap();
+    let (identity_account, identity_pubkey) =
+        signer_of(matches, "identity_account", wallet_manager)?;
     let commission = value_t_or_exit!(matches, "commission", u8);
-    let authorized_voter = pubkey_of(matches, "authorized_voter");
-    let authorized_withdrawer = pubkey_of(matches, "authorized_withdrawer");
+    let authorized_voter = pubkey_of_signer(matches, "authorized_voter", wallet_manager)?;
+    let authorized_withdrawer = pubkey_of_signer(matches, "authorized_withdrawer", wallet_manager)?;
 
     let payer_provided = None;
-    let CliSignerInfo { signers } = generate_unique_signers(
-        vec![payer_provided, vote_account],
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, vote_account, identity_account],
         matches,
         default_signer_path,
         wallet_manager,
@@ -209,12 +246,12 @@ pub fn parse_vote_create_account(
     Ok(CliCommandInfo {
         command: CliCommand::CreateVoteAccount {
             seed,
-            node_pubkey: identity_pubkey,
+            identity_account: signer_info.index_of(identity_pubkey).unwrap(),
             authorized_voter,
             authorized_withdrawer,
             commission,
         },
-        signers,
+        signers: signer_info.signers,
     })
 }
 
@@ -224,8 +261,10 @@ pub fn parse_vote_authorize(
     wallet_manager: Option<&Arc<RemoteWalletManager>>,
     vote_authorize: VoteAuthorize,
 ) -> Result<CliCommandInfo, CliError> {
-    let vote_account_pubkey = pubkey_of(matches, "vote_account_pubkey").unwrap();
-    let new_authorized_pubkey = pubkey_of(matches, "new_authorized_pubkey").unwrap();
+    let vote_account_pubkey =
+        pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
+    let new_authorized_pubkey =
+        pubkey_of_signer(matches, "new_authorized_pubkey", wallet_manager)?.unwrap();
 
     let authorized_voter_provided = None;
     let CliSignerInfo { signers } = generate_unique_signers(
@@ -250,13 +289,15 @@ pub fn parse_vote_update_validator(
     default_signer_path: &str,
     wallet_manager: Option<&Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let vote_account_pubkey = pubkey_of(matches, "vote_account_pubkey").unwrap();
-    let new_identity_pubkey = pubkey_of(matches, "new_identity_pubkey").unwrap();
-    let (authorized_voter, _) = signer_of(matches, "authorized_voter", wallet_manager)?;
+    let vote_account_pubkey =
+        pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
+    let (new_identity_account, new_identity_pubkey) =
+        signer_of(matches, "new_identity_account", wallet_manager)?;
+    let (authorized_withdrawer, _) = signer_of(matches, "authorized_withdrawer", wallet_manager)?;
 
     let payer_provided = None;
-    let CliSignerInfo { signers } = generate_unique_signers(
-        vec![payer_provided, authorized_voter],
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, authorized_withdrawer, new_identity_account],
         matches,
         default_signer_path,
         wallet_manager,
@@ -265,22 +306,20 @@ pub fn parse_vote_update_validator(
     Ok(CliCommandInfo {
         command: CliCommand::VoteUpdateValidator {
             vote_account_pubkey,
-            new_identity_pubkey,
+            new_identity_account: signer_info.index_of(new_identity_pubkey).unwrap(),
         },
-        signers,
+        signers: signer_info.signers,
     })
 }
 
 pub fn parse_vote_get_account_command(
     matches: &ArgMatches<'_>,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
-    let vote_account_pubkey = pubkey_of(matches, "vote_account_pubkey").unwrap();
+    let vote_account_pubkey =
+        pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
     let use_lamports_unit = matches.is_present("lamports");
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::ShowVoteAccount {
             pubkey: vote_account_pubkey,
@@ -291,11 +330,43 @@ pub fn parse_vote_get_account_command(
     })
 }
 
+pub fn parse_withdraw_from_vote_account(
+    matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let vote_account_pubkey =
+        pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
+    let destination_account_pubkey =
+        pubkey_of_signer(matches, "destination_account_pubkey", wallet_manager)?.unwrap();
+    let lamports = lamports_of_sol(matches, "amount").unwrap();
+    let (withdraw_authority, withdraw_authority_pubkey) =
+        signer_of(matches, "authorized_withdrawer", wallet_manager)?;
+
+    let payer_provided = None;
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, withdraw_authority],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
+
+    Ok(CliCommandInfo {
+        command: CliCommand::WithdrawFromVoteAccount {
+            vote_account_pubkey,
+            destination_account_pubkey,
+            withdraw_authority: signer_info.index_of(withdraw_authority_pubkey).unwrap(),
+            lamports,
+        },
+        signers: signer_info.signers,
+    })
+}
+
 pub fn process_create_vote_account(
     rpc_client: &RpcClient,
     config: &CliConfig,
     seed: &Option<String>,
-    identity_pubkey: &Pubkey,
+    identity_account: SignerIndex,
     authorized_voter: &Option<Pubkey>,
     authorized_withdrawer: &Option<Pubkey>,
     commission: u8,
@@ -312,6 +383,8 @@ pub fn process_create_vote_account(
         (&vote_account_address, "vote_account".to_string()),
     )?;
 
+    let identity_account = config.signers[identity_account];
+    let identity_pubkey = identity_account.pubkey();
     check_unique_pubkeys(
         (&vote_account_address, "vote_account".to_string()),
         (&identity_pubkey, "identity_pubkey".to_string()),
@@ -334,9 +407,9 @@ pub fn process_create_vote_account(
         .max(1);
 
     let vote_init = VoteInit {
-        node_pubkey: *identity_pubkey,
-        authorized_voter: authorized_voter.unwrap_or(vote_account_pubkey),
-        authorized_withdrawer: authorized_withdrawer.unwrap_or(vote_account_pubkey),
+        node_pubkey: identity_pubkey,
+        authorized_voter: authorized_voter.unwrap_or(identity_pubkey),
+        authorized_withdrawer: authorized_withdrawer.unwrap_or(identity_pubkey),
         commission,
     };
 
@@ -359,7 +432,7 @@ pub fn process_create_vote_account(
     };
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
 
-    let message = Message::new(ixs);
+    let message = Message::new(&ixs);
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
@@ -368,7 +441,7 @@ pub fn process_create_vote_account(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
     log_instruction_custom_error::<SystemError>(result)
 }
 
@@ -391,7 +464,7 @@ pub fn process_vote_authorize(
         vote_authorize,              // vote or withdraw
     )];
 
-    let message = Message::new_with_payer(ixs, Some(&config.signers[0].pubkey()));
+    let message = Message::new_with_payer(&ixs, Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
@@ -400,7 +473,8 @@ pub fn process_vote_authorize(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &[config.signers[0]]);
+    let result =
+        rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &[config.signers[0]]);
     log_instruction_custom_error::<VoteError>(result)
 }
 
@@ -408,21 +482,23 @@ pub fn process_vote_update_validator(
     rpc_client: &RpcClient,
     config: &CliConfig,
     vote_account_pubkey: &Pubkey,
-    new_identity_pubkey: &Pubkey,
+    new_identity_account: SignerIndex,
 ) -> ProcessResult {
-    let authorized_voter = config.signers[1];
+    let authorized_withdrawer = config.signers[1];
+    let new_identity_account = config.signers[new_identity_account];
+    let new_identity_pubkey = new_identity_account.pubkey();
     check_unique_pubkeys(
         (vote_account_pubkey, "vote_account_pubkey".to_string()),
-        (new_identity_pubkey, "new_identity_pubkey".to_string()),
+        (&new_identity_pubkey, "new_identity_account".to_string()),
     )?;
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
     let ixs = vec![vote_instruction::update_node(
         vote_account_pubkey,
-        &authorized_voter.pubkey(),
-        new_identity_pubkey,
+        &authorized_withdrawer.pubkey(),
+        &new_identity_pubkey,
     )];
 
-    let message = Message::new_with_payer(ixs, Some(&config.signers[0].pubkey()));
+    let message = Message::new_with_payer(&ixs, Some(&config.signers[0].pubkey()));
     let mut tx = Transaction::new_unsigned(message);
     tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
@@ -431,7 +507,7 @@ pub fn process_vote_update_validator(
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
+    let result = rpc_client.send_and_confirm_transaction_with_spinner(&mut tx, &config.signers);
     log_instruction_custom_error::<VoteError>(result)
 }
 
@@ -517,6 +593,38 @@ pub fn process_show_vote_account(
     Ok("".to_string())
 }
 
+pub fn process_withdraw_from_vote_account(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    vote_account_pubkey: &Pubkey,
+    withdraw_authority: SignerIndex,
+    lamports: u64,
+    destination_account_pubkey: &Pubkey,
+) -> ProcessResult {
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
+    let withdraw_authority = config.signers[withdraw_authority];
+
+    let ix = withdraw(
+        vote_account_pubkey,
+        &withdraw_authority.pubkey(),
+        lamports,
+        destination_account_pubkey,
+    );
+
+    let message = Message::new_with_payer(&[ix], Some(&config.signers[0].pubkey()));
+    let mut transaction = Transaction::new_unsigned(message);
+    transaction.try_sign(&config.signers, recent_blockhash)?;
+    check_account_for_fee(
+        rpc_client,
+        &config.signers[0].pubkey(),
+        &fee_calculator,
+        &transaction.message,
+    )?;
+    let result =
+        rpc_client.send_and_confirm_transaction_with_spinner(&mut transaction, &config.signers);
+    log_instruction_custom_error::<VoteError>(result)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -565,13 +673,14 @@ mod tests {
         let keypair = Keypair::new();
         write_keypair(&keypair, tmp_file.as_file_mut()).unwrap();
         // Test CreateVoteAccount SubCommand
-        let node_pubkey = Pubkey::new_rand();
-        let node_pubkey_string = format!("{}", node_pubkey);
+        let (identity_keypair_file, mut tmp_file) = make_tmp_file();
+        let identity_keypair = Keypair::new();
+        write_keypair(&identity_keypair, tmp_file.as_file_mut()).unwrap();
         let test_create_vote_account = test_commands.clone().get_matches_from(vec![
             "test",
             "create-vote-account",
             &keypair_file,
-            &node_pubkey_string,
+            &identity_keypair_file,
             "--commission",
             "10",
         ]);
@@ -580,14 +689,15 @@ mod tests {
             CliCommandInfo {
                 command: CliCommand::CreateVoteAccount {
                     seed: None,
-                    node_pubkey,
+                    identity_account: 2,
                     authorized_voter: None,
                     authorized_withdrawer: None,
                     commission: 10,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
-                    Box::new(keypair)
+                    Box::new(keypair),
+                    read_keypair_file(&identity_keypair_file).unwrap().into(),
                 ],
             }
         );
@@ -600,21 +710,22 @@ mod tests {
             "test",
             "create-vote-account",
             &keypair_file,
-            &node_pubkey_string,
+            &identity_keypair_file,
         ]);
         assert_eq!(
             parse_command(&test_create_vote_account2, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::CreateVoteAccount {
                     seed: None,
-                    node_pubkey,
+                    identity_account: 2,
                     authorized_voter: None,
                     authorized_withdrawer: None,
                     commission: 100,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
-                    Box::new(keypair)
+                    Box::new(keypair),
+                    read_keypair_file(&identity_keypair_file).unwrap().into(),
                 ],
             }
         );
@@ -629,7 +740,7 @@ mod tests {
             "test",
             "create-vote-account",
             &keypair_file,
-            &node_pubkey_string,
+            &identity_keypair_file,
             "--authorized-voter",
             &authed.to_string(),
         ]);
@@ -638,14 +749,15 @@ mod tests {
             CliCommandInfo {
                 command: CliCommand::CreateVoteAccount {
                     seed: None,
-                    node_pubkey,
+                    identity_account: 2,
                     authorized_voter: Some(authed),
                     authorized_withdrawer: None,
                     commission: 100
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
-                    Box::new(keypair)
+                    Box::new(keypair),
+                    read_keypair_file(&identity_keypair_file).unwrap().into(),
                 ],
             }
         );
@@ -658,7 +770,7 @@ mod tests {
             "test",
             "create-vote-account",
             &keypair_file,
-            &node_pubkey_string,
+            &identity_keypair_file,
             "--authorized-withdrawer",
             &authed.to_string(),
         ]);
@@ -667,14 +779,15 @@ mod tests {
             CliCommandInfo {
                 command: CliCommand::CreateVoteAccount {
                     seed: None,
-                    node_pubkey,
+                    identity_account: 2,
                     authorized_voter: None,
                     authorized_withdrawer: Some(authed),
                     commission: 100
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
-                    Box::new(keypair)
+                    Box::new(keypair),
+                    read_keypair_file(&identity_keypair_file).unwrap().into(),
                 ],
             }
         );
@@ -683,7 +796,7 @@ mod tests {
             "test",
             "vote-update-validator",
             &pubkey_string,
-            &pubkey2_string,
+            &identity_keypair_file,
             &keypair_file,
         ]);
         assert_eq!(
@@ -691,11 +804,72 @@ mod tests {
             CliCommandInfo {
                 command: CliCommand::VoteUpdateValidator {
                     vote_account_pubkey: pubkey,
-                    new_identity_pubkey: pubkey2,
+                    new_identity_account: 2,
                 },
                 signers: vec![
                     read_keypair_file(&default_keypair_file).unwrap().into(),
-                    Box::new(read_keypair_file(&keypair_file).unwrap())
+                    Box::new(read_keypair_file(&keypair_file).unwrap()),
+                    read_keypair_file(&identity_keypair_file).unwrap().into(),
+                ],
+            }
+        );
+
+        // Test WithdrawFromVoteAccount subcommand
+        let test_withdraw_from_vote_account = test_commands.clone().get_matches_from(vec![
+            "test",
+            "withdraw-from-vote-account",
+            &keypair_file,
+            &pubkey_string,
+            "42",
+        ]);
+        assert_eq!(
+            parse_command(
+                &test_withdraw_from_vote_account,
+                &default_keypair_file,
+                None
+            )
+            .unwrap(),
+            CliCommandInfo {
+                command: CliCommand::WithdrawFromVoteAccount {
+                    vote_account_pubkey: read_keypair_file(&keypair_file).unwrap().pubkey(),
+                    destination_account_pubkey: pubkey,
+                    withdraw_authority: 0,
+                    lamports: 42_000_000_000
+                },
+                signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
+            }
+        );
+
+        // Test WithdrawFromVoteAccount subcommand with authority
+        let withdraw_authority = Keypair::new();
+        let (withdraw_authority_file, mut tmp_file) = make_tmp_file();
+        write_keypair(&withdraw_authority, tmp_file.as_file_mut()).unwrap();
+        let test_withdraw_from_vote_account = test_commands.clone().get_matches_from(vec![
+            "test",
+            "withdraw-from-vote-account",
+            &keypair_file,
+            &pubkey_string,
+            "42",
+            "--authorized-withdrawer",
+            &withdraw_authority_file,
+        ]);
+        assert_eq!(
+            parse_command(
+                &test_withdraw_from_vote_account,
+                &default_keypair_file,
+                None
+            )
+            .unwrap(),
+            CliCommandInfo {
+                command: CliCommand::WithdrawFromVoteAccount {
+                    vote_account_pubkey: read_keypair_file(&keypair_file).unwrap().pubkey(),
+                    destination_account_pubkey: pubkey,
+                    withdraw_authority: 1,
+                    lamports: 42_000_000_000
+                },
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    read_keypair_file(&withdraw_authority_file).unwrap().into()
                 ],
             }
         );

cli/tests/nonce.rs

@@ -1,6 +1,13 @@
-use solana_cli::cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig};
+use solana_cli::{
+    cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
+    nonce,
+    offline::{
+        blockhash_query::{self, BlockhashQuery},
+        parse_sign_only_reply_string,
+    },
+};
 use solana_client::rpc_client::RpcClient;
-use solana_core::validator::TestValidator;
+use solana_core::validator::{TestValidator, TestValidatorOptions};
 use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
     hash::Hash,
@@ -247,3 +254,126 @@ fn full_battery_tests(
     check_balance(800, &rpc_client, &nonce_account);
     check_balance(200, &rpc_client, &payee_pubkey);
 }
+
+#[test]
+fn test_create_account_with_seed() {
+    let TestValidator {
+        server,
+        leader_data,
+        alice: mint_keypair,
+        ledger_path,
+        ..
+    } = TestValidator::run_with_options(TestValidatorOptions {
+        fees: 1,
+        bootstrap_validator_lamports: 42_000,
+    });
+
+    let (sender, receiver) = channel();
+    run_local_faucet(mint_keypair, sender, None);
+    let faucet_addr = receiver.recv().unwrap();
+
+    let offline_nonce_authority_signer = keypair_from_seed(&[1u8; 32]).unwrap();
+    let online_nonce_creator_signer = keypair_from_seed(&[2u8; 32]).unwrap();
+    let to_address = Pubkey::new(&[3u8; 32]);
+
+    // Setup accounts
+    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    request_and_confirm_airdrop(
+        &rpc_client,
+        &faucet_addr,
+        &offline_nonce_authority_signer.pubkey(),
+        42,
+    )
+    .unwrap();
+    request_and_confirm_airdrop(
+        &rpc_client,
+        &faucet_addr,
+        &online_nonce_creator_signer.pubkey(),
+        4242,
+    )
+    .unwrap();
+    check_balance(42, &rpc_client, &offline_nonce_authority_signer.pubkey());
+    check_balance(4242, &rpc_client, &online_nonce_creator_signer.pubkey());
+    check_balance(0, &rpc_client, &to_address);
+
+    // Create nonce account
+    let creator_pubkey = online_nonce_creator_signer.pubkey();
+    let authority_pubkey = offline_nonce_authority_signer.pubkey();
+    let seed = authority_pubkey.to_string()[0..32].to_string();
+    let nonce_address =
+        create_address_with_seed(&creator_pubkey, &seed, &system_program::id()).unwrap();
+    check_balance(0, &rpc_client, &nonce_address);
+
+    let mut creator_config = CliConfig::default();
+    creator_config.json_rpc_url =
+        format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    creator_config.signers = vec![&online_nonce_creator_signer];
+    creator_config.command = CliCommand::CreateNonceAccount {
+        nonce_account: 0,
+        seed: Some(seed),
+        nonce_authority: Some(authority_pubkey),
+        lamports: 241,
+    };
+    process_command(&creator_config).unwrap();
+    check_balance(241, &rpc_client, &nonce_address);
+    check_balance(42, &rpc_client, &offline_nonce_authority_signer.pubkey());
+    check_balance(4000, &rpc_client, &online_nonce_creator_signer.pubkey());
+    check_balance(0, &rpc_client, &to_address);
+
+    // Fetch nonce hash
+    let nonce_hash = nonce::get_account(&rpc_client, &nonce_address)
+        .and_then(|ref a| nonce::data_from_account(a))
+        .unwrap()
+        .blockhash;
+
+    // Test by creating transfer TX with nonce, fully offline
+    let mut authority_config = CliConfig::default();
+    authority_config.json_rpc_url = String::default();
+    authority_config.signers = vec![&offline_nonce_authority_signer];
+    // Verify we cannot contact the cluster
+    authority_config.command = CliCommand::ClusterVersion;
+    process_command(&authority_config).unwrap_err();
+    authority_config.command = CliCommand::Transfer {
+        lamports: 10,
+        to: to_address,
+        from: 0,
+        sign_only: true,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::None(nonce_hash),
+        nonce_account: Some(nonce_address),
+        nonce_authority: 0,
+        fee_payer: 0,
+    };
+    let sign_only_reply = process_command(&authority_config).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_only_reply);
+    let authority_presigner = sign_only.presigner_of(&authority_pubkey).unwrap();
+    assert_eq!(sign_only.blockhash, nonce_hash);
+
+    // And submit it
+    let mut submit_config = CliConfig::default();
+    submit_config.json_rpc_url =
+        format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    submit_config.signers = vec![&authority_presigner];
+    submit_config.command = CliCommand::Transfer {
+        lamports: 10,
+        to: to_address,
+        from: 0,
+        sign_only: false,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::FeeCalculator(
+            blockhash_query::Source::NonceAccount(nonce_address),
+            sign_only.blockhash,
+        ),
+        nonce_account: Some(nonce_address),
+        nonce_authority: 0,
+        fee_payer: 0,
+    };
+    process_command(&submit_config).unwrap();
+    check_balance(241, &rpc_client, &nonce_address);
+    check_balance(31, &rpc_client, &offline_nonce_authority_signer.pubkey());
+    check_balance(4000, &rpc_client, &online_nonce_creator_signer.pubkey());
+    check_balance(10, &rpc_client, &to_address);
+
+    server.close().unwrap();
+    remove_dir_all(ledger_path).unwrap();
+}

cli/tests/pay.rs

@@ -1,17 +1,18 @@
 use chrono::prelude::*;
 use serde_json::Value;
-use solana_clap_utils::keypair::presigner_from_pubkey_sigs;
 use solana_cli::{
     cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig, PayCommand},
-    offline::{parse_sign_only_reply_string, BlockhashQuery},
+    nonce,
+    offline::{
+        blockhash_query::{self, BlockhashQuery},
+        parse_sign_only_reply_string,
+    },
 };
 use solana_client::rpc_client::RpcClient;
 use solana_core::validator::TestValidator;
 use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
-    account_utils::StateMut,
-    fee_calculator::FeeCalculator,
-    nonce,
+    nonce::State as NonceState,
     pubkey::Pubkey,
     signature::{Keypair, Signer},
 };
@@ -323,7 +324,7 @@ fn test_offline_pay_tx() {
     config_offline.command = CliCommand::Pay(PayCommand {
         lamports: 10,
         to: bob_pubkey,
-        blockhash_query: BlockhashQuery::None(blockhash, FeeCalculator::default()),
+        blockhash_query: BlockhashQuery::None(blockhash),
         sign_only: true,
         ..PayCommand::default()
     });
@@ -333,15 +334,17 @@ fn test_offline_pay_tx() {
     check_balance(50, &rpc_client, &config_online.signers[0].pubkey());
     check_balance(0, &rpc_client, &bob_pubkey);
 
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner =
-        presigner_from_pubkey_sigs(&config_offline.signers[0].pubkey(), &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only
+        .presigner_of(&config_offline.signers[0].pubkey())
+        .unwrap();
     let online_pubkey = config_online.signers[0].pubkey();
     config_online.signers = vec![&offline_presigner];
     config_online.command = CliCommand::Pay(PayCommand {
         lamports: 10,
         to: bob_pubkey,
-        blockhash_query: BlockhashQuery::FeeCalculator(blockhash),
+        blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         ..PayCommand::default()
     });
     process_command(&config_online).unwrap();
@@ -377,7 +380,7 @@ fn test_nonced_pay_tx() {
     config.signers = vec![&default_signer];
 
     let minimum_nonce_balance = rpc_client
-        .get_minimum_balance_for_rent_exemption(nonce::State::size())
+        .get_minimum_balance_for_rent_exemption(NonceState::size())
         .unwrap();
 
     request_and_confirm_airdrop(
@@ -408,21 +411,20 @@ fn test_nonced_pay_tx() {
     check_balance(minimum_nonce_balance, &rpc_client, &nonce_account.pubkey());
 
     // Fetch nonce hash
-    let account = rpc_client.get_account(&nonce_account.pubkey()).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+    let nonce_hash = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    let nonce_hash = match nonce_state {
-        nonce::State::Initialized(ref data) => data.blockhash,
-        _ => panic!("Nonce is not initialized"),
-    };
+        .blockhash;
 
     let bob_pubkey = Pubkey::new_rand();
     config.signers = vec![&default_signer];
     config.command = CliCommand::Pay(PayCommand {
         lamports: 10,
         to: bob_pubkey,
-        blockhash_query: BlockhashQuery::FeeCalculator(nonce_hash),
+        blockhash_query: BlockhashQuery::FeeCalculator(
+            blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
+            nonce_hash,
+        ),
         nonce_account: Some(nonce_account.pubkey()),
         ..PayCommand::default()
     });
@@ -432,14 +434,11 @@ fn test_nonced_pay_tx() {
     check_balance(10, &rpc_client, &bob_pubkey);
 
     // Verify that nonce has been used
-    let account = rpc_client.get_account(&nonce_account.pubkey()).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+    let nonce_hash2 = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    match nonce_state {
-        nonce::State::Initialized(ref data) => assert_ne!(data.blockhash, nonce_hash),
-        _ => assert!(false, "Nonce is not initialized"),
-    }
+        .blockhash;
+    assert_ne!(nonce_hash, nonce_hash2);
 
     server.close().unwrap();
     remove_dir_all(ledger_path).unwrap();

cli/tests/stake.rs

@@ -1,15 +1,17 @@
-use solana_clap_utils::keypair::presigner_from_pubkey_sigs;
 use solana_cli::{
     cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
-    offline::{parse_sign_only_reply_string, BlockhashQuery},
+    nonce,
+    offline::{
+        blockhash_query::{self, BlockhashQuery},
+        parse_sign_only_reply_string,
+    },
 };
 use solana_client::rpc_client::RpcClient;
 use solana_core::validator::{TestValidator, TestValidatorOptions};
 use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
     account_utils::StateMut,
-    fee_calculator::FeeCalculator,
-    nonce,
+    nonce::State as NonceState,
     pubkey::Pubkey,
     signature::{keypair_from_seed, Keypair, Signer},
     system_instruction::create_address_with_seed,
@@ -66,7 +68,7 @@ fn test_stake_delegation_force() {
     config.signers = vec![&default_signer, &vote_keypair];
     config.command = CliCommand::CreateVoteAccount {
         seed: None,
-        node_pubkey: config.signers[0].pubkey(),
+        identity_account: 0,
         authorized_voter: None,
         authorized_withdrawer: None,
         commission: 0,
@@ -84,7 +86,7 @@ fn test_stake_delegation_force() {
         lockup: Lockup::default(),
         lamports: 50_000,
         sign_only: false,
-        blockhash_query: BlockhashQuery::All,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -175,7 +177,7 @@ fn test_seed_stake_delegation_and_deactivation() {
         lockup: Lockup::default(),
         lamports: 50_000,
         sign_only: false,
-        blockhash_query: BlockhashQuery::All,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -258,7 +260,7 @@ fn test_stake_delegation_and_deactivation() {
         lockup: Lockup::default(),
         lamports: 50_000,
         sign_only: false,
-        blockhash_query: BlockhashQuery::All,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -363,7 +365,7 @@ fn test_offline_stake_delegation_and_deactivation() {
         lockup: Lockup::default(),
         lamports: 50_000,
         sign_only: false,
-        blockhash_query: BlockhashQuery::All,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -379,15 +381,17 @@ fn test_offline_stake_delegation_and_deactivation() {
         stake_authority: 0,
         force: false,
         sign_only: true,
-        blockhash_query: BlockhashQuery::None(blockhash, FeeCalculator::default()),
+        blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner =
-        presigner_from_pubkey_sigs(&config_offline.signers[0].pubkey(), &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only
+        .presigner_of(&config_offline.signers[0].pubkey())
+        .unwrap();
     config_payer.signers = vec![&offline_presigner];
     config_payer.command = CliCommand::DelegateStake {
         stake_account_pubkey: stake_keypair.pubkey(),
@@ -395,7 +399,7 @@ fn test_offline_stake_delegation_and_deactivation() {
         stake_authority: 0,
         force: false,
         sign_only: false,
-        blockhash_query: BlockhashQuery::None(blockhash, FeeCalculator::default()),
+        blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -408,21 +412,23 @@ fn test_offline_stake_delegation_and_deactivation() {
         stake_account_pubkey: stake_keypair.pubkey(),
         stake_authority: 0,
         sign_only: true,
-        blockhash_query: BlockhashQuery::None(blockhash, FeeCalculator::default()),
+        blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner =
-        presigner_from_pubkey_sigs(&config_offline.signers[0].pubkey(), &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only
+        .presigner_of(&config_offline.signers[0].pubkey())
+        .unwrap();
     config_payer.signers = vec![&offline_presigner];
     config_payer.command = CliCommand::DeactivateStake {
         stake_account_pubkey: stake_keypair.pubkey(),
         stake_authority: 0,
         sign_only: false,
-        blockhash_query: BlockhashQuery::FeeCalculator(blockhash),
+        blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -457,7 +463,7 @@ fn test_nonced_stake_delegation_and_deactivation() {
     config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
 
     let minimum_nonce_balance = rpc_client
-        .get_minimum_balance_for_rent_exemption(nonce::State::size())
+        .get_minimum_balance_for_rent_exemption(NonceState::size())
         .unwrap();
 
     request_and_confirm_airdrop(
@@ -479,7 +485,7 @@ fn test_nonced_stake_delegation_and_deactivation() {
         lockup: Lockup::default(),
         lamports: 50_000,
         sign_only: false,
-        blockhash_query: BlockhashQuery::All,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -499,14 +505,10 @@ fn test_nonced_stake_delegation_and_deactivation() {
     process_command(&config).unwrap();
 
     // Fetch nonce hash
-    let account = rpc_client.get_account(&nonce_account.pubkey()).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+    let nonce_hash = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    let nonce_hash = match nonce_state {
-        nonce::State::Initialized(ref data) => data.blockhash,
-        _ => panic!("Nonce is not initialized"),
-    };
+        .blockhash;
 
     // Delegate stake
     config.signers = vec![&config_keypair];
@@ -516,7 +518,10 @@ fn test_nonced_stake_delegation_and_deactivation() {
         stake_authority: 0,
         force: false,
         sign_only: false,
-        blockhash_query: BlockhashQuery::None(nonce_hash, FeeCalculator::default()),
+        blockhash_query: BlockhashQuery::FeeCalculator(
+            blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
+            nonce_hash,
+        ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
@@ -524,21 +529,20 @@ fn test_nonced_stake_delegation_and_deactivation() {
     process_command(&config).unwrap();
 
     // Fetch nonce hash
-    let account = rpc_client.get_account(&nonce_account.pubkey()).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+    let nonce_hash = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    let nonce_hash = match nonce_state {
-        nonce::State::Initialized(ref data) => data.blockhash,
-        _ => panic!("Nonce is not initialized"),
-    };
+        .blockhash;
 
     // Deactivate stake
     config.command = CliCommand::DeactivateStake {
         stake_account_pubkey: stake_keypair.pubkey(),
         stake_authority: 0,
         sign_only: false,
-        blockhash_query: BlockhashQuery::FeeCalculator(nonce_hash),
+        blockhash_query: BlockhashQuery::FeeCalculator(
+            blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
+            nonce_hash,
+        ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
@@ -608,7 +612,7 @@ fn test_stake_authorize() {
         lockup: Lockup::default(),
         lamports: 50_000,
         sign_only: false,
-        blockhash_query: BlockhashQuery::All,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -622,9 +626,7 @@ fn test_stake_authorize() {
     config.signers.pop();
     config.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: online_authority_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 0,
+        new_authorizations: vec![(StakeAuthorize::Staker, online_authority_pubkey, 0)],
         sign_only: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
@@ -640,13 +642,40 @@ fn test_stake_authorize() {
     };
     assert_eq!(current_authority, online_authority_pubkey);
 
-    // Assign new offline stake authority
+    // Assign new online stake and withdraw authorities
+    let online_authority2 = Keypair::new();
+    let online_authority2_pubkey = online_authority2.pubkey();
+    let withdraw_authority = Keypair::new();
+    let withdraw_authority_pubkey = withdraw_authority.pubkey();
     config.signers.push(&online_authority);
     config.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: offline_authority_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 1,
+        new_authorizations: vec![
+            (StakeAuthorize::Staker, online_authority2_pubkey, 1),
+            (StakeAuthorize::Withdrawer, withdraw_authority_pubkey, 0),
+        ],
+        sign_only: false,
+        blockhash_query: BlockhashQuery::default(),
+        nonce_account: None,
+        nonce_authority: 0,
+        fee_payer: 0,
+    };
+    process_command(&config).unwrap();
+    let stake_account = rpc_client.get_account(&stake_account_pubkey).unwrap();
+    let stake_state: StakeState = stake_account.state().unwrap();
+    let (current_staker, current_withdrawer) = match stake_state {
+        StakeState::Initialized(meta) => (meta.authorized.staker, meta.authorized.withdrawer),
+        _ => panic!("Unexpected stake state!"),
+    };
+    assert_eq!(current_staker, online_authority2_pubkey);
+    assert_eq!(current_withdrawer, withdraw_authority_pubkey);
+
+    // Assign new offline stake authority
+    config.signers.pop();
+    config.signers.push(&online_authority2);
+    config.command = CliCommand::StakeAuthorize {
+        stake_account_pubkey,
+        new_authorizations: vec![(StakeAuthorize::Staker, offline_authority_pubkey, 1)],
         sign_only: false,
         blockhash_query: BlockhashQuery::default(),
         nonce_account: None,
@@ -668,27 +697,23 @@ fn test_stake_authorize() {
     let (blockhash, _) = rpc_client.get_recent_blockhash().unwrap();
     config_offline.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: nonced_authority_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 0,
+        new_authorizations: vec![(StakeAuthorize::Staker, nonced_authority_pubkey, 0)],
         sign_only: true,
-        blockhash_query: BlockhashQuery::None(blockhash, FeeCalculator::default()),
+        blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
     };
     let sign_reply = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sign_reply);
-    let offline_presigner =
-        presigner_from_pubkey_sigs(&offline_authority_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_reply);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_authority_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: nonced_authority_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 0,
+        new_authorizations: vec![(StakeAuthorize::Staker, nonced_authority_pubkey, 0)],
         sign_only: false,
-        blockhash_query: BlockhashQuery::FeeCalculator(blockhash),
+        blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -704,7 +729,7 @@ fn test_stake_authorize() {
 
     // Create nonce account
     let minimum_nonce_balance = rpc_client
-        .get_minimum_balance_for_rent_exemption(nonce::State::size())
+        .get_minimum_balance_for_rent_exemption(NonceState::size())
         .unwrap();
     let nonce_account = Keypair::new();
     config.signers = vec![&default_signer, &nonce_account];
@@ -717,14 +742,10 @@ fn test_stake_authorize() {
     process_command(&config).unwrap();
 
     // Fetch nonce hash
-    let account = rpc_client.get_account(&nonce_account.pubkey()).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+    let nonce_hash = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    let nonce_hash = match nonce_state {
-        nonce::State::Initialized(ref data) => data.blockhash,
-        _ => panic!("Nonce is not initialized"),
-    };
+        .blockhash;
 
     // Nonced assignment of new online stake authority
     let online_authority = Keypair::new();
@@ -732,30 +753,28 @@ fn test_stake_authorize() {
     config_offline.signers.push(&nonced_authority);
     config_offline.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: online_authority_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 1,
+        new_authorizations: vec![(StakeAuthorize::Staker, online_authority_pubkey, 1)],
         sign_only: true,
-        blockhash_query: BlockhashQuery::None(nonce_hash, FeeCalculator::default()),
+        blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
     };
     let sign_reply = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sign_reply);
-    assert_eq!(blockhash, nonce_hash);
-    let offline_presigner =
-        presigner_from_pubkey_sigs(&offline_authority_pubkey, &signers).unwrap();
-    let nonced_authority_presigner =
-        presigner_from_pubkey_sigs(&nonced_authority_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_reply);
+    assert!(sign_only.has_all_signers());
+    assert_eq!(sign_only.blockhash, nonce_hash);
+    let offline_presigner = sign_only.presigner_of(&offline_authority_pubkey).unwrap();
+    let nonced_authority_presigner = sign_only.presigner_of(&nonced_authority_pubkey).unwrap();
     config.signers = vec![&offline_presigner, &nonced_authority_presigner];
     config.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: online_authority_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 1,
+        new_authorizations: vec![(StakeAuthorize::Staker, online_authority_pubkey, 1)],
         sign_only: false,
-        blockhash_query: BlockhashQuery::FeeCalculator(blockhash),
+        blockhash_query: BlockhashQuery::FeeCalculator(
+            blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
+            sign_only.blockhash,
+        ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
@@ -768,14 +787,11 @@ fn test_stake_authorize() {
         _ => panic!("Unexpected stake state!"),
     };
     assert_eq!(current_authority, online_authority_pubkey);
-    let account = rpc_client.get_account(&nonce_account.pubkey()).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+
+    let new_nonce_hash = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    let new_nonce_hash = match nonce_state {
-        nonce::State::Initialized(ref data) => data.blockhash,
-        _ => panic!("Nonce is not initialized"),
-    };
+        .blockhash;
     assert_ne!(nonce_hash, new_nonce_hash);
 
     server.close().unwrap();
@@ -819,6 +835,7 @@ fn test_stake_authorize_with_fee_payer() {
     let mut config_offline = CliConfig::default();
     let offline_signer = Keypair::new();
     config_offline.signers = vec![&offline_signer];
+    config_offline.json_rpc_url = String::new();
     let offline_pubkey = config_offline.signers[0].pubkey();
     // Verify we're offline
     config_offline.command = CliCommand::ClusterVersion;
@@ -845,7 +862,7 @@ fn test_stake_authorize_with_fee_payer() {
         lockup: Lockup::default(),
         lamports: 50_000,
         sign_only: false,
-        blockhash_query: BlockhashQuery::All,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -859,11 +876,9 @@ fn test_stake_authorize_with_fee_payer() {
     config.signers = vec![&default_signer, &payer_keypair];
     config.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: offline_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 0,
+        new_authorizations: vec![(StakeAuthorize::Staker, offline_pubkey, 0)],
         sign_only: false,
-        blockhash_query: BlockhashQuery::All,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 1,
@@ -879,26 +894,23 @@ fn test_stake_authorize_with_fee_payer() {
     let (blockhash, _) = rpc_client.get_recent_blockhash().unwrap();
     config_offline.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: payer_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 0,
+        new_authorizations: vec![(StakeAuthorize::Staker, payer_pubkey, 0)],
         sign_only: true,
-        blockhash_query: BlockhashQuery::None(blockhash, FeeCalculator::default()),
+        blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
     };
     let sign_reply = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sign_reply);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_reply);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::StakeAuthorize {
         stake_account_pubkey,
-        new_authorized_pubkey: payer_pubkey,
-        stake_authorize: StakeAuthorize::Staker,
-        authority: 0,
+        new_authorizations: vec![(StakeAuthorize::Staker, payer_pubkey, 0)],
         sign_only: false,
-        blockhash_query: BlockhashQuery::FeeCalculator(blockhash),
+        blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -975,7 +987,7 @@ fn test_stake_split() {
         lockup: Lockup::default(),
         lamports: 10 * minimum_stake_balance,
         sign_only: false,
-        blockhash_query: BlockhashQuery::All,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -990,7 +1002,7 @@ fn test_stake_split() {
 
     // Create nonce account
     let minimum_nonce_balance = rpc_client
-        .get_minimum_balance_for_rent_exemption(nonce::State::size())
+        .get_minimum_balance_for_rent_exemption(NonceState::size())
         .unwrap();
     let nonce_account = keypair_from_seed(&[1u8; 32]).unwrap();
     config.signers = vec![&default_signer, &nonce_account];
@@ -1004,14 +1016,10 @@ fn test_stake_split() {
     check_balance(minimum_nonce_balance, &rpc_client, &nonce_account.pubkey());
 
     // Fetch nonce hash
-    let account = rpc_client.get_account(&nonce_account.pubkey()).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+    let nonce_hash = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    let nonce_hash = match nonce_state {
-        nonce::State::Initialized(ref data) => data.blockhash,
-        _ => panic!("Nonce is not initialized"),
-    };
+        .blockhash;
 
     // Nonced offline split
     let split_account = keypair_from_seed(&[2u8; 32]).unwrap();
@@ -1021,7 +1029,7 @@ fn test_stake_split() {
         stake_account_pubkey: stake_account_pubkey,
         stake_authority: 0,
         sign_only: true,
-        blockhash_query: BlockhashQuery::None(nonce_hash, FeeCalculator::default()),
+        blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         split_stake_account: 1,
@@ -1030,14 +1038,18 @@ fn test_stake_split() {
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner, &split_account];
     config.command = CliCommand::SplitStake {
         stake_account_pubkey: stake_account_pubkey,
         stake_authority: 0,
         sign_only: false,
-        blockhash_query: BlockhashQuery::FeeCalculator(blockhash),
+        blockhash_query: BlockhashQuery::FeeCalculator(
+            blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
+            sign_only.blockhash,
+        ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         split_stake_account: 1,
@@ -1127,7 +1139,7 @@ fn test_stake_set_lockup() {
         lockup,
         lamports: 10 * minimum_stake_balance,
         sign_only: false,
-        blockhash_query: BlockhashQuery::All,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -1242,7 +1254,7 @@ fn test_stake_set_lockup() {
 
     // Create nonce account
     let minimum_nonce_balance = rpc_client
-        .get_minimum_balance_for_rent_exemption(nonce::State::size())
+        .get_minimum_balance_for_rent_exemption(NonceState::size())
         .unwrap();
     let nonce_account = keypair_from_seed(&[1u8; 32]).unwrap();
     let nonce_account_pubkey = nonce_account.pubkey();
@@ -1257,14 +1269,10 @@ fn test_stake_set_lockup() {
     check_balance(minimum_nonce_balance, &rpc_client, &nonce_account_pubkey);
 
     // Fetch nonce hash
-    let account = rpc_client.get_account(&nonce_account_pubkey).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+    let nonce_hash = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    let nonce_hash = match nonce_state {
-        nonce::State::Initialized(ref data) => data.blockhash,
-        _ => panic!("Nonce is not initialized"),
-    };
+        .blockhash;
 
     // Nonced offline set lockup
     let lockup = LockupArgs {
@@ -1277,21 +1285,25 @@ fn test_stake_set_lockup() {
         lockup,
         custodian: 0,
         sign_only: true,
-        blockhash_query: BlockhashQuery::None(nonce_hash, FeeCalculator::default()),
+        blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_account_pubkey),
         nonce_authority: 0,
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::StakeSetLockup {
         stake_account_pubkey,
         lockup,
         custodian: 0,
         sign_only: false,
-        blockhash_query: BlockhashQuery::FeeCalculator(blockhash),
+        blockhash_query: BlockhashQuery::FeeCalculator(
+            blockhash_query::Source::NonceAccount(nonce_account_pubkey),
+            sign_only.blockhash,
+        ),
         nonce_account: Some(nonce_account_pubkey),
         nonce_authority: 0,
         fee_payer: 0,
@@ -1359,7 +1371,7 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
 
     // Create nonce account
     let minimum_nonce_balance = rpc_client
-        .get_minimum_balance_for_rent_exemption(nonce::State::size())
+        .get_minimum_balance_for_rent_exemption(NonceState::size())
         .unwrap();
     let nonce_account = keypair_from_seed(&[3u8; 32]).unwrap();
     let nonce_pubkey = nonce_account.pubkey();
@@ -1373,14 +1385,10 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     process_command(&config).unwrap();
 
     // Fetch nonce hash
-    let account = rpc_client.get_account(&nonce_account.pubkey()).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+    let nonce_hash = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    let nonce_hash = match nonce_state {
-        nonce::State::Initialized(ref data) => data.blockhash,
-        _ => panic!("Nonce is not initialized"),
-    };
+        .blockhash;
 
     // Create stake account offline
     let stake_keypair = keypair_from_seed(&[4u8; 32]).unwrap();
@@ -1394,16 +1402,17 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         lockup: Lockup::default(),
         lamports: 50_000,
         sign_only: true,
-        blockhash_query: BlockhashQuery::None(nonce_hash, FeeCalculator::default()),
+        blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
         fee_payer: 0,
         from: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
-    let stake_presigner = presigner_from_pubkey_sigs(&stake_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
+    let stake_presigner = sign_only.presigner_of(&stake_pubkey).unwrap();
     config.signers = vec![&offline_presigner, &stake_presigner];
     config.command = CliCommand::CreateStakeAccount {
         stake_account: 1,
@@ -1413,7 +1422,10 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         lockup: Lockup::default(),
         lamports: 50_000,
         sign_only: false,
-        blockhash_query: BlockhashQuery::FeeCalculator(blockhash),
+        blockhash_query: BlockhashQuery::FeeCalculator(
+            blockhash_query::Source::NonceAccount(nonce_pubkey),
+            sign_only.blockhash,
+        ),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
         fee_payer: 0,
@@ -1423,14 +1435,10 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     check_balance(50_000, &rpc_client, &stake_pubkey);
 
     // Fetch nonce hash
-    let account = rpc_client.get_account(&nonce_account.pubkey()).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+    let nonce_hash = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    let nonce_hash = match nonce_state {
-        nonce::State::Initialized(ref data) => data.blockhash,
-        _ => panic!("Nonce is not initialized"),
-    };
+        .blockhash;
 
     // Offline, nonced stake-withdraw
     let recipient = keypair_from_seed(&[5u8; 32]).unwrap();
@@ -1442,14 +1450,14 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         lamports: 42,
         withdraw_authority: 0,
         sign_only: true,
-        blockhash_query: BlockhashQuery::None(nonce_hash, FeeCalculator::default()),
+        blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
         fee_payer: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::WithdrawStake {
         stake_account_pubkey: stake_pubkey,
@@ -1457,7 +1465,10 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         lamports: 42,
         withdraw_authority: 0,
         sign_only: false,
-        blockhash_query: BlockhashQuery::FeeCalculator(blockhash),
+        blockhash_query: BlockhashQuery::FeeCalculator(
+            blockhash_query::Source::NonceAccount(nonce_pubkey),
+            sign_only.blockhash,
+        ),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
         fee_payer: 0,
@@ -1466,14 +1477,10 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
     check_balance(42, &rpc_client, &recipient_pubkey);
 
     // Fetch nonce hash
-    let account = rpc_client.get_account(&nonce_account.pubkey()).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+    let nonce_hash = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    let nonce_hash = match nonce_state {
-        nonce::State::Initialized(ref data) => data.blockhash,
-        _ => panic!("Nonce is not initialized"),
-    };
+        .blockhash;
 
     // Create another stake account. This time with seed
     let seed = "seedy";
@@ -1486,16 +1493,16 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         lockup: Lockup::default(),
         lamports: 50_000,
         sign_only: true,
-        blockhash_query: BlockhashQuery::None(nonce_hash, FeeCalculator::default()),
+        blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
         fee_payer: 0,
         from: 0,
     };
     let sig_response = process_command(&config_offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sig_response);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
-    let stake_presigner = presigner_from_pubkey_sigs(&stake_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sig_response);
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
+    let stake_presigner = sign_only.presigner_of(&stake_pubkey).unwrap();
     config.signers = vec![&offline_presigner, &stake_presigner];
     config.command = CliCommand::CreateStakeAccount {
         stake_account: 1,
@@ -1505,7 +1512,10 @@ fn test_offline_nonced_create_stake_account_and_withdraw() {
         lockup: Lockup::default(),
         lamports: 50_000,
         sign_only: false,
-        blockhash_query: BlockhashQuery::FeeCalculator(blockhash),
+        blockhash_query: BlockhashQuery::FeeCalculator(
+            blockhash_query::Source::NonceAccount(nonce_pubkey),
+            sign_only.blockhash,
+        ),
         nonce_account: Some(nonce_pubkey),
         nonce_authority: 0,
         fee_payer: 0,

cli/tests/transfer.rs

@@ -1,17 +1,18 @@
-use solana_clap_utils::keypair::presigner_from_pubkey_sigs;
 use solana_cli::{
     cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig},
-    offline::{parse_sign_only_reply_string, BlockhashQuery},
+    nonce,
+    offline::{
+        blockhash_query::{self, BlockhashQuery},
+        parse_sign_only_reply_string,
+    },
 };
 use solana_client::rpc_client::RpcClient;
 use solana_core::validator::{TestValidator, TestValidatorOptions};
 use solana_faucet::faucet::run_local_faucet;
 use solana_sdk::{
-    account_utils::StateMut,
-    fee_calculator::FeeCalculator,
-    nonce,
+    nonce::State as NonceState,
     pubkey::Pubkey,
-    signature::{keypair_from_seed, Keypair, Signer},
+    signature::{keypair_from_seed, Keypair, NullSigner, Signer},
 };
 use std::{fs::remove_dir_all, sync::mpsc::channel, thread::sleep, time::Duration};
 
@@ -67,7 +68,8 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
-        blockhash_query: BlockhashQuery::All,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::All(blockhash_query::Source::Cluster),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -94,21 +96,24 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: true,
-        blockhash_query: BlockhashQuery::None(blockhash, FeeCalculator::default()),
+        no_wait: false,
+        blockhash_query: BlockhashQuery::None(blockhash),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
     };
     let sign_only_reply = process_command(&offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sign_only_reply);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_only_reply);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::Transfer {
         lamports: 10,
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
-        blockhash_query: BlockhashQuery::FeeCalculator(blockhash),
+        no_wait: false,
+        blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
         nonce_account: None,
         nonce_authority: 0,
         fee_payer: 0,
@@ -120,7 +125,7 @@ fn test_transfer() {
     // Create nonce account
     let nonce_account = keypair_from_seed(&[3u8; 32]).unwrap();
     let minimum_nonce_balance = rpc_client
-        .get_minimum_balance_for_rent_exemption(nonce::State::size())
+        .get_minimum_balance_for_rent_exemption(NonceState::size())
         .unwrap();
     config.signers = vec![&default_signer, &nonce_account];
     config.command = CliCommand::CreateNonceAccount {
@@ -133,14 +138,10 @@ fn test_transfer() {
     check_balance(49_987 - minimum_nonce_balance, &rpc_client, &sender_pubkey);
 
     // Fetch nonce hash
-    let account = rpc_client.get_account(&nonce_account.pubkey()).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+    let nonce_hash = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    let nonce_hash = match nonce_state {
-        nonce::State::Initialized(ref data) => data.blockhash,
-        _ => panic!("Nonce is not initialized"),
-    };
+        .blockhash;
 
     // Nonced transfer
     config.signers = vec![&default_signer];
@@ -149,7 +150,11 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
-        blockhash_query: BlockhashQuery::FeeCalculator(nonce_hash),
+        no_wait: false,
+        blockhash_query: BlockhashQuery::FeeCalculator(
+            blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
+            nonce_hash,
+        ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
@@ -157,14 +162,10 @@ fn test_transfer() {
     process_command(&config).unwrap();
     check_balance(49_976 - minimum_nonce_balance, &rpc_client, &sender_pubkey);
     check_balance(30, &rpc_client, &recipient_pubkey);
-    let account = rpc_client.get_account(&nonce_account.pubkey()).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+    let new_nonce_hash = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    let new_nonce_hash = match nonce_state {
-        nonce::State::Initialized(ref data) => data.blockhash,
-        _ => panic!("Nonce is not initialized"),
-    };
+        .blockhash;
     assert_ne!(nonce_hash, new_nonce_hash);
 
     // Assign nonce authority to offline
@@ -178,14 +179,10 @@ fn test_transfer() {
     check_balance(49_975 - minimum_nonce_balance, &rpc_client, &sender_pubkey);
 
     // Fetch nonce hash
-    let account = rpc_client.get_account(&nonce_account.pubkey()).unwrap();
-    let nonce_state = StateMut::<nonce::state::Versions>::state(&account)
+    let nonce_hash = nonce::get_account(&rpc_client, &nonce_account.pubkey())
+        .and_then(|ref a| nonce::data_from_account(a))
         .unwrap()
-        .convert_to_current();
-    let nonce_hash = match nonce_state {
-        nonce::State::Initialized(ref data) => data.blockhash,
-        _ => panic!("Nonce is not initialized"),
-    };
+        .blockhash;
 
     // Offline, nonced transfer
     offline.signers = vec![&default_offline_signer];
@@ -194,21 +191,27 @@ fn test_transfer() {
         to: recipient_pubkey,
         from: 0,
         sign_only: true,
-        blockhash_query: BlockhashQuery::None(nonce_hash, FeeCalculator::default()),
+        no_wait: false,
+        blockhash_query: BlockhashQuery::None(nonce_hash),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
     };
     let sign_only_reply = process_command(&offline).unwrap();
-    let (blockhash, signers) = parse_sign_only_reply_string(&sign_only_reply);
-    let offline_presigner = presigner_from_pubkey_sigs(&offline_pubkey, &signers).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_only_reply);
+    assert!(sign_only.has_all_signers());
+    let offline_presigner = sign_only.presigner_of(&offline_pubkey).unwrap();
     config.signers = vec![&offline_presigner];
     config.command = CliCommand::Transfer {
         lamports: 10,
         to: recipient_pubkey,
         from: 0,
         sign_only: false,
-        blockhash_query: BlockhashQuery::FeeCalculator(blockhash),
+        no_wait: false,
+        blockhash_query: BlockhashQuery::FeeCalculator(
+            blockhash_query::Source::NonceAccount(nonce_account.pubkey()),
+            sign_only.blockhash,
+        ),
         nonce_account: Some(nonce_account.pubkey()),
         nonce_authority: 0,
         fee_payer: 0,
@@ -220,3 +223,117 @@ fn test_transfer() {
     server.close().unwrap();
     remove_dir_all(ledger_path).unwrap();
 }
+
+#[test]
+fn test_transfer_multisession_signing() {
+    let TestValidator {
+        server,
+        leader_data,
+        alice: mint_keypair,
+        ledger_path,
+        ..
+    } = TestValidator::run_with_options(TestValidatorOptions {
+        fees: 1,
+        bootstrap_validator_lamports: 42_000,
+    });
+
+    let (sender, receiver) = channel();
+    run_local_faucet(mint_keypair, sender, None);
+    let faucet_addr = receiver.recv().unwrap();
+
+    let to_pubkey = Pubkey::new(&[1u8; 32]);
+    let offline_from_signer = keypair_from_seed(&[2u8; 32]).unwrap();
+    let offline_fee_payer_signer = keypair_from_seed(&[3u8; 32]).unwrap();
+    let from_null_signer = NullSigner::new(&offline_from_signer.pubkey());
+
+    // Setup accounts
+    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    request_and_confirm_airdrop(&rpc_client, &faucet_addr, &offline_from_signer.pubkey(), 43)
+        .unwrap();
+    request_and_confirm_airdrop(
+        &rpc_client,
+        &faucet_addr,
+        &offline_fee_payer_signer.pubkey(),
+        3,
+    )
+    .unwrap();
+    check_balance(43, &rpc_client, &offline_from_signer.pubkey());
+    check_balance(3, &rpc_client, &offline_fee_payer_signer.pubkey());
+    check_balance(0, &rpc_client, &to_pubkey);
+
+    let (blockhash, _) = rpc_client.get_recent_blockhash().unwrap();
+
+    // Offline fee-payer signs first
+    let mut fee_payer_config = CliConfig::default();
+    fee_payer_config.json_rpc_url = String::default();
+    fee_payer_config.signers = vec![&offline_fee_payer_signer, &from_null_signer];
+    // Verify we cannot contact the cluster
+    fee_payer_config.command = CliCommand::ClusterVersion;
+    process_command(&fee_payer_config).unwrap_err();
+    fee_payer_config.command = CliCommand::Transfer {
+        lamports: 42,
+        to: to_pubkey,
+        from: 1,
+        sign_only: true,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::None(blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        fee_payer: 0,
+    };
+    let sign_only_reply = process_command(&fee_payer_config).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_only_reply);
+    assert!(!sign_only.has_all_signers());
+    let fee_payer_presigner = sign_only
+        .presigner_of(&offline_fee_payer_signer.pubkey())
+        .unwrap();
+
+    // Now the offline fund source
+    let mut from_config = CliConfig::default();
+    from_config.json_rpc_url = String::default();
+    from_config.signers = vec![&fee_payer_presigner, &offline_from_signer];
+    // Verify we cannot contact the cluster
+    from_config.command = CliCommand::ClusterVersion;
+    process_command(&from_config).unwrap_err();
+    from_config.command = CliCommand::Transfer {
+        lamports: 42,
+        to: to_pubkey,
+        from: 1,
+        sign_only: true,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::None(blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        fee_payer: 0,
+    };
+    let sign_only_reply = process_command(&from_config).unwrap();
+    let sign_only = parse_sign_only_reply_string(&sign_only_reply);
+    assert!(sign_only.has_all_signers());
+    let from_presigner = sign_only
+        .presigner_of(&offline_from_signer.pubkey())
+        .unwrap();
+
+    // Finally submit to the cluster
+    let mut config = CliConfig::default();
+    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.signers = vec![&fee_payer_presigner, &from_presigner];
+    config.command = CliCommand::Transfer {
+        lamports: 42,
+        to: to_pubkey,
+        from: 1,
+        sign_only: false,
+        no_wait: false,
+        blockhash_query: BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
+        nonce_account: None,
+        nonce_authority: 0,
+        fee_payer: 0,
+    };
+    process_command(&config).unwrap();
+
+    check_balance(1, &rpc_client, &offline_from_signer.pubkey());
+    check_balance(1, &rpc_client, &offline_fee_payer_signer.pubkey());
+    check_balance(42, &rpc_client, &to_pubkey);
+
+    server.close().unwrap();
+    remove_dir_all(ledger_path).unwrap();
+}

cli/tests/vote.rs

@@ -0,0 +1,118 @@
+use solana_cli::cli::{process_command, request_and_confirm_airdrop, CliCommand, CliConfig};
+use solana_client::rpc_client::RpcClient;
+use solana_core::validator::TestValidator;
+use solana_faucet::faucet::run_local_faucet;
+use solana_sdk::{
+    account_utils::StateMut,
+    pubkey::Pubkey,
+    signature::{Keypair, Signer},
+};
+use solana_vote_program::vote_state::{VoteAuthorize, VoteState, VoteStateVersions};
+use std::{fs::remove_dir_all, sync::mpsc::channel, thread::sleep, time::Duration};
+
+fn check_balance(expected_balance: u64, client: &RpcClient, pubkey: &Pubkey) {
+    (0..5).for_each(|tries| {
+        let balance = client.retry_get_balance(pubkey, 1).unwrap().unwrap();
+        if balance == expected_balance {
+            return;
+        }
+        if tries == 4 {
+            assert_eq!(balance, expected_balance);
+        }
+        sleep(Duration::from_millis(500));
+    });
+}
+
+#[test]
+fn test_vote_authorize_and_withdraw() {
+    let TestValidator {
+        server,
+        leader_data,
+        alice,
+        ledger_path,
+        ..
+    } = TestValidator::run();
+    let (sender, receiver) = channel();
+    run_local_faucet(alice, sender, None);
+    let faucet_addr = receiver.recv().unwrap();
+
+    let rpc_client = RpcClient::new_socket(leader_data.rpc);
+    let default_signer = Keypair::new();
+
+    let mut config = CliConfig::default();
+    config.json_rpc_url = format!("http://{}:{}", leader_data.rpc.ip(), leader_data.rpc.port());
+    config.signers = vec![&default_signer];
+
+    request_and_confirm_airdrop(
+        &rpc_client,
+        &faucet_addr,
+        &config.signers[0].pubkey(),
+        100_000,
+    )
+    .unwrap();
+
+    // Create vote account
+    let vote_account_keypair = Keypair::new();
+    let vote_account_pubkey = vote_account_keypair.pubkey();
+    config.signers = vec![&default_signer, &vote_account_keypair];
+    config.command = CliCommand::CreateVoteAccount {
+        seed: None,
+        identity_account: 0,
+        authorized_voter: None,
+        authorized_withdrawer: Some(config.signers[0].pubkey()),
+        commission: 0,
+    };
+    process_command(&config).unwrap();
+    let vote_account = rpc_client
+        .get_account(&vote_account_keypair.pubkey())
+        .unwrap();
+    let vote_state: VoteStateVersions = vote_account.state().unwrap();
+    let authorized_withdrawer = vote_state.convert_to_current().authorized_withdrawer;
+    assert_eq!(authorized_withdrawer, config.signers[0].pubkey());
+    let expected_balance = rpc_client
+        .get_minimum_balance_for_rent_exemption(VoteState::size_of())
+        .unwrap()
+        .max(1);
+    check_balance(expected_balance, &rpc_client, &vote_account_pubkey);
+
+    // Authorize vote account withdrawal to another signer
+    let withdraw_authority = Keypair::new();
+    config.signers = vec![&default_signer];
+    config.command = CliCommand::VoteAuthorize {
+        vote_account_pubkey,
+        new_authorized_pubkey: withdraw_authority.pubkey(),
+        vote_authorize: VoteAuthorize::Withdrawer,
+    };
+    process_command(&config).unwrap();
+    let vote_account = rpc_client
+        .get_account(&vote_account_keypair.pubkey())
+        .unwrap();
+    let vote_state: VoteStateVersions = vote_account.state().unwrap();
+    let authorized_withdrawer = vote_state.convert_to_current().authorized_withdrawer;
+    assert_eq!(authorized_withdrawer, withdraw_authority.pubkey());
+
+    // Withdraw from vote account
+    let destination_account = Pubkey::new_rand(); // Send withdrawal to new account to make balance check easy
+    config.signers = vec![&default_signer, &withdraw_authority];
+    config.command = CliCommand::WithdrawFromVoteAccount {
+        vote_account_pubkey,
+        withdraw_authority: 1,
+        lamports: 100,
+        destination_account_pubkey: destination_account,
+    };
+    process_command(&config).unwrap();
+    check_balance(expected_balance - 100, &rpc_client, &vote_account_pubkey);
+    check_balance(100, &rpc_client, &destination_account);
+
+    // Re-assign validator identity
+    let new_identity_keypair = Keypair::new();
+    config.signers.push(&new_identity_keypair);
+    config.command = CliCommand::VoteUpdateValidator {
+        vote_account_pubkey,
+        new_identity_account: 2,
+    };
+    process_command(&config).unwrap();
+
+    server.close().unwrap();
+    remove_dir_all(ledger_path).unwrap();
+}

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.0.5"
+version = "1.0.22"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,6 +11,7 @@ edition = "2018"
 [dependencies]
 bincode = "1.2.1"
 bs58 = "0.3.0"
+indicatif = "0.14.0"
 jsonrpc-core = "14.0.5"
 log = "0.4.8"
 rayon = "1.2.0"
@@ -18,8 +19,10 @@ reqwest = { version = "0.10.1", default-features = false, features = ["blocking"
 serde = "1.0.104"
 serde_derive = "1.0.103"
 serde_json = "1.0.46"
-solana-net-utils = { path = "../net-utils", version = "1.0.5" }
-solana-sdk = { path = "../sdk", version = "1.0.5" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
+solana-transaction-status = { path = "../transaction-status", version = "1.0.22" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.22" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -28,4 +31,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "14.0.5"
 jsonrpc-http-server = "14.0.6"
-solana-logger = { path = "../logger", version = "1.0.5" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

client/src/client_error.rs

@@ -1,20 +1,161 @@
 use crate::rpc_request;
-use solana_sdk::{signature::SignerError, transaction::TransactionError};
-use std::{fmt, io};
+use solana_sdk::{
+    signature::SignerError, transaction::TransactionError, transport::TransportError,
+};
+use std::io;
 use thiserror::Error;
 
 #[derive(Error, Debug)]
-pub enum ClientError {
+pub enum ClientErrorKind {
+    #[error(transparent)]
     Io(#[from] io::Error),
+    #[error(transparent)]
     Reqwest(#[from] reqwest::Error),
+    #[error(transparent)]
     RpcError(#[from] rpc_request::RpcError),
+    #[error(transparent)]
     SerdeJson(#[from] serde_json::error::Error),
+    #[error(transparent)]
     SigningError(#[from] SignerError),
+    #[error(transparent)]
     TransactionError(#[from] TransactionError),
+    #[error("Custom: {0}")]
+    Custom(String),
 }
 
-impl fmt::Display for ClientError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "solana client error")
+impl From<TransportError> for ClientErrorKind {
+    fn from(err: TransportError) -> Self {
+        match err {
+            TransportError::IoError(err) => Self::Io(err),
+            TransportError::TransactionError(err) => Self::TransactionError(err),
+            TransportError::Custom(err) => Self::Custom(err),
+        }
     }
 }
+
+impl Into<TransportError> for ClientErrorKind {
+    fn into(self) -> TransportError {
+        match self {
+            Self::Io(err) => TransportError::IoError(err),
+            Self::TransactionError(err) => TransportError::TransactionError(err),
+            Self::Reqwest(err) => TransportError::Custom(format!("{:?}", err)),
+            Self::RpcError(err) => TransportError::Custom(format!("{:?}", err)),
+            Self::SerdeJson(err) => TransportError::Custom(format!("{:?}", err)),
+            Self::SigningError(err) => TransportError::Custom(format!("{:?}", err)),
+            Self::Custom(err) => TransportError::Custom(format!("{:?}", err)),
+        }
+    }
+}
+
+#[derive(Error, Debug)]
+#[error("{kind}")]
+pub struct ClientError {
+    command: Option<&'static str>,
+    #[source]
+    #[error(transparent)]
+    kind: ClientErrorKind,
+}
+
+impl ClientError {
+    pub fn new_with_command(kind: ClientErrorKind, command: &'static str) -> Self {
+        Self {
+            command: Some(command),
+            kind,
+        }
+    }
+
+    pub fn into_with_command(self, command: &'static str) -> Self {
+        Self {
+            command: Some(command),
+            ..self
+        }
+    }
+
+    pub fn command(&self) -> Option<&'static str> {
+        self.command
+    }
+
+    pub fn kind(&self) -> &ClientErrorKind {
+        &self.kind
+    }
+}
+
+impl From<ClientErrorKind> for ClientError {
+    fn from(kind: ClientErrorKind) -> Self {
+        Self {
+            command: None,
+            kind,
+        }
+    }
+}
+
+impl From<TransportError> for ClientError {
+    fn from(err: TransportError) -> Self {
+        Self {
+            command: None,
+            kind: err.into(),
+        }
+    }
+}
+
+impl Into<TransportError> for ClientError {
+    fn into(self) -> TransportError {
+        self.kind.into()
+    }
+}
+
+impl From<std::io::Error> for ClientError {
+    fn from(err: std::io::Error) -> Self {
+        Self {
+            command: None,
+            kind: err.into(),
+        }
+    }
+}
+
+impl From<reqwest::Error> for ClientError {
+    fn from(err: reqwest::Error) -> Self {
+        Self {
+            command: None,
+            kind: err.into(),
+        }
+    }
+}
+
+impl From<rpc_request::RpcError> for ClientError {
+    fn from(err: rpc_request::RpcError) -> Self {
+        Self {
+            command: None,
+            kind: err.into(),
+        }
+    }
+}
+
+impl From<serde_json::error::Error> for ClientError {
+    fn from(err: serde_json::error::Error) -> Self {
+        Self {
+            command: None,
+            kind: err.into(),
+        }
+    }
+}
+
+impl From<SignerError> for ClientError {
+    fn from(err: SignerError) -> Self {
+        Self {
+            command: None,
+            kind: err.into(),
+        }
+    }
+}
+
+impl From<TransactionError> for ClientError {
+    fn from(err: TransactionError) -> Self {
+        Self {
+            command: None,
+            kind: err.into(),
+        }
+    }
+}
+
+pub type Result<T> = std::result::Result<T, ClientError>;

client/src/generic_rpc_client_request.rs

@@ -1,4 +1,4 @@
-use crate::{client_error::ClientError, rpc_request::RpcRequest};
+use crate::{client_error::Result, rpc_request::RpcRequest};
 
 pub(crate) trait GenericRpcClientRequest {
     fn send(
@@ -6,5 +6,5 @@ pub(crate) trait GenericRpcClientRequest {
         request: &RpcRequest,
         params: serde_json::Value,
         retries: usize,
-    ) -> Result<serde_json::Value, ClientError>;
+    ) -> Result<serde_json::Value>;
 }

client/src/mock_rpc_client_request.rs

@@ -1,5 +1,5 @@
 use crate::{
-    client_error::ClientError,
+    client_error::Result,
     generic_rpc_client_request::GenericRpcClientRequest,
     rpc_request::RpcRequest,
     rpc_response::{Response, RpcResponseContext},
@@ -10,6 +10,7 @@ use solana_sdk::{
     instruction::InstructionError,
     transaction::{self, TransactionError},
 };
+use solana_transaction_status::TransactionStatus;
 use std::{collections::HashMap, sync::RwLock};
 
 pub const PUBKEY: &str = "7RoSF9fUmdphVCpabEoefH81WwrW7orsWonXWqTXkKV8";
@@ -39,9 +40,9 @@ impl GenericRpcClientRequest for MockRpcClientRequest {
     fn send(
         &self,
         request: &RpcRequest,
-        params: serde_json::Value,
+        _params: serde_json::Value,
         _retries: usize,
-    ) -> Result<serde_json::Value, ClientError> {
+    ) -> Result<serde_json::Value> {
         if let Some(value) = self.mocks.write().unwrap().remove(request) {
             return Ok(value);
         }
@@ -49,17 +50,6 @@ impl GenericRpcClientRequest for MockRpcClientRequest {
             return Ok(Value::Null);
         }
         let val = match request {
-            RpcRequest::ConfirmTransaction => {
-                if let Some(params_array) = params.as_array() {
-                    if let Value::String(param_string) = &params_array[0] {
-                        Value::Bool(param_string == SIGNATURE)
-                    } else {
-                        Value::Null
-                    }
-                } else {
-                    Value::Null
-                }
-            }
             RpcRequest::GetBalance => serde_json::to_value(Response {
                 context: RpcResponseContext { slot: 1 },
                 value: Value::Number(Number::from(50)),
@@ -86,20 +76,32 @@ impl GenericRpcClientRequest for MockRpcClientRequest {
                 context: RpcResponseContext { slot: 1 },
                 value: serde_json::to_value(FeeRateGovernor::default()).unwrap(),
             })?,
-            RpcRequest::GetSignatureStatus => {
-                let response: Option<transaction::Result<()>> = if self.url == "account_in_use" {
-                    Some(Err(TransactionError::AccountInUse))
+            RpcRequest::GetSignatureStatuses => {
+                let status: transaction::Result<()> = if self.url == "account_in_use" {
+                    Err(TransactionError::AccountInUse)
                 } else if self.url == "instruction_error" {
-                    Some(Err(TransactionError::InstructionError(
+                    Err(TransactionError::InstructionError(
                         0,
                         InstructionError::UninitializedAccount,
-                    )))
-                } else if self.url == "sig_not_found" {
+                    ))
+                } else {
+                    Ok(())
+                };
+                let status = if self.url == "sig_not_found" {
                     None
                 } else {
-                    Some(Ok(()))
+                    let err = status.clone().err();
+                    Some(TransactionStatus {
+                        status,
+                        slot: 1,
+                        confirmations: None,
+                        err,
+                    })
                 };
-                serde_json::to_value(response).unwrap()
+                serde_json::to_value(Response {
+                    context: RpcResponseContext { slot: 1 },
+                    value: vec![status],
+                })?
             }
             RpcRequest::GetTransactionCount => Value::Number(Number::from(1234)),
             RpcRequest::GetSlot => Value::Number(Number::from(0)),

client/src/rpc_client_request.rs

@@ -1,5 +1,5 @@
 use crate::{
-    client_error::ClientError,
+    client_error::Result,
     generic_rpc_client_request::GenericRpcClientRequest,
     rpc_request::{RpcError, RpcRequest},
 };
@@ -34,7 +34,7 @@ impl GenericRpcClientRequest for RpcClientRequest {
         request: &RpcRequest,
         params: serde_json::Value,
         mut retries: usize,
-    ) -> Result<serde_json::Value, ClientError> {
+    ) -> Result<serde_json::Value> {
         // Concurrent requests are not supported so reuse the same request id for all requests
         let request_id = 1;
 

client/src/rpc_request.rs

@@ -1,9 +1,8 @@
 use serde_json::{json, Value};
-use std::{error, fmt};
+use thiserror::Error;
 
 #[derive(Debug, PartialEq, Eq, Hash)]
 pub enum RpcRequest {
-    ConfirmTransaction,
     DeregisterNode,
     ValidatorExit,
     GetAccountInfo,
@@ -12,18 +11,19 @@ pub enum RpcRequest {
     GetClusterNodes,
     GetConfirmedBlock,
     GetConfirmedBlocks,
+    GetConfirmedSignaturesForAddress,
+    GetConfirmedTransaction,
     GetEpochInfo,
     GetEpochSchedule,
     GetGenesisHash,
     GetIdentity,
     GetInflation,
     GetLeaderSchedule,
-    GetNumBlocksSinceSignatureConfirmation,
     GetProgramAccounts,
     GetRecentBlockhash,
     GetFeeCalculatorForBlockhash,
     GetFeeRateGovernor,
-    GetSignatureStatus,
+    GetSignatureStatuses,
     GetSlot,
     GetSlotLeader,
     GetStorageTurn,
@@ -45,7 +45,6 @@ impl RpcRequest {
     pub(crate) fn build_request_json(&self, id: u64, params: Value) -> Value {
         let jsonrpc = "2.0";
         let method = match self {
-            RpcRequest::ConfirmTransaction => "confirmTransaction",
             RpcRequest::DeregisterNode => "deregisterNode",
             RpcRequest::ValidatorExit => "validatorExit",
             RpcRequest::GetAccountInfo => "getAccountInfo",
@@ -54,20 +53,19 @@ impl RpcRequest {
             RpcRequest::GetClusterNodes => "getClusterNodes",
             RpcRequest::GetConfirmedBlock => "getConfirmedBlock",
             RpcRequest::GetConfirmedBlocks => "getConfirmedBlocks",
+            RpcRequest::GetConfirmedSignaturesForAddress => "getConfirmedSignaturesForAddress",
+            RpcRequest::GetConfirmedTransaction => "getConfirmedTransaction",
             RpcRequest::GetEpochInfo => "getEpochInfo",
             RpcRequest::GetEpochSchedule => "getEpochSchedule",
             RpcRequest::GetGenesisHash => "getGenesisHash",
             RpcRequest::GetIdentity => "getIdentity",
             RpcRequest::GetInflation => "getInflation",
             RpcRequest::GetLeaderSchedule => "getLeaderSchedule",
-            RpcRequest::GetNumBlocksSinceSignatureConfirmation => {
-                "getNumBlocksSinceSignatureConfirmation"
-            }
             RpcRequest::GetProgramAccounts => "getProgramAccounts",
             RpcRequest::GetRecentBlockhash => "getRecentBlockhash",
             RpcRequest::GetFeeCalculatorForBlockhash => "getFeeCalculatorForBlockhash",
             RpcRequest::GetFeeRateGovernor => "getFeeRateGovernor",
-            RpcRequest::GetSignatureStatus => "getSignatureStatus",
+            RpcRequest::GetSignatureStatuses => "getSignatureStatuses",
             RpcRequest::GetSlot => "getSlot",
             RpcRequest::GetSlotLeader => "getSlotLeader",
             RpcRequest::GetStorageTurn => "getStorageTurn",
@@ -93,26 +91,16 @@ impl RpcRequest {
     }
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Error)]
 pub enum RpcError {
+    #[error("rpc request error: {0}")]
     RpcRequestError(String),
-}
-
-impl fmt::Display for RpcError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "invalid")
-    }
-}
-
-impl error::Error for RpcError {
-    fn description(&self) -> &str {
-        "invalid"
-    }
-
-    fn cause(&self) -> Option<&dyn error::Error> {
-        // Generic error, underlying cause isn't tracked.
-        None
-    }
+    #[error("parse error: expected {0}")]
+    ParseError(String), /* "expected" */
+    // Anything in a `ForUser` needs to die.  The caller should be
+    // deciding what to tell their user
+    #[error("{0}")]
+    ForUser(String), /* "direct-to-user message" */
 }
 
 #[cfg(test)]

client/src/rpc_response.rs

@@ -1,18 +1,14 @@
-use crate::rpc_request::RpcError;
-use bincode::serialize;
-use jsonrpc_core::Result as JsonResult;
+use crate::{client_error, rpc_request::RpcError};
 use solana_sdk::{
     account::Account,
     clock::{Epoch, Slot},
     fee_calculator::{FeeCalculator, FeeRateGovernor},
-    message::MessageHeader,
     pubkey::Pubkey,
-    transaction::{Result, Transaction},
+    transaction::{Result, TransactionError},
 };
-use std::{collections::HashMap, io, net::SocketAddr, str::FromStr};
+use std::{collections::HashMap, net::SocketAddr, str::FromStr};
 
-pub type RpcResponseIn<T> = JsonResult<Response<T>>;
-pub type RpcResponse<T> = io::Result<Response<T>>;
+pub type RpcResult<T> = client_error::Result<Response<T>>;
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct RpcResponseContext {
@@ -32,119 +28,6 @@ pub struct RpcBlockCommitment<T> {
     pub total_stake: u64,
 }
 
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-pub struct RpcReward {
-    pub pubkey: String,
-    pub lamports: i64,
-}
-
-pub type RpcRewards = Vec<RpcReward>;
-
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcConfirmedBlock {
-    pub previous_blockhash: String,
-    pub blockhash: String,
-    pub parent_slot: Slot,
-    pub transactions: Vec<RpcTransactionWithStatusMeta>,
-    pub rewards: RpcRewards,
-}
-
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcTransactionWithStatusMeta {
-    pub transaction: RpcEncodedTransaction,
-    pub meta: Option<RpcTransactionStatus>,
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
-#[serde(rename_all = "camelCase")]
-pub enum RpcTransactionEncoding {
-    Binary,
-    Json,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase", untagged)]
-pub enum RpcEncodedTransaction {
-    Binary(String),
-    Json(RpcTransaction),
-}
-
-impl RpcEncodedTransaction {
-    pub fn encode(transaction: Transaction, encoding: RpcTransactionEncoding) -> Self {
-        if encoding == RpcTransactionEncoding::Json {
-            RpcEncodedTransaction::Json(RpcTransaction {
-                signatures: transaction
-                    .signatures
-                    .iter()
-                    .map(|sig| sig.to_string())
-                    .collect(),
-                message: RpcMessage {
-                    header: transaction.message.header,
-                    account_keys: transaction
-                        .message
-                        .account_keys
-                        .iter()
-                        .map(|pubkey| pubkey.to_string())
-                        .collect(),
-                    recent_blockhash: transaction.message.recent_blockhash.to_string(),
-                    instructions: transaction
-                        .message
-                        .instructions
-                        .iter()
-                        .map(|instruction| RpcCompiledInstruction {
-                            program_id_index: instruction.program_id_index,
-                            accounts: instruction.accounts.clone(),
-                            data: bs58::encode(instruction.data.clone()).into_string(),
-                        })
-                        .collect(),
-                },
-            })
-        } else {
-            RpcEncodedTransaction::Binary(
-                bs58::encode(serialize(&transaction).unwrap()).into_string(),
-            )
-        }
-    }
-}
-
-/// A duplicate representation of a Transaction for pretty JSON serialization
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcTransaction {
-    pub signatures: Vec<String>,
-    pub message: RpcMessage,
-}
-
-/// A duplicate representation of a Message for pretty JSON serialization
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcMessage {
-    pub header: MessageHeader,
-    pub account_keys: Vec<String>,
-    pub recent_blockhash: String,
-    pub instructions: Vec<RpcCompiledInstruction>,
-}
-
-/// A duplicate representation of a Message for pretty JSON serialization
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcCompiledInstruction {
-    pub program_id_index: u8,
-    pub accounts: Vec<u8>,
-    pub data: String,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct RpcTransactionStatus {
-    pub status: Result<()>,
-    pub fee: u64,
-    pub pre_balances: Vec<u64>,
-    pub post_balances: Vec<u64>,
-}
-
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct RpcBlockhashFeeCalculator {
@@ -171,6 +54,12 @@ pub struct RpcKeyedAccount {
     pub account: RpcAccount,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSignatureResult {
+    pub err: Option<TransactionError>,
+}
+
 /// A duplicate representation of a Message for pretty JSON serialization
 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
@@ -303,3 +192,19 @@ pub struct RpcStorageTurn {
     pub blockhash: String,
     pub slot: Slot,
 }
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcAccountBalance {
+    pub address: String,
+    pub lamports: u64,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSupply {
+    pub total: u64,
+    pub circulating: u64,
+    pub non_circulating: u64,
+    pub non_circulating_accounts: Vec<String>,
+}

client/src/thin_client.rs

@@ -188,7 +188,7 @@ impl ThinClient {
         transaction: &mut Transaction,
         tries: usize,
         min_confirmed_blocks: usize,
-    ) -> io::Result<Signature> {
+    ) -> TransportResult<Signature> {
         self.send_and_confirm_transaction(&[keypair], transaction, tries, min_confirmed_blocks)
     }
 
@@ -198,7 +198,7 @@ impl ThinClient {
         keypair: &Keypair,
         transaction: &mut Transaction,
         tries: usize,
-    ) -> io::Result<Signature> {
+    ) -> TransportResult<Signature> {
         self.send_and_confirm_transaction(&[keypair], transaction, tries, 0)
     }
 
@@ -209,7 +209,7 @@ impl ThinClient {
         transaction: &mut Transaction,
         tries: usize,
         pending_confirmations: usize,
-    ) -> io::Result<Signature> {
+    ) -> TransportResult<Signature> {
         for x in 0..tries {
             let now = Instant::now();
             let mut buf = vec![0; serialized_size(&transaction).unwrap() as usize];
@@ -243,13 +243,14 @@ impl ThinClient {
                 }
             }
             info!("{} tries failed transfer to {}", x, self.tpu_addr());
-            let (blockhash, _fee_calculator) = self.rpc_client().get_recent_blockhash()?;
+            let (blockhash, _fee_calculator) = self.get_recent_blockhash()?;
             transaction.sign(keypairs, blockhash);
         }
         Err(io::Error::new(
             io::ErrorKind::Other,
             format!("retry_transfer failed in {} retries", tries),
-        ))
+        )
+        .into())
     }
 
     pub fn poll_balance_with_timeout_and_commitment(
@@ -258,13 +259,15 @@ impl ThinClient {
         polling_frequency: &Duration,
         timeout: &Duration,
         commitment_config: CommitmentConfig,
-    ) -> io::Result<u64> {
-        self.rpc_client().poll_balance_with_timeout_and_commitment(
-            pubkey,
-            polling_frequency,
-            timeout,
-            commitment_config,
-        )
+    ) -> TransportResult<u64> {
+        self.rpc_client()
+            .poll_balance_with_timeout_and_commitment(
+                pubkey,
+                polling_frequency,
+                timeout,
+                commitment_config,
+            )
+            .map_err(|e| e.into())
     }
 
     pub fn poll_balance_with_timeout(
@@ -272,8 +275,8 @@ impl ThinClient {
         pubkey: &Pubkey,
         polling_frequency: &Duration,
         timeout: &Duration,
-    ) -> io::Result<u64> {
-        self.rpc_client().poll_balance_with_timeout_and_commitment(
+    ) -> TransportResult<u64> {
+        self.poll_balance_with_timeout_and_commitment(
             pubkey,
             polling_frequency,
             timeout,
@@ -281,18 +284,18 @@ impl ThinClient {
         )
     }
 
-    pub fn poll_get_balance(&self, pubkey: &Pubkey) -> io::Result<u64> {
-        self.rpc_client()
-            .poll_get_balance_with_commitment(pubkey, CommitmentConfig::default())
+    pub fn poll_get_balance(&self, pubkey: &Pubkey) -> TransportResult<u64> {
+        self.poll_get_balance_with_commitment(pubkey, CommitmentConfig::default())
     }
 
     pub fn poll_get_balance_with_commitment(
         &self,
         pubkey: &Pubkey,
         commitment_config: CommitmentConfig,
-    ) -> io::Result<u64> {
+    ) -> TransportResult<u64> {
         self.rpc_client()
             .poll_get_balance_with_commitment(pubkey, commitment_config)
+            .map_err(|e| e.into())
     }
 
     pub fn wait_for_balance(&self, pubkey: &Pubkey, expected_balance: Option<u64>) -> Option<u64> {
@@ -321,9 +324,9 @@ impl ThinClient {
         signature: &Signature,
         commitment_config: CommitmentConfig,
     ) -> TransportResult<()> {
-        Ok(self
-            .rpc_client()
-            .poll_for_signature_with_commitment(signature, commitment_config)?)
+        self.rpc_client()
+            .poll_for_signature_with_commitment(signature, commitment_config)
+            .map_err(|e| e.into())
     }
 
     /// Check a signature in the bank. This method blocks
@@ -332,16 +335,17 @@ impl ThinClient {
         self.rpc_client().check_signature(signature)
     }
 
-    pub fn validator_exit(&self) -> io::Result<bool> {
-        self.rpc_client().validator_exit()
+    pub fn validator_exit(&self) -> TransportResult<bool> {
+        self.rpc_client().validator_exit().map_err(|e| e.into())
     }
 
     pub fn get_num_blocks_since_signature_confirmation(
         &mut self,
         sig: &Signature,
-    ) -> io::Result<usize> {
+    ) -> TransportResult<usize> {
         self.rpc_client()
             .get_num_blocks_since_signature_confirmation(sig)
+            .map_err(|e| e.into())
     }
 }
 
@@ -368,7 +372,7 @@ impl SyncClient for ThinClient {
         keypair: &Keypair,
         instruction: Instruction,
     ) -> TransportResult<Signature> {
-        let message = Message::new(vec![instruction]);
+        let message = Message::new(&[instruction]);
         self.send_message(&[keypair], message)
     }
 
@@ -400,14 +404,14 @@ impl SyncClient for ThinClient {
         pubkey: &Pubkey,
         commitment_config: CommitmentConfig,
     ) -> TransportResult<Option<Account>> {
-        Ok(self
-            .rpc_client()
-            .get_account_with_commitment(pubkey, commitment_config)?
-            .value)
+        self.rpc_client()
+            .get_account_with_commitment(pubkey, commitment_config)
+            .map_err(|e| e.into())
+            .map(|r| r.value)
     }
 
     fn get_balance(&self, pubkey: &Pubkey) -> TransportResult<u64> {
-        Ok(self.rpc_client().get_balance(pubkey)?)
+        self.rpc_client().get_balance(pubkey).map_err(|e| e.into())
     }
 
     fn get_balance_with_commitment(
@@ -415,10 +419,10 @@ impl SyncClient for ThinClient {
         pubkey: &Pubkey,
         commitment_config: CommitmentConfig,
     ) -> TransportResult<u64> {
-        let balance = self
-            .rpc_client()
-            .get_balance_with_commitment(pubkey, commitment_config)?;
-        Ok(balance.value)
+        self.rpc_client()
+            .get_balance_with_commitment(pubkey, commitment_config)
+            .map_err(|e| e.into())
+            .map(|r| r.value)
     }
 
     fn get_recent_blockhash(&self) -> TransportResult<(Hash, FeeCalculator)> {
@@ -449,15 +453,16 @@ impl SyncClient for ThinClient {
         &self,
         blockhash: &Hash,
     ) -> TransportResult<Option<FeeCalculator>> {
-        let fee_calculator = self
-            .rpc_client()
-            .get_fee_calculator_for_blockhash(blockhash)?;
-        Ok(fee_calculator)
+        self.rpc_client()
+            .get_fee_calculator_for_blockhash(blockhash)
+            .map_err(|e| e.into())
     }
 
     fn get_fee_rate_governor(&self) -> TransportResult<FeeRateGovernor> {
-        let fee_rate_governor = self.rpc_client().get_fee_rate_governor()?;
-        Ok(fee_rate_governor.value)
+        self.rpc_client()
+            .get_fee_rate_governor()
+            .map_err(|e| e.into())
+            .map(|r| r.value)
     }
 
     fn get_signature_status(
@@ -466,7 +471,7 @@ impl SyncClient for ThinClient {
     ) -> TransportResult<Option<transaction::Result<()>>> {
         let status = self
             .rpc_client()
-            .get_signature_status(&signature.to_string())
+            .get_signature_status(&signature)
             .map_err(|err| {
                 io::Error::new(
                     io::ErrorKind::Other,
@@ -483,7 +488,7 @@ impl SyncClient for ThinClient {
     ) -> TransportResult<Option<transaction::Result<()>>> {
         let status = self
             .rpc_client()
-            .get_signature_status_with_commitment(&signature.to_string(), commitment_config)
+            .get_signature_status_with_commitment(&signature, commitment_config)
             .map_err(|err| {
                 io::Error::new(
                     io::ErrorKind::Other,
@@ -555,23 +560,26 @@ impl SyncClient for ThinClient {
         signature: &Signature,
         min_confirmed_blocks: usize,
     ) -> TransportResult<usize> {
-        Ok(self
-            .rpc_client()
-            .poll_for_signature_confirmation(signature, min_confirmed_blocks)?)
+        self.rpc_client()
+            .poll_for_signature_confirmation(signature, min_confirmed_blocks)
+            .map_err(|e| e.into())
     }
 
     fn poll_for_signature(&self, signature: &Signature) -> TransportResult<()> {
-        Ok(self.rpc_client().poll_for_signature(signature)?)
+        self.rpc_client()
+            .poll_for_signature(signature)
+            .map_err(|e| e.into())
     }
 
     fn get_new_blockhash(&self, blockhash: &Hash) -> TransportResult<(Hash, FeeCalculator)> {
-        let new_blockhash = self.rpc_client().get_new_blockhash(blockhash)?;
-        Ok(new_blockhash)
+        self.rpc_client()
+            .get_new_blockhash(blockhash)
+            .map_err(|e| e.into())
     }
 }
 
 impl AsyncClient for ThinClient {
-    fn async_send_transaction(&self, transaction: Transaction) -> io::Result<Signature> {
+    fn async_send_transaction(&self, transaction: Transaction) -> TransportResult<Signature> {
         let mut buf = vec![0; serialized_size(&transaction).unwrap() as usize];
         let mut wr = std::io::Cursor::new(&mut buf[..]);
         serialize_into(&mut wr, &transaction)
@@ -586,7 +594,7 @@ impl AsyncClient for ThinClient {
         keypairs: &T,
         message: Message,
         recent_blockhash: Hash,
-    ) -> io::Result<Signature> {
+    ) -> TransportResult<Signature> {
         let transaction = Transaction::new(keypairs, message, recent_blockhash);
         self.async_send_transaction(transaction)
     }
@@ -595,8 +603,8 @@ impl AsyncClient for ThinClient {
         keypair: &Keypair,
         instruction: Instruction,
         recent_blockhash: Hash,
-    ) -> io::Result<Signature> {
-        let message = Message::new(vec![instruction]);
+    ) -> TransportResult<Signature> {
+        let message = Message::new(&[instruction]);
         self.async_send_message(&[keypair], message, recent_blockhash)
     }
     fn async_transfer(
@@ -605,7 +613,7 @@ impl AsyncClient for ThinClient {
         keypair: &Keypair,
         pubkey: &Pubkey,
         recent_blockhash: Hash,
-    ) -> io::Result<Signature> {
+    ) -> TransportResult<Signature> {
         let transfer_instruction =
             system_instruction::transfer(&keypair.pubkey(), pubkey, lamports);
         self.async_send_instruction(keypair, transfer_instruction, recent_blockhash)

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.5"
+version = "1.0.22"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -15,6 +15,7 @@ codecov = { repository = "solana-labs/solana", branch = "master", service = "git
 
 [dependencies]
 bincode = "1.2.1"
+bv = { version = "0.11.1", features = ["serde"] }
 bs58 = "0.3.0"
 byteorder = "1.3.2"
 chrono = { version = "0.4.10", features = ["serde"] }
@@ -33,42 +34,43 @@ jsonrpc-ws-server = "14.0.6"
 libc = "0.2.66"
 log = "0.4.8"
 nix = "0.17.0"
+num_cpus = "1.0.0"
 num-traits = "0.2"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
+rand = "0.7.0"
+rand_chacha = "0.2.2"
 rayon = "1.2.0"
 regex = "1.3.4"
 serde = "1.0.104"
 serde_derive = "1.0.103"
 serde_json = "1.0.46"
-solana-budget-program = { path = "../programs/budget", version = "1.0.5" }
-solana-clap-utils = { path = "../clap-utils", version = "1.0.5" }
-solana-client = { path = "../client", version = "1.0.5" }
-solana-faucet = { path = "../faucet", version = "1.0.5" }
-ed25519-dalek = "=1.0.0-pre.1"
-solana-ledger = { path = "../ledger", version = "1.0.5" }
-solana-logger = { path = "../logger", version = "1.0.5" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.0.5" }
-solana-metrics = { path = "../metrics", version = "1.0.5" }
-solana-measure = { path = "../measure", version = "1.0.5" }
-solana-net-utils = { path = "../net-utils", version = "1.0.5" }
-solana-chacha-cuda = { path = "../chacha-cuda", version = "1.0.5" }
-solana-perf = { path = "../perf", version = "1.0.5" }
-solana-runtime = { path = "../runtime", version = "1.0.5" }
-solana-sdk = { path = "../sdk", version = "1.0.5" }
-solana-stake-program = { path = "../programs/stake", version = "1.0.5" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.5" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.5" }
-solana-vote-signer = { path = "../vote-signer", version = "1.0.5" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.0.5" }
-sys-info = "0.5.9"
+solana-budget-program = { path = "../programs/budget", version = "1.0.22" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.22" }
+solana-client = { path = "../client", version = "1.0.22" }
+solana-transaction-status = { path = "../transaction-status", version = "1.0.22" }
+solana-faucet = { path = "../faucet", version = "1.0.22" }
+ed25519-dalek = "=1.0.0-pre.3"
+solana-ledger = { path = "../ledger", version = "1.0.22" }
+solana-logger = { path = "../logger", version = "1.0.22" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.0.22" }
+solana-metrics = { path = "../metrics", version = "1.0.22" }
+solana-measure = { path = "../measure", version = "1.0.22" }
+solana-net-utils = { path = "../net-utils", version = "1.0.22" }
+solana-chacha-cuda = { path = "../chacha-cuda", version = "1.0.22" }
+solana-perf = { path = "../perf", version = "1.0.22" }
+solana-runtime = { path = "../runtime", version = "1.0.22" }
+solana-sdk = { path = "../sdk", version = "1.0.22" }
+solana-stake-program = { path = "../programs/stake", version = "1.0.22" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.22" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.22" }
+solana-vote-signer = { path = "../vote-signer", version = "1.0.22" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.0.22" }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = "0.1"
 tokio-codec = "0.1"
 tokio-fs = "0.1"
 tokio-io = "0.1"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.0.5" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.0.22" }
 trees = "0.2.1"
 
 [dev-dependencies]
@@ -102,3 +104,6 @@ name = "cluster_info"
 [[bench]]
 name = "chacha"
 required-features = ["chacha"]
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

core/src/accounts_hash_verifier.rs

@@ -0,0 +1,263 @@
+// Service to verify accounts hashes with other trusted validator nodes.
+//
+// Each interval, publish the snapshat hash which is the full accounts state
+// hash on gossip. Monitor gossip for messages from validators in the --trusted-validators
+// set and halt the node if a mismatch is detected.
+
+use crate::cluster_info::{ClusterInfo, MAX_SNAPSHOT_HASHES};
+use solana_ledger::{
+    snapshot_package::SnapshotPackage, snapshot_package::SnapshotPackageReceiver,
+    snapshot_package::SnapshotPackageSender,
+};
+use solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey};
+use std::collections::{HashMap, HashSet};
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        mpsc::RecvTimeoutError,
+        Arc, RwLock,
+    },
+    thread::{self, Builder, JoinHandle},
+    time::Duration,
+};
+
+pub struct AccountsHashVerifier {
+    t_accounts_hash_verifier: JoinHandle<()>,
+}
+
+impl AccountsHashVerifier {
+    pub fn new(
+        snapshot_package_receiver: SnapshotPackageReceiver,
+        snapshot_package_sender: Option<SnapshotPackageSender>,
+        exit: &Arc<AtomicBool>,
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        trusted_validators: Option<HashSet<Pubkey>>,
+        halt_on_trusted_validators_accounts_hash_mismatch: bool,
+        fault_injection_rate_slots: u64,
+    ) -> Self {
+        let exit = exit.clone();
+        let cluster_info = cluster_info.clone();
+        let t_accounts_hash_verifier = Builder::new()
+            .name("solana-accounts-hash".to_string())
+            .spawn(move || {
+                let mut hashes = vec![];
+                loop {
+                    if exit.load(Ordering::Relaxed) {
+                        break;
+                    }
+
+                    match snapshot_package_receiver.recv_timeout(Duration::from_secs(1)) {
+                        Ok(snapshot_package) => {
+                            Self::process_snapshot(
+                                snapshot_package,
+                                &cluster_info,
+                                &trusted_validators,
+                                halt_on_trusted_validators_accounts_hash_mismatch,
+                                &snapshot_package_sender,
+                                &mut hashes,
+                                &exit,
+                                fault_injection_rate_slots,
+                            );
+                        }
+                        Err(RecvTimeoutError::Disconnected) => break,
+                        Err(RecvTimeoutError::Timeout) => (),
+                    }
+                }
+            })
+            .unwrap();
+        Self {
+            t_accounts_hash_verifier,
+        }
+    }
+
+    fn process_snapshot(
+        snapshot_package: SnapshotPackage,
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        trusted_validators: &Option<HashSet<Pubkey>>,
+        halt_on_trusted_validator_accounts_hash_mismatch: bool,
+        snapshot_package_sender: &Option<SnapshotPackageSender>,
+        hashes: &mut Vec<(Slot, Hash)>,
+        exit: &Arc<AtomicBool>,
+        fault_injection_rate_slots: u64,
+    ) {
+        if fault_injection_rate_slots != 0
+            && snapshot_package.root % fault_injection_rate_slots == 0
+        {
+            // For testing, publish an invalid hash to gossip.
+            use rand::{thread_rng, Rng};
+            use solana_sdk::hash::extend_and_hash;
+            warn!("inserting fault at slot: {}", snapshot_package.root);
+            let rand = thread_rng().gen_range(0, 10);
+            let hash = extend_and_hash(&snapshot_package.hash, &[rand]);
+            hashes.push((snapshot_package.root, hash));
+        } else {
+            hashes.push((snapshot_package.root, snapshot_package.hash));
+        }
+
+        while hashes.len() > MAX_SNAPSHOT_HASHES {
+            hashes.remove(0);
+        }
+
+        if halt_on_trusted_validator_accounts_hash_mismatch {
+            let mut slot_to_hash = HashMap::new();
+            for (slot, hash) in hashes.iter() {
+                slot_to_hash.insert(*slot, *hash);
+            }
+            if Self::should_halt(&cluster_info, trusted_validators, &mut slot_to_hash) {
+                exit.store(true, Ordering::Relaxed);
+            }
+        }
+        if let Some(sender) = snapshot_package_sender.as_ref() {
+            if sender.send(snapshot_package).is_err() {}
+        }
+
+        cluster_info
+            .write()
+            .unwrap()
+            .push_accounts_hashes(hashes.clone());
+    }
+
+    fn should_halt(
+        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        trusted_validators: &Option<HashSet<Pubkey>>,
+        slot_to_hash: &mut HashMap<Slot, Hash>,
+    ) -> bool {
+        let mut verified_count = 0;
+        let mut highest_slot = 0;
+        if let Some(trusted_validators) = trusted_validators.as_ref() {
+            for trusted_validator in trusted_validators {
+                let cluster_info_r = cluster_info.read().unwrap();
+                if let Some(accounts_hashes) =
+                    cluster_info_r.get_accounts_hash_for_node(trusted_validator)
+                {
+                    for (slot, hash) in accounts_hashes {
+                        if let Some(reference_hash) = slot_to_hash.get(slot) {
+                            if *hash != *reference_hash {
+                                error!("Trusted validator {} produced conflicting hashes for slot: {} ({} != {})",
+                                    trusted_validator,
+                                    slot,
+                                    hash,
+                                    reference_hash,
+                                );
+
+                                return true;
+                            } else {
+                                verified_count += 1;
+                            }
+                        } else {
+                            highest_slot = std::cmp::max(*slot, highest_slot);
+                            slot_to_hash.insert(*slot, *hash);
+                        }
+                    }
+                }
+            }
+        }
+        inc_new_counter_info!("accounts_hash_verifier-hashes_verified", verified_count);
+        datapoint_info!(
+            "accounts_hash_verifier",
+            ("highest_slot_verified", highest_slot, i64),
+        );
+        false
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.t_accounts_hash_verifier.join()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::cluster_info::make_accounts_hashes_message;
+    use crate::contact_info::ContactInfo;
+    use solana_sdk::{
+        hash::hash,
+        signature::{Keypair, Signer},
+    };
+
+    #[test]
+    fn test_should_halt() {
+        let keypair = Keypair::new();
+
+        let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0);
+        let cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
+        let cluster_info = Arc::new(RwLock::new(cluster_info));
+
+        let mut trusted_validators = HashSet::new();
+        let mut slot_to_hash = HashMap::new();
+        assert!(!AccountsHashVerifier::should_halt(
+            &cluster_info,
+            &Some(trusted_validators.clone()),
+            &mut slot_to_hash,
+        ));
+
+        let validator1 = Keypair::new();
+        let hash1 = hash(&[1]);
+        let hash2 = hash(&[2]);
+        {
+            let message = make_accounts_hashes_message(&validator1, vec![(0, hash1)]).unwrap();
+            let mut cluster_info_w = cluster_info.write().unwrap();
+            cluster_info_w.push_message(message);
+        }
+        slot_to_hash.insert(0, hash2);
+        trusted_validators.insert(validator1.pubkey());
+        assert!(AccountsHashVerifier::should_halt(
+            &cluster_info,
+            &Some(trusted_validators.clone()),
+            &mut slot_to_hash,
+        ));
+    }
+
+    #[test]
+    fn test_max_hashes() {
+        solana_logger::setup();
+        use std::path::PathBuf;
+        use tempfile::TempDir;
+        let keypair = Keypair::new();
+
+        let contact_info = ContactInfo::new_localhost(&keypair.pubkey(), 0);
+        let cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
+        let cluster_info = Arc::new(RwLock::new(cluster_info));
+
+        let trusted_validators = HashSet::new();
+        let exit = Arc::new(AtomicBool::new(false));
+        let mut hashes = vec![];
+        for i in 0..MAX_SNAPSHOT_HASHES + 1 {
+            let snapshot_links = TempDir::new().unwrap();
+            let snapshot_package = SnapshotPackage {
+                hash: hash(&[i as u8]),
+                root: 100 + i as u64,
+                slot_deltas: vec![],
+                snapshot_links,
+                tar_output_file: PathBuf::from("."),
+                storages: vec![],
+            };
+
+            AccountsHashVerifier::process_snapshot(
+                snapshot_package,
+                &cluster_info,
+                &Some(trusted_validators.clone()),
+                false,
+                &None,
+                &mut hashes,
+                &exit,
+                0,
+            );
+        }
+        let cluster_info_r = cluster_info.read().unwrap();
+        let cluster_hashes = cluster_info_r
+            .get_accounts_hash_for_node(&keypair.pubkey())
+            .unwrap();
+        info!("{:?}", cluster_hashes);
+        assert_eq!(hashes.len(), MAX_SNAPSHOT_HASHES);
+        assert_eq!(cluster_hashes.len(), MAX_SNAPSHOT_HASHES);
+        assert_eq!(cluster_hashes[0], (101, hash(&[1])));
+        assert_eq!(
+            cluster_hashes[MAX_SNAPSHOT_HASHES - 1],
+            (
+                100 + MAX_SNAPSHOT_HASHES as u64,
+                hash(&[MAX_SNAPSHOT_HASHES as u8])
+            )
+        );
+    }
+}

core/src/banking_stage.rs

@@ -1017,7 +1017,6 @@ mod tests {
     };
     use crossbeam_channel::unbounded;
     use itertools::Itertools;
-    use solana_client::rpc_response::{RpcEncodedTransaction, RpcTransactionWithStatusMeta};
     use solana_ledger::{
         blockstore::entries_to_test_shreds,
         entry::{next_entry, Entry, EntrySlice},
@@ -1030,6 +1029,7 @@ mod tests {
         system_transaction,
         transaction::TransactionError,
     };
+    use solana_transaction_status::{EncodedTransaction, TransactionWithStatusMeta};
     use std::{sync::atomic::Ordering, thread::sleep};
 
     #[test]
@@ -1975,18 +1975,28 @@ mod tests {
             let confirmed_block = blockstore.get_confirmed_block(bank.slot(), None).unwrap();
             assert_eq!(confirmed_block.transactions.len(), 3);
 
-            for RpcTransactionWithStatusMeta { transaction, meta } in
+            for TransactionWithStatusMeta { transaction, meta } in
                 confirmed_block.transactions.into_iter()
             {
-                if let RpcEncodedTransaction::Json(transaction) = transaction {
+                if let EncodedTransaction::Json(transaction) = transaction {
                     if transaction.signatures[0] == success_signature.to_string() {
-                        assert_eq!(meta.unwrap().status, Ok(()));
+                        let meta = meta.unwrap();
+                        assert_eq!(meta.err, None);
+                        assert_eq!(meta.status, Ok(()));
                     } else if transaction.signatures[0] == ix_error_signature.to_string() {
+                        let meta = meta.unwrap();
                         assert_eq!(
-                            meta.unwrap().status,
+                            meta.err,
+                            Some(TransactionError::InstructionError(
+                                0,
+                                InstructionError::Custom(1)
+                            ))
+                        );
+                        assert_eq!(
+                            meta.status,
                             Err(TransactionError::InstructionError(
                                 0,
-                                InstructionError::CustomError(1)
+                                InstructionError::Custom(1)
                             ))
                         );
                     } else {

core/src/cluster_info.rs

@@ -12,8 +12,6 @@
 //! * layer 2 - Everyone else, if layer 1 is `2^10`, layer 2 should be able to fit `2^20` number of nodes.
 //!
 //! Bank needs to provide an interface for us to query the stake weight
-use crate::crds_value::CompressionType::*;
-use crate::crds_value::EpochIncompleteSlots;
 use crate::packet::limited_deserialize;
 use crate::streamer::{PacketReceiver, PacketSender};
 use crate::{
@@ -21,20 +19,28 @@ use crate::{
     crds_gossip::CrdsGossip,
     crds_gossip_error::CrdsGossipError,
     crds_gossip_pull::{CrdsFilter, CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS},
-    crds_value::{self, CrdsData, CrdsValue, CrdsValueLabel, EpochSlots, SnapshotHash, Vote},
+    crds_value::{
+        self, CrdsData, CrdsValue, CrdsValueLabel, EpochSlots, SnapshotHash, Vote, MAX_WALLCLOCK,
+    },
     packet::{Packet, PACKET_DATA_SIZE},
     result::{Error, Result},
     sendmmsg::{multicast, send_mmsg},
     weighted_shuffle::{weighted_best, weighted_shuffle},
 };
+
+use rand::distributions::{Distribution, WeightedIndex};
+use rand::SeedableRng;
+use rand_chacha::ChaChaRng;
+use solana_sdk::sanitize::{Sanitize, SanitizeError};
+
 use bincode::{serialize, serialized_size};
-use compression::prelude::*;
 use core::cmp;
 use itertools::Itertools;
 use rayon::iter::IntoParallelIterator;
 use rayon::iter::ParallelIterator;
 use rayon::ThreadPool;
 use solana_ledger::{bank_forks::BankForks, staking_utils};
+use solana_measure::measure::Measure;
 use solana_measure::thread_mem_usage;
 use solana_metrics::{datapoint_debug, inc_new_counter_debug, inc_new_counter_error};
 use solana_net_utils::{
@@ -48,7 +54,7 @@ use solana_sdk::timing::duration_as_s;
 use solana_sdk::{
     clock::{Slot, DEFAULT_MS_PER_SLOT},
     pubkey::Pubkey,
-    signature::{Keypair, Signable, Signature},
+    signature::{Keypair, Signable, Signature, Signer},
     timing::{duration_as_ms, timestamp},
     transaction::Transaction,
 };
@@ -81,9 +87,6 @@ const MAX_PROTOCOL_HEADER_SIZE: u64 = 214;
 /// 128MB/PACKET_DATA_SIZE
 const MAX_GOSSIP_TRAFFIC: usize = 128_000_000 / PACKET_DATA_SIZE;
 
-const NUM_BITS_PER_BYTE: u64 = 8;
-const MIN_SIZE_TO_COMPRESS_GZIP: u64 = 64;
-
 /// Keep the number of snapshot hashes a node publishes under MAX_PROTOCOL_PAYLOAD_SIZE
 pub const MAX_SNAPSHOT_HASHES: usize = 16;
 
@@ -95,6 +98,12 @@ pub enum ClusterInfoError {
     BadGossipAddress,
 }
 #[derive(Clone)]
+pub struct DataBudget {
+    bytes: usize, // amount of bytes we have in the budget to send
+    last_timestamp_ms: u64, // Last time that we upped the bytes count,
+                  // used to detect when to up the bytes budget again
+}
+#[derive(Clone)]
 pub struct ClusterInfo {
     /// The network
     pub gossip: CrdsGossip,
@@ -103,6 +112,8 @@ pub struct ClusterInfo {
     /// The network entrypoint
     entrypoint: Option<ContactInfo>,
     last_datapoint_submit: Instant,
+
+    outbound_budget: DataBudget,
 }
 
 #[derive(Default, Clone)]
@@ -143,6 +154,15 @@ pub struct PruneData {
     pub wallclock: u64,
 }
 
+impl Sanitize for PruneData {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        Ok(())
+    }
+}
+
 impl Signable for PruneData {
     fn pubkey(&self) -> Pubkey {
         self.pubkey
@@ -180,6 +200,22 @@ struct PullData {
     pub filter: CrdsFilter,
 }
 
+pub fn make_accounts_hashes_message(
+    keypair: &Keypair,
+    accounts_hashes: Vec<(Slot, Hash)>,
+) -> Option<CrdsValue> {
+    let message = CrdsData::AccountsHashes(SnapshotHash::new(keypair.pubkey(), accounts_hashes));
+    Some(CrdsValue::new_signed(message, keypair))
+}
+
+fn distance(a: u64, b: u64) -> u64 {
+    if a > b {
+        a - b
+    } else {
+        b - a
+    }
+}
+
 // TODO These messages should go through the gpu pipeline for spam filtering
 #[derive(Serialize, Deserialize, Debug)]
 #[allow(clippy::large_enum_variant)]
@@ -191,6 +227,31 @@ enum Protocol {
     PruneMessage(Pubkey, PruneData),
 }
 
+impl Sanitize for Protocol {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        match self {
+            Protocol::PullRequest(filter, val) => {
+                filter.sanitize()?;
+                val.sanitize()
+            }
+            Protocol::PullResponse(_, val) => val.sanitize(),
+            Protocol::PushMessage(_, val) => val.sanitize(),
+            Protocol::PruneMessage(_, val) => val.sanitize(),
+        }
+    }
+}
+
+// Rating for pull requests
+// A response table is generated as a
+// 2-d table arranged by target nodes and a
+// list of responses for that node,
+// to/responses_index is a location in that table.
+struct ResponseScore {
+    to: usize,              // to, index of who the response is to
+    responses_index: usize, // index into the list of responses for a given to
+    score: u64,             // Relative score of the response
+}
+
 impl ClusterInfo {
     /// Without a valid keypair gossip will not function. Only useful for tests.
     pub fn new_with_invalid_keypair(contact_info: ContactInfo) -> Self {
@@ -203,9 +264,13 @@ impl ClusterInfo {
             keypair,
             entrypoint: None,
             last_datapoint_submit: Instant::now(),
+            outbound_budget: DataBudget {
+                bytes: 0,
+                last_timestamp_ms: 0,
+            },
         };
-        let id = contact_info.id;
-        me.gossip.set_self(&id);
+        me.gossip.set_self(&contact_info.id);
+        me.gossip.set_shred_version(contact_info.shred_version);
         me.insert_self(contact_info);
         me.push_self(&HashMap::new());
         me
@@ -258,51 +323,59 @@ impl ClusterInfo {
         let now = timestamp();
         let mut spy_nodes = 0;
         let mut archivers = 0;
+        let mut different_shred_nodes = 0;
         let my_pubkey = self.my_data().id;
+        let my_shred_version = self.my_data().shred_version;
         let nodes: Vec<_> = self
             .all_peers()
             .into_iter()
-            .map(|(node, last_updated)| {
+            .filter_map(|(node, last_updated)| {
                 if Self::is_spy_node(&node) {
                     spy_nodes += 1;
                 } else if Self::is_archiver(&node) {
                     archivers += 1;
                 }
-                fn addr_to_string(default_ip: &IpAddr, addr: &SocketAddr) -> String {
-                    if ContactInfo::is_valid_address(addr) {
-                        if &addr.ip() == default_ip {
-                            addr.port().to_string()
-                        } else {
-                            addr.to_string()
-                        }
-                    } else {
-                        "none".to_string()
-                    }
-                }
 
-                let ip_addr = node.gossip.ip();
-                format!(
-                    "{:15} {:2}| {:5} | {:44} | {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {}\n",
-                    if ContactInfo::is_valid_address(&node.gossip) {
-                        ip_addr.to_string()
-                    } else {
-                        "none".to_string()
-                    },
-                    if node.id == my_pubkey { "me" } else { "" }.to_string(),
-                    now.saturating_sub(last_updated),
-                    node.id.to_string(),
-                    addr_to_string(&ip_addr, &node.gossip),
-                    addr_to_string(&ip_addr, &node.tpu),
-                    addr_to_string(&ip_addr, &node.tpu_forwards),
-                    addr_to_string(&ip_addr, &node.tvu),
-                    addr_to_string(&ip_addr, &node.tvu_forwards),
-                    addr_to_string(&ip_addr, &node.repair),
-                    addr_to_string(&ip_addr, &node.serve_repair),
-                    addr_to_string(&ip_addr, &node.storage_addr),
-                    addr_to_string(&ip_addr, &node.rpc),
-                    addr_to_string(&ip_addr, &node.rpc_pubsub),
-                    node.shred_version,
-                )
+                if my_shred_version != 0 && (node.shred_version != 0 && node.shred_version != my_shred_version) {
+                    different_shred_nodes += 1;
+                    None
+                } else {
+                    fn addr_to_string(default_ip: &IpAddr, addr: &SocketAddr) -> String {
+                        if ContactInfo::is_valid_address(addr) {
+                            if &addr.ip() == default_ip {
+                                addr.port().to_string()
+                            } else {
+                                addr.to_string()
+                            }
+                        } else {
+                            "none".to_string()
+                        }
+                    }
+
+                    let ip_addr = node.gossip.ip();
+                    Some(format!(
+                        "{:15} {:2}| {:5} | {:44} | {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {}\n",
+                        if ContactInfo::is_valid_address(&node.gossip) {
+                            ip_addr.to_string()
+                        } else {
+                            "none".to_string()
+                        },
+                        if node.id == my_pubkey { "me" } else { "" }.to_string(),
+                        now.saturating_sub(last_updated),
+                        node.id.to_string(),
+                        addr_to_string(&ip_addr, &node.gossip),
+                        addr_to_string(&ip_addr, &node.tpu),
+                        addr_to_string(&ip_addr, &node.tpu_forwards),
+                        addr_to_string(&ip_addr, &node.tvu),
+                        addr_to_string(&ip_addr, &node.tvu_forwards),
+                        addr_to_string(&ip_addr, &node.repair),
+                        addr_to_string(&ip_addr, &node.serve_repair),
+                        addr_to_string(&ip_addr, &node.storage_addr),
+                        addr_to_string(&ip_addr, &node.rpc),
+                        addr_to_string(&ip_addr, &node.rpc_pubsub),
+                        node.shred_version,
+                    ))
+                }
             })
             .collect();
 
@@ -312,7 +385,7 @@ impl ClusterInfo {
              ------------------+-------+----------------------------------------------+\
              ------+------+------+------+------+------+------+------+------+------+--------\n\
              {}\
-             Nodes: {}{}{}",
+             Nodes: {}{}{}{}",
             nodes.join(""),
             nodes.len() - spy_nodes - archivers,
             if archivers > 0 {
@@ -324,141 +397,65 @@ impl ClusterInfo {
                 format!("\nSpies: {}", spy_nodes)
             } else {
                 "".to_string()
+            },
+            if spy_nodes > 0 {
+                format!(
+                    "\nNodes with different shred version: {}",
+                    different_shred_nodes
+                )
+            } else {
+                "".to_string()
             }
         )
     }
 
-    pub fn compress_incomplete_slots(incomplete_slots: &BTreeSet<Slot>) -> EpochIncompleteSlots {
-        if !incomplete_slots.is_empty() {
-            let first_slot = incomplete_slots
-                .iter()
-                .next()
-                .expect("expected to find at least one slot");
-            let last_slot = incomplete_slots
-                .iter()
-                .next_back()
-                .expect("expected to find last slot");
-            let num_uncompressed_bits = last_slot.saturating_sub(*first_slot) + 1;
-            let num_uncompressed_bytes = if num_uncompressed_bits % NUM_BITS_PER_BYTE > 0 {
-                1
-            } else {
-                0
-            } + num_uncompressed_bits / NUM_BITS_PER_BYTE;
-            let mut uncompressed = vec![0u8; num_uncompressed_bytes as usize];
-            incomplete_slots.iter().for_each(|slot| {
-                let offset_from_first_slot = slot.saturating_sub(*first_slot);
-                let index = offset_from_first_slot / NUM_BITS_PER_BYTE;
-                let bit_index = offset_from_first_slot % NUM_BITS_PER_BYTE;
-                uncompressed[index as usize] |= 1 << bit_index;
-            });
-            if num_uncompressed_bytes >= MIN_SIZE_TO_COMPRESS_GZIP {
-                if let Ok(compressed) = uncompressed
-                    .iter()
-                    .cloned()
-                    .encode(&mut GZipEncoder::new(), Action::Finish)
-                    .collect::<std::result::Result<Vec<u8>, _>>()
-                {
-                    return EpochIncompleteSlots {
-                        first: *first_slot,
-                        compression: GZip,
-                        compressed_list: compressed,
-                    };
-                }
-            } else {
-                return EpochIncompleteSlots {
-                    first: *first_slot,
-                    compression: Uncompressed,
-                    compressed_list: uncompressed,
-                };
-            }
-        }
-        EpochIncompleteSlots::default()
-    }
-
-    fn bitmap_to_slot_list(first: Slot, bitmap: &[u8]) -> BTreeSet<Slot> {
-        let mut old_incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
-        bitmap.iter().enumerate().for_each(|(i, val)| {
-            if *val != 0 {
-                (0..8).for_each(|bit_index| {
-                    if (1 << bit_index & *val) != 0 {
-                        let slot = first + i as u64 * NUM_BITS_PER_BYTE + bit_index as u64;
-                        old_incomplete_slots.insert(slot);
-                    }
-                })
-            }
-        });
-        old_incomplete_slots
-    }
-
-    pub fn decompress_incomplete_slots(slots: &EpochIncompleteSlots) -> BTreeSet<Slot> {
-        match slots.compression {
-            Uncompressed => Self::bitmap_to_slot_list(slots.first, &slots.compressed_list),
-            GZip => {
-                if let Ok(decompressed) = slots
-                    .compressed_list
-                    .iter()
-                    .cloned()
-                    .decode(&mut GZipDecoder::new())
-                    .collect::<std::result::Result<Vec<u8>, _>>()
-                {
-                    Self::bitmap_to_slot_list(slots.first, &decompressed)
-                } else {
-                    BTreeSet::new()
-                }
-            }
-            BZip2 => {
-                if let Ok(decompressed) = slots
-                    .compressed_list
-                    .iter()
-                    .cloned()
-                    .decode(&mut BZip2Decoder::new())
-                    .collect::<std::result::Result<Vec<u8>, _>>()
-                {
-                    Self::bitmap_to_slot_list(slots.first, &decompressed)
-                } else {
-                    BTreeSet::new()
-                }
-            }
-        }
-    }
-
     pub fn push_epoch_slots(
         &mut self,
         id: Pubkey,
-        root: Slot,
+        _root: Slot,
         min: Slot,
-        slots: BTreeSet<Slot>,
-        incomplete_slots: &BTreeSet<Slot>,
+        _slots: BTreeSet<Slot>,
+        _incomplete_slots: &BTreeSet<Slot>,
     ) {
-        let compressed = Self::compress_incomplete_slots(incomplete_slots);
         let now = timestamp();
         let entry = CrdsValue::new_signed(
-            CrdsData::EpochSlots(
-                0,
-                EpochSlots::new(id, root, min, slots, vec![compressed], now),
-            ),
+            CrdsData::EpochSlots(0, EpochSlots::new(id, min, now)),
             &self.keypair,
         );
         self.gossip
             .process_push_message(&self.id(), vec![entry], now);
     }
 
-    pub fn push_snapshot_hashes(&mut self, snapshot_hashes: Vec<(Slot, Hash)>) {
-        if snapshot_hashes.len() > MAX_SNAPSHOT_HASHES {
+    pub fn push_message(&mut self, message: CrdsValue) {
+        let now = message.wallclock();
+        let id = message.pubkey();
+        self.gossip.process_push_message(&id, vec![message], now);
+    }
+
+    pub fn push_accounts_hashes(&mut self, accounts_hashes: Vec<(Slot, Hash)>) {
+        if accounts_hashes.len() > MAX_SNAPSHOT_HASHES {
             warn!(
-                "snapshot_hashes too large, ignored: {}",
-                snapshot_hashes.len()
+                "accounts hashes too large, ignored: {}",
+                accounts_hashes.len(),
             );
             return;
         }
 
-        let now = timestamp();
-        let entry = CrdsValue::new_signed(
-            CrdsData::SnapshotHash(SnapshotHash::new(self.id(), snapshot_hashes, now)),
-            &self.keypair,
-        );
-        self.gossip
-            .process_push_message(&self.id(), vec![entry], now);
+        let message = CrdsData::AccountsHashes(SnapshotHash::new(self.id(), accounts_hashes));
+        self.push_message(CrdsValue::new_signed(message, &self.keypair));
+    }
+
+    pub fn push_snapshot_hashes(&mut self, snapshot_hashes: Vec<(Slot, Hash)>) {
+        if snapshot_hashes.len() > MAX_SNAPSHOT_HASHES {
+            warn!(
+                "snapshot hashes too large, ignored: {}",
+                snapshot_hashes.len(),
+            );
+            return;
+        }
+
+        let message = CrdsData::SnapshotHashes(SnapshotHash::new(self.id(), snapshot_hashes));
+        self.push_message(CrdsValue::new_signed(message, &self.keypair));
     }
 
     pub fn push_vote(&mut self, tower_index: usize, vote: Transaction) {
@@ -518,11 +515,19 @@ impl ClusterInfo {
             .collect()
     }
 
+    pub fn get_accounts_hash_for_node(&self, pubkey: &Pubkey) -> Option<&Vec<(Slot, Hash)>> {
+        self.gossip
+            .crds
+            .table
+            .get(&CrdsValueLabel::AccountsHashes(*pubkey))
+            .map(|x| &x.value.accounts_hash().unwrap().hashes)
+    }
+
     pub fn get_snapshot_hash_for_node(&self, pubkey: &Pubkey) -> Option<&Vec<(Slot, Hash)>> {
         self.gossip
             .crds
             .table
-            .get(&CrdsValueLabel::SnapshotHash(*pubkey))
+            .get(&CrdsValueLabel::SnapshotHashes(*pubkey))
             .map(|x| &x.value.snapshot_hash().unwrap().hashes)
     }
 
@@ -973,7 +978,7 @@ impl ClusterInfo {
         let mut num_live_peers = 1i64;
         peers.iter().for_each(|p| {
             // A peer is considered live if they generated their contact info recently
-            if timestamp() - p.wallclock <= CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS {
+            if distance(timestamp(), p.wallclock) <= CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS {
                 num_live_peers += 1;
             }
         });
@@ -1252,6 +1257,10 @@ impl ClusterInfo {
                                         entrypoint.shred_version, entrypoint.id
                                     );
                                     self_info.shred_version = entrypoint.shred_version;
+                                    obj.write()
+                                        .unwrap()
+                                        .gossip
+                                        .set_shred_version(entrypoint.shred_version);
                                     obj.write().unwrap().insert_self(self_info);
                                     adopt_shred_version = false;
                                 }
@@ -1291,6 +1300,7 @@ impl ClusterInfo {
             let from_addr = packet.meta.addr();
             limited_deserialize(&packet.data[..packet.meta.size])
                 .into_iter()
+                .filter(|r: &Protocol| r.sanitize().is_ok())
                 .for_each(|request| match request {
                     Protocol::PullRequest(filter, caller) => {
                         let start = allocated.get();
@@ -1389,20 +1399,43 @@ impl ClusterInfo {
                 })
         });
         // process the collected pulls together
-        let rsp = Self::handle_pull_requests(me, recycler, gossip_pull_data);
+        let rsp = Self::handle_pull_requests(me, recycler, gossip_pull_data, stakes);
         if let Some(rsp) = rsp {
             let _ignore_disconnect = response_sender.send(rsp);
         }
     }
 
+    // Pull requests take an incoming bloom filter of contained entries from a node
+    // and tries to send back to them the values it detects are missing.
     fn handle_pull_requests(
         me: &Arc<RwLock<Self>>,
         recycler: &PacketsRecycler,
         requests: Vec<PullData>,
+        stakes: &HashMap<Pubkey, u64>,
     ) -> Option<Packets> {
         // split the requests into addrs and filters
         let mut caller_and_filters = vec![];
         let mut addrs = vec![];
+        let mut time = Measure::start("handle_pull_requests");
+        {
+            let mut cluster_info = me.write().unwrap();
+
+            let now = timestamp();
+            const INTERVAL_MS: u64 = 100;
+            // allow 50kBps per staked validator, epoch slots + votes ~= 1.5kB/slot ~= 4kB/s
+            const BYTES_PER_INTERVAL: usize = 5000;
+            const MAX_BUDGET_MULTIPLE: usize = 5; // allow budget build-up to 5x the interval default
+
+            if now - cluster_info.outbound_budget.last_timestamp_ms > INTERVAL_MS {
+                let len = std::cmp::max(stakes.len(), 2);
+                cluster_info.outbound_budget.bytes += len * BYTES_PER_INTERVAL;
+                cluster_info.outbound_budget.bytes = std::cmp::min(
+                    cluster_info.outbound_budget.bytes,
+                    MAX_BUDGET_MULTIPLE * len * BYTES_PER_INTERVAL,
+                );
+                cluster_info.outbound_budget.last_timestamp_ms = now;
+            }
+        }
         for pull_data in requests {
             caller_and_filters.push((pull_data.caller, pull_data.filter));
             addrs.push(pull_data.from_addr);
@@ -1414,30 +1447,101 @@ impl ClusterInfo {
             .unwrap()
             .gossip
             .process_pull_requests(caller_and_filters, now);
-        let mut packets = Packets::new_with_recycler(recycler.clone(), 64, "handle_pull_requests");
-        pull_responses
+
+        // Filter bad to addresses
+        let pull_responses: Vec<_> = pull_responses
             .into_iter()
             .zip(addrs.into_iter())
-            .for_each(|(response, from_addr)| {
-                if !from_addr.ip().is_unspecified() && from_addr.port() != 0 {
-                    let len = response.len();
-                    trace!("get updates since response {}", len);
-                    inc_new_counter_debug!("cluster_info-pull_request-rsp", len);
-                    Self::split_gossip_messages(response)
-                        .into_iter()
-                        .for_each(|payload| {
-                            let protocol = Protocol::PullResponse(self_id, payload);
-                            // The remote node may not know its public IP:PORT. Instead of responding to the caller's
-                            // gossip addr, respond to the origin addr. The last origin addr is picked from the list of
-                            // addrs.
-                            packets
-                                .packets
-                                .push(Packet::from_data(&from_addr, protocol))
-                        })
+            .filter_map(|(responses, from_addr)| {
+                if !from_addr.ip().is_unspecified()
+                    && from_addr.port() != 0
+                    && !responses.is_empty()
+                {
+                    Some((responses, from_addr))
                 } else {
-                    trace!("Dropping Gossip pull response, as destination is unknown");
+                    None
                 }
-            });
+            })
+            .collect();
+
+        if pull_responses.is_empty() {
+            return None;
+        }
+
+        let mut stats: Vec<_> = pull_responses
+            .iter()
+            .enumerate()
+            .map(|(i, (responses, _from_addr))| {
+                let score: u64 = if stakes.get(&responses[0].pubkey()).is_some() {
+                    2
+                } else {
+                    1
+                };
+                responses
+                    .iter()
+                    .enumerate()
+                    .map(|(j, _response)| ResponseScore {
+                        to: i,
+                        responses_index: j,
+                        score,
+                    })
+                    .collect::<Vec<ResponseScore>>()
+            })
+            .flatten()
+            .collect();
+
+        stats.sort_by(|a, b| a.score.cmp(&b.score));
+        let weights: Vec<_> = stats.iter().map(|stat| stat.score).collect();
+
+        let seed = [48u8; 32];
+        let rng = &mut ChaChaRng::from_seed(seed);
+        let weighted_index = WeightedIndex::new(weights).unwrap();
+
+        let mut packets = Packets::new_with_recycler(recycler.clone(), 64, "handle_pull_requests");
+        let mut total_bytes = 0;
+        let outbound_budget = me.read().unwrap().outbound_budget.bytes;
+        let mut sent = HashSet::new();
+        while sent.len() < stats.len() {
+            let index = weighted_index.sample(rng);
+            if sent.contains(&index) {
+                continue;
+            }
+            sent.insert(index);
+            let stat = &stats[index];
+            let from_addr = pull_responses[stat.to].1;
+            let response = pull_responses[stat.to].0[stat.responses_index].clone();
+            let protocol = Protocol::PullResponse(self_id, vec![response]);
+            packets
+                .packets
+                .push(Packet::from_data(&from_addr, protocol));
+            let len = packets.packets.len();
+            total_bytes += packets.packets[len - 1].meta.size;
+
+            if total_bytes > outbound_budget {
+                inc_new_counter_info!("gossip_pull_request-no_budget", 1);
+                break;
+            }
+        }
+        {
+            let mut cluster_info = me.write().unwrap();
+            cluster_info.outbound_budget.bytes = cluster_info
+                .outbound_budget
+                .bytes
+                .saturating_sub(total_bytes);
+        }
+        time.stop();
+        inc_new_counter_info!("gossip_pull_request-sent_requests", sent.len());
+        inc_new_counter_info!(
+            "gossip_pull_request-dropped_requests",
+            stats.len() - sent.len()
+        );
+        debug!(
+            "handle_pull_requests: {} sent: {} total: {} total_bytes: {}",
+            time,
+            sent.len(),
+            stats.len(),
+            total_bytes
+        );
         if packets.is_empty() {
             return None;
         }
@@ -1547,7 +1651,10 @@ impl ClusterInfo {
             requests.push(more_reqs)
         }
         if num_requests >= MAX_GOSSIP_TRAFFIC {
-            warn!("Too much gossip traffic, ignoring some messages");
+            warn!(
+                "Too much gossip traffic, ignoring some messages (requests={}, max requests={})",
+                num_requests, MAX_GOSSIP_TRAFFIC
+            );
         }
         let epoch_ms;
         let stakes: HashMap<_, _> = match bank_forks {
@@ -1615,7 +1722,7 @@ impl ClusterInfo {
             .unwrap()
     }
 
-    fn gossip_contact_info(id: &Pubkey, gossip: SocketAddr) -> ContactInfo {
+    pub fn gossip_contact_info(id: &Pubkey, gossip: SocketAddr) -> ContactInfo {
         ContactInfo {
             id: *id,
             gossip,
@@ -1920,7 +2027,7 @@ impl Node {
 
 fn report_time_spent(label: &str, time: &Duration, extra: &str) {
     let count = duration_as_ms(time);
-    if count > 5 {
+    if count > 100 {
         info!("{} took: {} ms {}", label, count, extra);
     }
 }
@@ -2339,14 +2446,7 @@ mod tests {
         }
         let value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
             0,
-            EpochSlots {
-                from: Pubkey::default(),
-                root: 0,
-                lowest: 0,
-                slots: btree_slots,
-                stash: vec![],
-                wallclock: 0,
-            },
+            EpochSlots::new(Pubkey::default(), 0, 0),
         ));
         test_split_messages(value);
     }
@@ -2358,39 +2458,19 @@ mod tests {
         let payload: Vec<CrdsValue> = vec![];
         let vec_size = serialized_size(&payload).unwrap();
         let desired_size = MAX_PROTOCOL_PAYLOAD_SIZE - vec_size;
-        let mut value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
-            0,
-            EpochSlots {
-                from: Pubkey::default(),
-                root: 0,
-                lowest: 0,
-                slots: BTreeSet::new(),
-                stash: vec![],
-                wallclock: 0,
-            },
-        ));
+        let mut value = CrdsValue::new_unsigned(CrdsData::SnapshotHashes(SnapshotHash {
+            from: Pubkey::default(),
+            hashes: vec![],
+            wallclock: 0,
+        }));
 
         let mut i = 0;
         while value.size() <= desired_size {
-            let slots = (0..i).collect::<BTreeSet<_>>();
-            if slots.len() > 200 {
-                panic!(
-                    "impossible to match size: last {:?} vs desired {:?}",
-                    serialized_size(&value).unwrap(),
-                    desired_size
-                );
-            }
-            value.data = CrdsData::EpochSlots(
-                0,
-                EpochSlots {
-                    from: Pubkey::default(),
-                    root: 0,
-                    lowest: 0,
-                    slots,
-                    stash: vec![],
-                    wallclock: 0,
-                },
-            );
+            value.data = CrdsData::SnapshotHashes(SnapshotHash {
+                from: Pubkey::default(),
+                hashes: vec![(0, Hash::default()); i],
+                wallclock: 0,
+            });
             i += 1;
         }
         let split = ClusterInfo::split_gossip_messages(vec![value.clone()]);
@@ -2520,11 +2600,9 @@ mod tests {
             node_keypair,
         );
         for i in 0..10 {
-            let mut peer_root = 5;
             let mut peer_lowest = 0;
             if i >= 5 {
                 // make these invalid for the upcoming repair request
-                peer_root = 15;
                 peer_lowest = 10;
             }
             let other_node_pubkey = Pubkey::new_rand();
@@ -2532,14 +2610,7 @@ mod tests {
             cluster_info.insert_info(other_node.clone());
             let value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
                 0,
-                EpochSlots::new(
-                    other_node_pubkey,
-                    peer_root,
-                    peer_lowest,
-                    BTreeSet::new(),
-                    vec![],
-                    timestamp(),
-                ),
+                EpochSlots::new(other_node_pubkey, peer_lowest, timestamp()),
             ));
             let _ = cluster_info.gossip.crds.insert(value, timestamp());
         }
@@ -2588,6 +2659,14 @@ mod tests {
         assert_eq!(MAX_PROTOCOL_HEADER_SIZE, max_protocol_size);
     }
 
+    #[test]
+    fn test_protocol_sanitize() {
+        let mut pd = PruneData::default();
+        pd.wallclock = MAX_WALLCLOCK;
+        let msg = Protocol::PruneMessage(Pubkey::default(), pd);
+        assert_eq!(msg.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+    }
+
     // computes the maximum size for pull request blooms
     fn max_bloom_size() -> usize {
         let filter_size = serialized_size(&CrdsFilter::default())
@@ -2600,38 +2679,4 @@ mod tests {
             serialized_size(&protocol).expect("unable to serialize gossip protocol") as usize;
         PACKET_DATA_SIZE - (protocol_size - filter_size)
     }
-
-    #[test]
-    fn test_compress_incomplete_slots() {
-        let mut incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
-
-        assert_eq!(
-            EpochIncompleteSlots::default(),
-            ClusterInfo::compress_incomplete_slots(&incomplete_slots)
-        );
-
-        incomplete_slots.insert(100);
-        let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
-        assert_eq!(100, compressed.first);
-        let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
-        assert_eq!(incomplete_slots, decompressed);
-
-        incomplete_slots.insert(104);
-        let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
-        assert_eq!(100, compressed.first);
-        let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
-        assert_eq!(incomplete_slots, decompressed);
-
-        incomplete_slots.insert(80);
-        let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
-        assert_eq!(80, compressed.first);
-        let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
-        assert_eq!(incomplete_slots, decompressed);
-
-        incomplete_slots.insert(10000);
-        let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
-        assert_eq!(80, compressed.first);
-        let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
-        assert_eq!(incomplete_slots, decompressed);
-    }
 }

core/src/cluster_info_vote_listener.rs

@@ -51,7 +51,8 @@ impl ClusterInfoVoteListener {
             if exit.load(Ordering::Relaxed) {
                 return Ok(());
             }
-            if let Some(bank) = poh_recorder.lock().unwrap().bank() {
+            let poh_bank = poh_recorder.lock().unwrap().bank();
+            if let Some(bank) = poh_bank {
                 let last_ts = bank.last_vote_sync.load(Ordering::Relaxed);
                 let (votes, new_ts) = cluster_info.read().unwrap().get_votes(last_ts);
                 bank.last_vote_sync

core/src/commitment.rs

@@ -1,3 +1,7 @@
+use crate::consensus::VOTE_THRESHOLD_SIZE;
+use solana_ledger::blockstore::Blockstore;
+use solana_measure::measure::Measure;
+use solana_metrics::inc_new_counter_info;
 use solana_runtime::bank::Bank;
 use solana_sdk::clock::Slot;
 use solana_vote_program::{vote_state::VoteState, vote_state::MAX_LOCKOUT_HISTORY};
@@ -10,9 +14,11 @@ use std::{
     time::Duration,
 };
 
+pub type BlockCommitmentArray = [u64; MAX_LOCKOUT_HISTORY + 1];
+
 #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
 pub struct BlockCommitment {
-    pub commitment: [u64; MAX_LOCKOUT_HISTORY],
+    pub commitment: BlockCommitmentArray,
 }
 
 impl BlockCommitment {
@@ -25,23 +31,71 @@ impl BlockCommitment {
         assert!(confirmation_count > 0 && confirmation_count <= MAX_LOCKOUT_HISTORY);
         self.commitment[confirmation_count - 1]
     }
+
+    pub fn increase_rooted_stake(&mut self, stake: u64) {
+        self.commitment[MAX_LOCKOUT_HISTORY] += stake;
+    }
+
+    pub fn get_rooted_stake(&self) -> u64 {
+        self.commitment[MAX_LOCKOUT_HISTORY]
+    }
+
     #[cfg(test)]
-    pub(crate) fn new(commitment: [u64; MAX_LOCKOUT_HISTORY]) -> Self {
+    pub(crate) fn new(commitment: BlockCommitmentArray) -> Self {
         Self { commitment }
     }
 }
 
-#[derive(Debug, Default)]
 pub struct BlockCommitmentCache {
     block_commitment: HashMap<Slot, BlockCommitment>,
+    largest_confirmed_root: Slot,
     total_stake: u64,
+    bank: Arc<Bank>,
+    blockstore: Arc<Blockstore>,
+    root: Slot,
+}
+
+impl std::fmt::Debug for BlockCommitmentCache {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("BlockCommitmentCache")
+            .field("block_commitment", &self.block_commitment)
+            .field("total_stake", &self.total_stake)
+            .field(
+                "bank",
+                &format_args!("Bank({{current_slot: {:?}}})", self.bank.slot()),
+            )
+            .field("root", &self.root)
+            .finish()
+    }
 }
 
 impl BlockCommitmentCache {
-    pub fn new(block_commitment: HashMap<Slot, BlockCommitment>, total_stake: u64) -> Self {
+    pub fn new(
+        block_commitment: HashMap<Slot, BlockCommitment>,
+        largest_confirmed_root: Slot,
+        total_stake: u64,
+        bank: Arc<Bank>,
+        blockstore: Arc<Blockstore>,
+        root: Slot,
+    ) -> Self {
         Self {
             block_commitment,
+            largest_confirmed_root,
             total_stake,
+            bank,
+            blockstore,
+            root,
+        }
+    }
+
+    pub fn default_with_blockstore(blockstore: Arc<Blockstore>) -> Self {
+        Self {
+            block_commitment: HashMap::default(),
+            largest_confirmed_root: Slot::default(),
+            total_stake: u64::default(),
+            bank: Arc::new(Bank::default()),
+            blockstore,
+            root: Slot::default(),
         }
     }
 
@@ -49,45 +103,99 @@ impl BlockCommitmentCache {
         self.block_commitment.get(&slot)
     }
 
+    pub fn largest_confirmed_root(&self) -> Slot {
+        self.largest_confirmed_root
+    }
+
     pub fn total_stake(&self) -> u64 {
         self.total_stake
     }
 
-    pub fn get_block_with_depth_commitment(
-        &self,
-        minimum_depth: usize,
-        minimum_stake_percentage: f64,
-    ) -> Option<Slot> {
-        self.block_commitment
-            .iter()
-            .filter(|&(_, block_commitment)| {
-                let fork_stake_minimum_depth: u64 = block_commitment.commitment[minimum_depth..]
-                    .iter()
-                    .cloned()
-                    .sum();
-                fork_stake_minimum_depth as f64 / self.total_stake as f64
-                    >= minimum_stake_percentage
-            })
-            .map(|(slot, _)| *slot)
-            .max()
+    pub fn bank(&self) -> Arc<Bank> {
+        self.bank.clone()
     }
 
-    pub fn get_rooted_block_with_commitment(&self, minimum_stake_percentage: f64) -> Option<u64> {
-        self.get_block_with_depth_commitment(MAX_LOCKOUT_HISTORY - 1, minimum_stake_percentage)
+    pub fn slot(&self) -> Slot {
+        self.bank.slot()
+    }
+
+    pub fn root(&self) -> Slot {
+        self.root
+    }
+
+    pub fn get_confirmation_count(&self, slot: Slot) -> Option<usize> {
+        self.get_lockout_count(slot, VOTE_THRESHOLD_SIZE)
+    }
+
+    // Returns the lowest level at which at least `minimum_stake_percentage` of the total epoch
+    // stake is locked out
+    fn get_lockout_count(&self, slot: Slot, minimum_stake_percentage: f64) -> Option<usize> {
+        self.get_block_commitment(slot).map(|block_commitment| {
+            let iterator = block_commitment.commitment.iter().enumerate().rev();
+            let mut sum = 0;
+            for (i, stake) in iterator {
+                sum += stake;
+                if (sum as f64 / self.total_stake as f64) > minimum_stake_percentage {
+                    return i + 1;
+                }
+            }
+            0
+        })
+    }
+
+    pub fn is_confirmed_rooted(&self, slot: Slot) -> bool {
+        slot <= self.largest_confirmed_root()
+            && (self.blockstore.is_root(slot) || self.bank.status_cache_ancestors().contains(&slot))
+    }
+
+    #[cfg(test)]
+    pub fn new_for_tests_with_blockstore(blockstore: Arc<Blockstore>) -> Self {
+        let mut block_commitment: HashMap<Slot, BlockCommitment> = HashMap::new();
+        block_commitment.insert(0, BlockCommitment::default());
+        Self {
+            block_commitment,
+            blockstore,
+            total_stake: 42,
+            largest_confirmed_root: Slot::default(),
+            bank: Arc::new(Bank::default()),
+            root: Slot::default(),
+        }
+    }
+
+    #[cfg(test)]
+    pub(crate) fn set_get_largest_confirmed_root(&mut self, root: Slot) {
+        self.largest_confirmed_root = root;
     }
 }
 
 pub struct CommitmentAggregationData {
     bank: Arc<Bank>,
+    root: Slot,
     total_staked: u64,
 }
 
 impl CommitmentAggregationData {
-    pub fn new(bank: Arc<Bank>, total_staked: u64) -> Self {
-        Self { bank, total_staked }
+    pub fn new(bank: Arc<Bank>, root: Slot, total_staked: u64) -> Self {
+        Self {
+            bank,
+            root,
+            total_staked,
+        }
     }
 }
 
+fn get_largest_confirmed_root(mut rooted_stake: Vec<(Slot, u64)>, total_stake: u64) -> Slot {
+    rooted_stake.sort_by(|a, b| a.0.cmp(&b.0).reverse());
+    let mut stake_sum = 0;
+    for (root, stake) in rooted_stake {
+        stake_sum += stake;
+        if (stake_sum as f64 / total_stake as f64) > VOTE_THRESHOLD_SIZE {
+            return root;
+        }
+    }
+    0
+}
+
 pub struct AggregateCommitmentService {
     t_commitment: JoinHandle<()>,
 }
@@ -144,18 +252,37 @@ impl AggregateCommitmentService {
                 continue;
             }
 
-            let block_commitment = Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
+            let mut aggregate_commitment_time = Measure::start("aggregate-commitment-ms");
+            let (block_commitment, rooted_stake) =
+                Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
 
-            let mut new_block_commitment =
-                BlockCommitmentCache::new(block_commitment, aggregation_data.total_staked);
+            let largest_confirmed_root =
+                get_largest_confirmed_root(rooted_stake, aggregation_data.total_staked);
+
+            let mut new_block_commitment = BlockCommitmentCache::new(
+                block_commitment,
+                largest_confirmed_root,
+                aggregation_data.total_staked,
+                aggregation_data.bank,
+                block_commitment_cache.read().unwrap().blockstore.clone(),
+                aggregation_data.root,
+            );
 
             let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
 
             std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
+            aggregate_commitment_time.stop();
+            inc_new_counter_info!(
+                "aggregate-commitment-ms",
+                aggregate_commitment_time.as_ms() as usize
+            );
         }
     }
 
-    pub fn aggregate_commitment(ancestors: &[Slot], bank: &Bank) -> HashMap<Slot, BlockCommitment> {
+    pub fn aggregate_commitment(
+        ancestors: &[Slot],
+        bank: &Bank,
+    ) -> (HashMap<Slot, BlockCommitment>, Vec<(Slot, u64)>) {
         assert!(!ancestors.is_empty());
 
         // Check ancestors is sorted
@@ -164,6 +291,7 @@ impl AggregateCommitmentService {
         }
 
         let mut commitment = HashMap::new();
+        let mut rooted_stake: Vec<(Slot, u64)> = Vec::new();
         for (_, (lamports, account)) in bank.vote_accounts().into_iter() {
             if lamports == 0 {
                 continue;
@@ -176,17 +304,19 @@ impl AggregateCommitmentService {
             let vote_state = vote_state.unwrap();
             Self::aggregate_commitment_for_vote_account(
                 &mut commitment,
+                &mut rooted_stake,
                 &vote_state,
                 ancestors,
                 lamports,
             );
         }
 
-        commitment
+        (commitment, rooted_stake)
     }
 
     fn aggregate_commitment_for_vote_account(
         commitment: &mut HashMap<Slot, BlockCommitment>,
+        rooted_stake: &mut Vec<(Slot, u64)>,
         vote_state: &VoteState,
         ancestors: &[Slot],
         lamports: u64,
@@ -199,12 +329,13 @@ impl AggregateCommitmentService {
                     commitment
                         .entry(*a)
                         .or_insert_with(BlockCommitment::default)
-                        .increase_confirmation_stake(MAX_LOCKOUT_HISTORY, lamports);
+                        .increase_rooted_stake(lamports);
                 } else {
                     ancestors_index = i;
                     break;
                 }
             }
+            rooted_stake.push((root, lamports));
         }
 
         for vote in &vote_state.votes {
@@ -231,6 +362,7 @@ impl AggregateCommitmentService {
 mod tests {
     use super::*;
     use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use solana_ledger::get_tmp_ledger_path;
     use solana_sdk::pubkey::Pubkey;
     use solana_stake_program::stake_state;
     use solana_vote_program::vote_state::{self, VoteStateVersions};
@@ -246,97 +378,98 @@ mod tests {
     }
 
     #[test]
-    fn test_get_block_with_depth_commitment() {
+    fn test_get_confirmations() {
+        let bank = Arc::new(Bank::default());
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         // Build BlockCommitmentCache with votes at depths 0 and 1 for 2 slots
         let mut cache0 = BlockCommitment::default();
-        cache0.increase_confirmation_stake(1, 15);
-        cache0.increase_confirmation_stake(2, 25);
+        cache0.increase_confirmation_stake(1, 5);
+        cache0.increase_confirmation_stake(2, 40);
 
         let mut cache1 = BlockCommitment::default();
-        cache1.increase_confirmation_stake(1, 10);
-        cache1.increase_confirmation_stake(2, 20);
+        cache1.increase_confirmation_stake(1, 40);
+        cache1.increase_confirmation_stake(2, 5);
+
+        let mut cache2 = BlockCommitment::default();
+        cache2.increase_confirmation_stake(1, 20);
+        cache2.increase_confirmation_stake(2, 5);
 
         let mut block_commitment = HashMap::new();
         block_commitment.entry(0).or_insert(cache0.clone());
         block_commitment.entry(1).or_insert(cache1.clone());
-        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 50);
+        block_commitment.entry(2).or_insert(cache2.clone());
+        let block_commitment_cache =
+            BlockCommitmentCache::new(block_commitment, 0, 50, bank, blockstore, 0);
 
-        // Neither slot has rooted votes
-        assert_eq!(
-            block_commitment_cache.get_rooted_block_with_commitment(0.1),
-            None
-        );
-        // Neither slot meets the minimum level of commitment 0.6 at depth 1
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(1, 0.6),
-            None
-        );
-        // Only slot 0 meets the minimum level of commitment 0.5 at depth 1
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(1, 0.5),
-            Some(0)
-        );
-        // If multiple slots meet the minimum level of commitment, method should return the most recent
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(1, 0.4),
-            Some(1)
-        );
-        // If multiple slots meet the minimum level of commitment, method should return the most recent
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(0, 0.6),
-            Some(1)
-        );
-        // Neither slot meets the minimum level of commitment 0.9 at depth 0
-        assert_eq!(
-            block_commitment_cache.get_block_with_depth_commitment(0, 0.9),
-            None
-        );
+        assert_eq!(block_commitment_cache.get_confirmation_count(0), Some(2));
+        assert_eq!(block_commitment_cache.get_confirmation_count(1), Some(1));
+        assert_eq!(block_commitment_cache.get_confirmation_count(2), Some(0),);
+        assert_eq!(block_commitment_cache.get_confirmation_count(3), None,);
     }
 
     #[test]
-    fn test_get_rooted_block_with_commitment() {
-        // Build BlockCommitmentCache with rooted votes
-        let mut cache0 = BlockCommitment::new([0; MAX_LOCKOUT_HISTORY]);
-        cache0.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, 40);
-        cache0.increase_confirmation_stake(MAX_LOCKOUT_HISTORY - 1, 10);
-        let mut cache1 = BlockCommitment::new([0; MAX_LOCKOUT_HISTORY]);
-        cache1.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, 30);
-        cache1.increase_confirmation_stake(MAX_LOCKOUT_HISTORY - 1, 10);
-        cache1.increase_confirmation_stake(MAX_LOCKOUT_HISTORY - 2, 10);
+    fn test_is_confirmed_rooted() {
+        let bank = Arc::new(Bank::default());
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        blockstore.set_roots(&[0, 1]).unwrap();
+        // Build BlockCommitmentCache with rooted slots
+        let mut cache0 = BlockCommitment::default();
+        cache0.increase_rooted_stake(50);
+        let mut cache1 = BlockCommitment::default();
+        cache1.increase_rooted_stake(40);
+        let mut cache2 = BlockCommitment::default();
+        cache2.increase_rooted_stake(20);
 
         let mut block_commitment = HashMap::new();
-        block_commitment.entry(0).or_insert(cache0.clone());
-        block_commitment.entry(1).or_insert(cache1.clone());
-        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 50);
+        block_commitment.entry(1).or_insert(cache0.clone());
+        block_commitment.entry(2).or_insert(cache1.clone());
+        block_commitment.entry(3).or_insert(cache2.clone());
+        let largest_confirmed_root = 1;
+        let block_commitment_cache = BlockCommitmentCache::new(
+            block_commitment,
+            largest_confirmed_root,
+            50,
+            bank,
+            blockstore,
+            0,
+        );
 
-        // Only slot 0 meets the minimum level of commitment 0.66 at root
-        assert_eq!(
-            block_commitment_cache.get_rooted_block_with_commitment(0.66),
-            Some(0)
-        );
-        // If multiple slots meet the minimum level of commitment, method should return the most recent
-        assert_eq!(
-            block_commitment_cache.get_rooted_block_with_commitment(0.6),
-            Some(1)
-        );
-        // Neither slot meets the minimum level of commitment 0.9 at root
-        assert_eq!(
-            block_commitment_cache.get_rooted_block_with_commitment(0.9),
-            None
-        );
+        assert!(block_commitment_cache.is_confirmed_rooted(0));
+        assert!(block_commitment_cache.is_confirmed_rooted(1));
+        assert!(!block_commitment_cache.is_confirmed_rooted(2));
+        assert!(!block_commitment_cache.is_confirmed_rooted(3));
+    }
+
+    #[test]
+    fn test_get_largest_confirmed_root() {
+        assert_eq!(get_largest_confirmed_root(vec![], 10), 0);
+        let mut rooted_stake = vec![];
+        rooted_stake.push((0, 5));
+        rooted_stake.push((1, 5));
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 10), 0);
+        let mut rooted_stake = vec![];
+        rooted_stake.push((1, 5));
+        rooted_stake.push((0, 10));
+        rooted_stake.push((2, 5));
+        rooted_stake.push((1, 4));
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 10), 1);
     }
 
     #[test]
     fn test_aggregate_commitment_for_vote_account_1() {
         let ancestors = vec![3, 4, 5, 7, 9, 11];
         let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
         let lamports = 5;
         let mut vote_state = VoteState::default();
 
-        let root = ancestors.last().unwrap();
-        vote_state.root_slot = Some(*root);
+        let root = ancestors.last().unwrap().clone();
+        vote_state.root_slot = Some(root);
         AggregateCommitmentService::aggregate_commitment_for_vote_account(
             &mut commitment,
+            &mut rooted_stake,
             &vote_state,
             &ancestors,
             lamports,
@@ -344,15 +477,17 @@ mod tests {
 
         for a in ancestors {
             let mut expected = BlockCommitment::default();
-            expected.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, lamports);
+            expected.increase_rooted_stake(lamports);
             assert_eq!(*commitment.get(&a).unwrap(), expected);
         }
+        assert_eq!(rooted_stake[0], (root, lamports));
     }
 
     #[test]
     fn test_aggregate_commitment_for_vote_account_2() {
         let ancestors = vec![3, 4, 5, 7, 9, 11];
         let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
         let lamports = 5;
         let mut vote_state = VoteState::default();
 
@@ -361,6 +496,7 @@ mod tests {
         vote_state.process_slot_vote_unchecked(*ancestors.last().unwrap());
         AggregateCommitmentService::aggregate_commitment_for_vote_account(
             &mut commitment,
+            &mut rooted_stake,
             &vote_state,
             &ancestors,
             lamports,
@@ -369,7 +505,7 @@ mod tests {
         for a in ancestors {
             if a <= root {
                 let mut expected = BlockCommitment::default();
-                expected.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, lamports);
+                expected.increase_rooted_stake(lamports);
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             } else {
                 let mut expected = BlockCommitment::default();
@@ -377,12 +513,14 @@ mod tests {
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             }
         }
+        assert_eq!(rooted_stake[0], (root, lamports));
     }
 
     #[test]
     fn test_aggregate_commitment_for_vote_account_3() {
         let ancestors = vec![3, 4, 5, 7, 9, 10, 11];
         let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
         let lamports = 5;
         let mut vote_state = VoteState::default();
 
@@ -393,6 +531,7 @@ mod tests {
         vote_state.process_slot_vote_unchecked(ancestors[6]);
         AggregateCommitmentService::aggregate_commitment_for_vote_account(
             &mut commitment,
+            &mut rooted_stake,
             &vote_state,
             &ancestors,
             lamports,
@@ -401,7 +540,7 @@ mod tests {
         for (i, a) in ancestors.iter().enumerate() {
             if *a <= root {
                 let mut expected = BlockCommitment::default();
-                expected.increase_confirmation_stake(MAX_LOCKOUT_HISTORY, lamports);
+                expected.increase_rooted_stake(lamports);
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             } else if i <= 4 {
                 let mut expected = BlockCommitment::default();
@@ -413,6 +552,7 @@ mod tests {
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             }
         }
+        assert_eq!(rooted_stake[0], (root, lamports));
     }
 
     #[test]
@@ -422,6 +562,8 @@ mod tests {
             mut genesis_config, ..
         } = create_genesis_config(10_000);
 
+        let rooted_stake_amount = 40;
+
         let sk1 = Pubkey::new_rand();
         let pk1 = Pubkey::new_rand();
         let mut vote_account1 = vote_state::create_account(&pk1, &Pubkey::new_rand(), 0, 100);
@@ -432,12 +574,36 @@ mod tests {
         let mut vote_account2 = vote_state::create_account(&pk2, &Pubkey::new_rand(), 0, 50);
         let stake_account2 =
             stake_state::create_account(&sk2, &pk2, &vote_account2, &genesis_config.rent, 50);
+        let sk3 = Pubkey::new_rand();
+        let pk3 = Pubkey::new_rand();
+        let mut vote_account3 = vote_state::create_account(&pk3, &Pubkey::new_rand(), 0, 1);
+        let stake_account3 = stake_state::create_account(
+            &sk3,
+            &pk3,
+            &vote_account3,
+            &genesis_config.rent,
+            rooted_stake_amount,
+        );
+        let sk4 = Pubkey::new_rand();
+        let pk4 = Pubkey::new_rand();
+        let mut vote_account4 = vote_state::create_account(&pk4, &Pubkey::new_rand(), 0, 1);
+        let stake_account4 = stake_state::create_account(
+            &sk4,
+            &pk4,
+            &vote_account4,
+            &genesis_config.rent,
+            rooted_stake_amount,
+        );
 
         genesis_config.accounts.extend(vec![
             (pk1, vote_account1.clone()),
             (sk1, stake_account1),
             (pk2, vote_account2.clone()),
             (sk2, stake_account2),
+            (pk3, vote_account3.clone()),
+            (sk3, stake_account3),
+            (pk4, vote_account4.clone()),
+            (sk4, stake_account4),
         ]);
 
         // Create bank
@@ -457,7 +623,20 @@ mod tests {
         VoteState::to(&versioned, &mut vote_account2).unwrap();
         bank.store_account(&pk2, &vote_account2);
 
-        let commitment = AggregateCommitmentService::aggregate_commitment(&ancestors, &bank);
+        let mut vote_state3 = VoteState::from(&vote_account3).unwrap();
+        vote_state3.root_slot = Some(1);
+        let versioned = VoteStateVersions::Current(Box::new(vote_state3));
+        VoteState::to(&versioned, &mut vote_account3).unwrap();
+        bank.store_account(&pk3, &vote_account3);
+
+        let mut vote_state4 = VoteState::from(&vote_account4).unwrap();
+        vote_state4.root_slot = Some(2);
+        let versioned = VoteStateVersions::Current(Box::new(vote_state4));
+        VoteState::to(&versioned, &mut vote_account4).unwrap();
+        bank.store_account(&pk4, &vote_account4);
+
+        let (commitment, rooted_stake) =
+            AggregateCommitmentService::aggregate_commitment(&ancestors, &bank);
 
         for a in ancestors {
             if a <= 3 {
@@ -481,5 +660,7 @@ mod tests {
                 assert!(commitment.get(&a).is_none());
             }
         }
+        assert_eq!(rooted_stake.len(), 2);
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 100), 1)
     }
 }

core/src/consensus.rs

@@ -621,7 +621,7 @@ pub mod test {
             }
             let vote = tower.new_vote_from_bank(&bank, &my_vote_pubkey).0;
             if let Some(new_root) = tower.record_bank_vote(vote) {
-                ReplayStage::handle_new_root(new_root, bank_forks, progress, &None);
+                ReplayStage::handle_new_root(new_root, bank_forks, progress, &None, None);
             }
 
             // Mark the vote for this bank under this node's pubkey so it will be

core/src/contact_info.rs

@@ -1,6 +1,8 @@
+use crate::crds_value::MAX_WALLCLOCK;
 use solana_sdk::pubkey::Pubkey;
 #[cfg(test)]
 use solana_sdk::rpc_port;
+use solana_sdk::sanitize::{Sanitize, SanitizeError};
 #[cfg(test)]
 use solana_sdk::signature::{Keypair, Signer};
 use solana_sdk::timing::timestamp;
@@ -37,6 +39,15 @@ pub struct ContactInfo {
     pub shred_version: u16,
 }
 
+impl Sanitize for ContactInfo {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        Ok(())
+    }
+}
+
 impl Ord for ContactInfo {
     fn cmp(&self, other: &Self) -> Ordering {
         self.id.cmp(&other.id)
@@ -314,4 +325,12 @@ mod tests {
         ci.rpc = socketaddr!("127.0.0.1:234");
         assert!(ci.valid_client_facing_addr().is_some());
     }
+
+    #[test]
+    fn test_sanitize() {
+        let mut ci = ContactInfo::default();
+        assert_eq!(ci.sanitize(), Ok(()));
+        ci.wallclock = MAX_WALLCLOCK;
+        assert_eq!(ci.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+    }
 }

core/src/crds_gossip.rs

@@ -20,6 +20,7 @@ pub const CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS: usize = 500;
 pub struct CrdsGossip {
     pub crds: Crds,
     pub id: Pubkey,
+    pub shred_version: u16,
     pub push: CrdsGossipPush,
     pub pull: CrdsGossipPull,
 }
@@ -29,6 +30,7 @@ impl Default for CrdsGossip {
         CrdsGossip {
             crds: Crds::default(),
             id: Pubkey::default(),
+            shred_version: 0,
             push: CrdsGossipPush::default(),
             pull: CrdsGossipPull::default(),
         }
@@ -39,6 +41,9 @@ impl CrdsGossip {
     pub fn set_self(&mut self, id: &Pubkey) {
         self.id = *id;
     }
+    pub fn set_shred_version(&mut self, shred_version: u16) {
+        self.shred_version = shred_version;
+    }
 
     /// process a push message to the network
     pub fn process_push_message(
@@ -122,6 +127,7 @@ impl CrdsGossip {
             &self.crds,
             stakes,
             &self.id,
+            self.shred_version,
             self.pull.pull_request_time.len(),
             CRDS_GOSSIP_NUM_ACTIVE,
         )
@@ -134,8 +140,14 @@ impl CrdsGossip {
         stakes: &HashMap<Pubkey, u64>,
         bloom_size: usize,
     ) -> Result<(Pubkey, Vec<CrdsFilter>, CrdsValue), CrdsGossipError> {
-        self.pull
-            .new_pull_request(&self.crds, &self.id, now, stakes, bloom_size)
+        self.pull.new_pull_request(
+            &self.crds,
+            &self.id,
+            self.shred_version,
+            now,
+            stakes,
+            bloom_size,
+        )
     }
 
     /// time when a request to `from` was initiated

core/src/crds_gossip_pull.rs

@@ -14,7 +14,6 @@ use crate::crds::Crds;
 use crate::crds_gossip::{get_stake, get_weight, CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS};
 use crate::crds_gossip_error::CrdsGossipError;
 use crate::crds_value::{CrdsValue, CrdsValueLabel};
-use rand;
 use rand::distributions::{Distribution, WeightedIndex};
 use rand::Rng;
 use solana_runtime::bloom::Bloom;
@@ -37,6 +36,13 @@ pub struct CrdsFilter {
     mask_bits: u32,
 }
 
+impl solana_sdk::sanitize::Sanitize for CrdsFilter {
+    fn sanitize(&self) -> std::result::Result<(), solana_sdk::sanitize::SanitizeError> {
+        self.filter.sanitize()?;
+        Ok(())
+    }
+}
+
 impl CrdsFilter {
     pub fn new_rand(num_items: usize, max_bytes: usize) -> Self {
         let max_bits = (max_bytes * 8) as f64;
@@ -138,11 +144,12 @@ impl CrdsGossipPull {
         &self,
         crds: &Crds,
         self_id: &Pubkey,
+        self_shred_version: u16,
         now: u64,
         stakes: &HashMap<Pubkey, u64>,
         bloom_size: usize,
     ) -> Result<(Pubkey, Vec<CrdsFilter>, CrdsValue), CrdsGossipError> {
-        let options = self.pull_options(crds, &self_id, now, stakes);
+        let options = self.pull_options(crds, &self_id, self_shred_version, now, stakes);
         if options.is_empty() {
             return Err(CrdsGossipError::NoPeers);
         }
@@ -159,13 +166,20 @@ impl CrdsGossipPull {
         &self,
         crds: &'a Crds,
         self_id: &Pubkey,
+        self_shred_version: u16,
         now: u64,
         stakes: &HashMap<Pubkey, u64>,
     ) -> Vec<(f32, &'a ContactInfo)> {
         crds.table
             .values()
             .filter_map(|v| v.value.contact_info())
-            .filter(|v| v.id != *self_id && ContactInfo::is_valid_address(&v.gossip))
+            .filter(|v| {
+                v.id != *self_id
+                    && ContactInfo::is_valid_address(&v.gossip)
+                    && (self_shred_version == 0
+                        || v.shred_version == 0
+                        || self_shred_version == v.shred_version)
+            })
             .map(|item| {
                 let max_weight = f32::from(u16::max_value()) - 1.0;
                 let req_time: u64 = *self.pull_request_time.get(&item.id).unwrap_or(&0);
@@ -396,7 +410,7 @@ mod test {
             stakes.insert(id, i * 100);
         }
         let now = 1024;
-        let mut options = node.pull_options(&crds, &me.label().pubkey(), now, &stakes);
+        let mut options = node.pull_options(&crds, &me.label().pubkey(), 0, now, &stakes);
         assert!(!options.is_empty());
         options.sort_by(|(weight_l, _), (weight_r, _)| weight_r.partial_cmp(weight_l).unwrap());
         // check that the highest stake holder is also the heaviest weighted.
@@ -406,6 +420,66 @@ mod test {
         );
     }
 
+    #[test]
+    fn test_no_pulls_from_different_shred_versions() {
+        let mut crds = Crds::default();
+        let stakes = HashMap::new();
+        let node = CrdsGossipPull::default();
+
+        let gossip = socketaddr!("127.0.0.1:1234");
+
+        let me = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let spy = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 0,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_123 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_456 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 456,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+
+        crds.insert(me.clone(), 0).unwrap();
+        crds.insert(spy.clone(), 0).unwrap();
+        crds.insert(node_123.clone(), 0).unwrap();
+        crds.insert(node_456.clone(), 0).unwrap();
+
+        // shred version 123 should ignore 456 nodes
+        let options = node
+            .pull_options(&crds, &me.label().pubkey(), 123, 0, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 2);
+        assert!(options.contains(&spy.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+
+        // spy nodes will see all
+        let options = node
+            .pull_options(&crds, &spy.label().pubkey(), 0, 0, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 3);
+        assert!(options.contains(&me.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+        assert!(options.contains(&node_456.pubkey()));
+    }
+
     #[test]
     fn test_new_pull_request() {
         let mut crds = Crds::default();
@@ -416,13 +490,13 @@ mod test {
         let id = entry.label().pubkey();
         let node = CrdsGossipPull::default();
         assert_eq!(
-            node.new_pull_request(&crds, &id, 0, &HashMap::new(), PACKET_DATA_SIZE),
+            node.new_pull_request(&crds, &id, 0, 0, &HashMap::new(), PACKET_DATA_SIZE),
             Err(CrdsGossipError::NoPeers)
         );
 
         crds.insert(entry.clone(), 0).unwrap();
         assert_eq!(
-            node.new_pull_request(&crds, &id, 0, &HashMap::new(), PACKET_DATA_SIZE),
+            node.new_pull_request(&crds, &id, 0, 0, &HashMap::new(), PACKET_DATA_SIZE),
             Err(CrdsGossipError::NoPeers)
         );
 
@@ -431,7 +505,7 @@ mod test {
             0,
         )));
         crds.insert(new.clone(), 0).unwrap();
-        let req = node.new_pull_request(&crds, &id, 0, &HashMap::new(), PACKET_DATA_SIZE);
+        let req = node.new_pull_request(&crds, &id, 0, 0, &HashMap::new(), PACKET_DATA_SIZE);
         let (to, _, self_info) = req.unwrap();
         assert_eq!(to, new.label().pubkey());
         assert_eq!(self_info, entry);
@@ -466,6 +540,7 @@ mod test {
             let req = node.new_pull_request(
                 &crds,
                 &node_pubkey,
+                0,
                 u64::max_value(),
                 &HashMap::new(),
                 PACKET_DATA_SIZE,
@@ -495,6 +570,7 @@ mod test {
             &node_crds,
             &node_pubkey,
             0,
+            0,
             &HashMap::new(),
             PACKET_DATA_SIZE,
         );
@@ -567,6 +643,7 @@ mod test {
                 &node_crds,
                 &node_pubkey,
                 0,
+                0,
                 &HashMap::new(),
                 PACKET_DATA_SIZE,
             );

core/src/crds_gossip_push.rs

@@ -236,13 +236,14 @@ impl CrdsGossipPush {
         crds: &Crds,
         stakes: &HashMap<Pubkey, u64>,
         self_id: &Pubkey,
+        self_shred_version: u16,
         network_size: usize,
         ratio: usize,
     ) {
         let need = Self::compute_need(self.num_active, self.active_set.len(), ratio);
         let mut new_items = HashMap::new();
 
-        let options: Vec<_> = self.push_options(crds, &self_id, stakes);
+        let options: Vec<_> = self.push_options(crds, &self_id, self_shred_version, stakes);
         if options.is_empty() {
             return;
         }
@@ -288,13 +289,20 @@ impl CrdsGossipPush {
         &self,
         crds: &'a Crds,
         self_id: &Pubkey,
+        self_shred_version: u16,
         stakes: &HashMap<Pubkey, u64>,
     ) -> Vec<(f32, &'a ContactInfo)> {
         crds.table
             .values()
             .filter(|v| v.value.contact_info().is_some())
             .map(|v| (v.value.contact_info().unwrap(), v))
-            .filter(|(info, _)| info.id != *self_id && ContactInfo::is_valid_address(&info.gossip))
+            .filter(|(info, _)| {
+                info.id != *self_id
+                    && ContactInfo::is_valid_address(&info.gossip)
+                    && (self_shred_version == 0
+                        || info.shred_version == 0
+                        || self_shred_version == info.shred_version)
+            })
             .map(|(info, value)| {
                 let max_weight = f32::from(u16::max_value()) - 1.0;
                 let last_updated: u64 = value.local_timestamp;
@@ -510,7 +518,7 @@ mod test {
         )));
 
         assert_eq!(crds.insert(value1.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         assert!(push.active_set.get(&value1.label().pubkey()).is_some());
         let value2 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
@@ -520,7 +528,7 @@ mod test {
         assert!(push.active_set.get(&value2.label().pubkey()).is_none());
         assert_eq!(crds.insert(value2.clone(), 0), Ok(None));
         for _ in 0..30 {
-            push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+            push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
             if push.active_set.get(&value2.label().pubkey()).is_some() {
                 break;
             }
@@ -533,7 +541,7 @@ mod test {
             ));
             assert_eq!(crds.insert(value2.clone(), 0), Ok(None));
         }
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
         assert_eq!(push.active_set.len(), push.num_active);
     }
     #[test]
@@ -551,7 +559,7 @@ mod test {
             crds.insert(peer.clone(), time).unwrap();
             stakes.insert(id, i * 100);
         }
-        let mut options = push.push_options(&crds, &Pubkey::default(), &stakes);
+        let mut options = push.push_options(&crds, &Pubkey::default(), 0, &stakes);
         assert!(!options.is_empty());
         options.sort_by(|(weight_l, _), (weight_r, _)| weight_r.partial_cmp(weight_l).unwrap());
         // check that the highest stake holder is also the heaviest weighted.
@@ -560,6 +568,66 @@ mod test {
             10_000_u64
         );
     }
+
+    #[test]
+    fn test_no_pushes_to_from_different_shred_versions() {
+        let mut crds = Crds::default();
+        let stakes = HashMap::new();
+        let node = CrdsGossipPush::default();
+
+        let gossip = socketaddr!("127.0.0.1:1234");
+
+        let me = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let spy = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 0,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_123 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_456 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 456,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+
+        crds.insert(me.clone(), 0).unwrap();
+        crds.insert(spy.clone(), 0).unwrap();
+        crds.insert(node_123.clone(), 0).unwrap();
+        crds.insert(node_456.clone(), 0).unwrap();
+
+        // shred version 123 should ignore 456 nodes
+        let options = node
+            .push_options(&crds, &me.label().pubkey(), 123, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 2);
+        assert!(options.contains(&spy.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+
+        // spy nodes will see all
+        let options = node
+            .push_options(&crds, &spy.label().pubkey(), 0, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 3);
+        assert!(options.contains(&me.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+        assert!(options.contains(&node_456.pubkey()));
+    }
     #[test]
     fn test_new_push_messages() {
         let mut crds = Crds::default();
@@ -569,7 +637,7 @@ mod test {
             0,
         )));
         assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         let new_msg = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
             &Pubkey::new_rand(),
@@ -606,7 +674,7 @@ mod test {
             push.process_push_message(&mut crds, &Pubkey::default(), peer_3.clone(), 0),
             Ok(None)
         );
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         // push 3's contact info to 1 and 2 and 3
         let new_msg = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
@@ -628,7 +696,7 @@ mod test {
             0,
         )));
         assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         let new_msg = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
             &Pubkey::new_rand(),
@@ -651,7 +719,7 @@ mod test {
             0,
         )));
         assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         let mut ci = ContactInfo::new_localhost(&Pubkey::new_rand(), 0);
         ci.wallclock = 1;

core/src/crds_value.rs

@@ -1,5 +1,7 @@
 use crate::contact_info::ContactInfo;
 use bincode::{serialize, serialized_size};
+use solana_sdk::sanitize::{Sanitize, SanitizeError};
+use solana_sdk::timing::timestamp;
 use solana_sdk::{
     clock::Slot,
     hash::Hash,
@@ -13,10 +15,14 @@ use std::{
     fmt,
 };
 
+pub const MAX_WALLCLOCK: u64 = 1_000_000_000_000_000;
+pub const MAX_SLOT: u64 = 1_000_000_000_000_000;
+
 pub type VoteIndex = u8;
 pub const MAX_VOTES: VoteIndex = 32;
 
 pub type EpochSlotIndex = u8;
+pub const MAX_EPOCH_SLOTS: EpochSlotIndex = 1;
 
 /// CrdsValue that is replicated across the cluster
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -25,6 +31,13 @@ pub struct CrdsValue {
     pub data: CrdsData,
 }
 
+impl Sanitize for CrdsValue {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        self.signature.sanitize()?;
+        self.data.sanitize()
+    }
+}
+
 impl Signable for CrdsValue {
     fn pubkey(&self) -> Pubkey {
         self.pubkey()
@@ -43,14 +56,8 @@ impl Signable for CrdsValue {
     }
 
     fn verify(&self) -> bool {
-        let sig_check = self
-            .get_signature()
-            .verify(&self.pubkey().as_ref(), self.signable_data().borrow());
-        let data_check = match &self.data {
-            CrdsData::Vote(ix, _) => *ix < MAX_VOTES,
-            _ => true,
-        };
-        sig_check && data_check
+        self.get_signature()
+            .verify(&self.pubkey().as_ref(), self.signable_data().borrow())
     }
 }
 
@@ -62,7 +69,8 @@ pub enum CrdsData {
     ContactInfo(ContactInfo),
     Vote(VoteIndex, Vote),
     EpochSlots(EpochSlotIndex, EpochSlots),
-    SnapshotHash(SnapshotHash),
+    SnapshotHashes(SnapshotHash),
+    AccountsHashes(SnapshotHash),
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -85,6 +93,39 @@ pub struct EpochIncompleteSlots {
     pub compressed_list: Vec<u8>,
 }
 
+impl Sanitize for EpochIncompleteSlots {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.first >= MAX_SLOT {
+            return Err(SanitizeError::InvalidValue);
+        }
+        //rest of the data doesn't matter since we no longer decompress
+        //these values
+        Ok(())
+    }
+}
+
+impl Sanitize for CrdsData {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        match self {
+            CrdsData::ContactInfo(val) => val.sanitize(),
+            CrdsData::Vote(ix, val) => {
+                if *ix >= MAX_VOTES {
+                    return Err(SanitizeError::ValueOutOfBounds);
+                }
+                val.sanitize()
+            }
+            CrdsData::SnapshotHashes(val) => val.sanitize(),
+            CrdsData::AccountsHashes(val) => val.sanitize(),
+            CrdsData::EpochSlots(ix, val) => {
+                if *ix as usize >= MAX_EPOCH_SLOTS as usize {
+                    return Err(SanitizeError::ValueOutOfBounds);
+                }
+                val.sanitize()
+            }
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub struct SnapshotHash {
     pub from: Pubkey,
@@ -92,12 +133,26 @@ pub struct SnapshotHash {
     pub wallclock: u64,
 }
 
+impl Sanitize for SnapshotHash {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        for (slot, _) in &self.hashes {
+            if *slot >= MAX_SLOT {
+                return Err(SanitizeError::ValueOutOfBounds);
+            }
+        }
+        self.from.sanitize()
+    }
+}
+
 impl SnapshotHash {
-    pub fn new(from: Pubkey, hashes: Vec<(Slot, Hash)>, wallclock: u64) -> Self {
+    pub fn new(from: Pubkey, hashes: Vec<(Slot, Hash)>) -> Self {
         Self {
             from,
             hashes,
-            wallclock,
+            wallclock: timestamp(),
         }
     }
 }
@@ -105,33 +160,47 @@ impl SnapshotHash {
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub struct EpochSlots {
     pub from: Pubkey,
-    pub root: Slot,
+    root: Slot,
     pub lowest: Slot,
-    pub slots: BTreeSet<Slot>,
-    pub stash: Vec<EpochIncompleteSlots>,
+    slots: BTreeSet<Slot>,
+    stash: Vec<EpochIncompleteSlots>,
     pub wallclock: u64,
 }
 
 impl EpochSlots {
-    pub fn new(
-        from: Pubkey,
-        root: Slot,
-        lowest: Slot,
-        slots: BTreeSet<Slot>,
-        stash: Vec<EpochIncompleteSlots>,
-        wallclock: u64,
-    ) -> Self {
+    pub fn new(from: Pubkey, lowest: Slot, wallclock: u64) -> Self {
         Self {
             from,
-            root,
+            root: 0,
             lowest,
-            slots,
-            stash,
+            slots: BTreeSet::new(),
+            stash: vec![],
             wallclock,
         }
     }
 }
 
+impl Sanitize for EpochSlots {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        if self.lowest >= MAX_SLOT {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        if self.root >= MAX_SLOT {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        for slot in &self.slots {
+            if *slot >= MAX_SLOT {
+                return Err(SanitizeError::ValueOutOfBounds);
+            }
+        }
+        self.stash.sanitize()?;
+        self.from.sanitize()
+    }
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub struct Vote {
     pub from: Pubkey,
@@ -139,6 +208,16 @@ pub struct Vote {
     pub wallclock: u64,
 }
 
+impl Sanitize for Vote {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        self.from.sanitize()?;
+        self.transaction.sanitize()
+    }
+}
+
 impl Vote {
     pub fn new(from: &Pubkey, transaction: Transaction, wallclock: u64) -> Self {
         Self {
@@ -156,7 +235,8 @@ pub enum CrdsValueLabel {
     ContactInfo(Pubkey),
     Vote(VoteIndex, Pubkey),
     EpochSlots(Pubkey),
-    SnapshotHash(Pubkey),
+    SnapshotHashes(Pubkey),
+    AccountsHashes(Pubkey),
 }
 
 impl fmt::Display for CrdsValueLabel {
@@ -165,7 +245,8 @@ impl fmt::Display for CrdsValueLabel {
             CrdsValueLabel::ContactInfo(_) => write!(f, "ContactInfo({})", self.pubkey()),
             CrdsValueLabel::Vote(ix, _) => write!(f, "Vote({}, {})", ix, self.pubkey()),
             CrdsValueLabel::EpochSlots(_) => write!(f, "EpochSlots({})", self.pubkey()),
-            CrdsValueLabel::SnapshotHash(_) => write!(f, "SnapshotHash({})", self.pubkey()),
+            CrdsValueLabel::SnapshotHashes(_) => write!(f, "SnapshotHashes({})", self.pubkey()),
+            CrdsValueLabel::AccountsHashes(_) => write!(f, "AccountsHashes({})", self.pubkey()),
         }
     }
 }
@@ -176,7 +257,8 @@ impl CrdsValueLabel {
             CrdsValueLabel::ContactInfo(p) => *p,
             CrdsValueLabel::Vote(_, p) => *p,
             CrdsValueLabel::EpochSlots(p) => *p,
-            CrdsValueLabel::SnapshotHash(p) => *p,
+            CrdsValueLabel::SnapshotHashes(p) => *p,
+            CrdsValueLabel::AccountsHashes(p) => *p,
         }
     }
 }
@@ -202,7 +284,8 @@ impl CrdsValue {
             CrdsData::ContactInfo(contact_info) => contact_info.wallclock,
             CrdsData::Vote(_, vote) => vote.wallclock,
             CrdsData::EpochSlots(_, vote) => vote.wallclock,
-            CrdsData::SnapshotHash(hash) => hash.wallclock,
+            CrdsData::SnapshotHashes(hash) => hash.wallclock,
+            CrdsData::AccountsHashes(hash) => hash.wallclock,
         }
     }
     pub fn pubkey(&self) -> Pubkey {
@@ -210,7 +293,8 @@ impl CrdsValue {
             CrdsData::ContactInfo(contact_info) => contact_info.id,
             CrdsData::Vote(_, vote) => vote.from,
             CrdsData::EpochSlots(_, slots) => slots.from,
-            CrdsData::SnapshotHash(hash) => hash.from,
+            CrdsData::SnapshotHashes(hash) => hash.from,
+            CrdsData::AccountsHashes(hash) => hash.from,
         }
     }
     pub fn label(&self) -> CrdsValueLabel {
@@ -218,7 +302,8 @@ impl CrdsValue {
             CrdsData::ContactInfo(_) => CrdsValueLabel::ContactInfo(self.pubkey()),
             CrdsData::Vote(ix, _) => CrdsValueLabel::Vote(*ix, self.pubkey()),
             CrdsData::EpochSlots(_, _) => CrdsValueLabel::EpochSlots(self.pubkey()),
-            CrdsData::SnapshotHash(_) => CrdsValueLabel::SnapshotHash(self.pubkey()),
+            CrdsData::SnapshotHashes(_) => CrdsValueLabel::SnapshotHashes(self.pubkey()),
+            CrdsData::AccountsHashes(_) => CrdsValueLabel::AccountsHashes(self.pubkey()),
         }
     }
     pub fn contact_info(&self) -> Option<&ContactInfo> {
@@ -250,7 +335,14 @@ impl CrdsValue {
 
     pub fn snapshot_hash(&self) -> Option<&SnapshotHash> {
         match &self.data {
-            CrdsData::SnapshotHash(slots) => Some(slots),
+            CrdsData::SnapshotHashes(slots) => Some(slots),
+            _ => None,
+        }
+    }
+
+    pub fn accounts_hash(&self) -> Option<&SnapshotHash> {
+        match &self.data {
+            CrdsData::AccountsHashes(slots) => Some(slots),
             _ => None,
         }
     }
@@ -260,7 +352,8 @@ impl CrdsValue {
         let mut labels = vec![
             CrdsValueLabel::ContactInfo(*key),
             CrdsValueLabel::EpochSlots(*key),
-            CrdsValueLabel::SnapshotHash(*key),
+            CrdsValueLabel::SnapshotHashes(*key),
+            CrdsValueLabel::AccountsHashes(*key),
         ];
         labels.extend((0..MAX_VOTES).map(|ix| CrdsValueLabel::Vote(ix, *key)));
         labels
@@ -310,14 +403,15 @@ mod test {
 
     #[test]
     fn test_labels() {
-        let mut hits = [false; 3 + MAX_VOTES as usize];
+        let mut hits = [false; 4 + MAX_VOTES as usize];
         // this method should cover all the possible labels
         for v in &CrdsValue::record_labels(&Pubkey::default()) {
             match v {
                 CrdsValueLabel::ContactInfo(_) => hits[0] = true,
                 CrdsValueLabel::EpochSlots(_) => hits[1] = true,
-                CrdsValueLabel::SnapshotHash(_) => hits[2] = true,
-                CrdsValueLabel::Vote(ix, _) => hits[*ix as usize + 3] = true,
+                CrdsValueLabel::SnapshotHashes(_) => hits[2] = true,
+                CrdsValueLabel::AccountsHashes(_) => hits[3] = true,
+                CrdsValueLabel::Vote(ix, _) => hits[*ix as usize + 4] = true,
             }
         }
         assert!(hits.iter().all(|x| *x));
@@ -339,7 +433,7 @@ mod test {
 
         let v = CrdsValue::new_unsigned(CrdsData::EpochSlots(
             0,
-            EpochSlots::new(Pubkey::default(), 0, 0, BTreeSet::new(), vec![], 0),
+            EpochSlots::new(Pubkey::default(), 0, 0),
         ));
         assert_eq!(v.wallclock(), 0);
         let key = v.clone().epoch_slots().unwrap().from;
@@ -360,10 +454,9 @@ mod test {
             Vote::new(&keypair.pubkey(), test_tx(), timestamp()),
         ));
         verify_signatures(&mut v, &keypair, &wrong_keypair);
-        let btreeset: BTreeSet<Slot> = vec![1, 2, 3, 6, 8].into_iter().collect();
         v = CrdsValue::new_unsigned(CrdsData::EpochSlots(
             0,
-            EpochSlots::new(keypair.pubkey(), 0, 0, btreeset, vec![], timestamp()),
+            EpochSlots::new(keypair.pubkey(), 0, timestamp()),
         ));
         verify_signatures(&mut v, &keypair, &wrong_keypair);
     }
@@ -378,9 +471,21 @@ mod test {
             ),
             &keypair,
         );
-        assert!(!vote.verify());
+        assert!(vote.sanitize().is_err());
     }
 
+    #[test]
+    fn test_max_epoch_slots_index() {
+        let keypair = Keypair::new();
+        let item = CrdsValue::new_signed(
+            CrdsData::Vote(
+                MAX_VOTES,
+                Vote::new(&keypair.pubkey(), test_tx(), timestamp()),
+            ),
+            &keypair,
+        );
+        assert_eq!(item.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+    }
     #[test]
     fn test_compute_vote_index_empty() {
         for i in 0..MAX_VOTES {

core/src/ledger_cleanup_service.rs

@@ -1,6 +1,8 @@
 //! The `ledger_cleanup_service` drops older ledger data to limit disk space usage
 
 use solana_ledger::blockstore::Blockstore;
+use solana_ledger::blockstore_db::Result as BlockstoreResult;
+use solana_measure::measure::Measure;
 use solana_metrics::datapoint_debug;
 use solana_sdk::clock::Slot;
 use std::string::ToString;
@@ -11,13 +13,25 @@ use std::thread;
 use std::thread::{Builder, JoinHandle};
 use std::time::Duration;
 
+// - To try and keep the RocksDB size under 400GB:
+//   Seeing about 1600b/shred, using 2000b/shred for margin, so 200m shreds can be stored in 400gb.
+//   at 5k shreds/slot at 50k tps, this is 500k slots (~5 hours).
+//   At idle, 60 shreds/slot this is about 4m slots (18 days)
 // This is chosen to allow enough time for
-// - To try and keep the RocksDB size under 512GB at 50k tps (100 slots take ~2GB).
 // - A validator to download a snapshot from a peer and boot from it
 // - To make sure that if a validator needs to reboot from its own snapshot, it has enough slots locally
 //   to catch back up to where it was when it stopped
-pub const DEFAULT_MAX_LEDGER_SLOTS: u64 = 270_000;
-// Remove a fixed number of slots at a time, it's more efficient than doing it one-by-one
+pub const DEFAULT_MAX_LEDGER_SHREDS: u64 = 200_000_000;
+
+// Allow down to 50m, or 3.5 days at idle, 1hr at 50k load, around ~100GB
+pub const DEFAULT_MIN_MAX_LEDGER_SHREDS: u64 = 50_000_000;
+
+// Check for removing slots at this interval so we don't purge too often
+// and starve other blockstore users.
+pub const DEFAULT_PURGE_SLOT_INTERVAL: u64 = 512;
+
+// Remove a limited number of slots at a time, so the operation
+// does not take too long and block other blockstore users.
 pub const DEFAULT_PURGE_BATCH_SIZE: u64 = 256;
 
 pub struct LedgerCleanupService {
@@ -36,7 +50,7 @@ impl LedgerCleanupService {
             max_ledger_slots
         );
         let exit = exit.clone();
-        let mut next_purge_batch = max_ledger_slots;
+        let mut last_purge_slot = 0;
         let t_cleanup = Builder::new()
             .name("solana-ledger-cleanup".to_string())
             .spawn(move || loop {
@@ -47,7 +61,8 @@ impl LedgerCleanupService {
                     &new_root_receiver,
                     &blockstore,
                     max_ledger_slots,
-                    &mut next_purge_batch,
+                    &mut last_purge_slot,
+                    DEFAULT_PURGE_SLOT_INTERVAL,
                 ) {
                     match e {
                         RecvTimeoutError::Disconnected => break,
@@ -59,45 +74,123 @@ impl LedgerCleanupService {
         Self { t_cleanup }
     }
 
+    fn find_slots_to_clean(
+        blockstore: &Arc<Blockstore>,
+        root: Slot,
+        max_ledger_shreds: u64,
+    ) -> (u64, Slot, Slot) {
+        let mut shreds = Vec::new();
+        let mut iterate_time = Measure::start("iterate_time");
+        let mut total_shreds = 0;
+        let mut first_slot = 0;
+        for (i, (slot, meta)) in blockstore.slot_meta_iterator(0).unwrap().enumerate() {
+            if i == 0 {
+                first_slot = slot;
+                debug!("purge: searching from slot: {}", slot);
+            }
+            // Not exact since non-full slots will have holes
+            total_shreds += meta.received;
+            shreds.push((slot, meta.received));
+            if slot > root {
+                break;
+            }
+        }
+        iterate_time.stop();
+        info!(
+            "checking for ledger purge: max_shreds: {} slots: {} total_shreds: {} {}",
+            max_ledger_shreds,
+            shreds.len(),
+            total_shreds,
+            iterate_time
+        );
+        if (total_shreds as u64) < max_ledger_shreds {
+            return (0, 0, 0);
+        }
+        let mut cur_shreds = 0;
+        let mut lowest_slot_to_clean = shreds[0].0;
+        for (slot, num_shreds) in shreds.iter().rev() {
+            cur_shreds += *num_shreds as u64;
+            if cur_shreds > max_ledger_shreds {
+                lowest_slot_to_clean = *slot;
+                break;
+            }
+        }
+
+        (cur_shreds, lowest_slot_to_clean, first_slot)
+    }
+
     fn cleanup_ledger(
         new_root_receiver: &Receiver<Slot>,
         blockstore: &Arc<Blockstore>,
-        max_ledger_slots: u64,
-        next_purge_batch: &mut u64,
+        max_ledger_shreds: u64,
+        last_purge_slot: &mut u64,
+        purge_interval: u64,
     ) -> Result<(), RecvTimeoutError> {
-        let disk_utilization_pre = blockstore.storage_size();
-
-        let root = new_root_receiver.recv_timeout(Duration::from_secs(1))?;
-
-        // Notify blockstore of impending purge
-        if root > *next_purge_batch {
-            //cleanup
-            let lowest_slot = root - max_ledger_slots;
-            *blockstore.lowest_cleanup_slot.write().unwrap() = lowest_slot;
-            blockstore.purge_slots(0, Some(lowest_slot));
-            *next_purge_batch += DEFAULT_PURGE_BATCH_SIZE;
+        let mut root = new_root_receiver.recv_timeout(Duration::from_secs(1))?;
+        // Get the newest root
+        while let Ok(new_root) = new_root_receiver.try_recv() {
+            root = new_root;
         }
 
-        let disk_utilization_post = blockstore.storage_size();
-
-        if let (Ok(disk_utilization_pre), Ok(disk_utilization_post)) =
-            (disk_utilization_pre, disk_utilization_post)
-        {
-            datapoint_debug!(
-                "ledger_disk_utilization",
-                ("disk_utilization_pre", disk_utilization_pre as i64, i64),
-                ("disk_utilization_post", disk_utilization_post as i64, i64),
-                (
-                    "disk_utilization_delta",
-                    (disk_utilization_pre as i64 - disk_utilization_post as i64),
-                    i64
-                )
+        if root - *last_purge_slot > purge_interval {
+            let disk_utilization_pre = blockstore.storage_size();
+            info!(
+                "purge: new root: {} last_purge: {} purge_interval: {} disk: {:?}",
+                root, last_purge_slot, purge_interval, disk_utilization_pre
             );
+            *last_purge_slot = root;
+
+            let (num_shreds_to_clean, lowest_slot_to_clean, mut first_slot) =
+                Self::find_slots_to_clean(blockstore, root, max_ledger_shreds);
+
+            if num_shreds_to_clean > 0 {
+                debug!(
+                    "cleaning up to: {} shreds: {} first: {}",
+                    lowest_slot_to_clean, num_shreds_to_clean, first_slot
+                );
+                loop {
+                    let current_lowest =
+                        std::cmp::min(lowest_slot_to_clean, first_slot + DEFAULT_PURGE_BATCH_SIZE);
+
+                    let mut slot_update_time = Measure::start("slot_update");
+                    *blockstore.lowest_cleanup_slot.write().unwrap() = current_lowest;
+                    slot_update_time.stop();
+
+                    let mut clean_time = Measure::start("ledger_clean");
+                    blockstore.purge_slots(first_slot, Some(current_lowest));
+                    clean_time.stop();
+
+                    debug!(
+                        "ledger purge {} -> {}: {} {}",
+                        first_slot, current_lowest, slot_update_time, clean_time
+                    );
+                    first_slot += DEFAULT_PURGE_BATCH_SIZE;
+                    if current_lowest == lowest_slot_to_clean {
+                        break;
+                    }
+                    thread::sleep(Duration::from_millis(500));
+                }
+            }
+
+            let disk_utilization_post = blockstore.storage_size();
+
+            Self::report_disk_metrics(disk_utilization_pre, disk_utilization_post);
         }
 
         Ok(())
     }
 
+    fn report_disk_metrics(pre: BlockstoreResult<u64>, post: BlockstoreResult<u64>) {
+        if let (Ok(pre), Ok(post)) = (pre, post) {
+            datapoint_debug!(
+                "ledger_disk_utilization",
+                ("disk_utilization_pre", pre as i64, i64),
+                ("disk_utilization_post", post as i64, i64),
+                ("disk_utilization_delta", (pre as i64 - post as i64), i64)
+            );
+        }
+    }
+
     pub fn join(self) -> thread::Result<()> {
         self.t_cleanup.join()
     }
@@ -111,6 +204,7 @@ mod tests {
 
     #[test]
     fn test_cleanup() {
+        solana_logger::setup();
         let blockstore_path = get_tmp_ledger_path!();
         let blockstore = Blockstore::open(&blockstore_path).unwrap();
         let (shreds, _) = make_many_slot_entries(0, 50, 5);
@@ -118,10 +212,10 @@ mod tests {
         let blockstore = Arc::new(blockstore);
         let (sender, receiver) = channel();
 
-        //send a signal to kill slots 0-40
-        let mut next_purge_slot = 0;
+        //send a signal to kill all but 5 shreds, which will be in the newest slots
+        let mut last_purge_slot = 0;
         sender.send(50).unwrap();
-        LedgerCleanupService::cleanup_ledger(&receiver, &blockstore, 10, &mut next_purge_slot)
+        LedgerCleanupService::cleanup_ledger(&receiver, &blockstore, 5, &mut last_purge_slot, 10)
             .unwrap();
 
         //check that 0-40 don't exist
@@ -134,6 +228,62 @@ mod tests {
         Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
     }
 
+    #[test]
+    fn test_cleanup_speed() {
+        solana_logger::setup();
+        let blockstore_path = get_tmp_ledger_path!();
+        let mut blockstore = Blockstore::open(&blockstore_path).unwrap();
+        blockstore.set_no_compaction(true);
+        let blockstore = Arc::new(blockstore);
+        let (sender, receiver) = channel();
+
+        let mut first_insert = Measure::start("first_insert");
+        let initial_slots = 50;
+        let initial_entries = 5;
+        let (shreds, _) = make_many_slot_entries(0, initial_slots, initial_entries);
+        blockstore.insert_shreds(shreds, None, false).unwrap();
+        first_insert.stop();
+        info!("{}", first_insert);
+
+        let mut last_purge_slot = 0;
+        let mut slot = initial_slots;
+        let mut num_slots = 6;
+        for _ in 0..5 {
+            let mut insert_time = Measure::start("insert time");
+            let batch_size = 2;
+            let batches = num_slots / batch_size;
+            for i in 0..batches {
+                let (shreds, _) = make_many_slot_entries(slot + i * batch_size, batch_size, 5);
+                blockstore.insert_shreds(shreds, None, false).unwrap();
+                if i % 100 == 0 {
+                    info!("inserting..{} of {}", i, batches);
+                }
+            }
+            insert_time.stop();
+
+            let mut time = Measure::start("purge time");
+            sender.send(slot + num_slots).unwrap();
+            LedgerCleanupService::cleanup_ledger(
+                &receiver,
+                &blockstore,
+                initial_slots,
+                &mut last_purge_slot,
+                10,
+            )
+            .unwrap();
+            time.stop();
+            info!(
+                "slot: {} size: {} {} {}",
+                slot, num_slots, insert_time, time
+            );
+            slot += num_slots;
+            num_slots *= 2;
+        }
+
+        drop(blockstore);
+        Blockstore::destroy(&blockstore_path).expect("Expected successful database destruction");
+    }
+
     #[test]
     fn test_compaction() {
         let blockstore_path = get_tmp_ledger_path!();
@@ -142,7 +292,7 @@ mod tests {
         let n = 10_000;
         let batch_size = 100;
         let batches = n / batch_size;
-        let max_ledger_slots = 100;
+        let max_ledger_shreds = 100;
 
         for i in 0..batches {
             let (shreds, _) = make_many_slot_entries(i * batch_size, batch_size, 1);
@@ -158,8 +308,9 @@ mod tests {
         LedgerCleanupService::cleanup_ledger(
             &receiver,
             &blockstore,
-            max_ledger_slots,
+            max_ledger_shreds,
             &mut next_purge_batch,
+            10,
         )
         .unwrap();
 
@@ -170,7 +321,7 @@ mod tests {
         assert!(u2 < u1, "insufficient compaction! pre={},post={}", u1, u2,);
 
         // check that early slots don't exist
-        let max_slot = n - max_ledger_slots;
+        let max_slot = n - max_ledger_shreds - 1;
         blockstore
             .slot_meta_iterator(0)
             .unwrap()

core/src/lib.rs

@@ -5,6 +5,7 @@
 //! command-line tools to spin up validators and a Rust library
 //!
 
+pub mod accounts_hash_verifier;
 pub mod banking_stage;
 pub mod broadcast_stage;
 pub mod cluster_info_vote_listener;
@@ -28,6 +29,7 @@ pub mod genesis_utils;
 pub mod gossip_service;
 pub mod ledger_cleanup_service;
 pub mod local_vote_signer_service;
+pub mod non_circulating_supply;
 pub mod packet;
 pub mod poh_recorder;
 pub mod poh_service;

core/src/non_circulating_supply.rs

@@ -0,0 +1,193 @@
+use solana_runtime::bank::Bank;
+use solana_sdk::pubkey::Pubkey;
+use solana_stake_program::stake_state::StakeState;
+use std::{collections::HashSet, sync::Arc};
+
+pub struct NonCirculatingSupply {
+    pub lamports: u64,
+    pub accounts: Vec<Pubkey>,
+}
+
+pub fn calculate_non_circulating_supply(bank: Arc<Bank>) -> NonCirculatingSupply {
+    debug!("Updating Bank supply, epoch: {}", bank.epoch());
+    let mut non_circulating_accounts_set: HashSet<Pubkey> = HashSet::new();
+
+    for key in non_circulating_accounts() {
+        non_circulating_accounts_set.insert(key);
+    }
+
+    let clock = bank.clock();
+    let stake_accounts = bank.get_program_accounts(Some(&solana_stake_program::id()));
+    for (pubkey, account) in stake_accounts.iter() {
+        let stake_account = StakeState::from(&account).unwrap_or_default();
+        match stake_account {
+            StakeState::Initialized(meta) => {
+                if meta.lockup.is_in_force(&clock, &HashSet::default())
+                    || meta.authorized.withdrawer == withdraw_authority()
+                {
+                    non_circulating_accounts_set.insert(*pubkey);
+                }
+            }
+            StakeState::Stake(meta, _stake) => {
+                if meta.lockup.is_in_force(&clock, &HashSet::default())
+                    || meta.authorized.withdrawer == withdraw_authority()
+                {
+                    non_circulating_accounts_set.insert(*pubkey);
+                }
+            }
+            _ => {}
+        }
+    }
+
+    let lamports = non_circulating_accounts_set
+        .iter()
+        .fold(0, |acc, pubkey| acc + bank.get_balance(&pubkey));
+
+    NonCirculatingSupply {
+        lamports,
+        accounts: non_circulating_accounts_set.into_iter().collect(),
+    }
+}
+
+// Mainnet-beta accounts that should be considered non-circulating
+solana_sdk::pubkeys!(
+    non_circulating_accounts,
+    [
+        "9huDUZfxoJ7wGMTffUE7vh1xePqef7gyrLJu9NApncqA",
+        "GK2zqSsXLA2rwVZk347RYhh6jJpRsCA69FjLW93ZGi3B",
+        "HCV5dGFJXRrJ3jhDYA4DCeb9TEDTwGGYXtT3wHksu2Zr",
+        "25odAafVXnd63L6Hq5Cx6xGmhKqkhE2y6UrLVuqUfWZj",
+        "14FUT96s9swbmH7ZjpDvfEDywnAYy9zaNhv4xvezySGu",
+        "HbZ5FfmKWNHC7uwk6TF1hVi6TCs7dtYfdjEcuPGgzFAg",
+        "C7C8odR8oashR5Feyrq2tJKaXL18id1dSj2zbkDGL2C2",
+        "APnSR52EC1eH676m7qTBHUJ1nrGpHYpV7XKPxgRDD8gX",
+        "9ibqedFVnu5k4wo1mJRbH6KJ5HLBCyjpA9omPYkDeeT5",
+        "FopBKzQkG9pkyQqjdMFBLMQ995pSkjy83ziR4aism4c6",
+        "AiUHvJhTbMCcgFE2K26Ea9qCe74y3sFwqUt38iD5sfoR",
+        "3DndE3W53QdHSfBJiSJgzDKGvKJBoQLVmRHvy5LtqYfG",
+        "Eyr9P5XsjK2NUKNCnfu39eqpGoiLFgVAv1LSQgMZCwiQ",
+        "DE1bawNcRJB9rVm3buyMVfr8mBEoyyu73NBovf2oXJsJ",
+        "CakcnaRDHka2gXyfbEd2d3xsvkJkqsLw2akB3zsN1D2S",
+        "7Np41oeYqPefeNQEHSv1UDhYrehxin3NStELsSKCT4K2",
+        "GdnSyH3YtwcxFvQrVVJMm1JhTS4QVX7MFsX56uJLUfiZ",
+        "Mc5XB47H3DKJHym5RLa9mPzWv5snERsF3KNv5AauXK8",
+        "7cvkjYAkUYs4W8XcXsca7cBrEGFeSUjeZmKoNBvEwyri",
+        "AG3m2bAibcY8raMt4oXEGqRHwX4FWKPPJVjZxn1LySDX",
+        "5XdtyEDREHJXXW1CTtCsVjJRjBapAwK78ZquzvnNVRrV",
+        "6yKHERk8rsbmJxvMpPuwPs1ct3hRiP7xaJF2tvnGU6nK",
+        "CHmdL15akDcJgBkY6BP3hzs98Dqr6wbdDC5p8odvtSbq",
+        "FR84wZQy3Y3j2gWz6pgETUiUoJtreMEuWfbg6573UCj9",
+        "5q54XjQ7vDx4y6KphPeE97LUNiYGtP55spjvXAWPGBuf",
+    ]
+);
+
+// Withdraw authority for autostaked accounts on mainnet-beta
+solana_sdk::pubkeys!(
+    withdraw_authority,
+    "8CUUMKYNGxdgYio5CLHRHyzMEhhVRMcqefgE6dLqnVRK"
+);
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_sdk::{
+        account::Account, epoch_schedule::EpochSchedule, genesis_config::GenesisConfig,
+    };
+    use solana_stake_program::stake_state::{Authorized, Lockup, Meta, StakeState};
+    use std::{collections::BTreeMap, sync::Arc};
+
+    fn new_from_parent(parent: &Arc<Bank>) -> Bank {
+        Bank::new_from_parent(parent, &Pubkey::default(), parent.slot() + 1)
+    }
+
+    #[test]
+    fn test_calculate_non_circulating_supply() {
+        let mut accounts: BTreeMap<Pubkey, Account> = BTreeMap::new();
+        let balance = 10;
+        let num_genesis_accounts = 10;
+        for _ in 0..num_genesis_accounts {
+            accounts.insert(
+                Pubkey::new_rand(),
+                Account::new(balance, 0, &Pubkey::default()),
+            );
+        }
+        let non_circulating_accounts = non_circulating_accounts();
+        let num_non_circulating_accounts = non_circulating_accounts.len() as u64;
+        for key in non_circulating_accounts.clone() {
+            accounts.insert(key, Account::new(balance, 0, &Pubkey::default()));
+        }
+
+        let num_stake_accounts = 3;
+        for _ in 0..num_stake_accounts {
+            let pubkey = Pubkey::new_rand();
+            let meta = Meta {
+                authorized: Authorized::auto(&pubkey),
+                lockup: Lockup {
+                    epoch: 1,
+                    ..Lockup::default()
+                },
+                ..Meta::default()
+            };
+            let stake_account = Account::new_data_with_space(
+                balance,
+                &StakeState::Initialized(meta),
+                std::mem::size_of::<StakeState>(),
+                &solana_stake_program::id(),
+            )
+            .unwrap();
+            accounts.insert(pubkey, stake_account);
+        }
+
+        let slots_per_epoch = 32;
+        let genesis_config = GenesisConfig {
+            accounts,
+            epoch_schedule: EpochSchedule::new(slots_per_epoch),
+            ..GenesisConfig::default()
+        };
+        let mut bank = Arc::new(Bank::new(&genesis_config));
+        assert_eq!(
+            bank.capitalization(),
+            (num_genesis_accounts + num_non_circulating_accounts + num_stake_accounts) * balance
+        );
+
+        let non_circulating_supply = calculate_non_circulating_supply(bank.clone());
+        assert_eq!(
+            non_circulating_supply.lamports,
+            (num_non_circulating_accounts + num_stake_accounts) * balance
+        );
+        assert_eq!(
+            non_circulating_supply.accounts.len(),
+            num_non_circulating_accounts as usize + num_stake_accounts as usize
+        );
+
+        bank = Arc::new(new_from_parent(&bank));
+        let new_balance = 11;
+        for key in non_circulating_accounts {
+            bank.store_account(&key, &Account::new(new_balance, 0, &Pubkey::default()));
+        }
+        let non_circulating_supply = calculate_non_circulating_supply(bank.clone());
+        assert_eq!(
+            non_circulating_supply.lamports,
+            (num_non_circulating_accounts * new_balance) + (num_stake_accounts * balance)
+        );
+        assert_eq!(
+            non_circulating_supply.accounts.len(),
+            num_non_circulating_accounts as usize + num_stake_accounts as usize
+        );
+
+        // Advance bank an epoch, which should unlock stakes
+        for _ in 0..slots_per_epoch {
+            bank = Arc::new(new_from_parent(&bank));
+        }
+        assert_eq!(bank.epoch(), 1);
+        let non_circulating_supply = calculate_non_circulating_supply(bank.clone());
+        assert_eq!(
+            non_circulating_supply.lamports,
+            num_non_circulating_accounts * new_balance
+        );
+        assert_eq!(
+            non_circulating_supply.accounts.len(),
+            num_non_circulating_accounts as usize
+        );
+    }
+}

core/src/poh_service.rs

@@ -1,10 +1,8 @@
 //! The `poh_service` module implements a service that records the passing of
 //! "ticks", a measure of time in the PoH stream
 use crate::poh_recorder::PohRecorder;
-use core_affinity;
 use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;
 use solana_sdk::poh_config::PohConfig;
-use solana_sys_tuner;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread::{self, sleep, Builder, JoinHandle};

core/src/recvmmsg.rs

@@ -213,49 +213,4 @@ mod tests {
             assert_eq!(packets[i].meta.addr(), saddr2);
         }
     }
-
-    #[cfg(target_os = "linux")]
-    #[test]
-    pub fn test_recv_mmsg_batch_size() {
-        let reader = UdpSocket::bind("127.0.0.1:0").expect("bind");
-        let addr = reader.local_addr().unwrap();
-        let sender = UdpSocket::bind("127.0.0.1:0").expect("bind");
-
-        const TEST_BATCH_SIZE: usize = 64;
-        let sent = TEST_BATCH_SIZE;
-
-        let mut elapsed_in_max_batch = 0;
-        (0..1000).for_each(|_| {
-            for _ in 0..sent {
-                let data = [0; PACKET_DATA_SIZE];
-                sender.send_to(&data[..], &addr).unwrap();
-            }
-            let mut packets = vec![Packet::default(); TEST_BATCH_SIZE];
-            let now = Instant::now();
-            let recv = recv_mmsg(&reader, &mut packets[..]).unwrap().1;
-            elapsed_in_max_batch += now.elapsed().as_nanos();
-            assert_eq!(TEST_BATCH_SIZE, recv);
-        });
-
-        let mut elapsed_in_small_batch = 0;
-        (0..1000).for_each(|_| {
-            for _ in 0..sent {
-                let data = [0; PACKET_DATA_SIZE];
-                sender.send_to(&data[..], &addr).unwrap();
-            }
-            let mut packets = vec![Packet::default(); 4];
-            let mut recv = 0;
-            let now = Instant::now();
-            while let Ok(num) = recv_mmsg(&reader, &mut packets[..]) {
-                recv += num.1;
-                if recv >= TEST_BATCH_SIZE {
-                    break;
-                }
-            }
-            elapsed_in_small_batch += now.elapsed().as_nanos();
-            assert_eq!(TEST_BATCH_SIZE, recv);
-        });
-
-        assert!(elapsed_in_max_batch <= elapsed_in_small_batch);
-    }
 }

core/src/replay_stage.rs

@@ -11,6 +11,7 @@ use crate::{
 };
 use solana_ledger::{
     bank_forks::BankForks,
+    block_error::BlockError,
     blockstore::Blockstore,
     blockstore_processor::{
         self, BlockstoreProcessorError, ConfirmationProgress, ConfirmationTiming,
@@ -25,6 +26,7 @@ use solana_metrics::inc_new_counter_info;
 use solana_runtime::bank::Bank;
 use solana_sdk::{
     clock::Slot,
+    genesis_config::GenesisConfig,
     hash::Hash,
     pubkey::Pubkey,
     signature::{Keypair, Signer},
@@ -65,7 +67,6 @@ impl Drop for Finalizer {
     }
 }
 
-#[derive(Default)]
 pub struct ReplayStageConfig {
     pub my_pubkey: Pubkey,
     pub vote_account: Pubkey,
@@ -75,10 +76,11 @@ pub struct ReplayStageConfig {
     pub leader_schedule_cache: Arc<LeaderScheduleCache>,
     pub slot_full_senders: Vec<Sender<(u64, Pubkey)>>,
     pub latest_root_senders: Vec<Sender<Slot>>,
-    pub snapshot_package_sender: Option<SnapshotPackageSender>,
+    pub accounts_hash_sender: Option<SnapshotPackageSender>,
     pub block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
     pub transaction_status_sender: Option<TransactionStatusSender>,
     pub rewards_recorder_sender: Option<RewardsRecorderSender>,
+    pub genesis_config: GenesisConfig,
 }
 
 pub struct ReplayStage {
@@ -178,10 +180,11 @@ impl ReplayStage {
             leader_schedule_cache,
             slot_full_senders,
             latest_root_senders,
-            snapshot_package_sender,
+            accounts_hash_sender,
             block_commitment_cache,
             transaction_status_sender,
             rewards_recorder_sender,
+            genesis_config,
         } = config;
 
         let (root_bank_sender, root_bank_receiver) = channel();
@@ -191,7 +194,7 @@ impl ReplayStage {
         // Start the replay stage loop
 
         let (lockouts_sender, commitment_service) =
-            AggregateCommitmentService::new(&exit, block_commitment_cache);
+            AggregateCommitmentService::new(&exit, block_commitment_cache.clone());
 
         #[allow(clippy::cognitive_complexity)]
         let t_replay = Builder::new()
@@ -245,6 +248,7 @@ impl ReplayStage {
                         &slot_full_senders,
                         transaction_status_sender.clone(),
                         &verify_recyclers,
+                        &genesis_config,
                     );
                     datapoint_debug!(
                         "replay_stage-memory",
@@ -252,13 +256,15 @@ impl ReplayStage {
                     );
 
                     let ancestors = Arc::new(bank_forks.read().unwrap().ancestors());
+                    let forks_root = bank_forks.read().unwrap().root();
                     let start = allocated.get();
                     let mut frozen_banks: Vec<_> = bank_forks
                         .read()
                         .unwrap()
                         .frozen_banks()
-                        .values()
-                        .cloned()
+                        .into_iter()
+                        .filter(|(slot, _)| *slot >= forks_root)
+                        .map(|(_, bank)| bank)
                         .collect();
                     let newly_computed_slot_stats = Self::compute_bank_stats(
                         &my_pubkey,
@@ -308,7 +314,10 @@ impl ReplayStage {
                     let start = allocated.get();
                     if !is_locked_out && vote_threshold {
                         info!("voting: {} {}", bank.slot(), fork_weight);
-                        subscriptions.notify_subscribers(bank.slot(), &bank_forks);
+                        subscriptions.notify_subscribers(
+                            block_commitment_cache.read().unwrap().slot(),
+                            &bank_forks,
+                        );
                         if let Some(votable_leader) =
                             leader_schedule_cache.slot_leader_at(bank.slot(), Some(&bank))
                         {
@@ -333,8 +342,10 @@ impl ReplayStage {
                             &root_bank_sender,
                             total_staked,
                             &lockouts_sender,
-                            &snapshot_package_sender,
+                            &accounts_hash_sender,
                             &latest_root_senders,
+                            &subscriptions,
+                            &block_commitment_cache,
                         )?;
                     }
                     datapoint_debug!(
@@ -395,7 +406,8 @@ impl ReplayStage {
                             rewards_recorder_sender.clone(),
                         );
 
-                        if let Some(bank) = poh_recorder.lock().unwrap().bank() {
+                        let poh_bank = poh_recorder.lock().unwrap().bank();
+                        if let Some(bank) = poh_bank {
                             Self::log_leader_change(
                                 &my_pubkey,
                                 bank.slot(),
@@ -545,6 +557,7 @@ impl ReplayStage {
         bank_progress: &mut ForkProgress,
         transaction_status_sender: Option<TransactionStatusSender>,
         verify_recyclers: &VerifyRecyclers,
+        genesis_config: &GenesisConfig,
     ) -> result::Result<usize, BlockstoreProcessorError> {
         let tx_count_before = bank_progress.replay_progress.num_txs;
         let confirm_result = blockstore_processor::confirm_slot(
@@ -556,6 +569,7 @@ impl ReplayStage {
             transaction_status_sender,
             None,
             verify_recyclers,
+            Some(genesis_config),
         );
         let tx_count_after = bank_progress.replay_progress.num_txs;
         let tx_count = tx_count_after - tx_count_before;
@@ -566,11 +580,19 @@ impl ReplayStage {
             // errors related to the slot being purged
             let slot = bank.slot();
             warn!("Fatal replay error in slot: {}, err: {:?}", slot, err);
-            datapoint_error!(
-                "replay-stage-mark_dead_slot",
-                ("error", format!("error: {:?}", err), String),
-                ("slot", slot, i64)
-            );
+            if let BlockstoreProcessorError::InvalidBlock(BlockError::InvalidTickCount) = err {
+                datapoint_info!(
+                    "replay-stage-mark_dead_slot",
+                    ("error", format!("error: {:?}", err), String),
+                    ("slot", slot, i64)
+                );
+            } else {
+                datapoint_error!(
+                    "replay-stage-mark_dead_slot",
+                    ("error", format!("error: {:?}", err), String),
+                    ("slot", slot, i64)
+                );
+            }
             bank_progress.is_dead = true;
             blockstore
                 .set_dead_slot(slot)
@@ -595,8 +617,10 @@ impl ReplayStage {
         root_bank_sender: &Sender<Vec<Arc<Bank>>>,
         total_staked: u64,
         lockouts_sender: &Sender<CommitmentAggregationData>,
-        snapshot_package_sender: &Option<SnapshotPackageSender>,
+        accounts_hash_sender: &Option<SnapshotPackageSender>,
         latest_root_senders: &[Sender<Slot>],
+        subscriptions: &Arc<RpcSubscriptions>,
+        block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
     ) -> Result<()> {
         if bank.is_empty() {
             inc_new_counter_info!("replay_stage-voted_empty_bank", 1);
@@ -622,7 +646,21 @@ impl ReplayStage {
             blockstore
                 .set_roots(&rooted_slots)
                 .expect("Ledger set roots failed");
-            Self::handle_new_root(new_root, &bank_forks, progress, snapshot_package_sender);
+            let largest_confirmed_root = Some(
+                block_commitment_cache
+                    .read()
+                    .unwrap()
+                    .largest_confirmed_root(),
+            );
+
+            Self::handle_new_root(
+                new_root,
+                &bank_forks,
+                progress,
+                accounts_hash_sender,
+                largest_confirmed_root,
+            );
+            subscriptions.notify_roots(rooted_slots);
             latest_root_senders.iter().for_each(|s| {
                 if let Err(e) = s.send(new_root) {
                     trace!("latest root send failed: {:?}", e);
@@ -634,7 +672,13 @@ impl ReplayStage {
                 return Err(e.into());
             }
         }
-        Self::update_commitment_cache(bank.clone(), total_staked, lockouts_sender);
+
+        Self::update_commitment_cache(
+            bank.clone(),
+            bank_forks.read().unwrap().root(),
+            total_staked,
+            lockouts_sender,
+        );
 
         if let Some(ref voting_keypair) = voting_keypair {
             let node_keypair = cluster_info.read().unwrap().keypair.clone();
@@ -662,10 +706,13 @@ impl ReplayStage {
 
     fn update_commitment_cache(
         bank: Arc<Bank>,
+        root: Slot,
         total_staked: u64,
         lockouts_sender: &Sender<CommitmentAggregationData>,
     ) {
-        if let Err(e) = lockouts_sender.send(CommitmentAggregationData::new(bank, total_staked)) {
+        if let Err(e) =
+            lockouts_sender.send(CommitmentAggregationData::new(bank, root, total_staked))
+        {
             trace!("lockouts_sender failed: {:?}", e);
         }
     }
@@ -712,6 +759,7 @@ impl ReplayStage {
         slot_full_senders: &[Sender<(u64, Pubkey)>],
         transaction_status_sender: Option<TransactionStatusSender>,
         verify_recyclers: &VerifyRecyclers,
+        genesis_config: &GenesisConfig,
     ) -> bool {
         let mut did_complete_bank = false;
         let mut tx_count = 0;
@@ -740,6 +788,7 @@ impl ReplayStage {
                     bank_progress,
                     transaction_status_sender.clone(),
                     verify_recyclers,
+                    genesis_config,
                 );
                 match replay_result {
                     Ok(replay_tx_count) => tx_count += replay_tx_count,
@@ -946,15 +995,17 @@ impl ReplayStage {
     }
 
     pub(crate) fn handle_new_root(
-        new_root: u64,
+        new_root: Slot,
         bank_forks: &RwLock<BankForks>,
         progress: &mut HashMap<u64, ForkProgress>,
-        snapshot_package_sender: &Option<SnapshotPackageSender>,
+        accounts_hash_sender: &Option<SnapshotPackageSender>,
+        largest_confirmed_root: Option<Slot>,
     ) {
-        bank_forks
-            .write()
-            .unwrap()
-            .set_root(new_root, snapshot_package_sender);
+        bank_forks.write().unwrap().set_root(
+            new_root,
+            accounts_hash_sender,
+            largest_confirmed_root,
+        );
         let r_bank_forks = bank_forks.read().unwrap();
         progress.retain(|k, _| r_bank_forks.get(*k).is_some());
     }
@@ -983,7 +1034,11 @@ impl ReplayStage {
         // Find the next slot that chains to the old slot
         let forks = forks_lock.read().unwrap();
         let frozen_banks = forks.frozen_banks();
-        let frozen_bank_slots: Vec<u64> = frozen_banks.keys().cloned().collect();
+        let frozen_bank_slots: Vec<u64> = frozen_banks
+            .keys()
+            .cloned()
+            .filter(|s| *s >= forks.root())
+            .collect();
         let next_slots = blockstore
             .get_slots_since(&frozen_bank_slots)
             .expect("Db error");
@@ -1056,9 +1111,7 @@ pub(crate) mod tests {
         transaction_status_service::TransactionStatusService,
     };
     use crossbeam_channel::unbounded;
-    use solana_client::rpc_response::{RpcEncodedTransaction, RpcTransactionWithStatusMeta};
     use solana_ledger::{
-        block_error::BlockError,
         blockstore::make_slot_entries,
         blockstore::{entries_to_test_shreds, BlockstoreError},
         create_new_tmp_ledger,
@@ -1081,6 +1134,7 @@ pub(crate) mod tests {
         transaction::TransactionError,
     };
     use solana_stake_program::stake_state;
+    use solana_transaction_status::{EncodedTransaction, TransactionWithStatusMeta};
     use solana_vote_program::{
         vote_state::{self, Vote, VoteState, VoteStateVersions},
         vote_transaction,
@@ -1385,7 +1439,12 @@ pub(crate) mod tests {
             let bank0 = Bank::new(&genesis_config);
             let leader_schedule_cache = Arc::new(LeaderScheduleCache::new_from_bank(&bank0));
             let exit = Arc::new(AtomicBool::new(false));
-            let subscriptions = Arc::new(RpcSubscriptions::new(&exit));
+            let subscriptions = Arc::new(RpcSubscriptions::new(
+                &exit,
+                Arc::new(RwLock::new(BlockCommitmentCache::default_with_blockstore(
+                    blockstore.clone(),
+                ))),
+            ));
             let bank_forks = BankForks::new(0, bank0);
             bank_forks.working_bank().freeze();
 
@@ -1437,12 +1496,58 @@ pub(crate) mod tests {
         for i in 0..=root {
             progress.insert(i, ForkProgress::new(Hash::default()));
         }
-        ReplayStage::handle_new_root(root, &bank_forks, &mut progress, &None);
+        ReplayStage::handle_new_root(root, &bank_forks, &mut progress, &None, None);
         assert_eq!(bank_forks.read().unwrap().root(), root);
         assert_eq!(progress.len(), 1);
         assert!(progress.get(&root).is_some());
     }
 
+    #[test]
+    fn test_handle_new_root_ahead_of_largest_confirmed_root() {
+        let genesis_config = create_genesis_config(10_000).genesis_config;
+        let bank0 = Bank::new(&genesis_config);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank0)));
+        let confirmed_root = 1;
+        let fork = 2;
+        let bank1 = Bank::new_from_parent(
+            bank_forks.read().unwrap().get(0).unwrap(),
+            &Pubkey::default(),
+            confirmed_root,
+        );
+        bank_forks.write().unwrap().insert(bank1);
+        let bank2 = Bank::new_from_parent(
+            bank_forks.read().unwrap().get(confirmed_root).unwrap(),
+            &Pubkey::default(),
+            fork,
+        );
+        bank_forks.write().unwrap().insert(bank2);
+        let root = 3;
+        let root_bank = Bank::new_from_parent(
+            bank_forks.read().unwrap().get(confirmed_root).unwrap(),
+            &Pubkey::default(),
+            root,
+        );
+        bank_forks.write().unwrap().insert(root_bank);
+        let mut progress = HashMap::new();
+        for i in 0..=root {
+            progress.insert(i, ForkProgress::new(Hash::default()));
+        }
+        ReplayStage::handle_new_root(
+            root,
+            &bank_forks,
+            &mut progress,
+            &None,
+            Some(confirmed_root),
+        );
+        assert_eq!(bank_forks.read().unwrap().root(), root);
+        assert!(bank_forks.read().unwrap().get(confirmed_root).is_some());
+        assert!(bank_forks.read().unwrap().get(fork).is_none());
+        assert_eq!(progress.len(), 2);
+        assert!(progress.get(&root).is_some());
+        assert!(progress.get(&confirmed_root).is_some());
+        assert!(progress.get(&fork).is_none());
+    }
+
     #[test]
     fn test_dead_fork_transaction_error() {
         let keypair1 = Keypair::new();
@@ -1682,6 +1787,7 @@ pub(crate) mod tests {
                 &mut bank0_progress,
                 None,
                 &VerifyRecyclers::default(),
+                &genesis_config,
             );
 
             // Check that the erroring bank was marked as dead in the progress map
@@ -1709,7 +1815,11 @@ pub(crate) mod tests {
             bank.store_account(&pubkey, &leader_vote_account);
         }
 
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::default_with_blockstore(blockstore.clone()),
+        ));
         let (lockouts_sender, _) = AggregateCommitmentService::new(
             &Arc::new(AtomicBool::new(false)),
             block_commitment_cache.clone(),
@@ -1754,7 +1864,12 @@ pub(crate) mod tests {
         bank_forks.write().unwrap().insert(bank1);
         let arc_bank1 = bank_forks.read().unwrap().get(1).unwrap().clone();
         leader_vote(&arc_bank1, &leader_voting_pubkey);
-        ReplayStage::update_commitment_cache(arc_bank1.clone(), leader_lamports, &lockouts_sender);
+        ReplayStage::update_commitment_cache(
+            arc_bank1.clone(),
+            0,
+            leader_lamports,
+            &lockouts_sender,
+        );
 
         let bank2 = Bank::new_from_parent(&arc_bank1, &Pubkey::default(), arc_bank1.slot() + 1);
         let _res = bank2.transfer(10, &genesis_config_info.mint_keypair, &Pubkey::new_rand());
@@ -1765,7 +1880,12 @@ pub(crate) mod tests {
         bank_forks.write().unwrap().insert(bank2);
         let arc_bank2 = bank_forks.read().unwrap().get(2).unwrap().clone();
         leader_vote(&arc_bank2, &leader_voting_pubkey);
-        ReplayStage::update_commitment_cache(arc_bank2.clone(), leader_lamports, &lockouts_sender);
+        ReplayStage::update_commitment_cache(
+            arc_bank2.clone(),
+            0,
+            leader_lamports,
+            &lockouts_sender,
+        );
         thread::sleep(Duration::from_millis(200));
 
         let mut expected0 = BlockCommitment::default();
@@ -1890,18 +2010,28 @@ pub(crate) mod tests {
             let confirmed_block = blockstore.get_confirmed_block(slot, None).unwrap();
             assert_eq!(confirmed_block.transactions.len(), 3);
 
-            for RpcTransactionWithStatusMeta { transaction, meta } in
+            for TransactionWithStatusMeta { transaction, meta } in
                 confirmed_block.transactions.into_iter()
             {
-                if let RpcEncodedTransaction::Json(transaction) = transaction {
+                if let EncodedTransaction::Json(transaction) = transaction {
                     if transaction.signatures[0] == signatures[0].to_string() {
-                        assert_eq!(meta.unwrap().status, Ok(()));
+                        let meta = meta.unwrap();
+                        assert_eq!(meta.err, None);
+                        assert_eq!(meta.status, Ok(()));
                     } else if transaction.signatures[0] == signatures[1].to_string() {
+                        let meta = meta.unwrap();
                         assert_eq!(
-                            meta.unwrap().status,
+                            meta.err,
+                            Some(TransactionError::InstructionError(
+                                0,
+                                InstructionError::Custom(1)
+                            ))
+                        );
+                        assert_eq!(
+                            meta.status,
                             Err(TransactionError::InstructionError(
                                 0,
-                                InstructionError::CustomError(1)
+                                InstructionError::Custom(1)
                             ))
                         );
                     } else {

core/src/rewards_recorder_service.rs

@@ -1,7 +1,7 @@
 use crossbeam_channel::{Receiver, RecvTimeoutError, Sender};
-use solana_client::rpc_response::RpcReward;
 use solana_ledger::blockstore::Blockstore;
 use solana_sdk::{clock::Slot, pubkey::Pubkey};
+use solana_transaction_status::Reward;
 use std::{
     sync::{
         atomic::{AtomicBool, Ordering},
@@ -49,7 +49,7 @@ impl RewardsRecorderService {
         let (slot, rewards) = rewards_receiver.recv_timeout(Duration::from_secs(1))?;
         let rpc_rewards = rewards
             .into_iter()
-            .map(|(pubkey, lamports)| RpcReward {
+            .map(|(pubkey, lamports)| Reward {
                 pubkey: pubkey.to_string(),
                 lamports,
             })

core/src/rpc_pubsub.rs

@@ -4,9 +4,16 @@ use crate::rpc_subscriptions::{Confirmations, RpcSubscriptions, SlotInfo};
 use jsonrpc_core::{Error, ErrorCode, Result};
 use jsonrpc_derive::rpc;
 use jsonrpc_pubsub::{typed::Subscriber, Session, SubscriptionId};
-use solana_client::rpc_response::{RpcAccount, RpcKeyedAccount};
-use solana_sdk::{pubkey::Pubkey, signature::Signature, transaction};
-use std::sync::{atomic, Arc};
+use solana_client::rpc_response::{
+    Response as RpcResponse, RpcAccount, RpcKeyedAccount, RpcSignatureResult,
+};
+#[cfg(test)]
+use solana_ledger::blockstore::Blockstore;
+use solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature};
+use std::{
+    str::FromStr,
+    sync::{atomic, Arc},
+};
 
 // Suppress needless_return due to
 //   https://github.com/paritytech/jsonrpc/blob/2d38e6424d8461cdf72e78425ce67d51af9c6586/derive/src/lib.rs#L204
@@ -26,7 +33,7 @@ pub trait RpcSolPubSub {
     fn account_subscribe(
         &self,
         meta: Self::Metadata,
-        subscriber: Subscriber<RpcAccount>,
+        subscriber: Subscriber<RpcResponse<RpcAccount>>,
         pubkey_str: String,
         confirmations: Option<Confirmations>,
     );
@@ -50,7 +57,7 @@ pub trait RpcSolPubSub {
     fn program_subscribe(
         &self,
         meta: Self::Metadata,
-        subscriber: Subscriber<RpcKeyedAccount>,
+        subscriber: Subscriber<RpcResponse<RpcKeyedAccount>>,
         pubkey_str: String,
         confirmations: Option<Confirmations>,
     );
@@ -74,7 +81,7 @@ pub trait RpcSolPubSub {
     fn signature_subscribe(
         &self,
         meta: Self::Metadata,
-        subscriber: Subscriber<transaction::Result<()>>,
+        subscriber: Subscriber<RpcResponse<RpcSignatureResult>>,
         signature_str: String,
         confirmations: Option<Confirmations>,
     );
@@ -102,9 +109,20 @@ pub trait RpcSolPubSub {
         name = "slotUnsubscribe"
     )]
     fn slot_unsubscribe(&self, meta: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool>;
+
+    // Get notification when a new root is set
+    #[pubsub(subscription = "rootNotification", subscribe, name = "rootSubscribe")]
+    fn root_subscribe(&self, meta: Self::Metadata, subscriber: Subscriber<Slot>);
+
+    // Unsubscribe from slot notification subscription.
+    #[pubsub(
+        subscription = "rootNotification",
+        unsubscribe,
+        name = "rootUnsubscribe"
+    )]
+    fn root_unsubscribe(&self, meta: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool>;
 }
 
-#[derive(Default)]
 pub struct RpcSolPubSubImpl {
     uid: Arc<atomic::AtomicUsize>,
     subscriptions: Arc<RpcSubscriptions>,
@@ -115,9 +133,14 @@ impl RpcSolPubSubImpl {
         let uid = Arc::new(atomic::AtomicUsize::default());
         Self { uid, subscriptions }
     }
-}
 
-use std::str::FromStr;
+    #[cfg(test)]
+    fn default_with_blockstore(blockstore: Arc<Blockstore>) -> Self {
+        let uid = Arc::new(atomic::AtomicUsize::default());
+        let subscriptions = Arc::new(RpcSubscriptions::default_with_blockstore(blockstore));
+        Self { uid, subscriptions }
+    }
+}
 
 fn param<T: FromStr>(param_str: &str, thing: &str) -> Result<T> {
     param_str.parse::<T>().map_err(|_e| Error {
@@ -133,7 +156,7 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
     fn account_subscribe(
         &self,
         _meta: Self::Metadata,
-        subscriber: Subscriber<RpcAccount>,
+        subscriber: Subscriber<RpcResponse<RpcAccount>>,
         pubkey_str: String,
         confirmations: Option<Confirmations>,
     ) {
@@ -142,10 +165,12 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
                 let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
                 let sub_id = SubscriptionId::Number(id as u64);
                 info!("account_subscribe: account={:?} id={:?}", pubkey, sub_id);
-                let sink = subscriber.assign_id(sub_id.clone()).unwrap();
-
-                self.subscriptions
-                    .add_account_subscription(&pubkey, confirmations, &sub_id, &sink)
+                self.subscriptions.add_account_subscription(
+                    pubkey,
+                    confirmations,
+                    sub_id,
+                    subscriber,
+                )
             }
             Err(e) => subscriber.reject(e).unwrap(),
         }
@@ -171,7 +196,7 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
     fn program_subscribe(
         &self,
         _meta: Self::Metadata,
-        subscriber: Subscriber<RpcKeyedAccount>,
+        subscriber: Subscriber<RpcResponse<RpcKeyedAccount>>,
         pubkey_str: String,
         confirmations: Option<Confirmations>,
     ) {
@@ -180,10 +205,12 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
                 let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
                 let sub_id = SubscriptionId::Number(id as u64);
                 info!("program_subscribe: account={:?} id={:?}", pubkey, sub_id);
-                let sink = subscriber.assign_id(sub_id.clone()).unwrap();
-
-                self.subscriptions
-                    .add_program_subscription(&pubkey, confirmations, &sub_id, &sink)
+                self.subscriptions.add_program_subscription(
+                    pubkey,
+                    confirmations,
+                    sub_id,
+                    subscriber,
+                )
             }
             Err(e) => subscriber.reject(e).unwrap(),
         }
@@ -209,7 +236,7 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
     fn signature_subscribe(
         &self,
         _meta: Self::Metadata,
-        subscriber: Subscriber<transaction::Result<()>>,
+        subscriber: Subscriber<RpcResponse<RpcSignatureResult>>,
         signature_str: String,
         confirmations: Option<Confirmations>,
     ) {
@@ -222,13 +249,11 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
                     "signature_subscribe: signature={:?} id={:?}",
                     signature, sub_id
                 );
-                let sink = subscriber.assign_id(sub_id.clone()).unwrap();
-
                 self.subscriptions.add_signature_subscription(
-                    &signature,
+                    signature,
                     confirmations,
-                    &sub_id,
-                    &sink,
+                    sub_id,
+                    subscriber,
                 );
             }
             Err(e) => subscriber.reject(e).unwrap(),
@@ -257,9 +282,7 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
         let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
         let sub_id = SubscriptionId::Number(id as u64);
         info!("slot_subscribe: id={:?}", sub_id);
-        let sink = subscriber.assign_id(sub_id.clone()).unwrap();
-
-        self.subscriptions.add_slot_subscription(&sub_id, &sink);
+        self.subscriptions.add_slot_subscription(sub_id, subscriber);
     }
 
     fn slot_unsubscribe(&self, _meta: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool> {
@@ -274,17 +297,42 @@ impl RpcSolPubSub for RpcSolPubSubImpl {
             })
         }
     }
+
+    fn root_subscribe(&self, _meta: Self::Metadata, subscriber: Subscriber<Slot>) {
+        info!("root_subscribe");
+        let id = self.uid.fetch_add(1, atomic::Ordering::Relaxed);
+        let sub_id = SubscriptionId::Number(id as u64);
+        info!("root_subscribe: id={:?}", sub_id);
+        self.subscriptions.add_root_subscription(sub_id, subscriber);
+    }
+
+    fn root_unsubscribe(&self, _meta: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool> {
+        info!("root_unsubscribe");
+        if self.subscriptions.remove_root_subscription(&id) {
+            Ok(true)
+        } else {
+            Err(Error {
+                code: ErrorCode::InvalidParams,
+                message: "Invalid Request: Subscription id does not exist".into(),
+                data: None,
+            })
+        }
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-    use crate::rpc_subscriptions::tests::robust_poll_or_panic;
+    use crate::{
+        commitment::{BlockCommitment, BlockCommitmentCache},
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        rpc_subscriptions::tests::robust_poll_or_panic,
+    };
     use jsonrpc_core::{futures::sync::mpsc, Response};
     use jsonrpc_pubsub::{PubSubHandler, Session};
+    use serial_test_derive::serial;
     use solana_budget_program::{self, budget_instruction};
-    use solana_ledger::bank_forks::BankForks;
+    use solana_ledger::{bank_forks::BankForks, get_tmp_ledger_path};
     use solana_runtime::bank::Bank;
     use solana_sdk::{
         pubkey::Pubkey,
@@ -292,7 +340,12 @@ mod tests {
         system_program, system_transaction,
         transaction::{self, Transaction},
     };
-    use std::{sync::RwLock, thread::sleep, time::Duration};
+    use std::{
+        collections::HashMap,
+        sync::{atomic::AtomicBool, RwLock},
+        thread::sleep,
+        time::Duration,
+    };
 
     fn process_transaction_and_notify(
         bank_forks: &Arc<RwLock<BankForks>>,
@@ -314,6 +367,7 @@ mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_signature_subscribe() {
         let GenesisConfigInfo {
             genesis_config,
@@ -325,8 +379,17 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
-
-        let rpc = RpcSolPubSubImpl::default();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let rpc = RpcSolPubSubImpl {
+            subscriptions: Arc::new(RpcSubscriptions::new(
+                &Arc::new(AtomicBool::new(false)),
+                Arc::new(RwLock::new(
+                    BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+                )),
+            )),
+            uid: Arc::new(atomic::AtomicUsize::default()),
+        };
 
         // Test signature subscriptions
         let tx = system_transaction::transfer(&alice, &bob_pubkey, 20, blockhash);
@@ -338,18 +401,24 @@ mod tests {
         process_transaction_and_notify(&bank_forks, &tx, &rpc.subscriptions).unwrap();
 
         // Test signature confirmation notification
-        let response = robust_poll_or_panic(receiver);
-        let expected_res: Option<transaction::Result<()>> = Some(Ok(()));
-        let expected_res_str =
-            serde_json::to_string(&serde_json::to_value(expected_res).unwrap()).unwrap();
-        let expected = format!(
-            r#"{{"jsonrpc":"2.0","method":"signatureNotification","params":{{"result":{},"subscription":0}}}}"#,
-            expected_res_str
-        );
-        assert_eq!(expected, response);
+        let (response, _) = robust_poll_or_panic(receiver);
+        let expected_res = RpcSignatureResult { err: None };
+        let expected = json!({
+           "jsonrpc": "2.0",
+           "method": "signatureNotification",
+           "params": {
+               "result": {
+                   "context": { "slot": 0 },
+                   "value": expected_res,
+               },
+               "subscription": 0,
+           }
+        });
+        assert_eq!(serde_json::to_string(&expected).unwrap(), response);
     }
 
     #[test]
+    #[serial]
     fn test_signature_unsubscribe() {
         let GenesisConfigInfo {
             genesis_config,
@@ -360,11 +429,13 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let arc_bank = Arc::new(bank);
         let blockhash = arc_bank.last_blockhash();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
         let session = create_session();
 
         let mut io = PubSubHandler::default();
-        let rpc = RpcSolPubSubImpl::default();
+        let rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore);
         io.extend_with(rpc.to_delegate());
 
         let tx = system_transaction::transfer(&alice, &bob_pubkey, 20, blockhash);
@@ -398,6 +469,7 @@ mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_account_subscribe() {
         let GenesisConfigInfo {
             mut genesis_config,
@@ -418,8 +490,18 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
-        let rpc = RpcSolPubSubImpl::default();
+        let rpc = RpcSolPubSubImpl {
+            subscriptions: Arc::new(RpcSubscriptions::new(
+                &Arc::new(AtomicBool::new(false)),
+                Arc::new(RwLock::new(
+                    BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+                )),
+            )),
+            uid: Arc::new(atomic::AtomicUsize::default()),
+        };
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification");
         rpc.account_subscribe(
@@ -462,17 +544,20 @@ mod tests {
            "method": "accountNotification",
            "params": {
                "result": {
-                   "owner": budget_program_id.to_string(),
-                   "lamports": 51,
-                   "data": bs58::encode(expected_data).into_string(),
-                   "executable": false,
-                   "rentEpoch": 1,
+                   "context": { "slot": 0 },
+                   "value": {
+                       "owner": budget_program_id.to_string(),
+                       "lamports": 51,
+                       "data": bs58::encode(expected_data).into_string(),
+                       "executable": false,
+                       "rentEpoch": 1,
+                   },
                },
                "subscription": 0,
            }
         });
 
-        let response = robust_poll_or_panic(receiver);
+        let (response, _) = robust_poll_or_panic(receiver);
         assert_eq!(serde_json::to_string(&expected).unwrap(), response);
 
         let tx = system_transaction::transfer(&alice, &witness.pubkey(), 1, blockhash);
@@ -499,12 +584,15 @@ mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_account_unsubscribe() {
         let bob_pubkey = Pubkey::new_rand();
         let session = create_session();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
         let mut io = PubSubHandler::default();
-        let rpc = RpcSolPubSubImpl::default();
+        let rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore);
 
         io.extend_with(rpc.to_delegate());
 
@@ -548,9 +636,19 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bob = Keypair::new();
 
-        let rpc = RpcSolPubSubImpl::default();
+        let mut rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore.clone());
+        let exit = Arc::new(AtomicBool::new(false));
+        let subscriptions = RpcSubscriptions::new(
+            &exit,
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
+        );
+        rpc.subscriptions = Arc::new(subscriptions);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification");
         rpc.account_subscribe(session, subscriber, bob.pubkey().to_string(), Some(2));
@@ -579,9 +677,18 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bob = Keypair::new();
 
-        let rpc = RpcSolPubSubImpl::default();
+        let mut rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore.clone());
+        let exit = Arc::new(AtomicBool::new(false));
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_blockstore(blockstore.clone()),
+        ));
+
+        let subscriptions = RpcSubscriptions::new(&exit, block_commitment_cache.clone());
+        rpc.subscriptions = Arc::new(subscriptions);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification");
         rpc.account_subscribe(session, subscriber, bob.pubkey().to_string(), Some(2));
@@ -599,39 +706,74 @@ mod tests {
         let bank0 = bank_forks.read().unwrap()[0].clone();
         let bank1 = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
         bank_forks.write().unwrap().insert(bank1);
-        rpc.subscriptions.notify_subscribers(1, &bank_forks);
         let bank1 = bank_forks.read().unwrap()[1].clone();
+
+        let mut cache0 = BlockCommitment::default();
+        cache0.increase_confirmation_stake(1, 10);
+        let mut block_commitment = HashMap::new();
+        block_commitment.entry(0).or_insert(cache0.clone());
+        let mut new_block_commitment = BlockCommitmentCache::new(
+            block_commitment,
+            0,
+            10,
+            bank1.clone(),
+            blockstore.clone(),
+            0,
+        );
+        let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
+        std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
+        drop(w_block_commitment_cache);
+
+        rpc.subscriptions.notify_subscribers(1, &bank_forks);
         let bank2 = Bank::new_from_parent(&bank1, &Pubkey::default(), 2);
         bank_forks.write().unwrap().insert(bank2);
+        let bank2 = bank_forks.read().unwrap()[2].clone();
+
+        let mut cache0 = BlockCommitment::default();
+        cache0.increase_confirmation_stake(2, 10);
+        let mut block_commitment = HashMap::new();
+        block_commitment.entry(0).or_insert(cache0.clone());
+        let mut new_block_commitment =
+            BlockCommitmentCache::new(block_commitment, 0, 10, bank2, blockstore.clone(), 0);
+        let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
+        std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
+        drop(w_block_commitment_cache);
+
         rpc.subscriptions.notify_subscribers(2, &bank_forks);
         let expected = json!({
            "jsonrpc": "2.0",
            "method": "accountNotification",
            "params": {
                "result": {
-                   "owner": system_program::id().to_string(),
-                   "lamports": 100,
-                   "data": "",
-                   "executable": false,
-                   "rentEpoch": 1,
+                   "context": { "slot": 0 },
+                   "value": {
+                       "owner": system_program::id().to_string(),
+                       "lamports": 100,
+                       "data": "",
+                       "executable": false,
+                       "rentEpoch": 1,
+                   },
                },
                "subscription": 0,
            }
         });
-        let response = robust_poll_or_panic(receiver);
+        let (response, _) = robust_poll_or_panic(receiver);
         assert_eq!(serde_json::to_string(&expected).unwrap(), response);
     }
 
     #[test]
+    #[serial]
     fn test_slot_subscribe() {
-        let rpc = RpcSolPubSubImpl::default();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification");
         rpc.slot_subscribe(session, subscriber);
 
         rpc.subscriptions.notify_slot(0, 0, 0);
         // Test slot confirmation notification
-        let response = robust_poll_or_panic(receiver);
+        let (response, _) = robust_poll_or_panic(receiver);
         let expected_res = SlotInfo {
             parent: 0,
             slot: 0,
@@ -647,13 +789,16 @@ mod tests {
     }
 
     #[test]
+    #[serial]
     fn test_slot_unsubscribe() {
-        let rpc = RpcSolPubSubImpl::default();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification");
         rpc.slot_subscribe(session, subscriber);
         rpc.subscriptions.notify_slot(0, 0, 0);
-        let response = robust_poll_or_panic(receiver);
+        let (response, _) = robust_poll_or_panic(receiver);
         let expected_res = SlotInfo {
             parent: 0,
             slot: 0,

core/src/rpc_pubsub_service.rs

@@ -1,14 +1,20 @@
 //! The `pubsub` module implements a threaded subscription service on client RPC request
 
-use crate::rpc_pubsub::{RpcSolPubSub, RpcSolPubSubImpl};
-use crate::rpc_subscriptions::RpcSubscriptions;
+use crate::{
+    rpc_pubsub::{RpcSolPubSub, RpcSolPubSubImpl},
+    rpc_subscriptions::RpcSubscriptions,
+};
 use jsonrpc_pubsub::{PubSubHandler, Session};
 use jsonrpc_ws_server::{RequestContext, ServerBuilder};
-use std::net::SocketAddr;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
-use std::thread::{self, sleep, Builder, JoinHandle};
-use std::time::Duration;
+use std::{
+    net::SocketAddr,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    thread::{self, sleep, Builder, JoinHandle},
+    time::Duration,
+};
 
 pub struct PubSubService {
     thread_hdl: JoinHandle<()>,
@@ -66,13 +72,25 @@ impl PubSubService {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::net::{IpAddr, Ipv4Addr};
+    use crate::commitment::BlockCommitmentCache;
+    use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
+    use std::{
+        net::{IpAddr, Ipv4Addr},
+        sync::RwLock,
+    };
 
     #[test]
     fn test_pubsub_new() {
         let pubsub_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0);
         let exit = Arc::new(AtomicBool::new(false));
-        let subscriptions = Arc::new(RpcSubscriptions::new(&exit));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let subscriptions = Arc::new(RpcSubscriptions::new(
+            &exit,
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
+        ));
         let pubsub_service = PubSubService::new(&subscriptions, pubsub_addr, &exit);
         let thread = pubsub_service.thread_hdl.thread();
         assert_eq!(thread.name().unwrap(), "solana-pubsub");

core/src/rpc_service.rs

@@ -15,8 +15,9 @@ use solana_ledger::{
     blockstore::Blockstore,
     snapshot_utils,
 };
-use solana_sdk::hash::Hash;
+use solana_sdk::{hash::Hash, pubkey::Pubkey};
 use std::{
+    collections::HashSet,
     net::SocketAddr,
     path::{Path, PathBuf},
     sync::{mpsc::channel, Arc, RwLock},
@@ -24,6 +25,10 @@ use std::{
 };
 use tokio::prelude::Future;
 
+// If trusted validators are specified, consider this validator healthy if its latest account hash
+// is no further behind than this distance from the latest trusted validator account hash
+const HEALTH_CHECK_SLOT_DISTANCE: u64 = 150;
+
 pub struct JsonRpcService {
     thread_hdl: JoinHandle<()>,
 
@@ -37,15 +42,24 @@ struct RpcRequestMiddleware {
     ledger_path: PathBuf,
     snapshot_archive_path_regex: Regex,
     snapshot_config: Option<SnapshotConfig>,
+    cluster_info: Arc<RwLock<ClusterInfo>>,
+    trusted_validators: Option<HashSet<Pubkey>>,
 }
 
 impl RpcRequestMiddleware {
-    pub fn new(ledger_path: PathBuf, snapshot_config: Option<SnapshotConfig>) -> Self {
+    pub fn new(
+        ledger_path: PathBuf,
+        snapshot_config: Option<SnapshotConfig>,
+        cluster_info: Arc<RwLock<ClusterInfo>>,
+        trusted_validators: Option<HashSet<Pubkey>>,
+    ) -> Self {
         Self {
             ledger_path,
             snapshot_archive_path_regex: Regex::new(r"/snapshot-\d+-[[:alnum:]]+\.tar\.bz2$")
                 .unwrap(),
             snapshot_config,
+            cluster_info,
+            trusted_validators,
         }
     }
 
@@ -104,6 +118,58 @@ impl RpcRequestMiddleware {
             ),
         }
     }
+
+    fn health_check(&self) -> &'static str {
+        let response = if let Some(trusted_validators) = &self.trusted_validators {
+            let (latest_account_hash_slot, latest_trusted_validator_account_hash_slot) = {
+                let cluster_info = self.cluster_info.read().unwrap();
+                (
+                    cluster_info
+                        .get_accounts_hash_for_node(&cluster_info.id())
+                        .map(|hashes| hashes.iter().max_by(|a, b| a.0.cmp(&b.0)))
+                        .flatten()
+                        .map(|slot_hash| slot_hash.0)
+                        .unwrap_or(0),
+                    trusted_validators
+                        .iter()
+                        .map(|trusted_validator| {
+                            cluster_info
+                                .get_accounts_hash_for_node(&trusted_validator)
+                                .map(|hashes| hashes.iter().max_by(|a, b| a.0.cmp(&b.0)))
+                                .flatten()
+                                .map(|slot_hash| slot_hash.0)
+                                .unwrap_or(0)
+                        })
+                        .max()
+                        .unwrap_or(0),
+                )
+            };
+
+            // This validator is considered healthy if its latest account hash slot is within
+            // `HEALTH_CHECK_SLOT_DISTANCE` of the latest trusted validator's account hash slot
+            if latest_account_hash_slot > 0
+                && latest_trusted_validator_account_hash_slot > 0
+                && latest_account_hash_slot
+                    > latest_trusted_validator_account_hash_slot
+                        .saturating_sub(HEALTH_CHECK_SLOT_DISTANCE)
+            {
+                "ok"
+            } else {
+                warn!(
+                    "health check: me={}, latest trusted_validator={}",
+                    latest_account_hash_slot, latest_trusted_validator_account_hash_slot
+                );
+                "behind"
+            }
+        } else {
+            // No trusted validator point of reference available, so this validator is healthy
+            // because it's running
+            "ok"
+        };
+
+        info!("health check: {}", response);
+        response
+    }
 }
 
 impl RequestMiddleware for RpcRequestMiddleware {
@@ -138,6 +204,16 @@ impl RequestMiddleware for RpcRequestMiddleware {
         }
         if self.is_get_path(request.uri().path()) {
             self.get(request.uri().path())
+        } else if request.uri().path() == "/health" {
+            RequestMiddlewareAction::Respond {
+                should_validate_hosts: true,
+                response: Box::new(jsonrpc_core::futures::future::ok(
+                    hyper::Response::builder()
+                        .status(hyper::StatusCode::OK)
+                        .body(hyper::Body::from(self.health_check()))
+                        .unwrap(),
+                )),
+            }
         } else {
             RequestMiddlewareAction::Proceed {
                 should_continue_on_invalid_cors: false,
@@ -161,6 +237,7 @@ impl JsonRpcService {
         ledger_path: &Path,
         storage_state: StorageState,
         validator_exit: Arc<RwLock<Option<ValidatorExit>>>,
+        trusted_validators: Option<HashSet<Pubkey>>,
     ) -> Self {
         info!("rpc bound to {:?}", rpc_addr);
         info!("rpc configuration: {:?}", config);
@@ -186,20 +263,35 @@ impl JsonRpcService {
                 let rpc = RpcSolImpl;
                 io.extend_with(rpc.to_delegate());
 
-                let server =
-                    ServerBuilder::with_meta_extractor(io, move |_req: &hyper::Request<hyper::Body>| Meta {
+                let request_middleware = RpcRequestMiddleware::new(
+                    ledger_path,
+                    snapshot_config,
+                    cluster_info.clone(),
+                    trusted_validators,
+                );
+                let server = ServerBuilder::with_meta_extractor(
+                    io,
+                    move |_req: &hyper::Request<hyper::Body>| Meta {
                         request_processor: request_processor.clone(),
                         cluster_info: cluster_info.clone(),
-                        genesis_hash
-                    }).threads(4)
-                        .cors(DomainsValidation::AllowOnly(vec![
-                            AccessControlAllowOrigin::Any,
-                        ]))
-                        .cors_max_age(86400)
-                        .request_middleware(RpcRequestMiddleware::new(ledger_path, snapshot_config))
-                        .start_http(&rpc_addr);
+                        genesis_hash,
+                    },
+                )
+                .threads(4)
+                .cors(DomainsValidation::AllowOnly(vec![
+                    AccessControlAllowOrigin::Any,
+                ]))
+                .cors_max_age(86400)
+                .request_middleware(request_middleware)
+                .start_http(&rpc_addr);
+
                 if let Err(e) = server {
-                    warn!("JSON RPC service unavailable error: {:?}. \nAlso, check that port {} is not already in use by another application", e, rpc_addr.port());
+                    warn!(
+                        "JSON RPC service unavailable error: {:?}. \n\
+                           Also, check that port {} is not already in use by another application",
+                        e,
+                        rpc_addr.port()
+                    );
                     return;
                 }
 
@@ -240,6 +332,7 @@ mod tests {
     use super::*;
     use crate::{
         contact_info::ContactInfo,
+        crds_value::{CrdsData, CrdsValue, SnapshotHash},
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
         rpc::tests::create_validator_exit,
     };
@@ -268,21 +361,24 @@ mod tests {
             solana_net_utils::find_available_port_in_range(ip_addr, (10000, 65535)).unwrap(),
         );
         let bank_forks = Arc::new(RwLock::new(BankForks::new(bank.slot(), bank)));
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
         let ledger_path = get_tmp_ledger_path!();
-        let blockstore = Blockstore::open(&ledger_path).unwrap();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::default_with_blockstore(blockstore.clone()),
+        ));
         let mut rpc_service = JsonRpcService::new(
             rpc_addr,
             JsonRpcConfig::default(),
             None,
             bank_forks,
             block_commitment_cache,
-            Arc::new(blockstore),
+            blockstore,
             cluster_info,
             Hash::default(),
             &PathBuf::from("farf"),
             StorageState::default(),
             validator_exit,
+            None,
         );
         let thread = rpc_service.thread_hdl.thread();
         assert_eq!(thread.name().unwrap(), "solana-jsonrpc");
@@ -303,7 +399,11 @@ mod tests {
 
     #[test]
     fn test_is_get_path() {
-        let rrm = RpcRequestMiddleware::new(PathBuf::from("/"), None);
+        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
+            ContactInfo::default(),
+        )));
+
+        let rrm = RpcRequestMiddleware::new(PathBuf::from("/"), None, cluster_info.clone(), None);
         let rrm_with_snapshot_config = RpcRequestMiddleware::new(
             PathBuf::from("/"),
             Some(SnapshotConfig {
@@ -311,6 +411,8 @@ mod tests {
                 snapshot_package_output_path: PathBuf::from("/"),
                 snapshot_path: PathBuf::from("/"),
             }),
+            cluster_info,
+            None,
         );
 
         assert!(rrm.is_get_path("/genesis.tar.bz2"));
@@ -332,4 +434,95 @@ mod tests {
         assert!(!rrm.is_get_path(".."));
         assert!(!rrm.is_get_path("🎣"));
     }
+
+    #[test]
+    fn test_health_check_with_no_trusted_validators() {
+        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
+            ContactInfo::default(),
+        )));
+
+        let rm = RpcRequestMiddleware::new(PathBuf::from("/"), None, cluster_info.clone(), None);
+        assert_eq!(rm.health_check(), "ok");
+    }
+
+    #[test]
+    fn test_health_check_with_trusted_validators() {
+        let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
+            ContactInfo::default(),
+        )));
+
+        let trusted_validators = vec![Pubkey::new_rand(), Pubkey::new_rand(), Pubkey::new_rand()];
+        let rm = RpcRequestMiddleware::new(
+            PathBuf::from("/"),
+            None,
+            cluster_info.clone(),
+            Some(trusted_validators.clone().into_iter().collect()),
+        );
+
+        // No account hashes for this node or any trusted validators == "behind"
+        assert_eq!(rm.health_check(), "behind");
+
+        // No account hashes for any trusted validators == "behind"
+        {
+            let mut cluster_info = cluster_info.write().unwrap();
+            cluster_info
+                .push_accounts_hashes(vec![(1000, Hash::default()), (900, Hash::default())]);
+        }
+        assert_eq!(rm.health_check(), "behind");
+
+        // This node is ahead of the trusted validators == "ok"
+        {
+            let mut cluster_info = cluster_info.write().unwrap();
+            cluster_info
+                .gossip
+                .crds
+                .insert(
+                    CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
+                        trusted_validators[0].clone(),
+                        vec![
+                            (1, Hash::default()),
+                            (1001, Hash::default()),
+                            (2, Hash::default()),
+                        ],
+                    ))),
+                    1,
+                )
+                .unwrap();
+        }
+        assert_eq!(rm.health_check(), "ok");
+
+        // Node is slightly behind the trusted validators == "ok"
+        {
+            let mut cluster_info = cluster_info.write().unwrap();
+            cluster_info
+                .gossip
+                .crds
+                .insert(
+                    CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
+                        trusted_validators[1].clone(),
+                        vec![(1000 + HEALTH_CHECK_SLOT_DISTANCE - 1, Hash::default())],
+                    ))),
+                    1,
+                )
+                .unwrap();
+        }
+        assert_eq!(rm.health_check(), "ok");
+
+        // Node is far behind the trusted validators == "behind"
+        {
+            let mut cluster_info = cluster_info.write().unwrap();
+            cluster_info
+                .gossip
+                .crds
+                .insert(
+                    CrdsValue::new_unsigned(CrdsData::AccountsHashes(SnapshotHash::new(
+                        trusted_validators[2].clone(),
+                        vec![(1000 + HEALTH_CHECK_SLOT_DISTANCE, Hash::default())],
+                    ))),
+                    1,
+                )
+                .unwrap();
+        }
+        assert_eq!(rm.health_check(), "behind");
+    }
 }

core/src/serve_repair.rs

@@ -9,6 +9,7 @@ use crate::{
 use bincode::serialize;
 use rand::{thread_rng, Rng};
 use solana_ledger::blockstore::Blockstore;
+use solana_measure::measure::Measure;
 use solana_measure::thread_mem_usage;
 use solana_metrics::{datapoint_debug, inc_new_counter_debug};
 use solana_perf::packet::{Packets, PacketsRecycler};
@@ -184,12 +185,33 @@ impl ServeRepair {
         blockstore: Option<&Arc<Blockstore>>,
         requests_receiver: &PacketReceiver,
         response_sender: &PacketSender,
+        max_packets: &mut usize,
     ) -> Result<()> {
         //TODO cache connections
         let timeout = Duration::new(1, 0);
-        let reqs = requests_receiver.recv_timeout(timeout)?;
+        let mut reqs_v = vec![requests_receiver.recv_timeout(timeout)?];
+        let mut total_packets = reqs_v[0].packets.len();
 
-        Self::handle_packets(obj, &recycler, blockstore, reqs, response_sender);
+        while let Ok(more) = requests_receiver.try_recv() {
+            total_packets += more.packets.len();
+            if total_packets < *max_packets {
+                // Drop the rest in the channel in case of dos
+                reqs_v.push(more);
+            }
+        }
+
+        let mut time = Measure::start("repair::handle_packets");
+        for reqs in reqs_v {
+            Self::handle_packets(obj, &recycler, blockstore, reqs, response_sender);
+        }
+        time.stop();
+        if total_packets >= *max_packets {
+            if time.as_ms() > 1000 {
+                *max_packets = (*max_packets * 9) / 10;
+            } else {
+                *max_packets = (*max_packets * 10) / 9;
+            }
+        }
         Ok(())
     }
 
@@ -204,22 +226,26 @@ impl ServeRepair {
         let recycler = PacketsRecycler::default();
         Builder::new()
             .name("solana-repair-listen".to_string())
-            .spawn(move || loop {
-                let result = Self::run_listen(
-                    &me,
-                    &recycler,
-                    blockstore.as_ref(),
-                    &requests_receiver,
-                    &response_sender,
-                );
-                match result {
-                    Err(Error::RecvTimeoutError(_)) | Ok(_) => {}
-                    Err(err) => info!("repair listener error: {:?}", err),
-                };
-                if exit.load(Ordering::Relaxed) {
-                    return;
+            .spawn(move || {
+                let mut max_packets = 1024;
+                loop {
+                    let result = Self::run_listen(
+                        &me,
+                        &recycler,
+                        blockstore.as_ref(),
+                        &requests_receiver,
+                        &response_sender,
+                        &mut max_packets,
+                    );
+                    match result {
+                        Err(Error::RecvTimeoutError(_)) | Ok(_) => {}
+                        Err(err) => info!("repair listener error: {:?}", err),
+                    };
+                    if exit.load(Ordering::Relaxed) {
+                        return;
+                    }
+                    thread_mem_usage::datapoint("solana-repair-listen");
                 }
-                thread_mem_usage::datapoint("solana-repair-listen");
             })
             .unwrap()
     }

core/src/shred_fetch_stage.rs

@@ -2,43 +2,128 @@
 
 use crate::packet::{Packet, PacketsRecycler};
 use crate::streamer::{self, PacketReceiver, PacketSender};
+use bv::BitVec;
+use solana_ledger::bank_forks::BankForks;
 use solana_ledger::blockstore::MAX_DATA_SHREDS_PER_SLOT;
-use solana_ledger::shred::{OFFSET_OF_SHRED_INDEX, SIZE_OF_SHRED_INDEX};
+use solana_ledger::shred::{
+    OFFSET_OF_SHRED_INDEX, OFFSET_OF_SHRED_SLOT, SIZE_OF_SHRED_INDEX, SIZE_OF_SHRED_SLOT,
+};
 use solana_perf::cuda_runtime::PinnedVec;
 use solana_perf::packet::limited_deserialize;
 use solana_perf::recycler::Recycler;
+use solana_sdk::clock::Slot;
+use std::collections::HashMap;
 use std::net::UdpSocket;
 use std::sync::atomic::AtomicBool;
 use std::sync::mpsc::channel;
 use std::sync::Arc;
+use std::sync::RwLock;
 use std::thread::{self, Builder, JoinHandle};
+use std::time::Instant;
+
+pub type ShredsReceived = HashMap<Slot, BitVec<u64>>;
 
 pub struct ShredFetchStage {
     thread_hdls: Vec<JoinHandle<()>>,
 }
 
 impl ShredFetchStage {
-    // updates packets received on a channel and sends them on another channel
-    fn modify_packets<F>(recvr: PacketReceiver, sendr: PacketSender, modify: F)
-    where
-        F: Fn(&mut Packet),
-    {
-        while let Some(mut p) = recvr.iter().next() {
-            let index_start = OFFSET_OF_SHRED_INDEX;
-            let index_end = index_start + SIZE_OF_SHRED_INDEX;
-            p.packets.iter_mut().for_each(|p| {
-                p.meta.discard = true;
-                if index_end <= p.meta.size {
-                    if let Ok(index) = limited_deserialize::<u32>(&p.data[index_start..index_end]) {
-                        if index < MAX_DATA_SHREDS_PER_SLOT as u32 {
-                            p.meta.discard = false;
-                            modify(p);
-                        } else {
-                            inc_new_counter_warn!("shred_fetch_stage-shred_index_overrun", 1);
-                        }
+    fn get_slot_index(p: &Packet, index_overrun: &mut usize) -> Option<(u64, u32)> {
+        let index_start = OFFSET_OF_SHRED_INDEX;
+        let index_end = index_start + SIZE_OF_SHRED_INDEX;
+        let slot_start = OFFSET_OF_SHRED_SLOT;
+        let slot_end = slot_start + SIZE_OF_SHRED_SLOT;
+
+        if index_end <= p.meta.size {
+            if let Ok(index) = limited_deserialize::<u32>(&p.data[index_start..index_end]) {
+                if index < MAX_DATA_SHREDS_PER_SLOT as u32 && slot_end <= p.meta.size {
+                    if let Ok(slot) = limited_deserialize::<Slot>(&p.data[slot_start..slot_end]) {
+                        return Some((slot, index));
                     }
                 }
+            }
+        } else {
+            *index_overrun += 1;
+        }
+        None
+    }
+
+    fn process_packet<F>(
+        p: &mut Packet,
+        shreds_received: &mut ShredsReceived,
+        index_overrun: &mut usize,
+        last_root: Slot,
+        last_slot: Slot,
+        slots_per_epoch: u64,
+        modify: &F,
+    ) where
+        F: Fn(&mut Packet),
+    {
+        p.meta.discard = true;
+        if let Some((slot, index)) = Self::get_slot_index(p, index_overrun) {
+            // Seems reasonable to limit shreds to 2 epochs away
+            if slot > last_root && slot < (last_slot + 2 * slots_per_epoch) {
+                // Shred filter
+                let slot_received = shreds_received
+                    .entry(slot)
+                    .or_insert_with(|| BitVec::new_fill(false, MAX_DATA_SHREDS_PER_SLOT as u64));
+                if !slot_received.get(index.into()) {
+                    p.meta.discard = false;
+                    modify(p);
+                    slot_received.set(index.into(), true);
+                }
+            }
+        }
+    }
+
+    // updates packets received on a channel and sends them on another channel
+    fn modify_packets<F>(
+        recvr: PacketReceiver,
+        sendr: PacketSender,
+        bank_forks: Option<Arc<RwLock<BankForks>>>,
+        modify: F,
+    ) where
+        F: Fn(&mut Packet),
+    {
+        let mut shreds_received = ShredsReceived::default();
+        let mut last_cleared = Instant::now();
+
+        // In the case of bank_forks=None, setup to accept any slot range
+        let mut last_root = 0;
+        let mut last_slot = std::u64::MAX;
+        let mut slots_per_epoch = 0;
+
+        while let Some(mut p) = recvr.iter().next() {
+            if last_cleared.elapsed().as_millis() > 200 {
+                shreds_received.clear();
+                last_cleared = Instant::now();
+                if let Some(bank_forks) = bank_forks.as_ref() {
+                    let bank_forks_r = bank_forks.read().unwrap();
+                    last_root = bank_forks_r.root();
+                    let working_bank = bank_forks_r.working_bank();
+                    last_slot = working_bank.slot();
+                    let root_bank = bank_forks_r
+                        .get(bank_forks_r.root())
+                        .expect("Root bank should exist");
+                    slots_per_epoch = root_bank.get_slots_in_epoch(root_bank.epoch());
+                }
+            }
+            let mut index_overrun = 0;
+            let mut shred_count = 0;
+            p.packets.iter_mut().for_each(|mut packet| {
+                shred_count += 1;
+                Self::process_packet(
+                    &mut packet,
+                    &mut shreds_received,
+                    &mut index_overrun,
+                    last_root,
+                    last_slot,
+                    slots_per_epoch,
+                    &modify,
+                );
             });
+            inc_new_counter_warn!("shred_fetch_stage-shred_index_overrun", index_overrun);
+            inc_new_counter_info!("shred_fetch_stage-shred_count", shred_count);
             if sendr.send(p).is_err() {
                 break;
             }
@@ -50,6 +135,7 @@ impl ShredFetchStage {
         exit: &Arc<AtomicBool>,
         sender: PacketSender,
         recycler: Recycler<PinnedVec<Packet>>,
+        bank_forks: Option<Arc<RwLock<BankForks>>>,
         modify: F,
     ) -> (Vec<JoinHandle<()>>, JoinHandle<()>)
     where
@@ -71,7 +157,7 @@ impl ShredFetchStage {
 
         let modifier_hdl = Builder::new()
             .name("solana-tvu-fetch-stage-packet-modifier".to_string())
-            .spawn(|| Self::modify_packets(packet_receiver, sender, modify))
+            .spawn(move || Self::modify_packets(packet_receiver, sender, bank_forks, modify))
             .unwrap();
         (streamers, modifier_hdl)
     }
@@ -81,6 +167,7 @@ impl ShredFetchStage {
         forward_sockets: Vec<Arc<UdpSocket>>,
         repair_socket: Arc<UdpSocket>,
         sender: &PacketSender,
+        bank_forks: Option<Arc<RwLock<BankForks>>>,
         exit: &Arc<AtomicBool>,
     ) -> Self {
         let recycler: PacketsRecycler = Recycler::warmed(100, 1024);
@@ -100,6 +187,7 @@ impl ShredFetchStage {
             &exit,
             sender.clone(),
             recycler.clone(),
+            bank_forks.clone(),
             |p| p.meta.forward = true,
         );
 
@@ -108,6 +196,7 @@ impl ShredFetchStage {
             &exit,
             sender.clone(),
             recycler.clone(),
+            bank_forks,
             |p| p.meta.repair = true,
         );
 
@@ -128,3 +217,111 @@ impl ShredFetchStage {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_ledger::shred::Shred;
+
+    #[test]
+    fn test_shred_filter() {
+        solana_logger::setup();
+        let mut shreds_received = ShredsReceived::default();
+        let mut packet = Packet::default();
+        let mut index_overrun = 0;
+        let last_root = 0;
+        let last_slot = 100;
+        let slots_per_epoch = 10;
+        // packet size is 0, so cannot get index
+        ShredFetchStage::process_packet(
+            &mut packet,
+            &mut shreds_received,
+            &mut index_overrun,
+            last_root,
+            last_slot,
+            slots_per_epoch,
+            &|_p| {},
+        );
+        assert_eq!(index_overrun, 1);
+        assert!(packet.meta.discard);
+        let shred = Shred::new_from_data(1, 3, 0, None, true, true, 0, 0, 0);
+        shred.copy_to_packet(&mut packet);
+
+        // rejected slot is 1, root is 3
+        ShredFetchStage::process_packet(
+            &mut packet,
+            &mut shreds_received,
+            &mut index_overrun,
+            3,
+            last_slot,
+            slots_per_epoch,
+            &|_p| {},
+        );
+        assert!(packet.meta.discard);
+
+        // Accepted for 1,3
+        ShredFetchStage::process_packet(
+            &mut packet,
+            &mut shreds_received,
+            &mut index_overrun,
+            last_root,
+            last_slot,
+            slots_per_epoch,
+            &|_p| {},
+        );
+        assert!(!packet.meta.discard);
+
+        // shreds_received should filter duplicate
+        ShredFetchStage::process_packet(
+            &mut packet,
+            &mut shreds_received,
+            &mut index_overrun,
+            last_root,
+            last_slot,
+            slots_per_epoch,
+            &|_p| {},
+        );
+        assert!(packet.meta.discard);
+
+        let shred = Shred::new_from_data(1_000_000, 3, 0, None, true, true, 0, 0, 0);
+        shred.copy_to_packet(&mut packet);
+
+        // Slot 1 million is too high
+        ShredFetchStage::process_packet(
+            &mut packet,
+            &mut shreds_received,
+            &mut index_overrun,
+            last_root,
+            last_slot,
+            slots_per_epoch,
+            &|_p| {},
+        );
+        assert!(packet.meta.discard);
+
+        let index = MAX_DATA_SHREDS_PER_SLOT as u32;
+        let shred = Shred::new_from_data(5, index, 0, None, true, true, 0, 0, 0);
+        shred.copy_to_packet(&mut packet);
+        ShredFetchStage::process_packet(
+            &mut packet,
+            &mut shreds_received,
+            &mut index_overrun,
+            last_root,
+            last_slot,
+            slots_per_epoch,
+            &|_p| {},
+        );
+        assert!(packet.meta.discard);
+    }
+
+    #[test]
+    fn test_shred_offsets() {
+        let shred = Shred::new_from_data(1, 3, 0, None, true, true, 0, 0, 0);
+        let mut packet = Packet::default();
+        shred.copy_to_packet(&mut packet);
+        let mut index_overrun = 0;
+        assert_eq!(
+            Some((1, 3)),
+            ShredFetchStage::get_slot_index(&packet, &mut index_overrun)
+        );
+    }
+}

core/src/storage_stage.rs

@@ -4,6 +4,7 @@
 
 use crate::{
     cluster_info::ClusterInfo,
+    commitment::BlockCommitmentCache,
     contact_info::ContactInfo,
     result::{Error, Result},
 };
@@ -28,6 +29,7 @@ use solana_storage_program::{
     storage_instruction,
     storage_instruction::proof_validation,
 };
+use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
 use std::{
     cmp,
     collections::HashMap,
@@ -183,6 +185,7 @@ impl StorageStage {
         exit: &Arc<AtomicBool>,
         bank_forks: &Arc<RwLock<BankForks>>,
         cluster_info: &Arc<RwLock<ClusterInfo>>,
+        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
     ) -> Self {
         let (instruction_sender, instruction_receiver) = channel();
 
@@ -254,6 +257,7 @@ impl StorageStage {
                                     &keypair,
                                     &storage_keypair,
                                     &transactions_socket,
+                                    &block_commitment_cache,
                                 )
                                 .unwrap_or_else(|err| {
                                     info!("failed to send storage transaction: {:?}", err)
@@ -287,6 +291,7 @@ impl StorageStage {
         keypair: &Arc<Keypair>,
         storage_keypair: &Arc<Keypair>,
         transactions_socket: &UdpSocket,
+        block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
     ) -> io::Result<()> {
         let working_bank = bank_forks.read().unwrap().working_bank();
         let blockhash = working_bank.confirmed_last_blockhash().0;
@@ -312,7 +317,7 @@ impl StorageStage {
         }
 
         let signer_keys = vec![keypair.as_ref(), storage_keypair.as_ref()];
-        let message = Message::new_with_payer(vec![instruction], Some(&signer_keys[0].pubkey()));
+        let message = Message::new_with_payer(&[instruction], Some(&signer_keys[0].pubkey()));
         let transaction = Transaction::new(&signer_keys, message, blockhash);
         // try sending the transaction upto 5 times
         for _ in 0..5 {
@@ -321,8 +326,13 @@ impl StorageStage {
                 cluster_info.read().unwrap().my_data().tpu,
             )?;
             sleep(Duration::from_millis(100));
-            if Self::poll_for_signature_confirmation(bank_forks, &transaction.signatures[0], 0)
-                .is_ok()
+            if Self::poll_for_signature_confirmation(
+                bank_forks,
+                block_commitment_cache,
+                &transaction.signatures[0],
+                0,
+            )
+            .is_ok()
             {
                 break;
             };
@@ -332,19 +342,25 @@ impl StorageStage {
 
     fn poll_for_signature_confirmation(
         bank_forks: &Arc<RwLock<BankForks>>,
+        block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
         signature: &Signature,
         min_confirmed_blocks: usize,
     ) -> Result<()> {
         let mut now = Instant::now();
         let mut confirmed_blocks = 0;
         loop {
-            let response = bank_forks
-                .read()
-                .unwrap()
-                .working_bank()
-                .get_signature_confirmation_status(signature);
-            if let Some((confirmations, res)) = response {
-                if res.is_ok() {
+            let working_bank = bank_forks.read().unwrap().working_bank();
+            let response = working_bank.get_signature_status_slot(signature);
+            if let Some((slot, status)) = response {
+                let confirmations = if working_bank.src.roots().contains(&slot) {
+                    MAX_LOCKOUT_HISTORY + 1
+                } else {
+                    let r_block_commitment_cache = block_commitment_cache.read().unwrap();
+                    r_block_commitment_cache
+                        .get_confirmation_count(slot)
+                        .unwrap_or(0)
+                };
+                if status.is_ok() {
                     if confirmed_blocks != confirmations {
                         now = Instant::now();
                         confirmed_blocks = confirmations;
@@ -647,13 +663,20 @@ mod tests {
     use super::*;
     use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use rayon::prelude::*;
+    use solana_ledger::get_tmp_ledger_path;
     use solana_runtime::bank::Bank;
-    use solana_sdk::hash::Hasher;
-    use solana_sdk::signature::{Keypair, Signer};
-    use std::cmp::{max, min};
-    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
-    use std::sync::mpsc::channel;
-    use std::sync::{Arc, RwLock};
+    use solana_sdk::{
+        hash::Hasher,
+        signature::{Keypair, Signer},
+    };
+    use std::{
+        cmp::{max, min},
+        sync::{
+            atomic::{AtomicBool, AtomicUsize, Ordering},
+            mpsc::channel,
+            Arc, RwLock,
+        },
+    };
 
     #[test]
     fn test_storage_stage_none_ledger() {
@@ -668,6 +691,11 @@ mod tests {
             &[bank.clone()],
             vec![0],
         )));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::default_with_blockstore(blockstore),
+        ));
         let (_slot_sender, slot_receiver) = channel();
         let storage_state = StorageState::new(
             &bank.last_blockhash(),
@@ -683,6 +711,7 @@ mod tests {
             &exit.clone(),
             &bank_forks,
             &cluster_info,
+            block_commitment_cache,
         );
         exit.store(true, Ordering::Relaxed);
         storage_stage.join().unwrap();

core/src/transaction_status_service.rs

@@ -1,10 +1,10 @@
 use crossbeam_channel::{Receiver, RecvTimeoutError};
-use solana_client::rpc_response::RpcTransactionStatus;
 use solana_ledger::{blockstore::Blockstore, blockstore_processor::TransactionStatusBatch};
 use solana_runtime::{
     bank::{Bank, HashAgeKind},
     nonce_utils,
 };
+use solana_transaction_status::TransactionStatusMeta;
 use std::{
     sync::{
         atomic::{AtomicBool, Ordering},
@@ -70,10 +70,15 @@ impl TransactionStatusService {
                 }
                 .expect("FeeCalculator must exist");
                 let fee = fee_calculator.calculate_fee(transaction.message());
+                let (writable_keys, readonly_keys) =
+                    transaction.message.get_account_keys_by_lock_type();
                 blockstore
                     .write_transaction_status(
-                        (slot, transaction.signatures[0]),
-                        &RpcTransactionStatus {
+                        slot,
+                        transaction.signatures[0],
+                        writable_keys,
+                        readonly_keys,
+                        &TransactionStatusMeta {
                             status,
                             fee,
                             pre_balances,