Quietly re-introduce legacy --voting-keypair/--identity-keypair args for v1.0.6 compatibility

(cherry picked from commit 49706172f3)

.gitbook.yaml

@@ -1,4 +1,4 @@
-root: ./book/src
+root: ./docs/src
 
 structure:
     readme: introduction.md

.gitignore

@@ -1,6 +1,6 @@
-/book/html/
-/book/src/tests.ok
-/book/src/.gitbook/assets/*.svg
+/docs/html/
+/docs/src/tests.ok
+/docs/src/.gitbook/assets/*.svg
 /farf/
 /solana-release/
 /solana-release.tar.bz2

.mergify.yml

@@ -19,22 +19,6 @@ pull_request_rules:
       label:
         add:
           - automerge
-  - name: v0.21 backport
-    conditions:
-      - base=master
-      - label=v0.21
-    actions:
-      backport:
-        branches:
-          - v0.21
-  - name: v0.22 backport
-    conditions:
-      - base=master
-      - label=v0.22
-    actions:
-      backport:
-        branches:
-          - v0.22
   - name: v0.23 backport
     conditions:
       - base=master
@@ -43,3 +27,27 @@ pull_request_rules:
       backport:
         branches:
           - v0.23
+  - name: v1.0 backport
+    conditions:
+      - base=master
+      - label=v1.0
+    actions:
+      backport:
+        branches:
+          - v1.0
+  - name: v1.1 backport
+    conditions:
+      - base=master
+      - label=v1.1
+    actions:
+      backport:
+        branches:
+          - v1.1
+  - name: v1.2 backport
+    conditions:
+      - base=master
+      - label=v1.2
+    actions:
+      backport:
+        branches:
+          - v1.2

CONTRIBUTING.md

@@ -45,7 +45,7 @@ $ git pull --rebase upstream master
 
 If there are no functional changes, PRs can be very large and that's no
 problem. If, however, your changes are making meaningful changes or additions,
-then about 1,000 lines of changes is about the most you should ask a Solana
+then about 1000 lines of changes is about the most you should ask a Solana
 maintainer to review.
 
 ### Should I send small PRs as I develop large, new components?
@@ -224,21 +224,20 @@ Inventing new terms is allowed, but should only be done when the term is widely
 used and understood. Avoid introducing new 3-letter terms, which can be
 confused with 3-letter acronyms.
 
-[Terms currently in use](book/src/terminology.md)
+[Terms currently in use](docs/src/terminology.md)
 
 
 ## Design Proposals
 
-Solana's architecture is described by a book generated from markdown files in
-the `book/src/` directory, maintained by an *editor* (currently @garious). To
-add a design proposal, you'll need to at least propose a change the content
-under the [Accepted Design
-Proposals](https://docs.solana.com/book/v/master/proposals) chapter.  Here's
-the full process:
+Solana's architecture is described by docs generated from markdown files in
+the `docs/src/` directory, maintained by an *editor* (currently @garious). To
+add a design proposal, you'll need to include it in the
+[Accepted Design Proposals](https://docs.solana.com/proposals)
+section of the Solana docs.  Here's the full process:
 
 1. Propose a design by creating a PR that adds a markdown document to the
-   directory `book/src/` and references it from the [table of
-   contents](book/src/SUMMARY.md). Add any relevant *maintainers* to the PR
+   `docs/src/proposals` directory and references it from the [table of
+   contents](docs/src/SUMMARY.md). Add any relevant *maintainers* to the PR
    review.
 2. The PR being merged indicates your proposed change was accepted and that the
    maintainers support your plan of attack.

Cargo.toml

@@ -4,7 +4,10 @@ members = [
     "bench-streamer",
     "bench-tps",
     "banking-bench",
+    "chacha",
+    "chacha-cuda",
     "chacha-sys",
+    "cli-config",
     "client",
     "core",
     "faucet",
@@ -38,6 +41,9 @@ members = [
     "programs/vest",
     "programs/vote",
     "archiver",
+    "archiver-lib",
+    "archiver-utils",
+    "remote-wallet",
     "runtime",
     "sdk",
     "sdk-c",
@@ -45,7 +51,6 @@ members = [
     "sys-tuner",
     "upload-perf",
     "net-utils",
-    "fixed-buf",
     "vote-signer",
     "cli",
     "rayon-threadlimit",

README.md

@@ -9,26 +9,12 @@ Blockchain Rebuilt for Scale
 Solana™ is a new blockchain architecture built from the ground up for scale. The architecture supports
 up to 710 thousand transactions per second on a gigabit network.
 
-Disclaimer
+Documentation
 ===
 
-All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort.  It is up to the reader to check and validate their accuracy and truthfulness.  Furthermore nothing in this project constitutes a solicitation for investment.
+Before you jump into the code, review the documentation [Solana: Blockchain Rebuilt for Scale](https://docs.solana.com).
 
-Introduction
-===
-
-It's possible for a centralized database to process 710,000 transactions per second on a standard gigabit network if the transactions are, on average, no more than 176 bytes. A centralized database can also replicate itself and maintain high availability without significantly compromising that transaction rate using the distributed system technique known as Optimistic Concurrency Control [\[H.T.Kung, J.T.Robinson (1981)\]](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.65.4735). At Solana, we're demonstrating that these same theoretical limits apply just as well to blockchain on an adversarial network. The key ingredient? Finding a way to share time when nodes can't trust one-another. Once nodes can trust time, suddenly ~40 years of distributed systems research becomes applicable to blockchain!
-
-> Perhaps the most striking difference between algorithms obtained by our method and ones based upon timeout is that using timeout produces a traditional distributed algorithm in which the processes operate asynchronously, while our method produces a globally synchronous one in which every process does the same thing at (approximately) the same time. Our method seems to contradict the whole purpose of distributed processing, which is to permit different processes to operate independently and perform different functions. However, if a distributed system is really a single system, then the processes must be synchronized in some way. Conceptually, the easiest way to synchronize processes is to get them all to do the same thing at the same time. Therefore, our method is used to implement a kernel that performs the necessary synchronization--for example, making sure that two different processes do not try to modify a file at the same time. Processes might spend only a small fraction of their time executing the synchronizing kernel; the rest of the time, they can operate independently--e.g., accessing different files. This is an approach we have advocated even when fault-tolerance is not required. The method's basic simplicity makes it easier to understand the precise properties of a system, which is crucial if one is to know just how fault-tolerant the system is. [\[L.Lamport (1984)\]](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.71.1078)
-
-Furthermore, and much to our surprise, it can be implemented using a mechanism that has existed in Bitcoin since day one. The Bitcoin feature is called nLocktime and it can be used to postdate transactions using block height instead of a timestamp. As a Bitcoin client, you'd use block height instead of a timestamp if you don't trust the network. Block height turns out to be an instance of what's being called a Verifiable Delay Function in cryptography circles. It's a cryptographically secure way to say time has passed. In Solana, we use a far more granular verifiable delay function, a SHA 256 hash chain, to checkpoint the ledger and coordinate consensus. With it, we implement Optimistic Concurrency Control and are now well en route towards that theoretical limit of 710,000 transactions per second.
-
-Architecture
-===
-
-Before you jump into the code, review the online book [Solana: Blockchain Rebuilt for Scale](https://docs.solana.com/book/).
-
-(The _latest_ development version of the online book is also [available here](https://docs.solana.com/book/v/master/).)
+(The _latest_ development version of the docs is [available here](https://docs.solana.com/v/master).)
 
 Release Binaries
 ===
@@ -87,7 +73,8 @@ $ rustup update
 On Linux systems you may need to install libssl-dev, pkg-config, zlib1g-dev, etc.  On Ubuntu:
 
 ```bash
-$ sudo apt-get install libssl-dev pkg-config zlib1g-dev llvm clang
+$ sudo apt-get update
+$ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang
 ```
 
 Download the source code:
@@ -120,16 +107,13 @@ $ cargo test
 Local Testnet
 ---
 
-Start your own testnet locally, instructions are in the book [Solana: Blockchain Rebuild for Scale: Getting Started](https://docs.solana.com/book/getting-started).
+Start your own testnet locally, instructions are in the online docs [Solana: Blockchain Rebuild for Scale: Getting Started](https://docs.solana.com/building-from-source).
 
 Remote Testnets
 ---
 
-We maintain several testnets:
+* `testnet` - public stable testnet accessible via devnet.solana.com. Runs 24/7
 
-* `testnet` - public stable testnet accessible via testnet.solana.com. Runs 24/7
-* `testnet-beta` - public beta channel testnet accessible via beta.testnet.solana.com. Runs 24/7
-* `testnet-edge` - public edge channel testnet accessible via edge.testnet.solana.com. Runs 24/7
 
 ## Deploy process
 
@@ -240,3 +224,8 @@ problem is solved by this code?" On the other hand, if a test does fail and you
 better way to solve the same problem, a Pull Request with your solution would most certainly be
 welcome! Likewise, if rewriting a test can better communicate what code it's protecting, please
 send us that patch!
+
+Disclaimer
+===
+
+All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort.  It is up to the reader to check and validate their accuracy and truthfulness.  Furthermore nothing in this project constitutes a solicitation for investment.

RELEASE.md

@@ -138,30 +138,11 @@ There are three release channels that map to branches as follows:
 ### Update documentation
 TODO: Documentation update procedure is WIP as we move to gitbook
 
-Document the new recommended version by updating `book/src/running-archiver.md` and `book/src/validator-testnet.md` on the release (beta) branch to point at the `solana-install` for the upcoming release version.
+Document the new recommended version by updating `docs/src/running-archiver.md` and `docs/src/validator-testnet.md` on the release (beta) branch to point at the `solana-install` for the upcoming release version.
 
-#### Publish updated Book
-We maintain three copies of the "book" as official documentation:
+### Update software on devnet.solana.com
 
-1) "Book" is the documentation for the latest official release.  This should get manually updated whenever a new release is made.  It is published here:
-https://solana-labs.github.io/book/
-
-2) "Book-edge" tracks the tip of the master branch and updates automatically.
-https://solana-labs.github.io/book-edge/
-
-3) "Book-beta" tracks the tip of the beta branch and updates automatically.
-https://solana-labs.github.io/book-beta/
-
-To manually trigger an update of the "Book", create a new job of the manual-update-book pipeline.
-Set the tag of the latest release as the PUBLISH_BOOK_TAG environment variable.
-```bash
-PUBLISH_BOOK_TAG=v0.16.6
-```
-https://buildkite.com/solana-labs/manual-update-book
-
-### Update software on testnet.solana.com
-
-The testnet running on testnet.solana.com is set to use a fixed release tag
+The testnet running on devnet.solana.com is set to use a fixed release tag
 which is set in the Buildkite testnet-management pipeline.
 This tag needs to be updated and the testnet restarted after a new release
 tag is created.
@@ -201,4 +182,4 @@ TESTNET_OP=create-and-start
 ### Alert the community
 
 Notify Discord users on #validator-support that a new release for
-testnet.solana.com is available
+devnet.solana.com is available

archiver-lib/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "solana-archiver-lib"
+version = "1.0.7"
+description = "Solana Archiver Library"
+authors = ["Solana Maintainers <maintainers@solana.com>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+bincode = "1.2.1"
+crossbeam-channel = "0.3"
+ed25519-dalek = "=1.0.0-pre.1"
+log = "0.4.8"
+rand = "0.6.5"
+rand_chacha = "0.1.1"
+solana-client = { path = "../client", version = "1.0.7" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.7" }
+thiserror = "1.0"
+serde = "1.0.104"
+serde_json = "1.0.46"
+serde_derive = "1.0.103"
+solana-net-utils = { path = "../net-utils", version = "1.0.7" }
+solana-chacha = { path = "../chacha", version = "1.0.7" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.0.7" }
+solana-ledger = { path = "../ledger", version = "1.0.7" }
+solana-logger = { path = "../logger", version = "1.0.7" }
+solana-perf = { path = "../perf", version = "1.0.7" }
+solana-sdk = { path = "../sdk", version = "1.0.7" }
+solana-core = { path = "../core", version = "1.0.7" }
+solana-archiver-utils = { path = "../archiver-utils", version = "1.0.7" }
+solana-metrics = { path = "../metrics", version = "1.0.7" }
+
+[dev-dependencies]
+hex = "0.4.0"
+
+[lib]
+name = "solana_archiver_lib"

archiver-lib/src/archiver.rs

@@ -1,38 +1,42 @@
-use crate::{
-    chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE},
+use crate::result::ArchiverError;
+use crossbeam_channel::unbounded;
+use rand::{thread_rng, Rng, SeedableRng};
+use rand_chacha::ChaChaRng;
+use solana_archiver_utils::sample_file;
+use solana_chacha::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE};
+use solana_client::{
+    rpc_client::RpcClient, rpc_request::RpcRequest, rpc_response::RpcStorageTurn,
+    thin_client::ThinClient,
+};
+use solana_core::{
     cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE},
     contact_info::ContactInfo,
     gossip_service::GossipService,
     packet::{limited_deserialize, PACKET_DATA_SIZE},
     repair_service,
     repair_service::{RepairService, RepairSlotRange, RepairStrategy},
-    result::{Error, Result},
+    serve_repair::ServeRepair,
     shred_fetch_stage::ShredFetchStage,
     sigverify_stage::{DisabledSigVerifier, SigVerifyStage},
     storage_stage::NUM_STORAGE_SAMPLES,
     streamer::{receiver, responder, PacketReceiver},
     window_service::WindowService,
 };
-use crossbeam_channel::unbounded;
-use ed25519_dalek;
-use rand::{thread_rng, Rng, SeedableRng};
-use rand_chacha::ChaChaRng;
-use solana_client::{rpc_client::RpcClient, rpc_request::RpcRequest, thin_client::ThinClient};
 use solana_ledger::{
-    blocktree::Blocktree, leader_schedule_cache::LeaderScheduleCache, shred::Shred,
+    blockstore::Blockstore, leader_schedule_cache::LeaderScheduleCache, shred::Shred,
 };
 use solana_net_utils::bind_in_range;
 use solana_perf::packet::Packets;
 use solana_perf::recycler::Recycler;
 use solana_sdk::packet::Packet;
 use solana_sdk::{
-    account_utils::State,
+    account_utils::StateMut,
     client::{AsyncClient, SyncClient},
     clock::{get_complete_segment_from_slot, get_segment_from_slot, Slot},
     commitment_config::CommitmentConfig,
-    hash::{Hash, Hasher},
+    hash::Hash,
     message::Message,
-    signature::{Keypair, KeypairUtil, Signature},
+    signature::{Keypair, Signature, Signer},
     timing::timestamp,
     transaction::Transaction,
     transport::TransportError,
@@ -42,10 +46,8 @@ use solana_storage_program::{
     storage_instruction::{self, StorageAccountType},
 };
 use std::{
-    fs::File,
-    io::{self, BufReader, ErrorKind, Read, Seek, SeekFrom},
-    mem::size_of,
-    net::{SocketAddr, UdpSocket},
+    io::{self, ErrorKind},
+    net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
     path::{Path, PathBuf},
     result,
     sync::atomic::{AtomicBool, Ordering},
@@ -55,6 +57,8 @@ use std::{
     time::Duration,
 };
 
+type Result<T> = std::result::Result<T, ArchiverError>;
+
 static ENCRYPTED_FILENAME: &str = "ledger.enc";
 
 #[derive(Serialize, Deserialize)]
@@ -82,47 +86,12 @@ struct ArchiverMeta {
     client_commitment: CommitmentConfig,
 }
 
-pub(crate) fn sample_file(in_path: &Path, sample_offsets: &[u64]) -> io::Result<Hash> {
-    let in_file = File::open(in_path)?;
-    let metadata = in_file.metadata()?;
-    let mut buffer_file = BufReader::new(in_file);
-
-    let mut hasher = Hasher::default();
-    let sample_size = size_of::<Hash>();
-    let sample_size64 = sample_size as u64;
-    let mut buf = vec![0; sample_size];
-
-    let file_len = metadata.len();
-    if file_len < sample_size64 {
-        return Err(io::Error::new(ErrorKind::Other, "file too short!"));
-    }
-    for offset in sample_offsets {
-        if *offset > (file_len - sample_size64) / sample_size64 {
-            return Err(io::Error::new(ErrorKind::Other, "offset too large"));
-        }
-        buffer_file.seek(SeekFrom::Start(*offset * sample_size64))?;
-        trace!("sampling @ {} ", *offset);
-        match buffer_file.read(&mut buf) {
-            Ok(size) => {
-                assert_eq!(size, buf.len());
-                hasher.hash(&buf);
-            }
-            Err(e) => {
-                warn!("Error sampling file");
-                return Err(e);
-            }
-        }
-    }
-
-    Ok(hasher.result())
-}
-
 fn get_slot_from_signature(
-    signature: &ed25519_dalek::Signature,
+    signature: &Signature,
     storage_turn: u64,
     slots_per_segment: u64,
 ) -> u64 {
-    let signature_vec = signature.to_bytes();
+    let signature_vec = signature.as_ref();
     let mut segment_index = u64::from(signature_vec[0])
         | (u64::from(signature_vec[1]) << 8)
         | (u64::from(signature_vec[1]) << 16)
@@ -222,38 +191,29 @@ impl Archiver {
         // Note for now, this ledger will not contain any of the existing entries
         // in the ledger located at ledger_path, and will only append on newly received
         // entries after being passed to window_service
-        let blocktree = Arc::new(
-            Blocktree::open(ledger_path).expect("Expected to be able to open database ledger"),
+        let blockstore = Arc::new(
+            Blockstore::open(ledger_path).expect("Expected to be able to open database ledger"),
         );
 
-        let gossip_service = GossipService::new(
-            &cluster_info,
-            Some(blocktree.clone()),
-            None,
-            node.sockets.gossip,
-            &exit,
-        );
+        let gossip_service = GossipService::new(&cluster_info, None, node.sockets.gossip, &exit);
 
         info!("Connecting to the cluster via {:?}", cluster_entrypoint);
         let (nodes, _) =
-            match crate::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 1) {
+            match solana_core::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 1) {
                 Ok(nodes_and_archivers) => nodes_and_archivers,
                 Err(e) => {
                     //shutdown services before exiting
                     exit.store(true, Ordering::Relaxed);
                     gossip_service.join()?;
-                    return Err(Error::from(e));
+                    return Err(e.into());
                 }
             };
-        let client = crate::gossip_service::get_client(&nodes);
+        let client = solana_core::gossip_service::get_client(&nodes);
 
         info!("Setting up mining account...");
-        if let Err(e) = Self::setup_mining_account(
-            &client,
-            &keypair,
-            &storage_keypair,
-            client_commitment.clone(),
-        ) {
+        if let Err(e) =
+            Self::setup_mining_account(&client, &keypair, &storage_keypair, client_commitment)
+        {
             //shutdown services before exiting
             exit.store(true, Ordering::Relaxed);
             gossip_service.join()?;
@@ -294,7 +254,7 @@ impl Archiver {
                 let window_service = match Self::setup(
                     &mut meta,
                     cluster_info.clone(),
-                    &blocktree,
+                    &blockstore,
                     &exit,
                     &node_info,
                     &storage_keypair,
@@ -320,7 +280,7 @@ impl Archiver {
                 // run archiver
                 Self::run(
                     &mut meta,
-                    &blocktree,
+                    &blockstore,
                     cluster_info,
                     &keypair,
                     &storage_keypair,
@@ -344,14 +304,14 @@ impl Archiver {
 
     fn run(
         meta: &mut ArchiverMeta,
-        blocktree: &Arc<Blocktree>,
+        blockstore: &Arc<Blockstore>,
         cluster_info: Arc<RwLock<ClusterInfo>>,
         archiver_keypair: &Arc<Keypair>,
         storage_keypair: &Arc<Keypair>,
         exit: &Arc<AtomicBool>,
     ) {
         // encrypt segment
-        Self::encrypt_ledger(meta, blocktree).expect("ledger encrypt not successful");
+        Self::encrypt_ledger(meta, blockstore).expect("ledger encrypt not successful");
         let enc_file_path = meta.ledger_data_file_encrypted.clone();
         // do replicate
         loop {
@@ -395,7 +355,7 @@ impl Archiver {
                 &cluster_info,
                 archiver_keypair,
                 storage_keypair,
-                meta.client_commitment.clone(),
+                meta.client_commitment,
             );
         }
         exit.store(true, Ordering::Relaxed);
@@ -408,10 +368,10 @@ impl Archiver {
         client_commitment: CommitmentConfig,
     ) {
         let nodes = cluster_info.read().unwrap().tvu_peers();
-        let client = crate::gossip_service::get_client(&nodes);
+        let client = solana_core::gossip_service::get_client(&nodes);
 
         if let Ok(Some(account)) =
-            client.get_account_with_commitment(&storage_keypair.pubkey(), client_commitment.clone())
+            client.get_account_with_commitment(&storage_keypair.pubkey(), client_commitment)
         {
             if let Ok(StorageContract::ArchiverStorage { validations, .. }) = account.state() {
                 if !validations.is_empty() {
@@ -419,9 +379,8 @@ impl Archiver {
                         &archiver_keypair.pubkey(),
                         &storage_keypair.pubkey(),
                     );
-                    let message =
-                        Message::new_with_payer(vec![ix], Some(&archiver_keypair.pubkey()));
-                    if let Err(e) = client.send_message(&[&archiver_keypair], message) {
+                    let message = Message::new_with_payer(&[ix], Some(&archiver_keypair.pubkey()));
+                    if let Err(e) = client.send_message(&[archiver_keypair.as_ref()], message) {
                         error!("unable to redeem reward, tx failed: {:?}", e);
                     } else {
                         info!(
@@ -443,7 +402,7 @@ impl Archiver {
     fn setup(
         meta: &mut ArchiverMeta,
         cluster_info: Arc<RwLock<ClusterInfo>>,
-        blocktree: &Arc<Blocktree>,
+        blockstore: &Arc<Blockstore>,
         exit: &Arc<AtomicBool>,
         node_info: &ContactInfo,
         storage_keypair: &Arc<Keypair>,
@@ -452,7 +411,7 @@ impl Archiver {
         slot_sender: Sender<u64>,
     ) -> Result<WindowService> {
         let slots_per_segment =
-            match Self::get_segment_config(&cluster_info, meta.client_commitment.clone()) {
+            match Self::get_segment_config(&cluster_info, meta.client_commitment) {
                 Ok(slots_per_segment) => slots_per_segment,
                 Err(e) => {
                     error!("unable to get segment size configuration, exiting...");
@@ -474,13 +433,13 @@ impl Archiver {
                 return Err(e);
             }
         };
-        let signature = storage_keypair.sign(segment_blockhash.as_ref());
+        let signature = storage_keypair.sign_message(segment_blockhash.as_ref());
         let slot = get_slot_from_signature(&signature, segment_slot, slots_per_segment);
         info!("replicating slot: {}", slot);
         slot_sender.send(slot)?;
         meta.slot = slot;
         meta.slots_per_segment = slots_per_segment;
-        meta.signature = Signature::new(&signature.to_bytes());
+        meta.signature = signature;
         meta.blockhash = segment_blockhash;
 
         let mut repair_slot_range = RepairSlotRange::default();
@@ -498,7 +457,7 @@ impl Archiver {
         );
 
         let window_service = WindowService::new(
-            blocktree.clone(),
+            blockstore.clone(),
             cluster_info.clone(),
             verified_receiver,
             retransmit_sender,
@@ -512,7 +471,7 @@ impl Archiver {
         Self::wait_for_segment_download(
             slot,
             slots_per_segment,
-            &blocktree,
+            &blockstore,
             &exit,
             &node_info,
             cluster_info,
@@ -523,7 +482,7 @@ impl Archiver {
     fn wait_for_segment_download(
         start_slot: Slot,
         slots_per_segment: u64,
-        blocktree: &Arc<Blocktree>,
+        blockstore: &Arc<Blockstore>,
         exit: &Arc<AtomicBool>,
         node_info: &ContactInfo,
         cluster_info: Arc<RwLock<ClusterInfo>>,
@@ -534,7 +493,7 @@ impl Archiver {
         );
         let mut current_slot = start_slot;
         'outer: loop {
-            while blocktree.is_full(current_slot) {
+            while blockstore.is_full(current_slot) {
                 current_slot += 1;
                 info!("current slot: {}", current_slot);
                 if current_slot >= start_slot + slots_per_segment {
@@ -553,13 +512,15 @@ impl Archiver {
         let mut contact_info = node_info.clone();
         contact_info.tvu = "0.0.0.0:0".parse().unwrap();
         contact_info.wallclock = timestamp();
+        // copy over the adopted shred_version from the entrypoint
+        contact_info.shred_version = cluster_info.read().unwrap().my_data().shred_version;
         {
             let mut cluster_info_w = cluster_info.write().unwrap();
             cluster_info_w.insert_self(contact_info);
         }
     }
 
-    fn encrypt_ledger(meta: &mut ArchiverMeta, blocktree: &Arc<Blocktree>) -> Result<()> {
+    fn encrypt_ledger(meta: &mut ArchiverMeta, blockstore: &Arc<Blockstore>) -> Result<()> {
         meta.ledger_data_file_encrypted = meta.ledger_path.join(ENCRYPTED_FILENAME);
 
         {
@@ -567,7 +528,7 @@ impl Archiver {
             ivec.copy_from_slice(&meta.signature.as_ref());
 
             let num_encrypted_bytes = chacha_cbc_encrypt_ledger(
-                blocktree,
+                blockstore,
                 meta.slot,
                 meta.slots_per_segment,
                 &meta.ledger_data_file_encrypted,
@@ -616,29 +577,23 @@ impl Archiver {
             &keypair.pubkey(),
             &Duration::from_millis(100),
             &Duration::from_secs(5),
-            client_commitment.clone(),
+            client_commitment,
         )? == 0
         {
-            return Err(
-                io::Error::new(io::ErrorKind::Other, "keypair account has no balance").into(),
-            );
+            return Err(ArchiverError::EmptyStorageAccountBalance);
         }
 
         info!("checking storage account keypair...");
         // check if the storage account exists
-        let balance = client
-            .poll_get_balance_with_commitment(&storage_keypair.pubkey(), client_commitment.clone());
+        let balance =
+            client.poll_get_balance_with_commitment(&storage_keypair.pubkey(), client_commitment);
         if balance.is_err() || balance.unwrap() == 0 {
-            let blockhash =
-                match client.get_recent_blockhash_with_commitment(client_commitment.clone()) {
-                    Ok((blockhash, _)) => blockhash,
-                    Err(_) => {
-                        return Err(Error::IO(<io::Error>::new(
-                            io::ErrorKind::Other,
-                            "unable to get recent blockhash, can't submit proof",
-                        )));
-                    }
-                };
+            let blockhash = match client.get_recent_blockhash_with_commitment(client_commitment) {
+                Ok((blockhash, _)) => blockhash,
+                Err(e) => {
+                    return Err(ArchiverError::TransportError(e));
+                }
+            };
 
             let ix = storage_instruction::create_storage_account(
                 &keypair.pubkey(),
@@ -657,6 +612,7 @@ impl Archiver {
                         ErrorKind::Other,
                         "setup_mining_account: signature not found",
                     ),
+                    TransportError::Custom(e) => io::Error::new(ErrorKind::Other, e),
                 })?;
         }
         Ok(())
@@ -670,33 +626,28 @@ impl Archiver {
     ) {
         // No point if we've got no storage account...
         let nodes = cluster_info.read().unwrap().tvu_peers();
-        let client = crate::gossip_service::get_client(&nodes);
-        let storage_balance = client.poll_get_balance_with_commitment(
-            &storage_keypair.pubkey(),
-            meta.client_commitment.clone(),
-        );
+        let client = solana_core::gossip_service::get_client(&nodes);
+        let storage_balance = client
+            .poll_get_balance_with_commitment(&storage_keypair.pubkey(), meta.client_commitment);
         if storage_balance.is_err() || storage_balance.unwrap() == 0 {
             error!("Unable to submit mining proof, no storage account");
             return;
         }
         // ...or no lamports for fees
-        let balance = client.poll_get_balance_with_commitment(
-            &archiver_keypair.pubkey(),
-            meta.client_commitment.clone(),
-        );
+        let balance = client
+            .poll_get_balance_with_commitment(&archiver_keypair.pubkey(), meta.client_commitment);
         if balance.is_err() || balance.unwrap() == 0 {
             error!("Unable to submit mining proof, insufficient Archiver Account balance");
             return;
         }
 
-        let blockhash =
-            match client.get_recent_blockhash_with_commitment(meta.client_commitment.clone()) {
-                Ok((blockhash, _)) => blockhash,
-                Err(_) => {
-                    error!("unable to get recent blockhash, can't submit proof");
-                    return;
-                }
-            };
+        let blockhash = match client.get_recent_blockhash_with_commitment(meta.client_commitment) {
+            Ok((blockhash, _)) => blockhash,
+            Err(_) => {
+                error!("unable to get recent blockhash, can't submit proof");
+                return;
+            }
+        };
         let instruction = storage_instruction::mining_proof(
             &storage_keypair.pubkey(),
             meta.sha_state,
@@ -704,14 +655,14 @@ impl Archiver {
             Signature::new(&meta.signature.as_ref()),
             meta.blockhash,
         );
-        let message = Message::new_with_payer(vec![instruction], Some(&archiver_keypair.pubkey()));
+        let message = Message::new_with_payer(&[instruction], Some(&archiver_keypair.pubkey()));
         let mut transaction = Transaction::new(
             &[archiver_keypair.as_ref(), storage_keypair.as_ref()],
             message,
             blockhash,
         );
         if let Err(err) = client.send_and_confirm_transaction(
-            &[&archiver_keypair, &storage_keypair],
+            &[archiver_keypair.as_ref(), storage_keypair.as_ref()],
             &mut transaction,
             10,
             0,
@@ -734,10 +685,10 @@ impl Archiver {
     fn get_segment_config(
         cluster_info: &Arc<RwLock<ClusterInfo>>,
         client_commitment: CommitmentConfig,
-    ) -> result::Result<u64, Error> {
+    ) -> Result<u64> {
         let rpc_peers = {
             let cluster_info = cluster_info.read().unwrap();
-            cluster_info.rpc_peers()
+            cluster_info.all_rpc_peers()
         };
         debug!("rpc peers: {:?}", rpc_peers);
         if !rpc_peers.is_empty() {
@@ -753,12 +704,12 @@ impl Archiver {
                 )
                 .map_err(|err| {
                     warn!("Error while making rpc request {:?}", err);
-                    Error::IO(io::Error::new(ErrorKind::Other, "rpc error"))
+                    ArchiverError::ClientError(err)
                 })?
                 .as_u64()
                 .unwrap())
         } else {
-            Err(io::Error::new(io::ErrorKind::Other, "No RPC peers...".to_string()).into())
+            Err(ArchiverError::NoRpcPeers)
         }
     }
 
@@ -768,7 +719,7 @@ impl Archiver {
         slots_per_segment: u64,
         previous_blockhash: &Hash,
         exit: &Arc<AtomicBool>,
-    ) -> result::Result<(Hash, u64), Error> {
+    ) -> Result<(Hash, u64)> {
         loop {
             let (blockhash, turn_slot) = Self::poll_for_blockhash_and_slot(
                 cluster_info,
@@ -788,12 +739,12 @@ impl Archiver {
         slots_per_segment: u64,
         previous_blockhash: &Hash,
         exit: &Arc<AtomicBool>,
-    ) -> result::Result<(Hash, u64), Error> {
+    ) -> Result<(Hash, u64)> {
         info!("waiting for the next turn...");
         loop {
             let rpc_peers = {
                 let cluster_info = cluster_info.read().unwrap();
-                cluster_info.rpc_peers()
+                cluster_info.all_rpc_peers()
             };
             debug!("rpc peers: {:?}", rpc_peers);
             if !rpc_peers.is_empty() {
@@ -809,15 +760,13 @@ impl Archiver {
                     )
                     .map_err(|err| {
                         warn!("Error while making rpc request {:?}", err);
-                        Error::IO(io::Error::new(ErrorKind::Other, "rpc error"))
-                    })?;
-                let (storage_blockhash, turn_slot) =
-                    serde_json::from_value::<(String, u64)>(response).map_err(|err| {
-                        io::Error::new(
-                            io::ErrorKind::Other,
-                            format!("Couldn't parse response: {:?}", err),
-                        )
+                        ArchiverError::ClientError(err)
                     })?;
+                let RpcStorageTurn {
+                    blockhash: storage_blockhash,
+                    slot: turn_slot,
+                } = serde_json::from_value::<RpcStorageTurn>(response)
+                    .map_err(ArchiverError::JsonError)?;
                 let turn_blockhash = storage_blockhash.parse().map_err(|err| {
                     io::Error::new(
                         io::ErrorKind::Other,
@@ -835,7 +784,7 @@ impl Archiver {
                 }
             }
             if exit.load(Ordering::Relaxed) {
-                return Err(Error::IO(io::Error::new(
+                return Err(ArchiverError::IO(io::Error::new(
                     ErrorKind::Other,
                     "exit signalled...",
                 )));
@@ -844,25 +793,26 @@ impl Archiver {
         }
     }
 
-    /// Ask an archiver to populate a given blocktree with its segment.
+    /// Ask an archiver to populate a given blockstore with its segment.
     /// Return the slot at the start of the archiver's segment
     ///
-    /// It is recommended to use a temporary blocktree for this since the download will not verify
+    /// It is recommended to use a temporary blockstore for this since the download will not verify
     /// shreds received and might impact the chaining of shreds across slots
     pub fn download_from_archiver(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
+        serve_repair: &ServeRepair,
         archiver_info: &ContactInfo,
-        blocktree: &Arc<Blocktree>,
+        blockstore: &Arc<Blockstore>,
         slots_per_segment: u64,
     ) -> Result<u64> {
+        let ip_addr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0));
         // Create a client which downloads from the archiver and see that it
         // can respond with shreds.
-        let start_slot = Self::get_archiver_segment_slot(archiver_info.storage_addr);
+        let start_slot = Self::get_archiver_segment_slot(ip_addr, archiver_info.storage_addr);
         info!("Archiver download: start at {}", start_slot);
 
         let exit = Arc::new(AtomicBool::new(false));
         let (s_reader, r_reader) = channel();
-        let repair_socket = Arc::new(bind_in_range(VALIDATOR_PORT_RANGE).unwrap().1);
+        let repair_socket = Arc::new(bind_in_range(ip_addr, VALIDATOR_PORT_RANGE).unwrap().1);
         let t_receiver = receiver(
             repair_socket.clone(),
             &exit,
@@ -870,10 +820,10 @@ impl Archiver {
             Recycler::default(),
             "archiver_reeciver",
         );
-        let id = cluster_info.read().unwrap().id();
+        let id = serve_repair.keypair().pubkey();
         info!(
             "Sending repair requests from: {} to: {}",
-            cluster_info.read().unwrap().my_data().id,
+            serve_repair.my_info().id,
             archiver_info.gossip
         );
         let repair_slot_range = RepairSlotRange {
@@ -884,7 +834,7 @@ impl Archiver {
         for _ in 0..120 {
             // Strategy used by archivers
             let repairs = RepairService::generate_repairs_in_range(
-                blocktree,
+                blockstore,
                 repair_service::MAX_REPAIR_LENGTH,
                 &repair_slot_range,
             );
@@ -893,9 +843,7 @@ impl Archiver {
                 let reqs: Vec<_> = repairs
                     .into_iter()
                     .filter_map(|repair_request| {
-                        cluster_info
-                            .read()
-                            .unwrap()
+                        serve_repair
                             .map_repair_request(&repair_request)
                             .map(|result| ((archiver_info.gossip, result), repair_request))
                             .ok()
@@ -930,10 +878,10 @@ impl Archiver {
                     .into_iter()
                     .filter_map(|p| Shred::new_from_serialized_shred(p.data.to_vec()).ok())
                     .collect();
-                blocktree.insert_shreds(shreds, None, false)?;
+                blockstore.insert_shreds(shreds, None, false)?;
             }
             // check if all the slots in the segment are complete
-            if Self::segment_complete(start_slot, slots_per_segment, blocktree) {
+            if Self::segment_complete(start_slot, slots_per_segment, blockstore) {
                 break;
             }
             sleep(Duration::from_millis(500));
@@ -942,10 +890,8 @@ impl Archiver {
         t_receiver.join().unwrap();
 
         // check if all the slots in the segment are complete
-        if !Self::segment_complete(start_slot, slots_per_segment, blocktree) {
-            return Err(
-                io::Error::new(ErrorKind::Other, "Unable to download the full segment").into(),
-            );
+        if !Self::segment_complete(start_slot, slots_per_segment, blockstore) {
+            return Err(ArchiverError::SegmentDownloadError);
         }
         Ok(start_slot)
     }
@@ -953,18 +899,18 @@ impl Archiver {
     fn segment_complete(
         start_slot: Slot,
         slots_per_segment: u64,
-        blocktree: &Arc<Blocktree>,
+        blockstore: &Arc<Blockstore>,
     ) -> bool {
         for slot in start_slot..(start_slot + slots_per_segment) {
-            if !blocktree.is_full(slot) {
+            if !blockstore.is_full(slot) {
                 return false;
             }
         }
         true
     }
 
-    fn get_archiver_segment_slot(to: SocketAddr) -> u64 {
-        let (_port, socket) = bind_in_range(VALIDATOR_PORT_RANGE).unwrap();
+    fn get_archiver_segment_slot(bind_ip_addr: IpAddr, to: SocketAddr) -> u64 {
+        let (_port, socket) = bind_in_range(bind_ip_addr, VALIDATOR_PORT_RANGE).unwrap();
         socket
             .set_read_timeout(Some(Duration::from_secs(5)))
             .unwrap();
@@ -988,74 +934,3 @@ impl Archiver {
         panic!("Couldn't get segment slot from archiver!");
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::fs::{create_dir_all, remove_file};
-    use std::io::Write;
-
-    fn tmp_file_path(name: &str) -> PathBuf {
-        use std::env;
-        let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
-        let keypair = Keypair::new();
-
-        let mut path = PathBuf::new();
-        path.push(out_dir);
-        path.push("tmp");
-        create_dir_all(&path).unwrap();
-
-        path.push(format!("{}-{}", name, keypair.pubkey()));
-        path
-    }
-
-    #[test]
-    fn test_sample_file() {
-        solana_logger::setup();
-        let in_path = tmp_file_path("test_sample_file_input.txt");
-        let num_strings = 4096;
-        let string = "12foobar";
-        {
-            let mut in_file = File::create(&in_path).unwrap();
-            for _ in 0..num_strings {
-                in_file.write(string.as_bytes()).unwrap();
-            }
-        }
-        let num_samples = (string.len() * num_strings / size_of::<Hash>()) as u64;
-        let samples: Vec<_> = (0..num_samples).collect();
-        let res = sample_file(&in_path, samples.as_slice());
-        let ref_hash: Hash = Hash::new(&[
-            173, 251, 182, 165, 10, 54, 33, 150, 133, 226, 106, 150, 99, 192, 179, 1, 230, 144,
-            151, 126, 18, 191, 54, 67, 249, 140, 230, 160, 56, 30, 170, 52,
-        ]);
-        let res = res.unwrap();
-        assert_eq!(res, ref_hash);
-
-        // Sample just past the end
-        assert!(sample_file(&in_path, &[num_samples]).is_err());
-        remove_file(&in_path).unwrap();
-    }
-
-    #[test]
-    fn test_sample_file_invalid_offset() {
-        let in_path = tmp_file_path("test_sample_file_invalid_offset_input.txt");
-        {
-            let mut in_file = File::create(&in_path).unwrap();
-            for _ in 0..4096 {
-                in_file.write("123456foobar".as_bytes()).unwrap();
-            }
-        }
-        let samples = [0, 200000];
-        let res = sample_file(&in_path, &samples);
-        assert!(res.is_err());
-        remove_file(in_path).unwrap();
-    }
-
-    #[test]
-    fn test_sample_file_missing_file() {
-        let in_path = tmp_file_path("test_sample_file_that_doesnt_exist.txt");
-        let samples = [0, 5];
-        let res = sample_file(&in_path, &samples);
-        assert!(res.is_err());
-    }
-}

archiver-lib/src/lib.rs

@@ -0,0 +1,11 @@
+#[macro_use]
+extern crate log;
+
+#[macro_use]
+extern crate serde_derive;
+
+#[macro_use]
+extern crate solana_metrics;
+
+pub mod archiver;
+mod result;

archiver-lib/src/result.rs

@@ -0,0 +1,48 @@
+use serde_json;
+use solana_client::client_error;
+use solana_ledger::blockstore;
+use solana_sdk::transport;
+use std::any::Any;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum ArchiverError {
+    #[error("IO error")]
+    IO(#[from] std::io::Error),
+
+    #[error("blockstore error")]
+    BlockstoreError(#[from] blockstore::BlockstoreError),
+
+    #[error("crossbeam error")]
+    CrossbeamSendError(#[from] crossbeam_channel::SendError<u64>),
+
+    #[error("send error")]
+    SendError(#[from] std::sync::mpsc::SendError<u64>),
+
+    #[error("join error")]
+    JoinError(Box<dyn Any + Send + 'static>),
+
+    #[error("transport error")]
+    TransportError(#[from] transport::TransportError),
+
+    #[error("client error")]
+    ClientError(#[from] client_error::ClientError),
+
+    #[error("Json parsing error")]
+    JsonError(#[from] serde_json::error::Error),
+
+    #[error("Storage account has no balance")]
+    EmptyStorageAccountBalance,
+
+    #[error("No RPC peers..")]
+    NoRpcPeers,
+
+    #[error("Couldn't download full segment")]
+    SegmentDownloadError,
+}
+
+impl std::convert::From<Box<dyn Any + Send + 'static>> for ArchiverError {
+    fn from(e: Box<dyn Any + Send + 'static>) -> ArchiverError {
+        ArchiverError::JoinError(e)
+    }
+}

archiver-utils/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "solana-archiver-utils"
+version = "1.0.7"
+description = "Solana Archiver Utils"
+authors = ["Solana Maintainers <maintainers@solana.com>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+log = "0.4.8"
+rand = "0.6.5"
+solana-chacha = { path = "../chacha", version = "1.0.7" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.0.7" }
+solana-ledger = { path = "../ledger", version = "1.0.7" }
+solana-logger = { path = "../logger", version = "1.0.7" }
+solana-perf = { path = "../perf", version = "1.0.7" }
+solana-sdk = { path = "../sdk", version = "1.0.7" }
+
+[dev-dependencies]
+hex = "0.4.0"
+
+[lib]
+name = "solana_archiver_utils"

archiver-utils/src/lib.rs

@@ -0,0 +1,120 @@
+#[macro_use]
+extern crate log;
+
+use solana_sdk::hash::{Hash, Hasher};
+use std::fs::File;
+use std::io::{self, BufReader, ErrorKind, Read, Seek, SeekFrom};
+use std::mem::size_of;
+use std::path::Path;
+
+pub fn sample_file(in_path: &Path, sample_offsets: &[u64]) -> io::Result<Hash> {
+    let in_file = File::open(in_path)?;
+    let metadata = in_file.metadata()?;
+    let mut buffer_file = BufReader::new(in_file);
+
+    let mut hasher = Hasher::default();
+    let sample_size = size_of::<Hash>();
+    let sample_size64 = sample_size as u64;
+    let mut buf = vec![0; sample_size];
+
+    let file_len = metadata.len();
+    if file_len < sample_size64 {
+        return Err(io::Error::new(ErrorKind::Other, "file too short!"));
+    }
+    for offset in sample_offsets {
+        if *offset > (file_len - sample_size64) / sample_size64 {
+            return Err(io::Error::new(ErrorKind::Other, "offset too large"));
+        }
+        buffer_file.seek(SeekFrom::Start(*offset * sample_size64))?;
+        trace!("sampling @ {} ", *offset);
+        match buffer_file.read(&mut buf) {
+            Ok(size) => {
+                assert_eq!(size, buf.len());
+                hasher.hash(&buf);
+            }
+            Err(e) => {
+                warn!("Error sampling file");
+                return Err(e);
+            }
+        }
+    }
+
+    Ok(hasher.result())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rand::{thread_rng, Rng};
+    use std::fs::{create_dir_all, remove_file};
+    use std::io::Write;
+    use std::path::PathBuf;
+
+    extern crate hex;
+
+    fn tmp_file_path(name: &str) -> PathBuf {
+        use std::env;
+        let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
+        let mut rand_bits = [0u8; 32];
+        thread_rng().fill(&mut rand_bits[..]);
+
+        let mut path = PathBuf::new();
+        path.push(out_dir);
+        path.push("tmp");
+        create_dir_all(&path).unwrap();
+
+        path.push(format!("{}-{:?}", name, hex::encode(rand_bits)));
+        println!("path: {:?}", path);
+        path
+    }
+
+    #[test]
+    fn test_sample_file() {
+        solana_logger::setup();
+        let in_path = tmp_file_path("test_sample_file_input.txt");
+        let num_strings = 4096;
+        let string = "12foobar";
+        {
+            let mut in_file = File::create(&in_path).unwrap();
+            for _ in 0..num_strings {
+                in_file.write(string.as_bytes()).unwrap();
+            }
+        }
+        let num_samples = (string.len() * num_strings / size_of::<Hash>()) as u64;
+        let samples: Vec<_> = (0..num_samples).collect();
+        let res = sample_file(&in_path, samples.as_slice());
+        let ref_hash: Hash = Hash::new(&[
+            173, 251, 182, 165, 10, 54, 33, 150, 133, 226, 106, 150, 99, 192, 179, 1, 230, 144,
+            151, 126, 18, 191, 54, 67, 249, 140, 230, 160, 56, 30, 170, 52,
+        ]);
+        let res = res.unwrap();
+        assert_eq!(res, ref_hash);
+
+        // Sample just past the end
+        assert!(sample_file(&in_path, &[num_samples]).is_err());
+        remove_file(&in_path).unwrap();
+    }
+
+    #[test]
+    fn test_sample_file_invalid_offset() {
+        let in_path = tmp_file_path("test_sample_file_invalid_offset_input.txt");
+        {
+            let mut in_file = File::create(&in_path).unwrap();
+            for _ in 0..4096 {
+                in_file.write("123456foobar".as_bytes()).unwrap();
+            }
+        }
+        let samples = [0, 200000];
+        let res = sample_file(&in_path, &samples);
+        assert!(res.is_err());
+        remove_file(in_path).unwrap();
+    }
+
+    #[test]
+    fn test_sample_file_missing_file() {
+        let in_path = tmp_file_path("test_sample_file_that_doesnt_exist.txt");
+        let samples = [0, 5];
+        let res = sample_file(&in_path, &samples);
+        assert!(res.is_err());
+    }
+}

archiver/Cargo.toml

@@ -2,18 +2,19 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-archiver"
-version = "0.22.0"
+version = "1.0.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-console = "0.9.1"
-solana-clap-utils = { path = "../clap-utils", version = "0.22.0" }
-solana-core = { path = "../core", version = "0.22.0" }
-solana-logger = { path = "../logger", version = "0.22.0" }
-solana-metrics = { path = "../metrics", version = "0.22.0" }
-solana-net-utils = { path = "../net-utils", version = "0.22.0" }
-solana-sdk = { path = "../sdk", version = "0.22.0" }
+console = "0.9.2"
+solana-clap-utils = { path = "../clap-utils", version = "1.0.7" }
+solana-core = { path = "../core", version = "1.0.7" }
+solana-logger = { path = "../logger", version = "1.0.7" }
+solana-metrics = { path = "../metrics", version = "1.0.7" }
+solana-archiver-lib = { path = "../archiver-lib", version = "1.0.7" }
+solana-net-utils = { path = "../net-utils", version = "1.0.7" }
+solana-sdk = { path = "../sdk", version = "1.0.7" }
 

archiver/src/main.rs

@@ -1,19 +1,23 @@
 use clap::{crate_description, crate_name, App, Arg};
 use console::style;
+use solana_archiver_lib::archiver::Archiver;
 use solana_clap_utils::{
-    input_validators::is_keypair,
-    keypair::{
-        self, keypair_input, KeypairWithSource, ASK_SEED_PHRASE_ARG,
-        SKIP_SEED_PHRASE_VALIDATION_ARG,
-    },
+    input_parsers::keypair_of, input_validators::is_keypair_or_ask_keyword,
+    keypair::SKIP_SEED_PHRASE_VALIDATION_ARG,
 };
 use solana_core::{
-    archiver::Archiver,
     cluster_info::{Node, VALIDATOR_PORT_RANGE},
     contact_info::ContactInfo,
 };
-use solana_sdk::{commitment_config::CommitmentConfig, signature::KeypairUtil};
-use std::{net::SocketAddr, path::PathBuf, process::exit, sync::Arc};
+use solana_sdk::{
+    commitment_config::CommitmentConfig,
+    signature::{Keypair, Signer},
+};
+use std::{
+    net::{IpAddr, Ipv4Addr, SocketAddr},
+    path::PathBuf,
+    sync::Arc,
+};
 
 fn main() {
     solana_logger::setup();
@@ -24,10 +28,10 @@ fn main() {
         .arg(
             Arg::with_name("identity_keypair")
                 .short("i")
-                .long("identity-keypair")
+                .long("identity")
                 .value_name("PATH")
                 .takes_value(true)
-                .validator(is_keypair)
+                .validator(is_keypair_or_ask_keyword)
                 .help("File containing an identity (keypair)"),
         )
         .arg(
@@ -55,48 +59,27 @@ fn main() {
                 .long("storage-keypair")
                 .value_name("PATH")
                 .takes_value(true)
-                .validator(is_keypair)
+                .validator(is_keypair_or_ask_keyword)
                 .help("File containing the storage account keypair"),
         )
-        .arg(
-            Arg::with_name(ASK_SEED_PHRASE_ARG.name)
-                .long(ASK_SEED_PHRASE_ARG.long)
-                .value_name("KEYPAIR NAME")
-                .multiple(true)
-                .takes_value(true)
-                .possible_values(&["identity-keypair", "storage-keypair"])
-                .help(ASK_SEED_PHRASE_ARG.help),
-        )
         .arg(
             Arg::with_name(SKIP_SEED_PHRASE_VALIDATION_ARG.name)
                 .long(SKIP_SEED_PHRASE_VALIDATION_ARG.long)
-                .requires(ASK_SEED_PHRASE_ARG.name)
                 .help(SKIP_SEED_PHRASE_VALIDATION_ARG.help),
         )
         .get_matches();
 
     let ledger_path = PathBuf::from(matches.value_of("ledger").unwrap());
 
-    let identity_keypair = keypair_input(&matches, "identity_keypair")
-        .unwrap_or_else(|err| {
-            eprintln!("Identity keypair input failed: {}", err);
-            exit(1);
-        })
-        .keypair;
-    let KeypairWithSource {
-        keypair: storage_keypair,
-        source: storage_keypair_source,
-    } = keypair_input(&matches, "storage_keypair").unwrap_or_else(|err| {
-        eprintln!("Storage keypair input failed: {}", err);
-        exit(1);
-    });
-    if storage_keypair_source == keypair::Source::Generated {
+    let identity_keypair = keypair_of(&matches, "identity_keypair").unwrap_or_else(Keypair::new);
+
+    let storage_keypair = keypair_of(&matches, "storage_keypair").unwrap_or_else(|| {
         clap::Error::with_description(
             "The `storage-keypair` argument was not found",
             clap::ErrorKind::ArgumentNotFound,
         )
         .exit();
-    }
+    });
 
     let entrypoint_addr = matches
         .value_of("entrypoint")
@@ -116,6 +99,7 @@ fn main() {
         &identity_keypair.pubkey(),
         &gossip_addr,
         VALIDATOR_PORT_RANGE,
+        IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
     );
 
     println!(

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "0.22.0"
+version = "1.0.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ homepage = "https://solana.com/"
 [dependencies]
 log = "0.4.6"
 rayon = "1.2.0"
-solana-core = { path = "../core", version = "0.22.0" }
-solana-ledger = { path = "../ledger", version = "0.22.0" }
-solana-logger = { path = "../logger", version = "0.22.0" }
-solana-runtime = { path = "../runtime", version = "0.22.0" }
-solana-measure = { path = "../measure", version = "0.22.0" }
-solana-sdk = { path = "../sdk", version = "0.22.0" }
+solana-core = { path = "../core", version = "1.0.7" }
+solana-ledger = { path = "../ledger", version = "1.0.7" }
+solana-logger = { path = "../logger", version = "1.0.7" }
+solana-runtime = { path = "../runtime", version = "1.0.7" }
+solana-measure = { path = "../measure", version = "1.0.7" }
+solana-sdk = { path = "../sdk", version = "1.0.7" }
 rand = "0.6.5"
 crossbeam-channel = "0.3"

banking-bench/src/main.rs

@@ -10,7 +10,7 @@ use solana_core::packet::to_packets_chunked;
 use solana_core::poh_recorder::PohRecorder;
 use solana_core::poh_recorder::WorkingBankEntry;
 use solana_ledger::bank_forks::BankForks;
-use solana_ledger::{blocktree::Blocktree, get_tmp_ledger_path};
+use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
 use solana_measure::measure::Measure;
 use solana_runtime::bank::Bank;
 use solana_sdk::hash::Hash;
@@ -139,11 +139,11 @@ fn main() {
     let mut verified: Vec<_> = to_packets_chunked(&transactions.clone(), PACKETS_PER_BATCH);
     let ledger_path = get_tmp_ledger_path!();
     {
-        let blocktree = Arc::new(
-            Blocktree::open(&ledger_path).expect("Expected to be able to open database ledger"),
+        let blockstore = Arc::new(
+            Blockstore::open(&ledger_path).expect("Expected to be able to open database ledger"),
         );
         let (exit, poh_recorder, poh_service, signal_receiver) =
-            create_test_recorder(&bank, &blocktree, None);
+            create_test_recorder(&bank, &blockstore, None);
         let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
         let cluster_info = Arc::new(RwLock::new(cluster_info));
         let banking_stage = BankingStage::new(
@@ -162,8 +162,8 @@ fn main() {
         // If it is dropped before poh_service, then poh_service will error when
         // calling send() on the channel.
         let signal_receiver = Arc::new(signal_receiver);
-        let mut total = 0;
-        let mut tx_total = 0;
+        let mut total_us = 0;
+        let mut tx_total_us = 0;
         let mut txs_processed = 0;
         let mut root = 1;
         let collector = Pubkey::new_rand();
@@ -173,6 +173,7 @@ fn main() {
             chunk_len,
             num_threads,
         };
+        let mut total_sent = 0;
         for _ in 0..ITERS {
             let now = Instant::now();
             let mut sent = 0;
@@ -223,7 +224,7 @@ fn main() {
                 );
                 assert!(txs_processed < bank.transaction_count());
                 txs_processed = bank.transaction_count();
-                tx_total += duration_as_us(&now.elapsed());
+                tx_total_us += duration_as_us(&now.elapsed());
 
                 let mut poh_time = Measure::start("poh_time");
                 poh_recorder.lock().unwrap().reset(
@@ -255,20 +256,21 @@ fn main() {
                     poh_time.as_us(),
                 );
             } else {
-                tx_total += duration_as_us(&now.elapsed());
+                tx_total_us += duration_as_us(&now.elapsed());
             }
 
             // This signature clear may not actually clear the signatures
             // in this chunk, but since we rotate between CHUNKS then
             // we should clear them by the time we come around again to re-use that chunk.
             bank.clear_signatures();
-            total += duration_as_us(&now.elapsed());
+            total_us += duration_as_us(&now.elapsed());
             debug!(
                 "time: {} us checked: {} sent: {}",
                 duration_as_us(&now.elapsed()),
                 txes / CHUNKS,
                 sent,
             );
+            total_sent += sent;
 
             if bank.slot() > 0 && bank.slot() % 16 == 0 {
                 for tx in transactions.iter_mut() {
@@ -284,11 +286,11 @@ fn main() {
         }
         eprintln!(
             "{{'name': 'banking_bench_total', 'median': '{}'}}",
-            total / ITERS as u64,
+            (1000.0 * 1000.0 * total_sent as f64) / (total_us as f64),
         );
         eprintln!(
             "{{'name': 'banking_bench_tx_total', 'median': '{}'}}",
-            tx_total / ITERS as u64,
+            (1000.0 * 1000.0 * total_sent as f64) / (tx_total_us as f64),
         );
 
         drop(verified_sender);
@@ -300,5 +302,5 @@ fn main() {
         sleep(Duration::from_secs(1));
         debug!("waited for poh_service");
     }
-    let _unused = Blocktree::destroy(&ledger_path);
+    let _unused = Blockstore::destroy(&ledger_path);
 }

bench-exchange/Cargo.toml

@@ -2,40 +2,33 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "0.22.0"
+version = "1.0.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 publish = false
 
 [dependencies]
-bincode = "1.2.1"
-bs58 = "0.3.0"
 clap = "2.32.0"
-env_logger = "0.7.1"
 itertools = "0.8.2"
 log = "0.4.8"
 num-derive = "0.3"
 num-traits = "0.2"
 rand = "0.6.5"
 rayon = "1.2.0"
-serde = "1.0.104"
-serde_derive = "1.0.103"
-serde_json = "1.0.44"
+serde_json = "1.0.46"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "0.22.0" }
-solana-core = { path = "../core", version = "0.22.0" }
-solana-genesis = { path = "../genesis", version = "0.22.0" }
-solana-client = { path = "../client", version = "0.22.0" }
-solana-faucet = { path = "../faucet", version = "0.22.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "0.22.0" }
-solana-logger = { path = "../logger", version = "0.22.0" }
-solana-metrics = { path = "../metrics", version = "0.22.0" }
-solana-net-utils = { path = "../net-utils", version = "0.22.0" }
-solana-runtime = { path = "../runtime", version = "0.22.0" }
-solana-sdk = { path = "../sdk", version = "0.22.0" }
-untrusted = "0.7.0"
-ws = "0.9.1"
+solana-clap-utils = { path = "../clap-utils", version = "1.0.7" }
+solana-core = { path = "../core", version = "1.0.7" }
+solana-genesis = { path = "../genesis", version = "1.0.7" }
+solana-client = { path = "../client", version = "1.0.7" }
+solana-faucet = { path = "../faucet", version = "1.0.7" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.0.7" }
+solana-logger = { path = "../logger", version = "1.0.7" }
+solana-metrics = { path = "../metrics", version = "1.0.7" }
+solana-net-utils = { path = "../net-utils", version = "1.0.7" }
+solana-runtime = { path = "../runtime", version = "1.0.7" }
+solana-sdk = { path = "../sdk", version = "1.0.7" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "0.22.0" }
+solana-local-cluster = { path = "../local-cluster", version = "1.0.7" }

bench-exchange/src/bench.rs

@@ -15,7 +15,7 @@ use solana_sdk::{
     client::{Client, SyncClient},
     commitment_config::CommitmentConfig,
     pubkey::Pubkey,
-    signature::{Keypair, KeypairUtil},
+    signature::{Keypair, Signer},
     timing::{duration_as_ms, duration_as_s},
     transaction::Transaction,
     {system_instruction, system_program},
@@ -701,7 +701,7 @@ fn verify_funding_transfer<T: SyncClient + ?Sized>(
     false
 }
 
-pub fn fund_keys(client: &dyn Client, source: &Keypair, dests: &[Arc<Keypair>], lamports: u64) {
+pub fn fund_keys<T: Client>(client: &T, source: &Keypair, dests: &[Arc<Keypair>], lamports: u64) {
     let total = lamports * (dests.len() as u64 + 1);
     let mut funded: Vec<(&Keypair, u64)> = vec![(source, total)];
     let mut notfunded: Vec<&Arc<Keypair>> = dests.iter().collect();
@@ -824,7 +824,11 @@ pub fn fund_keys(client: &dyn Client, source: &Keypair, dests: &[Arc<Keypair>],
     }
 }
 
-pub fn create_token_accounts(client: &dyn Client, signers: &[Arc<Keypair>], accounts: &[Keypair]) {
+pub fn create_token_accounts<T: Client>(
+    client: &T,
+    signers: &[Arc<Keypair>],
+    accounts: &[Keypair],
+) {
     let mut notfunded: Vec<(&Arc<Keypair>, &Keypair)> = signers.iter().zip(accounts).collect();
 
     while !notfunded.is_empty() {
@@ -968,7 +972,12 @@ fn generate_keypairs(num: u64) -> Vec<Keypair> {
     rnd.gen_n_keypairs(num)
 }
 
-pub fn airdrop_lamports(client: &dyn Client, faucet_addr: &SocketAddr, id: &Keypair, amount: u64) {
+pub fn airdrop_lamports<T: Client>(
+    client: &T,
+    faucet_addr: &SocketAddr,
+    id: &Keypair,
+    amount: u64,
+) {
     let balance = client.get_balance_with_commitment(&id.pubkey(), CommitmentConfig::recent());
     let balance = balance.unwrap_or(0);
     if balance >= amount {

bench-exchange/src/cli.rs

@@ -1,7 +1,7 @@
 use clap::{crate_description, crate_name, value_t, App, Arg, ArgMatches};
 use solana_core::gen_keys::GenKeys;
 use solana_faucet::faucet::FAUCET_PORT;
-use solana_sdk::signature::{read_keypair_file, Keypair, KeypairUtil};
+use solana_sdk::signature::{read_keypair_file, Keypair};
 use std::net::SocketAddr;
 use std::process::exit;
 use std::time::Duration;

bench-exchange/src/main.rs

@@ -5,7 +5,7 @@ pub mod order_book;
 use crate::bench::{airdrop_lamports, create_client_accounts_file, do_bench_exchange, Config};
 use log::*;
 use solana_core::gossip_service::{discover_cluster, get_multi_client};
-use solana_sdk::signature::KeypairUtil;
+use solana_sdk::signature::Signer;
 
 fn main() {
     solana_logger::setup();

bench-exchange/tests/bench_exchange.rs

@@ -10,12 +10,13 @@ use solana_local_cluster::local_cluster::{ClusterConfig, LocalCluster};
 use solana_runtime::bank::Bank;
 use solana_runtime::bank_client::BankClient;
 use solana_sdk::genesis_config::create_genesis_config;
-use solana_sdk::signature::{Keypair, KeypairUtil};
+use solana_sdk::signature::{Keypair, Signer};
 use std::process::exit;
 use std::sync::mpsc::channel;
 use std::time::Duration;
 
 #[test]
+#[ignore]
 fn test_exchange_local_cluster() {
     solana_logger::setup();
 

bench-streamer/Cargo.toml

@@ -2,14 +2,14 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "0.22.0"
+version = "1.0.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-solana-clap-utils = { path = "../clap-utils", version = "0.22.0" }
-solana-core = { path = "../core", version = "0.22.0" }
-solana-logger = { path = "../logger", version = "0.22.0" }
-solana-net-utils = { path = "../net-utils", version = "0.22.0" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.7" }
+solana-core = { path = "../core", version = "1.0.7" }
+solana-logger = { path = "../logger", version = "1.0.7" }
+solana-net-utils = { path = "../net-utils", version = "1.0.7" }

bench-streamer/src/main.rs

@@ -1,6 +1,5 @@
 use clap::{crate_description, crate_name, App, Arg};
 use solana_core::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
-use solana_core::result::Result;
 use solana_core::streamer::{receiver, PacketReceiver};
 use std::cmp::max;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
@@ -8,7 +7,7 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::sync::mpsc::channel;
 use std::sync::Arc;
 use std::thread::sleep;
-use std::thread::{spawn, JoinHandle};
+use std::thread::{spawn, JoinHandle, Result};
 use std::time::Duration;
 use std::time::SystemTime;
 
@@ -68,7 +67,8 @@ fn main() -> Result<()> {
     }
 
     let mut port = 0;
-    let mut addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0);
+    let ip_addr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0));
+    let mut addr = SocketAddr::new(ip_addr, 0);
 
     let exit = Arc::new(AtomicBool::new(false));
 
@@ -76,7 +76,7 @@ fn main() -> Result<()> {
     let mut read_threads = Vec::new();
     let recycler = PacketsRecycler::default();
     for _ in 0..num_sockets {
-        let read = solana_net_utils::bind_to(port, false).unwrap();
+        let read = solana_net_utils::bind_to(ip_addr, port, false).unwrap();
         read.set_read_timeout(Some(Duration::new(1, 0))).unwrap();
 
         addr = read.local_addr().unwrap();

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "0.22.0"
+version = "1.0.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -12,28 +12,26 @@ bincode = "1.2.1"
 clap = "2.33.0"
 log = "0.4.8"
 rayon = "1.2.0"
-serde = "1.0.104"
-serde_derive = "1.0.103"
-serde_json = "1.0.44"
+serde_json = "1.0.46"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "0.22.0" }
-solana-core = { path = "../core", version = "0.22.0" }
-solana-genesis = { path = "../genesis", version = "0.22.0" }
-solana-client = { path = "../client", version = "0.22.0" }
-solana-faucet = { path = "../faucet", version = "0.22.0" }
-solana-librapay = { path = "../programs/librapay", version = "0.22.0", optional = true }
-solana-logger = { path = "../logger", version = "0.22.0" }
-solana-metrics = { path = "../metrics", version = "0.22.0" }
-solana-measure = { path = "../measure", version = "0.22.0" }
-solana-net-utils = { path = "../net-utils", version = "0.22.0" }
-solana-runtime = { path = "../runtime", version = "0.22.0" }
-solana-sdk = { path = "../sdk", version = "0.22.0" }
-solana-move-loader-program = { path = "../programs/move_loader", version = "0.22.0", optional = true }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.7" }
+solana-core = { path = "../core", version = "1.0.7" }
+solana-genesis = { path = "../genesis", version = "1.0.7" }
+solana-client = { path = "../client", version = "1.0.7" }
+solana-faucet = { path = "../faucet", version = "1.0.7" }
+solana-librapay = { path = "../programs/librapay", version = "1.0.7", optional = true }
+solana-logger = { path = "../logger", version = "1.0.7" }
+solana-metrics = { path = "../metrics", version = "1.0.7" }
+solana-measure = { path = "../measure", version = "1.0.7" }
+solana-net-utils = { path = "../net-utils", version = "1.0.7" }
+solana-runtime = { path = "../runtime", version = "1.0.7" }
+solana-sdk = { path = "../sdk", version = "1.0.7" }
+solana-move-loader-program = { path = "../programs/move_loader", version = "1.0.7", optional = true }
 
 [dev-dependencies]
 serial_test = "0.3.2"
-serial_test_derive = "0.3.1"
-solana-local-cluster = { path = "../local-cluster", version = "0.22.0" }
+serial_test_derive = "0.4.0"
+solana-local-cluster = { path = "../local-cluster", version = "1.0.7" }
 
 [features]
 move = ["solana-librapay", "solana-move-loader-program"]

bench-tps/src/bench.rs

@@ -7,7 +7,7 @@ use solana_faucet::faucet::request_airdrop_transaction;
 #[cfg(feature = "move")]
 use solana_librapay::{create_genesis, upload_mint_script, upload_payment_script};
 use solana_measure::measure::Measure;
-use solana_metrics::{self, datapoint_debug};
+use solana_metrics::{self, datapoint_info};
 use solana_sdk::{
     client::Client,
     clock::{DEFAULT_TICKS_PER_SECOND, DEFAULT_TICKS_PER_SLOT, MAX_PROCESSING_AGE},
@@ -15,14 +15,13 @@ use solana_sdk::{
     fee_calculator::FeeCalculator,
     hash::Hash,
     pubkey::Pubkey,
-    signature::{Keypair, KeypairUtil},
+    signature::{Keypair, Signer},
     system_instruction, system_transaction,
     timing::{duration_as_ms, duration_as_s, duration_as_us, timestamp},
     transaction::Transaction,
 };
 use std::{
-    cmp,
-    collections::VecDeque,
+    collections::{HashSet, VecDeque},
     net::SocketAddr,
     process::exit,
     sync::{
@@ -66,10 +65,9 @@ fn get_recent_blockhash<T: Client>(client: &T) -> (Hash, FeeCalculator) {
 }
 
 pub fn do_bench_tps<T>(
-    clients: Vec<T>,
+    client: Arc<T>,
     config: Config,
     gen_keypairs: Vec<Keypair>,
-    keypair0_balance: u64,
     libra_args: Option<LibraKeys>,
 ) -> u64
 where
@@ -82,13 +80,9 @@ where
         duration,
         tx_count,
         sustained,
-        num_lamports_per_account,
         ..
     } = config;
 
-    let clients: Vec<_> = clients.into_iter().map(Arc::new).collect();
-    let client = &clients[0];
-
     let mut source_keypair_chunks: Vec<Vec<&Keypair>> = Vec::new();
     let mut dest_keypair_chunks: Vec<VecDeque<&Keypair>> = Vec::new();
     assert!(gen_keypairs.len() >= 2 * tx_count);
@@ -115,20 +109,17 @@ where
     let maxes = Arc::new(RwLock::new(Vec::new()));
     let sample_period = 1; // in seconds
     info!("Sampling TPS every {} second...", sample_period);
-    let v_threads: Vec<_> = clients
-        .iter()
-        .map(|client| {
-            let exit_signal = exit_signal.clone();
-            let maxes = maxes.clone();
-            let client = client.clone();
-            Builder::new()
-                .name("solana-client-sample".to_string())
-                .spawn(move || {
-                    sample_txs(&exit_signal, &maxes, sample_period, &client);
-                })
-                .unwrap()
-        })
-        .collect();
+    let sample_thread = {
+        let exit_signal = exit_signal.clone();
+        let maxes = maxes.clone();
+        let client = client.clone();
+        Builder::new()
+            .name("solana-client-sample".to_string())
+            .spawn(move || {
+                sample_txs(&exit_signal, &maxes, sample_period, &client);
+            })
+            .unwrap()
+    };
 
     let shared_txs: SharedTransactions = Arc::new(RwLock::new(VecDeque::new()));
 
@@ -174,11 +165,10 @@ where
 
     // generate and send transactions for the specified duration
     let start = Instant::now();
-    let keypair_chunks = source_keypair_chunks.len() as u64;
+    let keypair_chunks = source_keypair_chunks.len();
     let mut reclaim_lamports_back_to_source_account = false;
-    let mut i = keypair0_balance;
+    let mut chunk_index = 0;
     while start.elapsed() < duration {
-        let chunk_index = (i % keypair_chunks) as usize;
         generate_txs(
             &shared_txs,
             &recent_blockhash,
@@ -197,7 +187,9 @@ where
                 sleep(Duration::from_millis(1));
             }
         } else {
-            while shared_tx_active_thread_count.load(Ordering::Relaxed) > 0 {
+            while !shared_txs.read().unwrap().is_empty()
+                || shared_tx_active_thread_count.load(Ordering::Relaxed) > 0
+            {
                 sleep(Duration::from_millis(1));
             }
         }
@@ -206,8 +198,11 @@ where
         // transaction signatures even when blockhash is reused.
         dest_keypair_chunks[chunk_index].rotate_left(1);
 
-        i += 1;
-        if should_switch_directions(num_lamports_per_account, keypair_chunks, i) {
+        // Move on to next chunk
+        chunk_index = (chunk_index + 1) % keypair_chunks;
+
+        // Switch directions after transfering for each "chunk"
+        if chunk_index == 0 {
             reclaim_lamports_back_to_source_account = !reclaim_lamports_back_to_source_account;
         }
     }
@@ -215,11 +210,9 @@ where
     // Stop the sampling threads so it will collect the stats
     exit_signal.store(true, Ordering::Relaxed);
 
-    info!("Waiting for validator threads...");
-    for t in v_threads {
-        if let Err(err) = t.join() {
-            info!("  join() failed with: {:?}", err);
-        }
+    info!("Waiting for sampler threads...");
+    if let Err(err) = sample_thread.join() {
+        info!("  join() failed with: {:?}", err);
     }
 
     // join the tx send threads
@@ -251,7 +244,7 @@ where
 
 fn metrics_submit_lamport_balance(lamport_balance: u64) {
     info!("Token balance: {}", lamport_balance);
-    datapoint_debug!(
+    datapoint_info!(
         "bench-tps-lamport_balance",
         ("balance", lamport_balance, i64)
     );
@@ -382,7 +375,7 @@ fn generate_txs(
         duration_as_ms(&duration),
         blockhash,
     );
-    datapoint_debug!(
+    datapoint_info!(
         "bench-tps-generate_txs",
         ("duration", duration_as_us(&duration), i64)
     );
@@ -488,7 +481,7 @@ fn do_tx_transfers<T: Client>(
                 duration_as_ms(&transfer_start.elapsed()),
                 tx_len as f32 / duration_as_s(&transfer_start.elapsed()),
             );
-            datapoint_debug!(
+            datapoint_info!(
                 "bench-tps-do_tx_transfers",
                 ("duration", duration_as_us(&transfer_start.elapsed()), i64),
                 ("count", tx_len, i64)
@@ -500,177 +493,218 @@ fn do_tx_transfers<T: Client>(
     }
 }
 
-fn verify_funding_transfer<T: Client>(client: &T, tx: &Transaction, amount: u64) -> bool {
+fn verify_funding_transfer<T: Client>(client: &Arc<T>, tx: &Transaction, amount: u64) -> bool {
     for a in &tx.message().account_keys[1..] {
-        if client
-            .get_balance_with_commitment(a, CommitmentConfig::recent())
-            .unwrap_or(0)
-            >= amount
-        {
-            return true;
+        match client.get_balance_with_commitment(a, CommitmentConfig::recent()) {
+            Ok(balance) => return balance >= amount,
+            Err(err) => error!("failed to get balance {:?}", err),
         }
     }
     false
 }
 
+trait FundingTransactions<'a> {
+    fn fund<T: 'static + Client + Send + Sync>(
+        &mut self,
+        client: &Arc<T>,
+        to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)],
+        to_lamports: u64,
+    );
+    fn make(&mut self, to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)]);
+    fn sign(&mut self, blockhash: Hash);
+    fn send<T: Client>(&self, client: &Arc<T>);
+    fn verify<T: 'static + Client + Send + Sync>(&mut self, client: &Arc<T>, to_lamports: u64);
+}
+
+impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
+    fn fund<T: 'static + Client + Send + Sync>(
+        &mut self,
+        client: &Arc<T>,
+        to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)],
+        to_lamports: u64,
+    ) {
+        self.make(to_fund);
+
+        let mut tries = 0;
+        while !self.is_empty() {
+            info!(
+                "{} {} each to {} accounts in {} txs",
+                if tries == 0 {
+                    "transferring"
+                } else {
+                    " retrying"
+                },
+                to_lamports,
+                self.len() * MAX_SPENDS_PER_TX as usize,
+                self.len(),
+            );
+
+            let (blockhash, _fee_calculator) = get_recent_blockhash(client.as_ref());
+
+            // re-sign retained to_fund_txes with updated blockhash
+            self.sign(blockhash);
+            self.send(&client);
+
+            // Sleep a few slots to allow transactions to process
+            sleep(Duration::from_secs(1));
+
+            self.verify(&client, to_lamports);
+
+            // retry anything that seems to have dropped through cracks
+            //  again since these txs are all or nothing, they're fine to
+            //  retry
+            tries += 1;
+        }
+        info!("transferred");
+    }
+
+    fn make(&mut self, to_fund: &[(&'a Keypair, Vec<(Pubkey, u64)>)]) {
+        let mut make_txs = Measure::start("make_txs");
+        let to_fund_txs: Vec<(&Keypair, Transaction)> = to_fund
+            .par_iter()
+            .map(|(k, t)| {
+                let tx = Transaction::new_unsigned_instructions(system_instruction::transfer_many(
+                    &k.pubkey(),
+                    &t,
+                ));
+                (*k, tx)
+            })
+            .collect();
+        make_txs.stop();
+        debug!(
+            "make {} unsigned txs: {}us",
+            to_fund_txs.len(),
+            make_txs.as_us()
+        );
+        self.extend(to_fund_txs);
+    }
+
+    fn sign(&mut self, blockhash: Hash) {
+        let mut sign_txs = Measure::start("sign_txs");
+        self.par_iter_mut().for_each(|(k, tx)| {
+            tx.sign(&[*k], blockhash);
+        });
+        sign_txs.stop();
+        debug!("sign {} txs: {}us", self.len(), sign_txs.as_us());
+    }
+
+    fn send<T: Client>(&self, client: &Arc<T>) {
+        let mut send_txs = Measure::start("send_txs");
+        self.iter().for_each(|(_, tx)| {
+            client.async_send_transaction(tx.clone()).expect("transfer");
+        });
+        send_txs.stop();
+        debug!("send {} txs: {}us", self.len(), send_txs.as_us());
+    }
+
+    fn verify<T: 'static + Client + Send + Sync>(&mut self, client: &Arc<T>, to_lamports: u64) {
+        let starting_txs = self.len();
+        let verified_txs = Arc::new(AtomicUsize::new(0));
+        let too_many_failures = Arc::new(AtomicBool::new(false));
+        let loops = if starting_txs < 1000 { 3 } else { 1 };
+        // Only loop multiple times for small (quick) transaction batches
+        for _ in 0..loops {
+            let failed_verify = Arc::new(AtomicUsize::new(0));
+            let client = client.clone();
+            let verified_txs = &verified_txs;
+            let failed_verify = &failed_verify;
+            let too_many_failures = &too_many_failures;
+            let verified_set: HashSet<Pubkey> = self
+                .par_iter()
+                .filter_map(move |(k, tx)| {
+                    if too_many_failures.load(Ordering::Relaxed) {
+                        return None;
+                    }
+
+                    let verified = if verify_funding_transfer(&client, &tx, to_lamports) {
+                        verified_txs.fetch_add(1, Ordering::Relaxed);
+                        Some(k.pubkey())
+                    } else {
+                        failed_verify.fetch_add(1, Ordering::Relaxed);
+                        None
+                    };
+
+                    let verified_txs = verified_txs.load(Ordering::Relaxed);
+                    let failed_verify = failed_verify.load(Ordering::Relaxed);
+                    let remaining_count = starting_txs.saturating_sub(verified_txs + failed_verify);
+                    if failed_verify > 100 && failed_verify > verified_txs {
+                        too_many_failures.store(true, Ordering::Relaxed);
+                        warn!(
+                            "Too many failed transfers... {} remaining, {} verified, {} failures",
+                            remaining_count, verified_txs, failed_verify
+                        );
+                    }
+                    if remaining_count % 100 == 0 {
+                        info!(
+                            "Verifying transfers... {} remaining, {} verified, {} failures",
+                            remaining_count, verified_txs, failed_verify
+                        );
+                    }
+
+                    verified
+                })
+                .collect();
+
+            self.retain(|(k, _)| !verified_set.contains(&k.pubkey()));
+            if self.is_empty() {
+                break;
+            }
+            info!("Looping verifications");
+
+            let verified_txs = verified_txs.load(Ordering::Relaxed);
+            let failed_verify = failed_verify.load(Ordering::Relaxed);
+            let remaining_count = starting_txs.saturating_sub(verified_txs + failed_verify);
+            info!(
+                "Verifying transfers... {} remaining, {} verified, {} failures",
+                remaining_count, verified_txs, failed_verify
+            );
+            sleep(Duration::from_millis(100));
+        }
+    }
+}
+
 /// fund the dests keys by spending all of the source keys into MAX_SPENDS_PER_TX
 /// on every iteration.  This allows us to replay the transfers because the source is either empty,
 /// or full
-pub fn fund_keys<T: Client>(
-    client: &T,
+pub fn fund_keys<T: 'static + Client + Send + Sync>(
+    client: Arc<T>,
     source: &Keypair,
     dests: &[Keypair],
     total: u64,
     max_fee: u64,
-    mut extra: u64,
+    lamports_per_account: u64,
 ) {
-    let mut funded: Vec<(&Keypair, u64)> = vec![(source, total)];
-    let mut notfunded: Vec<&Keypair> = dests.iter().collect();
-    let lamports_per_account = (total - (extra * max_fee)) / (notfunded.len() as u64 + 1);
-
-    info!(
-        "funding keys {} with lamports: {:?} total: {}",
-        dests.len(),
-        client.get_balance(&source.pubkey()),
-        total
-    );
-    while !notfunded.is_empty() {
-        let mut new_funded: Vec<(&Keypair, u64)> = vec![];
-        let mut to_fund = vec![];
-        info!("creating from... {}", funded.len());
-        let mut build_to_fund = Measure::start("build_to_fund");
-        for f in &mut funded {
-            let max_units = cmp::min(notfunded.len() as u64, MAX_SPENDS_PER_TX);
-            if max_units == 0 {
-                break;
-            }
-            let start = notfunded.len() - max_units as usize;
-            let fees = if extra > 0 { max_fee } else { 0 };
-            let per_unit = (f.1 - lamports_per_account - fees) / max_units;
-            let moves: Vec<_> = notfunded[start..]
-                .iter()
-                .map(|k| (k.pubkey(), per_unit))
-                .collect();
-            notfunded[start..]
-                .iter()
-                .for_each(|k| new_funded.push((k, per_unit)));
-            notfunded.truncate(start);
-            if !moves.is_empty() {
-                to_fund.push((f.0, moves));
-            }
-            extra -= 1;
+    let mut funded: Vec<&Keypair> = vec![source];
+    let mut funded_funds = total;
+    let mut not_funded: Vec<&Keypair> = dests.iter().collect();
+    while !not_funded.is_empty() {
+        // Build to fund list and prepare funding sources for next iteration
+        let mut new_funded: Vec<&Keypair> = vec![];
+        let mut to_fund: Vec<(&Keypair, Vec<(Pubkey, u64)>)> = vec![];
+        let to_lamports = (funded_funds - lamports_per_account - max_fee) / MAX_SPENDS_PER_TX;
+        for f in funded {
+            let start = not_funded.len() - MAX_SPENDS_PER_TX as usize;
+            let dests: Vec<_> = not_funded.drain(start..).collect();
+            let spends: Vec<_> = dests.iter().map(|k| (k.pubkey(), to_lamports)).collect();
+            to_fund.push((f, spends));
+            new_funded.extend(dests.into_iter());
         }
-        build_to_fund.stop();
-        debug!("build to_fund vec: {}us", build_to_fund.as_us());
 
         // try to transfer a "few" at a time with recent blockhash
         //  assume 4MB network buffers, and 512 byte packets
         const FUND_CHUNK_LEN: usize = 4 * 1024 * 1024 / 512;
 
         to_fund.chunks(FUND_CHUNK_LEN).for_each(|chunk| {
-            let mut tries = 0;
-
-            let mut make_txs = Measure::start("make_txs");
-            // this set of transactions just initializes us for bookkeeping
-            #[allow(clippy::clone_double_ref)] // sigh
-            let mut to_fund_txs: Vec<_> = chunk
-                .par_iter()
-                .map(|(k, m)| {
-                    let tx = Transaction::new_unsigned_instructions(
-                        system_instruction::transfer_many(&k.pubkey(), &m),
-                    );
-                    (k.clone(), tx)
-                })
-                .collect();
-            make_txs.stop();
-            debug!(
-                "make {} unsigned txs: {}us",
-                to_fund_txs.len(),
-                make_txs.as_us()
+            Vec::<(&Keypair, Transaction)>::with_capacity(chunk.len()).fund(
+                &client,
+                chunk,
+                to_lamports,
             );
-
-            let amount = chunk[0].1[0].1;
-
-            while !to_fund_txs.is_empty() {
-                let receivers = to_fund_txs
-                    .iter()
-                    .fold(0, |len, (_, tx)| len + tx.message().instructions.len());
-
-                info!(
-                    "{} {} to {} in {} txs",
-                    if tries == 0 {
-                        "transferring"
-                    } else {
-                        " retrying"
-                    },
-                    amount,
-                    receivers,
-                    to_fund_txs.len(),
-                );
-
-                let (blockhash, _fee_calculator) = get_recent_blockhash(client);
-
-                // re-sign retained to_fund_txes with updated blockhash
-                let mut sign_txs = Measure::start("sign_txs");
-                to_fund_txs.par_iter_mut().for_each(|(k, tx)| {
-                    tx.sign(&[*k], blockhash);
-                });
-                sign_txs.stop();
-                debug!("sign {} txs: {}us", to_fund_txs.len(), sign_txs.as_us());
-
-                let mut send_txs = Measure::start("send_txs");
-                to_fund_txs.iter().for_each(|(_, tx)| {
-                    client.async_send_transaction(tx.clone()).expect("transfer");
-                });
-                send_txs.stop();
-                debug!("send {} txs: {}us", to_fund_txs.len(), send_txs.as_us());
-
-                let mut verify_txs = Measure::start("verify_txs");
-                let mut starting_txs = to_fund_txs.len();
-                let mut verified_txs = 0;
-                let mut failed_verify = 0;
-                // Only loop multiple times for small (quick) transaction batches
-                for _ in 0..(if starting_txs < 1000 { 3 } else { 1 }) {
-                    let mut timer = Instant::now();
-                    to_fund_txs.retain(|(_, tx)| {
-                        if timer.elapsed() >= Duration::from_secs(5) {
-                            if failed_verify > 0 {
-                                debug!("total txs failed verify: {}", failed_verify);
-                            }
-                            info!(
-                                "Verifying transfers... {} remaining",
-                                starting_txs - verified_txs
-                            );
-                            timer = Instant::now();
-                        }
-                        let verified = verify_funding_transfer(client, &tx, amount);
-                        if verified {
-                            verified_txs += 1;
-                        } else {
-                            failed_verify += 1;
-                        }
-                        !verified
-                    });
-                    if to_fund_txs.is_empty() {
-                        break;
-                    }
-                    debug!("Looping verifications");
-                    info!("Verifying transfers... {} remaining", to_fund_txs.len());
-                    sleep(Duration::from_millis(100));
-                }
-                starting_txs -= to_fund_txs.len();
-                verify_txs.stop();
-                debug!("verified {} txs: {}us", starting_txs, verify_txs.as_us());
-
-                // retry anything that seems to have dropped through cracks
-                //  again since these txs are all or nothing, they're fine to
-                //  retry
-                tries += 1;
-            }
-            info!("transferred");
         });
-        info!("funded: {} left: {}", new_funded.len(), notfunded.len());
+
+        info!("funded: {} left: {}", new_funded.len(), not_funded.len());
         funded = new_funded;
+        funded_funds = to_lamports;
     }
 }
 
@@ -678,14 +712,14 @@ pub fn airdrop_lamports<T: Client>(
     client: &T,
     faucet_addr: &SocketAddr,
     id: &Keypair,
-    tx_count: u64,
+    desired_balance: u64,
 ) -> Result<()> {
     let starting_balance = client.get_balance(&id.pubkey()).unwrap_or(0);
     metrics_submit_lamport_balance(starting_balance);
     info!("starting balance {}", starting_balance);
 
-    if starting_balance < tx_count {
-        let airdrop_amount = tx_count - starting_balance;
+    if starting_balance < desired_balance {
+        let airdrop_amount = desired_balance - starting_balance;
         info!(
             "Airdropping {:?} lamports from {} for {}",
             airdrop_amount,
@@ -810,17 +844,6 @@ fn compute_and_report_stats(
     );
 }
 
-// First transfer 2/3 of the lamports to the dest accounts
-// then ping-pong 1/3 of the lamports back to the other account
-// this leaves 1/3 lamport buffer in each account
-fn should_switch_directions(num_lamports_per_account: u64, keypair_chunks: u64, i: u64) -> bool {
-    if i < keypair_chunks * (2 * num_lamports_per_account) / 3 {
-        return false;
-    }
-
-    i % (keypair_chunks * num_lamports_per_account / 3) == 0
-}
-
 pub fn generate_keypairs(seed_keypair: &Keypair, count: u64) -> (Vec<Keypair>, u64) {
     let mut seed = [0u8; 32];
     seed.copy_from_slice(&seed_keypair.to_bytes()[..32]);
@@ -1004,23 +1027,25 @@ fn fund_move_keys<T: Client>(
     info!("done funding keys, took {} ms", funding_time.as_ms());
 }
 
-pub fn generate_and_fund_keypairs<T: Client>(
-    client: &T,
+pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
+    client: Arc<T>,
     faucet_addr: Option<SocketAddr>,
     funding_key: &Keypair,
     keypair_count: usize,
     lamports_per_account: u64,
     use_move: bool,
-) -> Result<(Vec<Keypair>, Option<LibraKeys>, u64)> {
+) -> Result<(Vec<Keypair>, Option<LibraKeys>)> {
     info!("Creating {} keypairs...", keypair_count);
     let (mut keypairs, extra) = generate_keypairs(funding_key, keypair_count as u64);
     info!("Get lamports...");
 
-    // Sample the first keypair, see if it has lamports, if so then resume.
-    // This logic is to prevent lamport loss on repeated solana-bench-tps executions
-    let last_keypair_balance = client
-        .get_balance(&keypairs[keypair_count - 1].pubkey())
-        .unwrap_or(0);
+    // Sample the first keypair, to prevent lamport loss on repeated solana-bench-tps executions
+    let first_key = keypairs[0].pubkey();
+    let first_keypair_balance = client.get_balance(&first_key).unwrap_or(0);
+
+    // Sample the last keypair, to check if funding was already completed
+    let last_key = keypairs[keypair_count - 1].pubkey();
+    let last_keypair_balance = client.get_balance(&last_key).unwrap_or(0);
 
     #[cfg(feature = "move")]
     let mut move_keypairs_ret = None;
@@ -1028,31 +1053,38 @@ pub fn generate_and_fund_keypairs<T: Client>(
     #[cfg(not(feature = "move"))]
     let move_keypairs_ret = None;
 
-    if lamports_per_account > last_keypair_balance {
-        let (_blockhash, fee_calculator) = get_recent_blockhash(client);
-        let account_desired_balance =
-            lamports_per_account - last_keypair_balance + fee_calculator.max_lamports_per_signature;
-        let extra_fees = extra * fee_calculator.max_lamports_per_signature;
-        let mut total = account_desired_balance * (1 + keypairs.len() as u64) + extra_fees;
+    // Repeated runs will eat up keypair balances from transaction fees. In order to quickly
+    //   start another bench-tps run without re-funding all of the keypairs, check if the
+    //   keypairs still have at least 80% of the expected funds. That should be enough to
+    //   pay for the transaction fees in a new run.
+    let enough_lamports = 8 * lamports_per_account / 10;
+    if first_keypair_balance < enough_lamports || last_keypair_balance < enough_lamports {
+        let fee_rate_governor = client.get_fee_rate_governor().unwrap();
+        let max_fee = fee_rate_governor.max_lamports_per_signature;
+        let extra_fees = extra * max_fee;
+        let total_keypairs = keypairs.len() as u64 + 1; // Add one for funding keypair
+        let mut total = lamports_per_account * total_keypairs + extra_fees;
         if use_move {
             total *= 3;
         }
 
-        info!("Previous key balance: {} max_fee: {} lamports_per_account: {} extra: {} desired_balance: {} total: {}",
-                 last_keypair_balance, fee_calculator.max_lamports_per_signature, lamports_per_account, extra,
-                 account_desired_balance, total
-                 );
+        let funding_key_balance = client.get_balance(&funding_key.pubkey()).unwrap_or(0);
+        info!(
+            "Funding keypair balance: {} max_fee: {} lamports_per_account: {} extra: {} total: {}",
+            funding_key_balance, max_fee, lamports_per_account, extra, total
+        );
 
         if client.get_balance(&funding_key.pubkey()).unwrap_or(0) < total {
-            airdrop_lamports(client, &faucet_addr.unwrap(), funding_key, total)?;
+            airdrop_lamports(client.as_ref(), &faucet_addr.unwrap(), funding_key, total)?;
         }
 
         #[cfg(feature = "move")]
         {
             if use_move {
-                let libra_genesis_keypair = create_genesis(&funding_key, client, 10_000_000);
-                let libra_mint_program_id = upload_mint_script(&funding_key, client);
-                let libra_pay_program_id = upload_payment_script(&funding_key, client);
+                let libra_genesis_keypair =
+                    create_genesis(&funding_key, client.as_ref(), 10_000_000);
+                let libra_mint_program_id = upload_mint_script(&funding_key, client.as_ref());
+                let libra_pay_program_id = upload_payment_script(&funding_key, client.as_ref());
 
                 // Generate another set of keypairs for move accounts.
                 // Still fund the solana ones which will be used for fees.
@@ -1060,7 +1092,7 @@ pub fn generate_and_fund_keypairs<T: Client>(
                 let mut rnd = GenKeys::new(seed);
                 let move_keypairs = rnd.gen_n_keypairs(keypair_count as u64);
                 fund_move_keys(
-                    client,
+                    client.as_ref(),
                     funding_key,
                     &move_keypairs,
                     total / 3,
@@ -1085,15 +1117,15 @@ pub fn generate_and_fund_keypairs<T: Client>(
             funding_key,
             &keypairs,
             total,
-            fee_calculator.max_lamports_per_signature,
-            extra,
+            max_fee,
+            lamports_per_account,
         );
     }
 
     // 'generate_keypairs' generates extra keys to be able to have size-aligned funding batches for fund_keys.
     keypairs.truncate(keypair_count);
 
-    Ok((keypairs, move_keypairs_ret, last_keypair_balance))
+    Ok((keypairs, move_keypairs_ret))
 }
 
 #[cfg(test)]
@@ -1102,33 +1134,14 @@ mod tests {
     use solana_runtime::bank::Bank;
     use solana_runtime::bank_client::BankClient;
     use solana_sdk::client::SyncClient;
-    use solana_sdk::fee_calculator::FeeCalculator;
+    use solana_sdk::fee_calculator::FeeRateGovernor;
     use solana_sdk::genesis_config::create_genesis_config;
 
-    #[test]
-    fn test_switch_directions() {
-        assert_eq!(should_switch_directions(30, 1, 0), false);
-        assert_eq!(should_switch_directions(30, 1, 1), false);
-        assert_eq!(should_switch_directions(30, 1, 20), true);
-        assert_eq!(should_switch_directions(30, 1, 21), false);
-        assert_eq!(should_switch_directions(30, 1, 30), true);
-        assert_eq!(should_switch_directions(30, 1, 90), true);
-        assert_eq!(should_switch_directions(30, 1, 91), false);
-
-        assert_eq!(should_switch_directions(30, 2, 0), false);
-        assert_eq!(should_switch_directions(30, 2, 1), false);
-        assert_eq!(should_switch_directions(30, 2, 20), false);
-        assert_eq!(should_switch_directions(30, 2, 40), true);
-        assert_eq!(should_switch_directions(30, 2, 90), false);
-        assert_eq!(should_switch_directions(30, 2, 100), true);
-        assert_eq!(should_switch_directions(30, 2, 101), false);
-    }
-
     #[test]
     fn test_bench_tps_bank_client() {
         let (genesis_config, id) = create_genesis_config(10_000);
         let bank = Bank::new(&genesis_config);
-        let clients = vec![BankClient::new(bank)];
+        let client = Arc::new(BankClient::new(bank));
 
         let mut config = Config::default();
         config.id = id;
@@ -1136,23 +1149,24 @@ mod tests {
         config.duration = Duration::from_secs(5);
 
         let keypair_count = config.tx_count * config.keypair_multiplier;
-        let (keypairs, _move_keypairs, _keypair_balance) =
-            generate_and_fund_keypairs(&clients[0], None, &config.id, keypair_count, 20, false)
+        let (keypairs, _move_keypairs) =
+            generate_and_fund_keypairs(client.clone(), None, &config.id, keypair_count, 20, false)
                 .unwrap();
 
-        do_bench_tps(clients, config, keypairs, 0, None);
+        do_bench_tps(client, config, keypairs, None);
     }
 
     #[test]
     fn test_bench_tps_fund_keys() {
         let (genesis_config, id) = create_genesis_config(10_000);
         let bank = Bank::new(&genesis_config);
-        let client = BankClient::new(bank);
+        let client = Arc::new(BankClient::new(bank));
         let keypair_count = 20;
         let lamports = 20;
 
-        let (keypairs, _move_keypairs, _keypair_balance) =
-            generate_and_fund_keypairs(&client, None, &id, keypair_count, lamports, false).unwrap();
+        let (keypairs, _move_keypairs) =
+            generate_and_fund_keypairs(client.clone(), None, &id, keypair_count, lamports, false)
+                .unwrap();
 
         for kp in &keypairs {
             assert_eq!(
@@ -1167,26 +1181,19 @@ mod tests {
     #[test]
     fn test_bench_tps_fund_keys_with_fees() {
         let (mut genesis_config, id) = create_genesis_config(10_000);
-        let fee_calculator = FeeCalculator::new(11, 0);
-        genesis_config.fee_calculator = fee_calculator;
+        let fee_rate_governor = FeeRateGovernor::new(11, 0);
+        genesis_config.fee_rate_governor = fee_rate_governor;
         let bank = Bank::new(&genesis_config);
-        let client = BankClient::new(bank);
+        let client = Arc::new(BankClient::new(bank));
         let keypair_count = 20;
         let lamports = 20;
 
-        let (keypairs, _move_keypairs, _keypair_balance) =
-            generate_and_fund_keypairs(&client, None, &id, keypair_count, lamports, false).unwrap();
+        let (keypairs, _move_keypairs) =
+            generate_and_fund_keypairs(client.clone(), None, &id, keypair_count, lamports, false)
+                .unwrap();
 
-        let max_fee = client
-            .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
-            .unwrap()
-            .1
-            .max_lamports_per_signature;
         for kp in &keypairs {
-            assert_eq!(
-                client.get_balance(&kp.pubkey()).unwrap(),
-                lamports + max_fee
-            );
+            assert_eq!(client.get_balance(&kp.pubkey()).unwrap(), lamports);
         }
     }
 }

bench-tps/src/cli.rs

@@ -1,10 +1,10 @@
 use clap::{crate_description, crate_name, App, Arg, ArgMatches};
 use solana_faucet::faucet::FAUCET_PORT;
-use solana_sdk::fee_calculator::FeeCalculator;
-use solana_sdk::signature::{read_keypair_file, Keypair, KeypairUtil};
+use solana_sdk::fee_calculator::FeeRateGovernor;
+use solana_sdk::signature::{read_keypair_file, Keypair};
 use std::{net::SocketAddr, process::exit, time::Duration};
 
-const NUM_LAMPORTS_PER_ACCOUNT_DEFAULT: u64 = solana_sdk::native_token::SOL_LAMPORTS;
+const NUM_LAMPORTS_PER_ACCOUNT_DEFAULT: u64 = solana_sdk::native_token::LAMPORTS_PER_SOL;
 
 /// Holds the configuration for a single run of the benchmark
 pub struct Config {
@@ -43,7 +43,7 @@ impl Default for Config {
             client_ids_and_stake_file: String::new(),
             write_to_client_file: false,
             read_from_client_file: false,
-            target_lamports_per_signature: FeeCalculator::default().target_lamports_per_signature,
+            target_lamports_per_signature: FeeRateGovernor::default().target_lamports_per_signature,
             multi_client: true,
             use_move: false,
             num_lamports_per_account: NUM_LAMPORTS_PER_ACCOUNT_DEFAULT,

bench-tps/src/main.rs

@@ -3,16 +3,16 @@ use solana_bench_tps::bench::{do_bench_tps, generate_and_fund_keypairs, generate
 use solana_bench_tps::cli;
 use solana_core::gossip_service::{discover_cluster, get_client, get_multi_client};
 use solana_genesis::Base64Account;
-use solana_sdk::fee_calculator::FeeCalculator;
-use solana_sdk::signature::{Keypair, KeypairUtil};
+use solana_sdk::fee_calculator::FeeRateGovernor;
+use solana_sdk::signature::{Keypair, Signer};
 use solana_sdk::system_program;
-use std::{collections::HashMap, fs::File, io::prelude::*, path::Path, process::exit};
+use std::{collections::HashMap, fs::File, io::prelude::*, path::Path, process::exit, sync::Arc};
 
 /// Number of signatures for all transactions in ~1 week at ~100K TPS
 pub const NUM_SIGNATURES_FOR_TXS: u64 = 100_000 * 60 * 60 * 24 * 7;
 
 fn main() {
-    solana_logger::setup_with_filter("solana=info");
+    solana_logger::setup_with_default("solana=info");
     solana_metrics::set_panic_hook("bench-tps");
 
     let matches = cli::build_args(solana_clap_utils::version!()).get_matches();
@@ -41,7 +41,7 @@ fn main() {
         let (keypairs, _) = generate_keypairs(&id, keypair_count as u64);
         let num_accounts = keypairs.len() as u64;
         let max_fee =
-            FeeCalculator::new(*target_lamports_per_signature, 0).max_lamports_per_signature;
+            FeeRateGovernor::new(*target_lamports_per_signature, 0).max_lamports_per_signature;
         let num_lamports_per_account = (num_accounts - 1 + NUM_SIGNATURES_FOR_TXS * max_fee)
             / num_accounts
             + num_lamports_per_account;
@@ -82,12 +82,12 @@ fn main() {
             );
             exit(1);
         }
-        client
+        Arc::new(client)
     } else {
-        get_client(&nodes)
+        Arc::new(get_client(&nodes))
     };
 
-    let (keypairs, move_keypairs, keypair_balance) = if *read_from_client_file && !use_move {
+    let (keypairs, move_keypairs) = if *read_from_client_file && !use_move {
         let path = Path::new(&client_ids_and_stake_file);
         let file = File::open(path).unwrap();
 
@@ -117,10 +117,10 @@ fn main() {
         // This prevents the amount of storage needed for bench-tps accounts from creeping up
         // across multiple runs.
         keypairs.sort_by(|x, y| x.pubkey().to_string().cmp(&y.pubkey().to_string()));
-        (keypairs, None, last_balance)
+        (keypairs, None)
     } else {
         generate_and_fund_keypairs(
-            &client,
+            client.clone(),
             Some(*faucet_addr),
             &id,
             keypair_count,
@@ -133,11 +133,5 @@ fn main() {
         })
     };
 
-    do_bench_tps(
-        vec![client],
-        cli_config,
-        keypairs,
-        keypair_balance,
-        move_keypairs,
-    );
+    do_bench_tps(client, cli_config, keypairs, move_keypairs);
 }

bench-tps/tests/bench_tps.rs

@@ -8,8 +8,8 @@ use solana_faucet::faucet::run_local_faucet;
 use solana_local_cluster::local_cluster::{ClusterConfig, LocalCluster};
 #[cfg(feature = "move")]
 use solana_sdk::move_loader::solana_move_loader_program;
-use solana_sdk::signature::{Keypair, KeypairUtil};
-use std::sync::mpsc::channel;
+use solana_sdk::signature::{Keypair, Signer};
+use std::sync::{mpsc::channel, Arc};
 use std::time::Duration;
 
 fn test_bench_tps_local_cluster(config: Config) {
@@ -36,10 +36,10 @@ fn test_bench_tps_local_cluster(config: Config) {
         100_000_000,
     );
 
-    let client = create_client(
+    let client = Arc::new(create_client(
         (cluster.entry_point_info.rpc, cluster.entry_point_info.tpu),
         VALIDATOR_PORT_RANGE,
-    );
+    ));
 
     let (addr_sender, addr_receiver) = channel();
     run_local_faucet(faucet_keypair, addr_sender, None);
@@ -48,8 +48,8 @@ fn test_bench_tps_local_cluster(config: Config) {
     let lamports_per_account = 100;
 
     let keypair_count = config.tx_count * config.keypair_multiplier;
-    let (keypairs, move_keypairs, _keypair_balance) = generate_and_fund_keypairs(
-        &client,
+    let (keypairs, move_keypairs) = generate_and_fund_keypairs(
+        client.clone(),
         Some(faucet_addr),
         &config.id,
         keypair_count,
@@ -58,7 +58,7 @@ fn test_bench_tps_local_cluster(config: Config) {
     )
     .unwrap();
 
-    let _total = do_bench_tps(vec![client], config, keypairs, 0, move_keypairs);
+    let _total = do_bench_tps(client, config, keypairs, move_keypairs);
 
     #[cfg(not(debug_assertions))]
     assert!(_total > 100);

book/art/spv-bank-merkle.bob

@@ -1,18 +0,0 @@
-                             +------------+
-                             | Bank-Merkle|
-                             +------------+
-                                ^      ^
-                               /        \
-               +-----------------+    +-------------+
-               | Bank-Diff-Merkle|    | Block-Merkle|
-               +-----------------+    +-------------+
-                     ^     ^
-                    /       \
-               +------+   +--------------------------+
-               | Hash |   | Previous Bank-Diff-Merkle|
-               +------+   +--------------------------+
-               ^      ^
-              /        \
- +---------------+   +---------------+
- | Hash(Account1)|   | Hash(Account2)|
- +---------------+   +---------------+

book/build.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-cd "$(dirname "$0")"
-
-make -j"$(nproc)" test

book/src/api-reference/README.md

@@ -1,4 +0,0 @@
-# API Reference
-
-The following sections contain API references material you may find useful when developing applications utilizing a Solana cluster.
-

book/src/api-reference/blockstreamer.md

@@ -1,28 +0,0 @@
-# Blockstreamer
-
-Solana supports a node type called an _blockstreamer_. This validator variation is intended for applications that need to observe the data plane without participating in transaction validation or ledger replication.
-
-A blockstreamer runs without a vote signer, and can optionally stream ledger entries out to a Unix domain socket as they are processed. The JSON-RPC service still functions as on any other node.
-
-To run a blockstreamer, include the argument `no-signer` and \(optional\) `blockstream` socket location:
-
-```bash
-$ ./multinode-demo/validator-x.sh --no-signer --blockstream <SOCKET>
-```
-
-The stream will output a series of JSON objects:
-
-* An Entry event JSON object is sent when each ledger entry is processed, with the following fields:
-  * `dt`, the system datetime, as RFC3339-formatted string
-  * `t`, the event type, always "entry"
-  * `s`, the slot height, as unsigned 64-bit integer
-  * `h`, the tick height, as unsigned 64-bit integer
-  * `entry`, the entry, as JSON object
-* A Block event JSON object is sent when a block is complete, with the following fields:
-  * `dt`, the system datetime, as RFC3339-formatted string
-  * `t`, the event type, always "block"
-  * `s`, the slot height, as unsigned 64-bit integer
-  * `h`, the tick height, as unsigned 64-bit integer
-  * `l`, the slot leader id, as base-58 encoded string
-  * `hash`, the [blockhash](terminology.md#blockhash), as base-58 encoded string
-

book/src/api-reference/instruction-api.md

@@ -1,38 +0,0 @@
-# Instruction
-
-For the purposes of building a [Transaction](../transaction.md), a more verbose instruction format is used:
-
-* **Instruction:**
-  * **program\_id:** The pubkey of the on-chain program that executes the
-
-    instruction
-
-  * **accounts:** An ordered list of accounts that should be passed to
-
-    the program processing the instruction, including metadata detailing
-
-    if an account is a signer of the transaction and if it is a credit
-
-    only account.
-
-  * **data:** A byte array that is passed to the program executing the
-
-    instruction
-
-A more compact form is actually included in a `Transaction`:
-
-* **CompiledInstruction:**
-  * **program\_id\_index:** The index of the `program_id` in the
-
-    `account_keys` list
-
-  * **accounts:** An ordered list of indices into `account_keys`
-
-    specifying the accounds that should be passed to the program
-
-    processing the instruction.
-
-  * **data:** A byte array that is passed to the program executing the
-
-    instruction
-

book/src/api-reference/jsonrpc-api.md

@@ -1,994 +0,0 @@
-# JSON RPC API
-
-Solana nodes accept HTTP requests using the [JSON-RPC 2.0](https://www.jsonrpc.org/specification) specification.
-
-To interact with a Solana node inside a JavaScript application, use the [solana-web3.js](https://github.com/solana-labs/solana-web3.js) library, which gives a convenient interface for the RPC methods.
-
-## RPC HTTP Endpoint
-
-**Default port:** 8899 eg. [http://localhost:8899](http://localhost:8899), [http://192.168.1.88:8899](http://192.168.1.88:8899)
-
-## RPC PubSub WebSocket Endpoint
-
-**Default port:** 8900 eg. ws://localhost:8900, [http://192.168.1.88:8900](http://192.168.1.88:8900)
-
-## Methods
-
-* [confirmTransaction](jsonrpc-api.md#confirmtransaction)
-* [getAccountInfo](jsonrpc-api.md#getaccountinfo)
-* [getBalance](jsonrpc-api.md#getbalance)
-* [getBlockCommitment](jsonrpc-api.md#getblockcommitment)
-* [getBlockTime](jsonrpc-api.md#getblocktime)
-* [getClusterNodes](jsonrpc-api.md#getclusternodes)
-* [getConfirmedBlock](jsonrpc-api.md#getconfirmedblock)
-* [getConfirmedBlocks](jsonrpc-api.md#getconfirmedblocks)
-* [getEpochInfo](jsonrpc-api.md#getepochinfo)
-* [getEpochSchedule](jsonrpc-api.md#getepochschedule)
-* [getGenesisHash](jsonrpc-api.md#getgenesishash)
-* [getLeaderSchedule](jsonrpc-api.md#getleaderschedule)
-* [getMinimumBalanceForRentExemption](jsonrpc-api.md#getminimumbalanceforrentexemption)
-* [getNumBlocksSinceSignatureConfirmation](jsonrpc-api.md#getnumblockssincesignatureconfirmation)
-* [getProgramAccounts](jsonrpc-api.md#getprogramaccounts)
-* [getRecentBlockhash](jsonrpc-api.md#getrecentblockhash)
-* [getSignatureStatus](jsonrpc-api.md#getsignaturestatus)
-* [getSlot](jsonrpc-api.md#getslot)
-* [getSlotLeader](jsonrpc-api.md#getslotleader)
-* [getSlotsPerSegment](jsonrpc-api.md#getslotspersegment)
-* [getStorageTurn](jsonrpc-api.md#getstorageturn)
-* [getStorageTurnRate](jsonrpc-api.md#getstorageturnrate)
-* [getTransactionCount](jsonrpc-api.md#gettransactioncount)
-* [getTotalSupply](jsonrpc-api.md#gettotalsupply)
-* [getVersion](jsonrpc-api.md#getversion)
-* [getVoteAccounts](jsonrpc-api.md#getvoteaccounts)
-* [requestAirdrop](jsonrpc-api.md#requestairdrop)
-* [sendTransaction](jsonrpc-api.md#sendtransaction)
-* [startSubscriptionChannel](jsonrpc-api.md#startsubscriptionchannel)
-* [Subscription Websocket](jsonrpc-api.md#subscription-websocket)
-  * [accountSubscribe](jsonrpc-api.md#accountsubscribe)
-  * [accountUnsubscribe](jsonrpc-api.md#accountunsubscribe)
-  * [programSubscribe](jsonrpc-api.md#programsubscribe)
-  * [programUnsubscribe](jsonrpc-api.md#programunsubscribe)
-  * [signatureSubscribe](jsonrpc-api.md#signaturesubscribe)
-  * [signatureUnsubscribe](jsonrpc-api.md#signatureunsubscribe)
-
-## Request Formatting
-
-To make a JSON-RPC request, send an HTTP POST request with a `Content-Type: application/json` header. The JSON request data should contain 4 fields:
-
-* `jsonrpc`, set to `"2.0"`
-* `id`, a unique client-generated identifying integer
-* `method`, a string containing the method to be invoked
-* `params`, a JSON array of ordered parameter values
-
-Example using curl:
-
-```bash
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getBalance", "params":["83astBRguLMdt2h5U1Tpdq5tjFoJ6noeGwaY3mDLVcri"]}' 192.168.1.88:8899
-```
-
-The response output will be a JSON object with the following fields:
-
-* `jsonrpc`, matching the request specification
-* `id`, matching the request identifier
-* `result`, requested data or success confirmation
-
-Requests can be sent in batches by sending an array of JSON-RPC request objects as the data for a single POST.
-
-## Definitions
-
-* Hash: A SHA-256 hash of a chunk of data.
-* Pubkey: The public key of a Ed25519 key-pair.
-* Signature: An Ed25519 signature of a chunk of data.
-* Transaction: A Solana instruction signed by a client key-pair.
-
-## Configuring State Commitment
-
-Solana nodes choose which bank state to query based on a commitment requirement
-set by the client. Clients may specify either:
-* `{"commitment":"max"}` - the node will query the most recent bank having reached `MAX_LOCKOUT_HISTORY` confirmations
-* `{"commitment":"recent"}` - the node will query its most recent bank state
-
-The commitment parameter should be included as the last element in the `params` array:
-
-```bash
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getBalance", "params":["83astBRguLMdt2h5U1Tpdq5tjFoJ6noeGwaY3mDLVcri",{"commitment":"max"}]}' 192.168.1.88:8899
-```
-
-#### Default:
-If commitment configuration is not provided, the node will default to `"commitment":"max"`
-
-Only methods that query bank state accept the commitment parameter. They are indicated in the API Reference below.
-
-#### RpcResponse Structure
-Many methods that take a commitment parameter return an RpcResponse JSON object comprised of two parts:
-
-* `context` : An RpcResponseContext JSON structure including a `slot` field at which the operation was evaluated.
-* `value` : The value returned by the operation itself.
-
-
-## JSON RPC API Reference
-
-### confirmTransaction
-
-Returns a transaction receipt
-
-#### Parameters:
-
-* `string` - Signature of Transaction to confirm, as base-58 encoded string
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-* `RpcResponse<boolean>` - RpcResponse JSON object with `value` field set to Transaction status, boolean true if Transaction is confirmed
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"confirmTransaction", "params":["5VERv8NMvzbJMEkV8xnrLkEaWRtSz9CosKDYjCJjBRnbJLgp8uirBgmQpjKhoR4tjF3ZpRzrFmBV6UjKdiSZkQUW"]}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":{"context":{"slot":1},"value":true},"id":1}
-```
-
-### getAccountInfo
-
-Returns all information associated with the account of provided Pubkey
-
-#### Parameters:
-
-* `string` - Pubkey of account to query, as base-58 encoded string
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-The result value will be an RpcResponse JSON object containing an AccountInfo JSON object.
-
-* `RpcResponse<AccountInfo>`, RpcResponse JSON object with `value` field set to AccountInfo, a JSON object containing:
-* `lamports`, number of lamports assigned to this account, as a u64
-* `owner`, array of 32 bytes representing the program this account has been assigned to
-* `data`, array of bytes representing any data associated with the account
-* `executable`, boolean indicating if the account contains a program \(and is strictly read-only\)
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getAccountInfo", "params":["2gVkYWexTHR5Hb2aLeQN3tnngvWzisFKXDUPrgMHpdST"]}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":{"context":{"slot":1},"value":{"executable":false,"owner":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"lamports":1,"data":[3,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0.22.0,0,0,0,0,0,0,50,48,53,48,45,48,49,45,48,49,84,48,48,58,48,48,58,48,48,90,252,10,7,28,246,140,88,177,98,82,10,227,89,81,18,30,194,101,199,16,11,73,133,20,246,62,114,39,20,113,189,32,50,0,0,0,0,0,0,0,247,15,36,102,167,83,225,42,133,127,82,34,36,224,207,130,109,230,224,188,163,33,213,13,5,117,211,251,65,159,197,51,0,0,0,0,0,0]}},"id":1}
-```
-
-### getBalance
-
-Returns the balance of the account of provided Pubkey
-
-#### Parameters:
-
-* `string` - Pubkey of account to query, as base-58 encoded string
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-* `RpcResponse<u64>` - RpcResponse JSON object with `value` field set to quantity
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getBalance", "params":["83astBRguLMdt2h5U1Tpdq5tjFoJ6noeGwaY3mDLVcri"]}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":{"context":{"slot":1},"value":0},"id":1}
-```
-
-### getBlockCommitment
-
-Returns commitment for particular block
-
-#### Parameters:
-
-* `u64` - block, identified by Slot
-
-#### Results:
-
-The result field will be an array with two fields:
-
-* Commitment
-  * `null` - Unknown block
-  * `object` - BlockCommitment
-    * `array` - commitment, array of u64 integers logging the amount of cluster stake in lamports that has voted on the block at each depth from 0 to `MAX_LOCKOUT_HISTORY`
-* 'integer' - total active stake, in lamports, of the current epoch
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getBlockCommitment","params":[5]}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":[{"commitment":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10,32]},42],"id":1}
-```
-
-### getBlockTime
-
-Returns the estimated production time of a block. Validators report their UTC
-time to the ledger on a regular interval. A block's time is calculated as an
-offset from the median value of the most recent validator time report.
-
-#### Parameters:
-
-* `u64` - block, identified by Slot
-
-#### Results:
-
-* `null` - block has not yet been produced
-* `i64` - estimated production time, as Unix timestamp (seconds since the Unix epoch)
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getBlockTime","params":[5]}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":1574721591,"id":1}
-```
-
-### getClusterNodes
-
-Returns information about all the nodes participating in the cluster
-
-#### Parameters:
-
-None
-
-#### Results:
-
-The result field will be an array of JSON objects, each with the following sub fields:
-
-* `pubkey` - Node public key, as base-58 encoded string
-* `gossip` - Gossip network address for the node
-* `tpu` - TPU network address for the node
-* `rpc` - JSON RPC network address for the node, or `null` if the JSON RPC service is not enabled
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getClusterNodes"}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":[{"gossip":"10.239.6.48:8001","pubkey":"9QzsJf7LPLj8GkXbYT3LFDKqsj2hHG7TA3xinJHu8epQ","rpc":"10.239.6.48:8899","tpu":"10.239.6.48:8856"}],"id":1}
-```
-
-### getConfirmedBlock
-
-Returns identity and transaction information about a confirmed block in the ledger
-
-#### Parameters:
-
-* `integer` - slot, as u64 integer
-
-#### Results:
-
-The result field will be an object with the following fields:
-
-* `blockhash` - the blockhash of this block
-* `previousBlockhash` - the blockhash of this block's parent
-* `parentSlot` - the slot index of this block's parent
-* `transactions` - an array of tuples containing:
-  * [Transaction](transaction-api.md) object, in JSON format
-  * Transaction status object, containing:
-     * `status` - Transaction status:
-       * `"Ok": null` - Transaction was successful
-       * `"Err": <ERR>` - Transaction failed with TransactionError  [TransactionError definitions](https://github.com/solana-labs/solana/blob/master/sdk/src/transaction.rs#L14)
-     * `fee` - fee this transaction was charged, as u64 integer
-     * `preBalances` - array of u64 account balances from before the transaction was processed
-     * `postBalances` - array of u64 account balances after the transaction was processed
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc": "2.0","id":1,"method":"getConfirmedBlock","params":[430]}' localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":{"blockhash":[165,245,120,183,32,205,89,222,249,114,229,49,250,231,149,122,156,232,181,83,238,194,157,153,7,213,180,54,177,6,25,101],"parentSlot":429,"previousBlockhash":[21,108,181,90,139,241,212,203,45,78,232,29,161,31,159,188,110,82,81,11,250,74,47,140,188,28,23,96,251,164,208,166],"transactions":[[{"message":{"accountKeys":[[5],[219,181,202,40,52,148,34,136,186,59,137,160,250,225,234,17,244,160,88,116,24,176,30,227,68,11,199,38,141,68,131,228],[233,48,179,56,91,40,254,206,53,48,196,176,119,248,158,109,121,77,11,69,108,160,128,27,228,122,146,249,53,184,68,87],[6,167,213,23,25,47,10,175,198,242,101,227,251,119,204,122,218,130,197,41,208,190,59,19,110,45,0,85,32,0,0,0],[6,167,213,23,24,199,116,201,40,86,99,152,105,29,94,182,139,94,184,163,155,75,109,92,115,85,91,33,0,0,0,0],[7,97,72,29,53,116,116,187,124,77,118,36,235,211,189,179,216,53,94,115,209,16,67,252,13,163,83,128,0,0,0,0]],"header":{"numReadonlySignedAccounts":0,"numReadonlyUnsignedAccounts":3,"numRequiredSignatures":2},"instructions":[[1],{"accounts":[[3],1,2,3],"data":[[52],2,0,0,0,1,0,0,0,0,0,0,0,173,1,0,0,0,0,0,0,86,55,9,248,142,238,135,114,103,83,247,124,67,68,163,233,55,41,59,129,64,50,110,221,234,234,27,213,205,193,219,50],"program_id_index":4}],"recentBlockhash":[21,108,181,90,139,241,212,203,45,78,232,29,161,31,159,188,110,82,81,11,250,74,47,140,188,28,23,96,251,164,208,166]},"signatures":[[2],[119,9,95,108,35,95,7,1,69,101,65,45,5,204,61,114,172,88,123,238,32,201,135,229,57,50,13,21,106,216,129,183,238,43,37,101,148,81,56,232,88,136,80,65,46,189,39,106,94,13,238,54,186,48,118,186,0,62,121,122,172,171,66,5],[78,40,77,250,10,93,6,157,48,173,100,40,251,9,7,218,7,184,43,169,76,240,254,34,235,48,41,175,119,126,75,107,106,248,45,161,119,48,174,213,57,69,111,225,245,60,148,73,124,82,53,6,203,126,120,180,111,169,89,64,29,23,237,13]]},{"fee":100000,"status":{"Ok":null},"preBalances":[499998337500,15298080,1,1,1],"postBalances":[499998237500,15298080,1,1,1]}]]},"id":1}
-```
-
-### getConfirmedBlocks
-
-Returns a list of confirmed blocks
-
-#### Parameters:
-
-* `integer` - start_slot, as u64 integer
-* `integer` - (optional) end_slot, as u64 integer
-
-#### Results:
-
-The result field will be an array of u64 integers listing confirmed blocks
-between start_slot and either end_slot, if provided, or latest confirmed block,
-inclusive.
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc": "2.0","id":1,"method":"getConfirmedBlocks","params":[5, 10]}' localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":[5,6,7,8,9,10],"id":1}
-```
-
-### getEpochInfo
-
-Returns information about the current epoch
-
-#### Parameters:
-
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-The result field will be an object with the following fields:
-
-* `epoch`, the current epoch
-* `slotIndex`, the current slot relative to the start of the current epoch
-* `slotsInEpoch`, the number of slots in this epoch
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getEpochInfo"}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":{"epoch":3,"slotIndex":126,"slotsInEpoch":256},"id":1}
-```
-
-### getEpochSchedule
-
-Returns epoch schedule information from this cluster's genesis config
-
-#### Parameters:
-
-None
-
-#### Results:
-
-The result field will be an object with the following fields:
-
-* `slots_per_epoch`, the maximum number of slots in each epoch
-* `leader_schedule_slot_offset`, the number of slots before beginning of an epoch to calculate a leader schedule for that epoch
-* `warmup`, whether epochs start short and grow
-* `first_normal_epoch`, first normal-length epoch, log2(slots_per_epoch) - log2(MINIMUM_SLOTS_PER_EPOCH)
-* `first_normal_slot`, MINIMUM_SLOTS_PER_EPOCH * (2.pow(first_normal_epoch) - 1)
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getEpochSchedule"}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":{"first_normal_epoch":8,"first_normal_slot":8160,"leader_schedule_slot_offset":8192,"slots_per_epoch":8192,"warmup":true},"id":1}
-```
-
-### getGenesisHash
-
-Returns the genesis hash
-
-#### Parameters:
-
-None
-
-#### Results:
-
-* `string` - a Hash as base-58 encoded string
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getGenesisHash"}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":"GH7ome3EiwEr7tu9JuTh2dpYWBJK3z69Xm1ZE3MEE6JC","id":1}
-```
-
-### getLeaderSchedule
-
-Returns the leader schedule for an epoch
-
-#### Parameters:
-
-* `slot` - (optional) Fetch the leader schedule for the epoch that corresponds to the provided slot.  If unspecified, the leader schedule for the current epoch is fetched
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-The result field will be a dictionary of leader public keys \(as base-58 encoded
-strings\) and their corresponding leader slot indices as values (indices are to
-the first slot in the requested epoch)
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getLeaderSchedule"}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":{"4Qkev8aNZcqFNSRhQzwyLMFSsi94jHqE8WNVTJzTP99F":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63]},"id":1}
-```
-
-### getMinimumBalanceForRentExemption
-
-Returns minimum balance required to make account rent exempt.
-
-#### Parameters:
-
-* `u64` - account data length
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-* `u64` - minimum lamports required in account
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getMinimumBalanceForRentExemption", "params":[50]}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":500,"id":1}
-```
-
-### getNumBlocksSinceSignatureConfirmation
-
-Returns the current number of blocks since signature has been confirmed.
-
-#### Parameters:
-
-* `string` - Signature of Transaction to confirm, as base-58 encoded string
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-* `u64` - count
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getNumBlocksSinceSignatureConfirmation", "params":["5VERv8NMvzbJMEkV8xnrLkEaWRtSz9CosKDYjCJjBRnbJLgp8uirBgmQpjKhoR4tjF3ZpRzrFmBV6UjKdiSZkQUW"]}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":8,"id":1}
-```
-
-### getProgramAccounts
-
-Returns all accounts owned by the provided program Pubkey
-
-#### Parameters:
-
-* `string` - Pubkey of program, as base-58 encoded string
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-The result field will be an array of arrays. Each sub array will contain:
-
-* `string` - the account Pubkey as base-58 encoded string and a JSON object, with the following sub fields:
-* `lamports`, number of lamports assigned to this account, as a u64
-* `owner`, array of 32 bytes representing the program this account has been assigned to
-* `data`, array of bytes representing any data associated with the account
-* `executable`, boolean indicating if the account contains a program \(and is strictly read-only\)
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getProgramAccounts", "params":["8nQwAgzN2yyUzrukXsCa3JELBYqDQrqJ3UyHiWazWxHR"]}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":[["BqGKYtAKu69ZdWEBtZHh4xgJY1BYa2YBiBReQE3pe383", {"executable":false,"owner":[50,28,250,90,221,24,94,136,147,165,253,136,1,62,196,215,225,34,222,212,99,84,202,223,245,13,149,99,149,231,91,96],"lamports":1,"data":[]], ["4Nd1mBQtrMJVYVfKf2PJy9NZUZdTAsp7D4xWLs4gDB4T", {"executable":false,"owner":[50,28,250,90,221,24,94,136,147,165,253,136,1,62,196,215,225,34,222,212,99,84,202,223,245,13,149,99,149,231,91,96],"lamports":10,"data":[]]]},"id":1}
-```
-
-### getRecentBlockhash
-
-Returns a recent block hash from the ledger, and a fee schedule that can be used to compute the cost of submitting a transaction using it.
-
-#### Parameters:
-
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-An RpcResponse containing an array consisting of a string blockhash and FeeCalculator JSON object.
-
-* `RpcResponse<array>` - RpcResponse JSON object with `value` field set to an array including:
-* `string` - a Hash as base-58 encoded string
-* `FeeCalculator object` - the fee schedule for this block hash
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getRecentBlockhash"}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":{"context":{"slot":1},"value":["GH7ome3EiwEr7tu9JuTh2dpYWBJK3z69Xm1ZE3MEE6JC",{"lamportsPerSignature": 0}]},"id":1}
-```
-
-### getSignatureStatus
-
-Returns the status of a given signature. This method is similar to [confirmTransaction](jsonrpc-api.md#confirmtransaction) but provides more resolution for error events.
-
-#### Parameters:
-
-* `string` - Signature of Transaction to confirm, as base-58 encoded string
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-* `null` - Unknown transaction
-* `object` - Transaction status:
-  * `"Ok": null` - Transaction was successful
-  * `"Err": <ERR>` - Transaction failed with TransactionError  [TransactionError definitions](https://github.com/solana-labs/solana/blob/master/sdk/src/transaction.rs#L14)
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getSignatureStatus", "params":["5VERv8NMvzbJMEkV8xnrLkEaWRtSz9CosKDYjCJjBRnbJLgp8uirBgmQpjKhoR4tjF3ZpRzrFmBV6UjKdiSZkQUW"]}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":"SignatureNotFound","id":1}
-```
-
-### getSlot
-
-Returns the current slot the node is processing
-
-#### Parameters:
-
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-* `u64` - Current slot
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getSlot"}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":"1234","id":1}
-```
-
-### getSlotLeader
-
-Returns the current slot leader
-
-#### Parameters:
-
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-* `string` - Node Id as base-58 encoded string
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getSlotLeader"}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":"ENvAW7JScgYq6o4zKZwewtkzzJgDzuJAFxYasvmEQdpS","id":1}
-```
-
-### getSlotsPerSegment
-
-Returns the current storage segment size in terms of slots
-
-#### Parameters:
-
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-* `u64` - Number of slots in a storage segment
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getSlotsPerSegment"}' http://localhost:8899
-// Result
-{"jsonrpc":"2.0","result":"1024","id":1}
-```
-
-### getStorageTurn
-
-Returns the current storage turn's blockhash and slot
-
-#### Parameters:
-
-None
-
-#### Results:
-
-An array consisting of
-
-* `string` - a Hash as base-58 encoded string indicating the blockhash of the turn slot
-* `u64` - the current storage turn slot
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getStorageTurn"}' http://localhost:8899
- // Result
-{"jsonrpc":"2.0","result":["GH7ome3EiwEr7tu9JuTh2dpYWBJK3z69Xm1ZE3MEE6JC", "2048"],"id":1}
-```
-
-### getStorageTurnRate
-
-Returns the current storage turn rate in terms of slots per turn
-
-#### Parameters:
-
-None
-
-#### Results:
-
-* `u64` - Number of slots in storage turn
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getStorageTurnRate"}' http://localhost:8899
- // Result
-{"jsonrpc":"2.0","result":"1024","id":1}
-```
-
-### getTransactionCount
-
-Returns the current Transaction count from the ledger
-
-#### Parameters:
-
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-* `u64` - count
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getTransactionCount"}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":268,"id":1}
-```
-
-### getTotalSupply
-
-Returns the current total supply in Lamports
-
-#### Parameters:
-
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-* `u64` - Total supply
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getTotalSupply"}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":10126,"id":1}
-```
-
-### getVersion
-
-Returns the current solana versions running on the node
-
-#### Parameters:
-
-None
-
-#### Results:
-
-The result field will be a JSON object with the following sub fields:
-
-* `solana-core`, software version of solana-core
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getVersion"}' http://localhost:8899
-// Result
-{"jsonrpc":"2.0","result":{"solana-core": "0.17.2"},"id":1}
-```
-
-### getVoteAccounts
-
-Returns the account info and associated stake for all the voting accounts in the current bank.
-
-#### Parameters:
-
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-The result field will be a JSON object of `current` and `delinquent` accounts, each containing an array of JSON objects with the following sub fields:
-
-* `votePubkey` - Vote account public key, as base-58 encoded string
-* `nodePubkey` - Node public key, as base-58 encoded string
-* `activatedStake` - the stake, in lamports, delegated to this vote account and active in this epoch
-* `epochVoteAccount` - bool, whether the vote account is staked for this epoch
-* `commission`, percentage (0-100) of rewards payout owed to the vote account
-* `lastVote` - Most recent slot voted on by this vote account
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getVoteAccounts"}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":{"current":[{"commission":0,"epochVoteAccount":true,"nodePubkey":"B97CCUW3AEZFGy6uUg6zUdnNYvnVq5VG8PUtb2HayTDD","lastVote":147,"activatedStake":42,"votePubkey":"3ZT31jkAGhUaw8jsy4bTknwBMP8i4Eueh52By4zXcsVw"}],"delinquent":[{"commission":127,"epochVoteAccount":false,"nodePubkey":"6ZPxeQaDo4bkZLRsdNrCzchNQr5LN9QMc9sipXv9Kw8f","lastVote":0,"activatedStake":0,"votePubkey":"CmgCk4aMS7KW1SHX3s9K5tBJ6Yng2LBaC8MFov4wx9sm"}]},"id":1}
-```
-
-### requestAirdrop
-
-Requests an airdrop of lamports to a Pubkey
-
-#### Parameters:
-
-* `string` - Pubkey of account to receive lamports, as base-58 encoded string
-* `integer` - lamports, as a u64
-* `object` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment) (used for retrieving blockhash and verifying airdrop success)
-
-#### Results:
-
-* `string` - Transaction Signature of airdrop, as base-58 encoded string
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"requestAirdrop", "params":["83astBRguLMdt2h5U1Tpdq5tjFoJ6noeGwaY3mDLVcri", 50]}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":"5VERv8NMvzbJMEkV8xnrLkEaWRtSz9CosKDYjCJjBRnbJLgp8uirBgmQpjKhoR4tjF3ZpRzrFmBV6UjKdiSZkQUW","id":1}
-```
-
-### sendTransaction
-
-Creates new transaction
-
-#### Parameters:
-
-* `array` - array of octets containing a fully-signed Transaction
-
-#### Results:
-
-* `string` - Transaction Signature, as base-58 encoded string
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"sendTransaction", "params":[[61, 98, 55, 49, 15, 187, 41, 215, 176, 49, 234, 229, 228, 77, 129, 221, 239, 88, 145, 227, 81, 158, 223, 123, 14, 229, 235, 247, 191, 115, 199, 71, 121, 17, 32, 67, 63, 209, 239, 160, 161, 2, 94, 105, 48, 159, 235, 235, 93, 98, 172, 97, 63, 197, 160, 164, 192, 20, 92, 111, 57, 145, 251, 6, 40, 240, 124, 194, 149, 155, 16, 138, 31, 113, 119, 101, 212, 128, 103, 78, 191, 80, 182, 234, 216, 21, 121, 243, 35, 100, 122, 68, 47, 57, 13, 39, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 40, 240, 124, 194, 149, 155, 16, 138, 31, 113, 119, 101, 212, 128, 103, 78, 191, 80, 182, 234, 216, 21, 121, 243, 35, 100, 122, 68, 47, 57, 11, 12, 106, 49, 74, 226, 201, 16, 161, 192, 28, 84, 124, 97, 190, 201, 171, 186, 6, 18, 70, 142, 89, 185, 176, 154, 115, 61, 26, 163, 77, 1, 88, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":"2EBVM6cB8vAAD93Ktr6Vd8p67XPbQzCJX47MpReuiCXJAtcjaxpvWpcg9Ege1Nr5Tk3a2GFrByT7WPBjdsTycY9b","id":1}
-```
-
-### Subscription Websocket
-
-After connect to the RPC PubSub websocket at `ws://<ADDRESS>/`:
-
-* Submit subscription requests to the websocket using the methods below
-* Multiple subscriptions may be active at once
-* All subscriptions take an optional `confirmations` parameter, which defines
-
-  how many confirmed blocks the node should wait before sending a notification.
-
-  The greater the number, the more likely the notification is to represent
-
-  consensus across the cluster, and the less likely it is to be affected by
-
-  forking or rollbacks. If unspecified, the default value is 0; the node will
-
-  send a notification as soon as it witnesses the event. The maximum
-
-  `confirmations` wait length is the cluster's `MAX_LOCKOUT_HISTORY`, which
-
-  represents the economic finality of the chain.
-
-### accountSubscribe
-
-Subscribe to an account to receive notifications when the lamports or data for a given account public key changes
-
-#### Parameters:
-
-* `string` - account Pubkey, as base-58 encoded string
-* `integer` - optional, number of confirmed blocks to wait before notification.
-
-  Default: 0, Max: `MAX_LOCKOUT_HISTORY` \(greater integers rounded down\)
-
-#### Results:
-
-* `integer` - Subscription id \(needed to unsubscribe\)
-
-#### Example:
-
-```bash
-// Request
-{"jsonrpc":"2.0", "id":1, "method":"accountSubscribe", "params":["CM78CPUeXjn8o3yroDHxUtKsZZgoy4GPkPPXfouKNH12"]}
-
-{"jsonrpc":"2.0", "id":1, "method":"accountSubscribe", "params":["CM78CPUeXjn8o3yroDHxUtKsZZgoy4GPkPPXfouKNH12", 15]}
-
-// Result
-{"jsonrpc": "2.0","result": 0,"id": 1}
-```
-
-#### Notification Format:
-
-```bash
-{"jsonrpc": "2.0","method": "accountNotification", "params": {"result": {"executable":false,"owner":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"lamports":1,"data":[3,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0.22.0,0,0,0,0,0,0,50,48,53,48,45,48,49,45,48,49,84,48,48,58,48,48,58,48,48,90,252,10,7,28,246,140,88,177,98,82,10,227,89,81,18,30,194,101,199,16,11,73,133,20,246,62,114,39,20,113,189,32,50,0,0,0,0,0,0,0,247,15,36,102,167,83,225,42,133,127,82,34,36,224,207,130,109,230,224,188,163,33,213,13,5,117,211,251,65,159,197,51,0,0,0,0,0,0]},"subscription":0}}
-```
-
-### accountUnsubscribe
-
-Unsubscribe from account change notifications
-
-#### Parameters:
-
-* `integer` - id of account Subscription to cancel
-
-#### Results:
-
-* `bool` - unsubscribe success message
-
-#### Example:
-
-```bash
-// Request
-{"jsonrpc":"2.0", "id":1, "method":"accountUnsubscribe", "params":[0]}
-
-// Result
-{"jsonrpc": "2.0","result": true,"id": 1}
-```
-
-### programSubscribe
-
-Subscribe to a program to receive notifications when the lamports or data for a given account owned by the program changes
-
-#### Parameters:
-
-* `string` - program\_id Pubkey, as base-58 encoded string
-* `integer` - optional, number of confirmed blocks to wait before notification.
-
-  Default: 0, Max: `MAX_LOCKOUT_HISTORY` \(greater integers rounded down\)
-
-#### Results:
-
-* `integer` - Subscription id \(needed to unsubscribe\)
-
-#### Example:
-
-```bash
-// Request
-{"jsonrpc":"2.0", "id":1, "method":"programSubscribe", "params":["9gZbPtbtHrs6hEWgd6MbVY9VPFtS5Z8xKtnYwA2NynHV"]}
-
-{"jsonrpc":"2.0", "id":1, "method":"programSubscribe", "params":["9gZbPtbtHrs6hEWgd6MbVY9VPFtS5Z8xKtnYwA2NynHV", 15]}
-
-// Result
-{"jsonrpc": "2.0","result": 0,"id": 1}
-```
-
-#### Notification Format:
-
-* `string` - account Pubkey, as base-58 encoded string
-* `object` - account info JSON object \(see [getAccountInfo](jsonrpc-api.md#getaccountinfo) for field details\)
-
-  ```bash
-  {"jsonrpc":"2.0","method":"programNotification","params":{{"result":["8Rshv2oMkPu5E4opXTRyuyBeZBqQ4S477VG26wUTFxUM",{"executable":false,"lamports":1,"owner":[129,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"data":[1,1,1,0,0,0,0,0,0,0.22.0,0,0,0,0,0,0,50,48,49,56,45,49,50,45,50,52,84,50,51,58,53,57,58,48,48,90,235,233,39,152,15,44,117,176,41,89,100,86,45,61,2,44,251,46,212,37,35,118,163,189,247,84,27,235,178,62,55,89,0,0,0,0,50,0,0,0,0,0,0,0,235,233,39,152,15,44,117,176,41,89,100,86,45,61,2,44,251,46,212,37,35,118,163,189,247,84,27,235,178,62,45,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}],"subscription":0}}
-  ```
-
-### programUnsubscribe
-
-Unsubscribe from program-owned account change notifications
-
-#### Parameters:
-
-* `integer` - id of account Subscription to cancel
-
-#### Results:
-
-* `bool` - unsubscribe success message
-
-#### Example:
-
-```bash
-// Request
-{"jsonrpc":"2.0", "id":1, "method":"programUnsubscribe", "params":[0]}
-
-// Result
-{"jsonrpc": "2.0","result": true,"id": 1}
-```
-
-### signatureSubscribe
-
-Subscribe to a transaction signature to receive notification when the transaction is confirmed On `signatureNotification`, the subscription is automatically cancelled
-
-#### Parameters:
-
-* `string` - Transaction Signature, as base-58 encoded string
-* `integer` - optional, number of confirmed blocks to wait before notification.
-
-  Default: 0, Max: `MAX_LOCKOUT_HISTORY` \(greater integers rounded down\)
-
-#### Results:
-
-* `integer` - subscription id \(needed to unsubscribe\)
-
-#### Example:
-
-```bash
-// Request
-{"jsonrpc":"2.0", "id":1, "method":"signatureSubscribe", "params":["2EBVM6cB8vAAD93Ktr6Vd8p67XPbQzCJX47MpReuiCXJAtcjaxpvWpcg9Ege1Nr5Tk3a2GFrByT7WPBjdsTycY9b"]}
-
-{"jsonrpc":"2.0", "id":1, "method":"signatureSubscribe", "params":["2EBVM6cB8vAAD93Ktr6Vd8p67XPbQzCJX47MpReuiCXJAtcjaxpvWpcg9Ege1Nr5Tk3a2GFrByT7WPBjdsTycY9b", 15]}
-
-// Result
-{"jsonrpc": "2.0","result": 0,"id": 1}
-```
-
-#### Notification Format:
-
-```bash
-{"jsonrpc": "2.0","method": "signatureNotification", "params": {"result": "Confirmed","subscription":0}}
-```
-
-### signatureUnsubscribe
-
-Unsubscribe from signature confirmation notification
-
-#### Parameters:
-
-* `integer` - subscription id to cancel
-
-#### Results:
-
-* `bool` - unsubscribe success message
-
-#### Example:
-
-```bash
-// Request
-{"jsonrpc":"2.0", "id":1, "method":"signatureUnsubscribe", "params":[0]}
-
-// Result
-{"jsonrpc": "2.0","result": true,"id": 1}
-```

book/src/api-reference/transaction-api.md

@@ -1,62 +0,0 @@
-# Transaction
-
-## Components of a `Transaction`
-
-* **Transaction:**
-  * **message:** Defines the transaction
-    * **header:** Details the account types of and signatures required by
-
-      the transaction
-
-      * **num\_required\_signatures:** The total number of signatures
-
-        required to make the transaction valid.
-
-      * **num\_credit\_only\_signed\_accounts:** The last
-
-        `num_readonly_signed_accounts` signatures refer to signing
-
-        credit only accounts. Credit only accounts can be used concurrently
-
-        by multiple parallel transactions, but their balance may only be
-
-        increased, and their account data is read-only.
-
-      * **num\_credit\_only\_unsigned\_accounts:** The last
-
-        `num_readonly_unsigned_accounts` public keys in `account_keys` refer
-
-        to non-signing credit only accounts
-
-    * **account\_keys:** List of public keys used by the transaction, including
-
-      by the instructions and for signatures. The first
-
-      `num_required_signatures` public keys must sign the transaction.
-
-    * **recent\_blockhash:** The ID of a recent ledger entry. Validators will
-
-      reject transactions with a `recent_blockhash` that is too old.
-
-    * **instructions:** A list of [instructions](https://github.com/solana-labs/solana/tree/aacead62c0eb052068172eba6b53fc85874d6d54/book/src/instruction.md) that are
-
-      run sequentially and committed in one atomic transaction if all
-
-      succeed.
-  * **signatures:** A list of signatures applied to the transaction. The
-
-    list is always of length `num_required_signatures`, and the signature
-
-    at index `i` corresponds to the public key at index `i` in `account_keys`.
-
-    The list is initialized with empty signatures \(i.e. zeros\), and
-
-    populated as signatures are added.
-
-## Transaction Signing
-
-A `Transaction` is signed by using an ed25519 keypair to sign the serialization of the `message`. The resulting signature is placed at the index of `signatures` matching the index of the keypair's public key in `account_keys`.
-
-## Transaction Serialization
-
-`Transaction`s \(and their `message`s\) are serialized and deserialized using the [bincode](https://crates.io/crates/bincode) crate with a non-standard vector serialization that uses only one byte for the length if it can be encoded in 7 bits, 2 bytes if it fits in 14 bits, or 3 bytes if it requires 15 or 16 bits. The vector serialization is defined by Solana's [short-vec](https://github.com/solana-labs/solana/blob/master/sdk/src/short_vec.rs).

book/src/getting-started/testnet-participation.md

@@ -1,7 +0,0 @@
-# Testnet Participation
-
-Participate in our testnet:
-
-* [Running a Validator](../running-validator/)
-* [Running an Archiver](../running-archiver.md)
-

book/src/implemented-proposals/blocktree.md

@@ -1,90 +0,0 @@
-# Blocktree
-
-After a block reaches finality, all blocks from that one on down to the genesis block form a linear chain with the familiar name blockchain. Until that point, however, the validator must maintain all potentially valid chains, called _forks_. The process by which forks naturally form as a result of leader rotation is described in [fork generation](../cluster/fork-generation.md). The _blocktree_ data structure described here is how a validator copes with those forks until blocks are finalized.
-
-The blocktree allows a validator to record every shred it observes on the network, in any order, as long as the shred is signed by the expected leader for a given slot.
-
-Shreds are moved to a fork-able key space the tuple of `leader slot` + `shred index` \(within the slot\). This permits the skip-list structure of the Solana protocol to be stored in its entirety, without a-priori choosing which fork to follow, which Entries to persist or when to persist them.
-
-Repair requests for recent shreds are served out of RAM or recent files and out of deeper storage for less recent shreds, as implemented by the store backing Blocktree.
-
-## Functionalities of Blocktree
-
-1. Persistence: the Blocktree lives in the front of the nodes verification
-
-   pipeline, right behind network receive and signature verification. If the
-
-   shred received is consistent with the leader schedule \(i.e. was signed by the
-
-   leader for the indicated slot\), it is immediately stored.
-
-2. Repair: repair is the same as window repair above, but able to serve any
-
-   shred that's been received. Blocktree stores shreds with signatures,
-
-   preserving the chain of origination.
-
-3. Forks: Blocktree supports random access of shreds, so can support a
-
-   validator's need to rollback and replay from a Bank checkpoint.
-
-4. Restart: with proper pruning/culling, the Blocktree can be replayed by
-
-   ordered enumeration of entries from slot 0. The logic of the replay stage
-
-   \(i.e. dealing with forks\) will have to be used for the most recent entries in
-
-   the Blocktree.
-
-## Blocktree Design
-
-1. Entries in the Blocktree are stored as key-value pairs, where the key is the concatenated slot index and shred index for an entry, and the value is the entry data. Note shred indexes are zero-based for each slot \(i.e. they're slot-relative\).
-2. The Blocktree maintains metadata for each slot, in the `SlotMeta` struct containing:
-   * `slot_index` - The index of this slot
-   * `num_blocks` - The number of blocks in the slot \(used for chaining to a previous slot\)
-   * `consumed` - The highest shred index `n`, such that for all `m < n`, there exists a shred in this slot with shred index equal to `n` \(i.e. the highest consecutive shred index\).
-   * `received` - The highest received shred index for the slot
-   * `next_slots` - A list of future slots this slot could chain to. Used when rebuilding
-
-     the ledger to find possible fork points.
-
-   * `last_index` - The index of the shred that is flagged as the last shred for this slot. This flag on a shred will be set by the leader for a slot when they are transmitting the last shred for a slot.
-   * `is_rooted` - True iff every block from 0...slot forms a full sequence without any holes. We can derive is\_rooted for each slot with the following rules. Let slot\(n\) be the slot with index `n`, and slot\(n\).is\_full\(\) is true if the slot with index `n` has all the ticks expected for that slot. Let is\_rooted\(n\) be the statement that "the slot\(n\).is\_rooted is true". Then:
-
-     is\_rooted\(0\) is\_rooted\(n+1\) iff \(is\_rooted\(n\) and slot\(n\).is\_full\(\)
-3. Chaining - When a shred for a new slot `x` arrives, we check the number of blocks \(`num_blocks`\) for that new slot \(this information is encoded in the shred\). We then know that this new slot chains to slot `x - num_blocks`.
-4. Subscriptions - The Blocktree records a set of slots that have been "subscribed" to. This means entries that chain to these slots will be sent on the Blocktree channel for consumption by the ReplayStage. See the `Blocktree APIs` for details.
-5. Update notifications - The Blocktree notifies listeners when slot\(n\).is\_rooted is flipped from false to true for any `n`.
-
-## Blocktree APIs
-
-The Blocktree offers a subscription based API that ReplayStage uses to ask for entries it's interested in. The entries will be sent on a channel exposed by the Blocktree. These subscription API's are as follows: 1. `fn get_slots_since(slot_indexes: &[u64]) -> Vec<SlotMeta>`: Returns new slots connecting to any element of the list `slot_indexes`.
-
-1. `fn get_slot_entries(slot_index: u64, entry_start_index: usize, max_entries: Option<u64>) -> Vec<Entry>`: Returns the entry vector for the slot starting with `entry_start_index`, capping the result at `max` if `max_entries == Some(max)`, otherwise, no upper limit on the length of the return vector is imposed.
-
-Note: Cumulatively, this means that the replay stage will now have to know when a slot is finished, and subscribe to the next slot it's interested in to get the next set of entries. Previously, the burden of chaining slots fell on the Blocktree.
-
-## Interfacing with Bank
-
-The bank exposes to replay stage:
-
-1. `prev_hash`: which PoH chain it's working on as indicated by the hash of the last
-
-   entry it processed
-
-2. `tick_height`: the ticks in the PoH chain currently being verified by this
-
-   bank
-
-3. `votes`: a stack of records that contain: 1. `prev_hashes`: what anything after this vote must chain to in PoH 2. `tick_height`: the tick height at which this vote was cast 3. `lockout period`: how long a chain must be observed to be in the ledger to
-
-   be able to be chained below this vote
-
-Replay stage uses Blocktree APIs to find the longest chain of entries it can hang off a previous vote. If that chain of entries does not hang off the latest vote, the replay stage rolls back the bank to that vote and replays the chain from there.
-
-## Pruning Blocktree
-
-Once Blocktree entries are old enough, representing all the possible forks becomes less useful, perhaps even problematic for replay upon restart. Once a validator's votes have reached max lockout, however, any Blocktree contents that are not on the PoH chain for that vote for can be pruned, expunged.
-
-Archiver nodes will be responsible for storing really old ledger contents, and validators need only persist their bank periodically.
-

book/src/implemented-proposals/durable-tx-nonces.md

@@ -1,123 +0,0 @@
-# Durable Transaction Nonces
-
-## Problem
-
-To prevent replay, Solana transactions contain a nonce field populated with a
-"recent" blockhash value. A transaction containing a blockhash that is too old
-(~2min as of this writing) is rejected by the network as invalid. Unfortunately
-certain use cases, such as custodial services, require more time to produce a
-signature for the transaction. A mechanism is needed to enable these potentially
-offline network participants.
-
-## Requirements
-
-1) The transaction's signature needs to cover the nonce value
-2) The nonce must not be reusable, even in the case of signing key disclosure
-
-## A Contract-based Solution
-
-Here we describe a contract-based solution to the problem, whereby a client can
-"stash" a nonce value for future use in a transaction's `recent_blockhash`
-field. This approach is akin to the Compare and Swap atomic instruction,
-implemented by some CPU ISAs.
-
-When making use of a durable nonce, the client must first query its value from
-account data. A transaction is now constructed in the normal way, but with the
-following additional requirements:
-
-  1) The durable nonce value is used in the `recent_blockhash` field
-  2) A `Nonce` instruction is issued (first?)
-  3) The appropriate transaction flag is set, signaling that the usual
-hash age check should be skipped and the previous requirements enforced. This
-may be unnecessary, see [Runtime Support](#runtime-support) below
-
-### Contract Mechanics
-
-TODO: svgbob this into a flowchart
-
-```text
-Start
-Create Account
-  state = Uninitialized
-NonceInstruction
-  if state == Uninitialized
-    if account.balance < rent_exempt
-      error InsufficientFunds
-    state = Initialized
-  elif state != Initialized
-    error BadState
-  if sysvar.recent_blockhashes.is_empty()
-    error EmptyRecentBlockhashes
-  if !sysvar.recent_blockhashes.contains(stored_nonce)
-    error NotReady
-  stored_hash = sysvar.recent_blockhashes[0]
-  success
-WithdrawInstruction(to, lamports)
-  if state == Uninitialized
-    if !signers.contains(owner)
-      error MissingRequiredSignatures
-  elif state == Initialized
-    if !sysvar.recent_blockhashes.contains(stored_nonce)
-      error NotReady
-    if lamports != account.balance && lamports + rent_exempt > account.balance
-      error InsufficientFunds
-  account.balance -= lamports
-  to.balance += lamports
-  success
-```
-
-A client wishing to use this feature starts by creating a nonce account and
-depositing sufficient lamports as to make it rent-exempt. The resultant account
-will be in the `Uninitialized` state with no stored hash and thus unusable.
-
-The `Nonce` instruction is used to request that a new nonce be stored for the
-calling account. The first `Nonce` instruction run on a newly created account
-will drive the account's state to `Initialized`. As such, a `Nonce` instruction
-MUST be issued before the account can be used.
-
-To discard a `NonceAccount`, the client should issue a `Withdraw` instruction
-which withdraws all lamports, leaving a zero balance and making the account
-eligible for deletion.
-
-`Nonce` and `Withdraw` instructions each will only succeed if the stored
-blockhash is no longer resident in sysvar.recent_blockhashes.
-
-### Runtime Support
-
-The contract alone is not sufficient for implementing this feature. To enforce
-an extant `recent_blockhash` on the transaction and prevent fee theft via
-failed transaction replay, runtime modifications are necessary.
-
-Any transaction failing the usual `check_hash_age` validation will be tested
-for a Durable Transaction Nonce. This specifics of this test are undecided, some
-options:
-
-  1) Require that the `Nonce` instruction be the first in the transaction
-    * + No ABI changes
-    * + Fast and simple
-    * - Sets a precedent that may lead to incompatible instruction combinations
-  2) Blind search for a `Nonce` instruction over all instructions in the
-transaction
-    * + No ABI changes
-    * - Potentially slow
-  3) [2], but guarded by a transaction flag
-    * - ABI changes
-    * - Wire size increase
-    * + We'll probably end up with some sort of flags eventually anyway
-
-Current prototyping will use [1]. If it is determined that a Durable Transaction
-Nonce is in use, the runtime will take the following actions to validate the
-transaction:
-
-  1) The `NonceAccount` specified in the `Nonce` instruction is loaded.
-  2) The `NonceState` is deserialized from the `NonceAccount`'s data field and
-confirmed to be in the `Initialized` state.
-  3) The nonce value stored in the `NonceAccount` is tested to match against the
-one specified in the transaction's `recent_blockhash` field.
-
-If all three of the above checks succeed, the transaction is allowed to continue
-validation.
-
-### Open Questions
-
-* Should this feature be restricted in the number of uses per transaction?

book/src/implemented-proposals/ed_overview/ed_validation_client_economics/ed_vce_replication_validation_transaction_fees.md

@@ -1,12 +0,0 @@
-# Replication-validation Transaction Fees
-
-**Subject to change.**
-
-As previously mentioned, validator-clients will also be responsible for validating PoReps submitted into the PoH stream by archiver-clients. In this case, validators are providing compute \(CPU/GPU\) and light storage resources to confirm that these replication proofs could only be generated by a client that is storing the referenced PoH leger block.
-
-While replication-clients are incentivized and rewarded through protocol-based rewards schedule \(see [Replication-client Economics](../ed_replication_client_economics/)\), validator-clients will be incentivized to include and validate PoReps in PoH through collection of transaction fees associated with the submitted PoReps and distribution of protocol rewards proportional to the validated PoReps. As will be described in detail in the Section 3.1, replication-client rewards are protocol-based and designed to reward based on a global data redundancy factor. I.e. the protocol will incentivize replication-client participation through rewards based on a target ledger redundancy \(e.g. 10x data redundancy\).
-
-The validation of PoReps by validation-clients is computationally more expensive than state-validation \(detail in the [Economic Sustainability](../ed_economic_sustainability.md) chapter\), thus the transaction fees are expected to be proportionally higher.
-
-There are various attack vectors available for colluding validation and replication clients, also described in detail below in [Economic Sustainability](https://github.com/solana-labs/solana/tree/aacead62c0eb052068172eba6b53fc85874d6d54/book/src/ed_economic_sustainability/README.md). To protect against various collusion attack vectors, for a given epoch, validator rewards are distributed across participating validation-clients in proportion to the number of validated PoReps in the epoch less the number of PoReps that mismatch the archivers challenge. The PoRep challenge game is described in [Ledger Replication](https://github.com/solana-labs/solana/blob/master/book/src/ledger-replication.md#the-porep-game). This design rewards validators proportional to the number of PoReps they process and validate, while providing negative pressure for validation-clients to submit lazy or malicious invalid votes on submitted PoReps \(note that it is computationally prohibitive to determine whether a validator-client has marked a valid PoRep as invalid\).
-

book/src/implemented-proposals/repair-service.md

@@ -1,49 +0,0 @@
-# Repair Service
-
-## Repair Service
-
-The RepairService is in charge of retrieving missing shreds that failed to be delivered by primary communication protocols like Avalanche. It is in charge of managing the protocols described below in the `Repair Protocols` section below.
-
-## Challenges:
-
-1\) Validators can fail to receive particular shreds due to network failures
-
-2\) Consider a scenario where blocktree contains the set of slots {1, 3, 5}. Then Blocktree receives shreds for some slot 7, where for each of the shreds b, b.parent == 6, so then the parent-child relation 6 -> 7 is stored in blocktree. However, there is no way to chain these slots to any of the existing banks in Blocktree, and thus the `Shred Repair` protocol will not repair these slots. If these slots happen to be part of the main chain, this will halt replay progress on this node.
-
-3\) Validators that find themselves behind the cluster by an entire epoch struggle/fail to catch up because they do not have a leader schedule for future epochs. If nodes were to blindly accept repair shreds in these future epochs, this exposes nodes to spam.
-
-## Repair Protocols
-
-The repair protocol makes best attempts to progress the forking structure of Blocktree.
-
-The different protocol strategies to address the above challenges:
-
-1. Shred Repair \(Addresses Challenge \#1\): This is the most basic repair protocol, with the purpose of detecting and filling "holes" in the ledger. Blocktree tracks the latest root slot. RepairService will then periodically iterate every fork in blocktree starting from the root slot, sending repair requests to validators for any missing shreds. It will send at most some `N` repair reqeusts per iteration.
-
-   Note: Validators will only accept shreds within the current verifiable epoch \(epoch the validator has a leader schedule for\).
-
-2. Preemptive Slot Repair \(Addresses Challenge \#2\): The goal of this protocol is to discover the chaining relationship of "orphan" slots that do not currently chain to any known fork.
-   * Blocktree will track the set of "orphan" slots in a separate column family.
-   * RepairService will periodically make `RequestOrphan` requests for each of the orphans in blocktree.
-
-     `RequestOrphan(orphan)` request - `orphan` is the orphan slot that the requestor wants to know the parents of `RequestOrphan(orphan)` response - The highest shreds for each of the first `N` parents of the requested `orphan`
-
-     On receiving the responses `p`, where `p` is some shred in a parent slot, validators will:
-
-     * Insert an empty `SlotMeta` in blocktree for `p.slot` if it doesn't already exist.
-     * If `p.slot` does exist, update the parent of `p` based on `parents`
-
-     Note: that once these empty slots are added to blocktree, the `Shred Repair` protocol should attempt to fill those slots.
-
-     Note: Validators will only accept responses containing shreds within the current verifiable epoch \(epoch the validator has a leader schedule for\).
-3. Repairmen \(Addresses Challenge \#3\): This part of the repair protocol is the primary mechanism by which new nodes joining the cluster catch up after loading a snapshot. This protocol works in a "forward" fashion, so validators can verify every shred that they receive against a known leader schedule.
-
-   Each validator advertises in gossip:
-
-   * Current root
-   * The set of all completed slots in the confirmed epochs \(an epoch that was calculated based on a bank <= current root\) past the current root
-
-     Observers of this gossip message with higher epochs \(repairmen\) send shreds to catch the lagging node up with the rest of the cluster. The repairmen are responsible for sending the slots within the epochs that are confrimed by the advertised `root` in gossip. The repairmen divide the responsibility of sending each of the missing slots in these epochs based on a random seed \(simple shred.index iteration by N, seeded with the repairman's node\_pubkey\). Ideally, each repairman in an N node cluster \(N nodes whose epochs are higher than that of the repairee\) sends 1/N of the missing shreds. Both data and coding shreds for missing slots are sent. Repairmen do not send shreds again to the same validator until they see the message in gossip updated, at which point they perform another iteration of this protocol.
-
-     Gossip messages are updated every time a validator receives a complete slot within the epoch. Completed slots are detected by blocktree and sent over a channel to RepairService. It is important to note that we know that by the time a slot X is complete, the epoch schedule must exist for the epoch that contains slot X because WindowService will reject shreds for unconfirmed epochs. When a newly completed slot is detected, we also update the current root if it has changed since the last update. The root is made available to RepairService through Blocktree, which holds the latest root.
-

book/src/introduction.md

@@ -1,37 +0,0 @@
-# Introduction
-
-## What is Solana?
-
-Solana is an open source project implementing a new, high-performance, permissionless blockchain. Solana is also the name of a company headquartered in San Francisco that maintains the open source project.
-
-## About this Book
-
-This book describes the Solana open source project, a blockchain built from the ground up for scale. The book covers why Solana is useful, how to use it, how it works, and why it will continue to work long after the company Solana closes its doors. The goal of the Solana architecture is to demonstrate there exists a set of software algorithms that when used in combination to implement a blockchain, removes software as a performance bottleneck, allowing transaction throughput to scale proportionally with network bandwidth. The architecture goes on to satisfy all three desirable properties of a proper blockchain: it is scalable, secure and decentralized.
-
-The architecture describes a theoretical upper bound of 710 thousand transactions per second \(tps\) on a standard gigabit network and 28.4 million tps on 40 gigabit. Furthermore, the architecture supports safe, concurrent execution of programs authored in general purpose programming languages such as C or Rust.
-
-## Disclaimer
-
-All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort. It is up to the reader to check and validate their accuracy and truthfulness. Furthermore, nothing in this project constitutes a solicitation for investment.
-
-## History of the Solana Codebase
-
-In November of 2017, Anatoly Yakovenko published a whitepaper describing Proof of History, a technique for keeping time between computers that do not trust one another. From Anatoly's previous experience designing distributed systems at Qualcomm, Mesosphere and Dropbox, he knew that a reliable clock makes network synchronization very simple. When synchronization is simple the resulting network can be blazing fast, bound only by network bandwidth.
-
-Anatoly watched as blockchain systems without clocks, such as Bitcoin and Ethereum, struggled to scale beyond 15 transactions per second worldwide when centralized payment systems such as Visa required peaks of 65,000 tps. Without a clock, it was clear they'd never graduate to being the global payment system or global supercomputer most had dreamed them to be. When Anatoly solved the problem of getting computers that don’t trust each other to agree on time, he knew he had the key to bring 40 years of distributed systems research to the world of blockchain. The resulting cluster wouldn't be just 10 times faster, or a 100 times, or a 1,000 times, but 10,000 times faster, right out of the gate!
-
-Anatoly's implementation began in a private codebase and was implemented in the C programming language. Greg Fitzgerald, who had previously worked with Anatoly at semiconductor giant Qualcomm Incorporated, encouraged him to reimplement the project in the Rust programming language. Greg had worked on the LLVM compiler infrastructure, which underlies both the Clang C/C++ compiler as well as the Rust compiler. Greg claimed that the language's safety guarantees would improve software productivity and that its lack of a garbage collector would allow programs to perform as well as those written in C. Anatoly gave it a shot and just two weeks later, had migrated his entire codebase to Rust. Sold. With plans to weave all the world's transactions together on a single, scalable blockchain, Anatoly called the project Loom.
-
-On February 13th of 2018, Greg began prototyping the first open source implementation of Anatoly's whitepaper. The project was published to GitHub under the name Silk in the loomprotocol organization. On February 28th, Greg made his first release, demonstrating 10 thousand signed transactions could be verified and processed in just over half a second. Shortly after, another former Qualcomm cohort, Stephen Akridge, demonstrated throughput could be massively improved by offloading signature verification to graphics processors. Anatoly recruited Greg, Stephen and three others to co-found a company, then called Loom.
-
-Around the same time, Ethereum-based project Loom Network sprung up and many people were confused about whether they were the same project. The Loom team decided it would rebrand. They chose the name Solana, a nod to a small beach town North of San Diego called Solana Beach, where Anatoly, Greg and Stephen lived and surfed for three years when they worked for Qualcomm. On March 28th, the team created the Solana Labs GitHub organization and renamed Greg's prototype Silk to Solana.
-
-In June of 2018, the team scaled up the technology to run on cloud-based networks and on July 19th, published a 50-node, permissioned, public testnet consistently supporting bursts of 250,000 transactions per second. In a later release in December, called v0.10 Pillbox, the team published a permissioned testnet running 150 nodes on a gigabit network and demonstrated soak tests processing an _average_ of 200 thousand transactions per second with bursts over 500 thousand. The project was also extended to support on-chain programs written in the C programming language and run concurrently in a safe execution environment called BPF.
-
-## What is a Solana Cluster?
-
-A cluster is a set of computers that work together and can be viewed from the outside as a single system. A Solana cluster is a set of independently owned computers working together \(and sometimes against each other\) to verify the output of untrusted, user-submitted programs. A Solana cluster can be utilized any time a user wants to preserve an immutable record of events in time or programmatic interpretations of those events. One use is to track which of the computers did meaningful work to keep the cluster running. Another use might be to track the possession of real-world assets. In each case, the cluster produces a record of events called the ledger. It will be preserved for the lifetime of the cluster. As long as someone somewhere in the world maintains a copy of the ledger, the output of its programs \(which may contain a record of who possesses what\) will forever be reproducible, independent of the organization that launched it.
-
-## What are SOLs?
-
-A SOL is the name of Solana's native token, which can be passed to nodes in a Solana cluster in exchange for running an on-chain program or validating its output. The system may perform micropayments of fractional SOLs, which are called _lamports_. They are named in honor of Solana's biggest technical influence, [Leslie Lamport](https://en.wikipedia.org/wiki/Leslie_Lamport). A lamport has a value of 0.000000001 SOL.

book/src/paper-wallet/installation.md

@@ -1,51 +0,0 @@
-# Installation Guide
-Follow this guide to setup Solana's key generation tool called `solana-keygen`
-
-{% hint style="warn" %}
-After installation, ensure your version is `0.21.1` or higher by running `solana-keygen -V`
-{% endhint %}
-
-## Download
-First, download the latest release tarball from GitHub.
-
-1. Setup download url
-
-  ```bash
-  solana_downloads=https://github.com/solana-labs/solana/releases/latest/download
-  ```
-
-2. Specify the download file based on your machine
-
-  **MacOS**
-  ```bash
-  solana_release=solana-release-x86_64-apple-darwin.tar.bz2
-  ```
-
-  **Linux**
-  ```bash
-  solana_release=solana-release-x86_64-unknown-linux-gnu.tar.bz2
-  ```
-
-3. Download
-
-  ```bash
-  curl -L -sSf -o solana-release.tar.bz2 $solana_downloads/$solana_release
-  ```
-
-## Extract
-Next, extract the tarball
-```bash
-tar xf solana-release.tar.bz2
-```
-
-## Add to "PATH"
-Now add the tool to your PATH environment variable with the following command
-```bash
-export PATH="$(pwd)/solana-release/bin:${PATH}"
-```
-
-## Check
-Finally, check that `solana-keygen` can be run by running
-```bash
-solana-keygen -V
-```

book/src/proposals/simple-payment-and-state-verification.md

@@ -1,109 +0,0 @@
-# Simple Payment and State Verification
-
-It is often useful to allow low resourced clients to participate in a Solana cluster. Be this participation economic or contract execution, verification that a client's activity has been accepted by the network is typically expensive. This proposal lays out a mechanism for such clients to confirm that their actions have been committed to the ledger state with minimal resource expenditure and third-party trust.
-
-## A Naive Approach
-
-Validators store the signatures of recently confirmed transactions for a short period of time to ensure that they are not processed more than once. Validators provide a JSON RPC endpoint, which clients can use to query the cluster if a transaction has been recently processed. Validators also provide a PubSub notification, whereby a client registers to be notified when a given signature is observed by the validator. While these two mechanisms allow a client to verify a payment, they are not a proof and rely on completely trusting a validator.
-
-We will describe a way to minimize this trust using Merkle Proofs to anchor the validator's response in the ledger, allowing the client to confirm on their own that a sufficient number of their preferred validators have confirmed a transaction. Requiring multiple validator attestations further reduces trust in the validator, as it increases both the technical and economic difficulty of compromising several other network participants.
-
-## Light Clients
-
-A 'light client' is a cluster participant that does not itself run a validator. This light client would provide a level of security greater than trusting a remote validator, without requiring the light client to spend a lot of resources verifying the ledger.
-
-Rather than providing transaction signatures directly to a light client, the validator instead generates a Merkle Proof from the transaction of interest to the root of a Merkle Tree of all transactions in the including block. This Merkle Root is stored in a ledger entry which is voted on by validators, providing it consensus legitimacy. The additional level of security for a light client depends on an initial canonical set of validators the light client considers to be the stakeholders of the cluster. As that set is changed, the client can update its internal set of known validators with [receipts](simple-payment-and-state-verification.md#receipts). This may become challenging with a large number of delegated stakes.
-
-Validators themselves may want to use light client APIs for performance reasons. For example, during the initial launch of a validator, the validator may use a cluster provided checkpoint of the state and verify it with a receipt.
-
-## Receipts
-
-A receipt is a minimal proof that; a transaction has been included in a block, that the block has been voted on by the client's preferred set of validators and that the votes have reached the desired confirmation depth.
-
-The receipts for both state and payments start with a Merkle Path from the value into a Bank-Merkle that has been voted on and included in the ledger. A chain of PoH Entries containing subsequent validator votes, deriving from the Bank-Merkle, is the confirmation proof.
-
-Clients can examine this ledger data and compute the finality using Solana's fork selection rules.
-
-### Payment Merkle Path
-
-A payment receipt is a data structure that contains a Merkle Path from a transaction to the required set of validator votes.
-
-An Entry-Merkle is a Merkle Root including all transactions in the entry, sorted by signature.
-
-![Block Merkle Diagram](../.gitbook/assets/spv-block-merkle.svg)
-
-A Block-Merkle is a Merkle root of all the Entry-Merkles sequenced in the block. Transaction status is necessary for the receipt because the state receipt is constructed for the block. Two transactions over the same state can appear in the block, and therefore, there is no way to infer from just the state whether a transaction that is committed to the ledger has succeeded or failed in modifying the intended state. It may not be necessary to encode the full status code, but a single status bit to indicate the transaction's success.
-
-### State Merkle Path
-
-A state receipt provides a confirmation that a specific state is committed at the end of the block. Inter-block state transitions do not generate a receipt.
-
-For example:
-
-* A sends 5 Lamports to B
-* B spends 5 Lamports
-* C sends 5 Lamports to A
-
-At the end of the block, A and B are in the exact same starting state, and any state receipt would point to the same value for A or B.
-
-The Bank-Merkle is computed from the Merkle Tree of the new state changes, along with the Previous Bank-Merkle, and the Block-Merkle.
-
-![Bank Merkle Diagram](../.gitbook/assets/spv-bank-merkle.svg)
-
-A state receipt contains only the state changes occurring in the block. A direct Merkle Path to the current Bank-Merkle guarantees the state value at that bank hash, but it cannot be used to generate a “current” receipt to the latest state if the state modification occurred in some previous block. There is no guarantee that the path provided by the validator is the latest one available out of all the previous Bank-Merkles.
-
-Clients that want to query the chain for a receipt of the "latest" state would need to create a transaction that would update the Merkle Path for that account, such as a credit of 0 Lamports.
-
-### Validator Votes
-
-Leaders should coalesce the validator votes by stake weight into a single entry. This will reduce the number of entries necessary to create a receipt.
-
-### Chain of Entries
-
-A receipt has a PoH link from the payment or state Merkle Path root to a list of consecutive validation votes.
-
-It contains the following:
-
-* State -> Bank-Merkle
-
-  or
-
-* Transaction -> Entry-Merkle -> Block-Merkle -> Bank-Merkle
-
-And a vector of PoH entries:
-
-* Validator vote entries
-* Ticks
-* Light entries
-
-```text
-/// This Entry definition skips over the transactions and only contains the
-/// hash of the transactions used to modify PoH.
-LightEntry {
-    /// The number of hashes since the previous Entry ID.
-    pub num_hashes: u64,
-    /// The SHA-256 hash `num_hashes` after the previous Entry ID.
-    hash: Hash,
-    /// The Merkle Root of the transactions encoded into the Entry.
-    entry_hash: Hash,
-}
-```
-
-The light entries are reconstructed from Entries and simply show the entry Merkle Root that was mixed in to the PoH hash, instead of the full transaction set.
-
-Clients do not need the starting vote state. The [fork selection](https://github.com/solana-labs/solana/tree/aacead62c0eb052068172eba6b53fc85874d6d54/book/src/book/src/fork-selection.md) algorithm is defined such that only votes that appear after the transaction provide finality for the transaction, and finality is independent of the starting state.
-
-### Verification
-
-A light client that is aware of the supermajority set validators can verify a receipt by following the Merkle Path to the PoH chain. The Bank-Merkle is the Merkle Root and will appear in votes included in an Entry. The light client can simulate [fork selection](https://github.com/solana-labs/solana/tree/aacead62c0eb052068172eba6b53fc85874d6d54/book/src/book/src/fork-selection.md) for the consecutive votes and verify that the receipt is confirmed at the desired lockout threshold.
-
-### Synthetic State
-
-Synthetic state should be computed into the Bank-Merkle along with the bank generated state.
-
-For example:
-
-* Epoch validator accounts and their stakes and weights.
-* Computed fee rates
-
-These values should have an entry in the Bank-Merkle. They should live under known accounts, and therefore have an exact address in the Merkle Path.

book/src/proposals/validator-proposal.md

@@ -1,35 +0,0 @@
-# Validator
-
-## History
-
-When we first started Solana, the goal was to de-risk our TPS claims. We knew that between optimistic concurrency control and sufficiently long leader slots, that PoS consensus was not the biggest risk to TPS. It was GPU-based signature verification, software pipelining and concurrent banking. Thus, the TPU was born. After topping 100k TPS, we split the team into one group working toward 710k TPS and another to flesh out the validator pipeline. Hence, the TVU was born. The current architecture is a consequence of incremental development with that ordering and project priorities. It is not a reflection of what we ever believed was the most technically elegant cross-section of those technologies. In the context of leader rotation, the strong distinction between leading and validating is blurred.
-
-## Difference between validating and leading
-
-The fundamental difference between the pipelines is when the PoH is present. In a leader, we process transactions, removing bad ones, and then tag the result with a PoH hash. In the validator, we verify that hash, peel it off, and process the transactions in exactly the same way. The only difference is that if a validator sees a bad transaction, it can't simply remove it like the leader does, because that would cause the PoH hash to change. Instead, it rejects the whole block. The other difference between the pipelines is what happens _after_ banking. The leader broadcasts entries to downstream validators whereas the validator will have already done that in RetransmitStage, which is a confirmation time optimization. The validation pipeline, on the other hand, has one last step. Any time it finishes processing a block, it needs to weigh any forks it's observing, possibly cast a vote, and if so, reset its PoH hash to the block hash it just voted on.
-
-## Proposed Design
-
-We unwrap the many abstraction layers and build a single pipeline that can toggle leader mode on whenever the validator's ID shows up in the leader schedule.
-
-![Validator block diagram](../.gitbook/assets/validator-proposal.svg)
-
-## Notable changes
-
-* No threads are shut down to switch out of leader mode. Instead, FetchStage
-
-  should forward transactions to the next leader.
-
-* Hoist FetchStage and BroadcastStage out of TPU
-* Blocktree renamed to Blockstore
-* BankForks renamed to Banktree
-* TPU moves to new socket-free crate called solana-tpu.
-* TPU's BankingStage absorbs ReplayStage
-* TVU goes away
-* New RepairStage absorbs Shred Fetch Stage and repair requests
-* JSON RPC Service is optional - used for debugging. It should instead be part
-
-  of a separate `solana-blockstreamer` executable.
-
-* New MulticastStage absorbs retransmit part of RetransmitStage
-* MulticastStage downstream of Blockstore

book/src/running-archiver.md

@@ -1,156 +0,0 @@
-# Running an Archiver
-
-This document describes how to setup an archiver in the testnet
-
-Please note some of the information and instructions described here may change in future releases.
-
-## Overview
-
-Archivers are specialized light clients. They download a part of the ledger \(a.k.a Segment\) and store it. They earn rewards for storing segments.
-
-The testnet features a validator running at testnet.solana.com, which serves as the entrypoint to the cluster for your archiver node.
-
-Additionally there is a blockexplorer available at [http://testnet.solana.com/](http://testnet.solana.com/).
-
-The testnet is configured to reset the ledger daily, or sooner should the hourly automated cluster sanity test fail.
-
-## Machine Requirements
-
-Archivers don't need specialized hardware. Anything with more than 128GB of disk space will be able to participate in the cluster as an archiver node.
-
-Currently the disk space requirements are very low but we expect them to change in the future.
-
-Prebuilt binaries are available for Linux x86\_64 \(Ubuntu 18.04 recommended\), macOS, and Windows.
-
-### Confirm The Testnet Is Reachable
-
-Before starting an archiver node, sanity check that the cluster is accessible to your machine by running some simple commands. If any of the commands fail, please retry 5-10 minutes later to confirm the testnet is not just restarting itself before debugging further.
-
-Fetch the current transaction count over JSON RPC:
-
-```bash
-curl -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":1, "method":"getTransactionCount"}' http://testnet.solana.com:8899
-```
-
-Inspect the blockexplorer at [http://testnet.solana.com/](http://testnet.solana.com/) for activity.
-
-View the [metrics dashboard](https://metrics.solana.com:3000/d/testnet-beta/testnet-monitor-beta?var-testnet=testnet) for more detail on cluster activity.
-
-## Archiver Setup
-
-#### Obtaining The Software
-
-#### Bootstrap with `solana-install`
-
-The `solana-install` tool can be used to easily install and upgrade the cluster software.
-
-#### Linux and mac OS
-
-```bash
-curl -sSf https://raw.githubusercontent.com/solana-labs/solana/v0.18.0/install/solana-install-init.sh | sh -s
-```
-
-Alternatively build the `solana-install` program from source and run the following command to obtain the same result:
-
-```bash
-solana-install init
-```
-
-#### Windows
-
-Download and install **solana-install-init** from [https://github.com/solana-labs/solana/releases/latest](https://github.com/solana-labs/solana/releases/latest)
-
-After a successful install, `solana-install update` may be used to easily update the software to a newer version at any time.
-
-#### Download Prebuilt Binaries
-
-If you would rather not use `solana-install` to manage the install, you can manually download and install the binaries.
-
-#### Linux
-
-Download the binaries by navigating to [https://github.com/solana-labs/solana/releases/latest](https://github.com/solana-labs/solana/releases/latest), download **solana-release-x86\_64-unknown-linux-gnu.tar.bz2**, then extract the archive:
-
-```bash
-tar jxf solana-release-x86_64-unknown-linux-gnu.tar.bz2
-cd solana-release/
-export PATH=$PWD/bin:$PATH
-```
-
-#### mac OS
-
-Download the binaries by navigating to [https://github.com/solana-labs/solana/releases/latest](https://github.com/solana-labs/solana/releases/latest), download **solana-release-x86\_64-apple-darwin.tar.bz2**, then extract the archive:
-
-```bash
-tar jxf solana-release-x86_64-apple-darwin.tar.bz2
-cd solana-release/
-export PATH=$PWD/bin:$PATH
-```
-
-#### Windows
-
-Download the binaries by navigating to [https://github.com/solana-labs/solana/releases/latest](https://github.com/solana-labs/solana/releases/latest), download **solana-release-x86\_64-pc-windows-msvc.tar.bz2**, then extract it into a folder. It is a good idea to add this extracted folder to your windows PATH.
-
-## Starting The Archiver
-
-Try running following command to join the gossip network and view all the other nodes in the cluster:
-
-```bash
-solana-gossip --entrypoint testnet.solana.com:8001 spy
-# Press ^C to exit
-```
-
-Now configure the keypairs for your archiver by running:
-
-Navigate to the solana install location and open a cmd prompt
-
-```bash
-solana-keygen new -o archiver-keypair.json
-solana-keygen new -o storage-keypair.json
-```
-
-Use solana-keygen to show the public keys for each of the keypairs, they will be needed in the next step:
-
-* Windows
-
-  ```bash
-  # The archiver's identity
-  solana-keygen pubkey archiver-keypair.json
-  solana-keygen pubkey storage-keypair.json
-  ```
-
-* Linux and mac OS
-
-  \`\`\`bash
-
-  export ARCHIVER\_IDENTITY=$\(solana-keygen pubkey archiver-keypair.json\)
-
-  export STORAGE\_IDENTITY=$\(solana-keygen pubkey storage-keypair.json\)
-
-```text
-Then set up the storage accounts for your archiver by running:
-```bash
-solana --keypair archiver-keypair.json airdrop 100000 lamports
-solana --keypair archiver-keypair.json create-archiver-storage-account $ARCHIVER_IDENTITY $STORAGE_IDENTITY
-```
-
-Note: Every time the testnet restarts, run the steps to setup the archiver accounts again.
-
-To start the archiver:
-
-```bash
-solana-archiver --entrypoint testnet.solana.com:8001 --identity-keypair archiver-keypair.json --storage-keypair storage-keypair.json --ledger archiver-ledger
-```
-
-## Verify Archiver Setup
-
-From another console, confirm the IP address and **identity pubkey** of your archiver is visible in the gossip network by running:
-
-```bash
-solana-gossip --entrypoint testnet.solana.com:8001 spy
-```
-
-Provide the **storage account pubkey** to the `solana show-storage-account` command to view the recent mining activity from your archiver:
-
-```bash
-solana --keypair storage-keypair.json show-storage-account $STORAGE_IDENTITY
-```

book/src/running-validator/validator-testnet.md

@@ -1,40 +0,0 @@
-# Choosing a Testnet
-
-Solana maintains several testnets, each featuring a Solana-owned validator
-that serves as an entrypoint to the cluster.
-
-Current testnet entrypoints:
-
-* Stable: testnet.solana.com
-* Beta: beta.testnet.solana.com
-
-Application developers should target the Stable testnet. Key differences
-between the Stable testnet and what will be mainnet:
-
-* Stable testnet tokens are not real
-* Stable testnet includes a token faucet for application testing
-* Stable testnet may be subject to ledger resets
-* Stable testnet typically runs a newer software version than mainnet
-* Stable testnet may be maintained by different validators than mainnet
-
-The Beta testnet is used to showcase and stabilize new features before they
-are tagged for release. Application developers are free to target the Beta
-testnet, but should expect instability and periodic ledger resets. Regarding
-stability, all that can be said is that CI automation was successful.
-
-### Get Testnet Version
-
-You can submit a JSON-RPC request to see the specific software version of the
-cluster. Use this to specify [the software version to install](validator-software.md).
-
-```bash
-curl -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":1, "method":"getVersion"}' testnet.solana.com:8899
-```
-Example result:
-`{"jsonrpc":"2.0","result":{"solana-core":"0.21.0"},"id":1}`
-
-## Using a Different Testnet
-
-This guide is written in the context of testnet.solana.com, our most stable
-cluster. To participate in another testnet, modify the commands in the following
-pages, replacing `testnet.solana.com` with your desired testnet.

book/src/transaction.md

@@ -1,17 +0,0 @@
-# Anatomy of a Transaction
-
-Transactions encode lists of instructions that are executed sequentially, and only committed if all the instructions complete successfully. All account updates are reverted upon the failure of a transaction. Each transaction details the accounts used, including which must sign and which are read only, a recent blockhash, the instructions, and any signatures.
-
-## Accounts and Signatures
-
-Each transaction explicitly lists all account public keys referenced by the transaction's instructions. A subset of those public keys are each accompanied by a transaction signature. Those signatures signal on-chain programs that the account holder has authorized the transaction. Typically, the program uses the authorization to permit debiting the account or modifying its data.
-
-The transaction also marks some accounts as _read-only accounts_. The runtime permits read-only accounts to be read concurrently. If a program attempts to modify a read-only account, the transaction is rejected by the runtime.
-
-## Recent Blockhash
-
-A Transaction includes a recent blockhash to prevent duplication and to give transactions lifetimes. Any transaction that is completely identical to a previous one is rejected, so adding a newer blockhash allows multiple transactions to repeat the exact same action. Transactions also have lifetimes that are defined by the blockhash, as any transaction whose blockhash is too old will be rejected.
-
-## Instructions
-
-Each instruction specifies a single program account \(which must be marked executable\), a subset of the transaction's accounts that should be passed to the program, and a data byte array instruction that is passed to the program. The program interprets the data array and operates on the accounts specified by the instructions. The program can return successfully, or with an error code. An error return causes the entire transaction to fail immediately.

chacha-cuda/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "solana-chacha-cuda"
+version = "1.0.7"
+description = "Solana Chacha Cuda APIs"
+authors = ["Solana Maintainers <maintainers@solana.com>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+log = "0.4.8"
+solana-archiver-utils = { path = "../archiver-utils", version = "1.0.7" }
+solana-chacha = { path = "../chacha", version = "1.0.7" }
+solana-ledger = { path = "../ledger", version = "1.0.7" }
+solana-logger = { path = "../logger", version = "1.0.7" }
+solana-perf = { path = "../perf", version = "1.0.7" }
+solana-sdk = { path = "../sdk", version = "1.0.7" }
+
+[dev-dependencies]
+hex-literal = "0.2.1"
+
+[lib]
+name = "solana_chacha_cuda"

chacha-cuda/src/chacha_cuda.rs

@@ -1,7 +1,7 @@
 // Module used by validators to approve storage mining proofs in parallel using the GPU
 
-use crate::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE};
-use solana_ledger::blocktree::Blocktree;
+use solana_chacha::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE};
+use solana_ledger::blockstore::Blockstore;
 use solana_perf::perf_libs;
 use solana_sdk::hash::Hash;
 use std::io;
@@ -13,7 +13,7 @@ use std::sync::Arc;
 // Then sample each block at the offsets provided by samples argument with sha256
 // and return the vec of sha states
 pub fn chacha_cbc_encrypt_file_many_keys(
-    blocktree: &Arc<Blocktree>,
+    blockstore: &Arc<Blockstore>,
     segment: u64,
     slots_per_segment: u64,
     ivecs: &mut [u8],
@@ -46,7 +46,7 @@ pub fn chacha_cbc_encrypt_file_many_keys(
         (api.chacha_init_sha_state)(int_sha_states.as_mut_ptr(), num_keys as u32);
     }
     loop {
-        match blocktree.get_data_shreds(current_slot, start_index, std::u64::MAX, &mut buffer) {
+        match blockstore.get_data_shreds(current_slot, start_index, std::u64::MAX, &mut buffer) {
             Ok((last_index, mut size)) => {
                 debug!(
                     "chacha_cuda: encrypting segment: {} num_shreds: {} data_len: {}",
@@ -113,12 +113,12 @@ pub fn chacha_cbc_encrypt_file_many_keys(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::archiver::sample_file;
-    use crate::chacha::chacha_cbc_encrypt_ledger;
+    use solana_archiver_utils::sample_file;
+    use solana_chacha::chacha::chacha_cbc_encrypt_ledger;
     use solana_ledger::entry::create_ticks;
     use solana_ledger::get_tmp_ledger_path;
     use solana_sdk::clock::DEFAULT_SLOTS_PER_SEGMENT;
-    use solana_sdk::signature::{Keypair, KeypairUtil};
+    use solana_sdk::signature::Keypair;
     use std::fs::{remove_dir_all, remove_file};
     use std::path::Path;
 
@@ -134,9 +134,9 @@ mod tests {
         let entries = create_ticks(slots_per_segment, 0, Hash::default());
         let ledger_path = get_tmp_ledger_path!();
         let ticks_per_slot = 16;
-        let blocktree = Arc::new(Blocktree::open(&ledger_path).unwrap());
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
-        blocktree
+        blockstore
             .write_entries(
                 0,
                 0,
@@ -160,7 +160,7 @@ mod tests {
 
         let mut cpu_iv = ivecs.clone();
         chacha_cbc_encrypt_ledger(
-            &blocktree,
+            &blockstore,
             0,
             slots_per_segment as u64,
             out_path,
@@ -171,7 +171,7 @@ mod tests {
         let ref_hash = sample_file(&out_path, &samples).unwrap();
 
         let hashes = chacha_cbc_encrypt_file_many_keys(
-            &blocktree,
+            &blockstore,
             0,
             slots_per_segment as u64,
             &mut ivecs,
@@ -196,8 +196,8 @@ mod tests {
         let ledger_path = get_tmp_ledger_path!();
         let ticks_per_slot = 90;
         let entries = create_ticks(2 * ticks_per_slot, 0, Hash::default());
-        let blocktree = Arc::new(Blocktree::open(&ledger_path).unwrap());
-        blocktree
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        blockstore
             .write_entries(
                 0,
                 0,
@@ -224,7 +224,7 @@ mod tests {
             ivec[0] = i;
             ivecs.extend(ivec.clone().iter());
             chacha_cbc_encrypt_ledger(
-                &blocktree.clone(),
+                &blockstore.clone(),
                 0,
                 DEFAULT_SLOTS_PER_SEGMENT,
                 out_path,
@@ -242,7 +242,7 @@ mod tests {
         }
 
         let hashes = chacha_cbc_encrypt_file_many_keys(
-            &blocktree,
+            &blockstore,
             0,
             DEFAULT_SLOTS_PER_SEGMENT,
             &mut ivecs,
@@ -267,9 +267,9 @@ mod tests {
         let mut keys = hex!("abc123");
         let ledger_path = get_tmp_ledger_path!();
         let samples = [0];
-        let blocktree = Arc::new(Blocktree::open(&ledger_path).unwrap());
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         assert!(chacha_cbc_encrypt_file_many_keys(
-            &blocktree,
+            &blockstore,
             0,
             DEFAULT_SLOTS_PER_SEGMENT,
             &mut keys,

chacha-cuda/src/lib.rs

@@ -0,0 +1,8 @@
+#[macro_use]
+extern crate log;
+
+#[cfg(test)]
+#[macro_use]
+extern crate hex_literal;
+
+pub mod chacha_cuda;

chacha-sys/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha-sys"
-version = "0.22.0"
+version = "1.0.7"
 description = "Solana chacha-sys"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,4 +9,4 @@ license = "Apache-2.0"
 edition = "2018"
 
 [build-dependencies]
-cc = "1.0.48"
+cc = "1.0.49"

chacha/.gitignore

@@ -0,0 +1 @@
+/farf/

chacha/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "solana-chacha"
+version = "1.0.7"
+description = "Solana Chacha APIs"
+authors = ["Solana Maintainers <maintainers@solana.com>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+log = "0.4.8"
+rand = "0.6.5"
+rand_chacha = "0.1.1"
+solana-chacha-sys = { path = "../chacha-sys", version = "1.0.7" }
+solana-ledger = { path = "../ledger", version = "1.0.7" }
+solana-logger = { path = "../logger", version = "1.0.7" }
+solana-perf = { path = "../perf", version = "1.0.7" }
+solana-sdk = { path = "../sdk", version = "1.0.7" }
+
+[dev-dependencies]
+hex-literal = "0.2.1"
+
+[lib]
+name = "solana_chacha"

chacha/src/chacha.rs

@@ -1,4 +1,4 @@
-use solana_ledger::blocktree::Blocktree;
+use solana_ledger::blockstore::Blockstore;
 use solana_sdk::clock::Slot;
 use std::fs::File;
 use std::io;
@@ -12,7 +12,7 @@ pub const CHACHA_BLOCK_SIZE: usize = 64;
 pub const CHACHA_KEY_SIZE: usize = 32;
 
 pub fn chacha_cbc_encrypt_ledger(
-    blocktree: &Arc<Blocktree>,
+    blockstore: &Arc<Blockstore>,
     start_slot: Slot,
     slots_per_segment: u64,
     out_path: &Path,
@@ -28,7 +28,7 @@ pub fn chacha_cbc_encrypt_ledger(
     let mut current_slot = start_slot;
     let mut start_index = 0;
     loop {
-        match blocktree.get_data_shreds(current_slot, start_index, std::u64::MAX, &mut buffer) {
+        match blockstore.get_data_shreds(current_slot, start_index, std::u64::MAX, &mut buffer) {
             Ok((last_index, mut size)) => {
                 debug!(
                     "chacha: encrypting slice: {} num_shreds: {} data_len: {}",
@@ -74,13 +74,14 @@ pub fn chacha_cbc_encrypt_ledger(
 #[cfg(test)]
 mod tests {
     use crate::chacha::chacha_cbc_encrypt_ledger;
-    use crate::gen_keys::GenKeys;
-    use solana_ledger::blocktree::Blocktree;
+    use rand::SeedableRng;
+    use rand_chacha::ChaChaRng;
+    use solana_ledger::blockstore::Blockstore;
     use solana_ledger::entry::Entry;
     use solana_ledger::get_tmp_ledger_path;
     use solana_sdk::hash::{hash, Hash, Hasher};
     use solana_sdk::pubkey::Pubkey;
-    use solana_sdk::signature::KeypairUtil;
+    use solana_sdk::signature::{Keypair, Signer};
     use solana_sdk::system_transaction;
     use std::fs::remove_file;
     use std::fs::File;
@@ -92,8 +93,9 @@ mod tests {
         let one = hash(&zero.as_ref());
 
         let seed = [2u8; 32];
-        let mut rnd = GenKeys::new(seed);
-        let keypair = rnd.gen_keypair();
+
+        let mut generator = ChaChaRng::from_seed(seed);
+        let keypair = Keypair::generate(&mut generator);
 
         let mut id = one;
         let mut num_hashes = 0;
@@ -131,15 +133,16 @@ mod tests {
         let ledger_path = get_tmp_ledger_path!();
         let ticks_per_slot = 16;
         let slots_per_segment = 32;
-        let blocktree = Arc::new(Blocktree::open(&ledger_path).unwrap());
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let out_path = tmp_file_path("test_encrypt_ledger");
 
         let seed = [2u8; 32];
-        let mut rnd = GenKeys::new(seed);
-        let keypair = rnd.gen_keypair();
+
+        let mut generator = ChaChaRng::from_seed(seed);
+        let keypair = Keypair::generate(&mut generator);
 
         let entries = make_tiny_deterministic_test_entries(slots_per_segment);
-        blocktree
+        blockstore
             .write_entries(
                 0,
                 0,
@@ -157,8 +160,14 @@ mod tests {
             "abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234
                             abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234"
         );
-        chacha_cbc_encrypt_ledger(&blocktree, 0, slots_per_segment as u64, &out_path, &mut key)
-            .unwrap();
+        chacha_cbc_encrypt_ledger(
+            &blockstore,
+            0,
+            slots_per_segment as u64,
+            &out_path,
+            &mut key,
+        )
+        .unwrap();
         let mut out_file = File::open(&out_path).unwrap();
         let mut buf = vec![];
         let size = out_file.read_to_end(&mut buf).unwrap();

chacha/src/lib.rs

@@ -0,0 +1,8 @@
+#[macro_use]
+extern crate log;
+
+#[cfg(test)]
+#[macro_use]
+extern crate hex_literal;
+
+pub mod chacha;

ci/affects-files.sh

@@ -9,7 +9,7 @@
 #     ./affects-files.sh ^snap/  -- anything under the snap/ subdirectory
 #     ./affects-files.sh snap/   -- also matches foo/snap/
 # Any pattern starting with the ! character will be negated:
-#     ./affects-files.sh !^book/  -- anything *not* under the book/ subdirectory
+#     ./affects-files.sh !^docs/  -- anything *not* under the docs/ subdirectory
 #
 set -e
 cd "$(dirname "$0")"/..

ci/buildkite-secondary.yml

@@ -6,7 +6,7 @@ steps:
     timeout_in_minutes: 60
     name: "publish docker"
   - command: "ci/publish-crate.sh"
-    timeout_in_minutes: 120
+    timeout_in_minutes: 240
     name: "publish crate"
     branches: "!master"
   - command: "ci/publish-bpf-sdk.sh"
@@ -15,6 +15,6 @@ steps:
   - command: "ci/publish-tarball.sh"
     timeout_in_minutes: 60
     name: "publish tarball"
-  - command: "ci/publish-book.sh"
+  - command: "ci/publish-docs.sh"
     timeout_in_minutes: 15
-    name: "publish book"
+    name: "publish docs"

ci/buildkite.yml

@@ -20,8 +20,10 @@ steps:
     timeout_in_minutes: 30
   - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh"
     name: "stable"
-    timeout_in_minutes: 40
+    timeout_in_minutes: 60
     artifact_paths: "log-*.txt"
+    agents:
+      - "queue=rpc-test-capable"
   - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
     name: "move"
     timeout_in_minutes: 20
@@ -32,6 +34,8 @@ steps:
   - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh"
     name: "coverage"
     timeout_in_minutes: 30
+    agents:
+      - "queue=rpc-test-capable"
   - wait
   - trigger: "solana-secondary"
     branches: "!pull/*"

ci/docker-run.sh

@@ -67,15 +67,22 @@ ARGS+=(
   --env BUILDKITE_JOB_ID
   --env CI
   --env CI_BRANCH
+  --env CI_TAG
   --env CI_BUILD_ID
   --env CI_COMMIT
   --env CI_JOB_ID
   --env CI_PULL_REQUEST
   --env CI_REPO_SLUG
-  --env CODECOV_TOKEN
   --env CRATES_IO_TOKEN
 )
 
+# Also propagate environment variables needed for codecov
+# https://docs.codecov.io/docs/testing-with-docker#section-codecov-inside-docker
+# We normalize CI to `1`; but codecov expects it to be `true` to detect Buildkite...
+# Unfortunately, codecov.io fails sometimes:
+#   curl: (7) Failed to connect to codecov.io port 443: Connection timed out
+CODECOV_ENVS=$(CI=true bash <(while ! curl -sS --retry 5 --retry-delay 2 --retry-connrefused https://codecov.io/env; do sleep 10; done))
+
 if $INTERACTIVE; then
   if [[ -n $1 ]]; then
     echo
@@ -83,8 +90,10 @@ if $INTERACTIVE; then
     echo
   fi
   set -x
-  exec docker run --interactive --tty "${ARGS[@]}" "$IMAGE" bash
+  # shellcheck disable=SC2086
+  exec docker run --interactive --tty "${ARGS[@]}" $CODECOV_ENVS "$IMAGE" bash
 fi
 
 set -x
-exec docker run "${ARGS[@]}" "$IMAGE" "$@"
+# shellcheck disable=SC2086
+exec docker run "${ARGS[@]}" $CODECOV_ENVS "$IMAGE" "$@"

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.40.0
+FROM solanalabs/rust:1.42.0
 ARG date
 
 RUN set -x \

ci/docker-rust-nightly/README.md

@@ -15,6 +15,8 @@ To update the pinned version:
 1. Run `ci/docker-rust-nightly/build.sh` to rebuild the nightly image locally,
    or potentially `ci/docker-rust-nightly/build.sh YYYY-MM-DD` if there's a
    specific YYYY-MM-DD that is desired (default is today's build).
+   Check https://rust-lang.github.io/rustup-components-history/ for build
+   status
 1. Update `ci/rust-version.sh` to reflect the new nightly `YYY-MM-DD`
 1. Run `SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-coverage.sh`
    to confirm the new nightly image builds.  Fix any issues as needed

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.40.0
+FROM rust:1.42.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0
@@ -17,6 +17,7 @@ RUN set -x \
       clang-7 \
       cmake \
       lcov \
+      libudev-dev \
       libclang-common-7-dev \
       mscgen \
       net-tools \

ci/localnet-sanity.sh

@@ -29,7 +29,7 @@ Start a local cluster and run sanity on it
    -x          - Add an extra validator (may be supplied multiple times)
    -r          - Select the RPC endpoint hosted by a node that starts as
                  a validator node.  If unspecified the RPC endpoint hosted by
-                 the bootstrap leader will be used.
+                 the bootstrap validator will be used.
    -c          - Reuse existing node/ledger configuration from a previous sanity
                  run
 
@@ -74,7 +74,7 @@ source multinode-demo/common.sh
 
 nodes=(
   "multinode-demo/faucet.sh"
-  "multinode-demo/bootstrap-leader.sh \
+  "multinode-demo/bootstrap-validator.sh \
     --no-restart \
     --init-complete-file init-complete-node1.log \
     --dynamic-port-range 8000-8050"
@@ -170,7 +170,7 @@ startNodes() {
       logs+=("$(getNodeLogFile "$i" "$cmd")")
     fi
 
-    # 1 == bootstrap leader, wait until it boots before starting
+    # 1 == bootstrap validator, wait until it boots before starting
     # other validators
     if [[ "$i" -eq 1 ]]; then
       SECONDS=
@@ -178,8 +178,8 @@ startNodes() {
 
       (
         set -x
-        $solana_cli --keypair config/bootstrap-leader/identity-keypair.json \
-          --url http://127.0.0.1:8899 get-genesis-hash
+        $solana_cli --keypair config/bootstrap-validator/identity.json \
+          --url http://127.0.0.1:8899 genesis-hash
       ) | tee genesis-hash.log
       maybeExpectedGenesisHash="--expected-genesis-hash $(tail -n1 genesis-hash.log)"
     fi
@@ -277,7 +277,7 @@ rollingNodeRestart() {
 }
 
 verifyLedger() {
-  for ledger in bootstrap-leader validator; do
+  for ledger in bootstrap-validator validator; do
     echo "--- $ledger ledger verification"
     (
       set -x
@@ -331,7 +331,7 @@ while [[ $iteration -le $iterations ]]; do
     rm -rf $client_keypair
   ) || flag_error
 
-  echo "--- RPC API: bootstrap-leader getTransactionCount ($iteration)"
+  echo "--- RPC API: bootstrap-validator getTransactionCount ($iteration)"
   (
     set -x
     curl --retry 5 --retry-delay 2 --retry-connrefused \
@@ -351,7 +351,7 @@ while [[ $iteration -le $iterations ]]; do
       http://localhost:18899
   ) || flag_error
 
-  # Verify transaction count as reported by the bootstrap-leader node is advancing
+  # Verify transaction count as reported by the bootstrap-validator node is advancing
   transactionCount=$(sed -e 's/{"jsonrpc":"2.0","result":\([0-9]*\),"id":1}/\1/' log-transactionCount.txt)
   if [[ -n $lastTransactionCount ]]; then
     echo "--- Transaction count check: $lastTransactionCount < $transactionCount"

ci/nits.sh

@@ -18,19 +18,22 @@ declare prints=(
 
 # Parts of the tree that are expected to be print free
 declare print_free_tree=(
-  'core/src'
-  'faucet/src'
-  'metrics/src'
-  'net-utils/src'
-  'runtime/src'
-  'sdk/bpf/rust/rust-utils'
-  'sdk/src'
-  'programs/bpf/rust'
-  'programs/stake/src'
-  'programs/vote/src'
+  ':core/src/**.rs'
+  ':faucet/src/**.rs'
+  ':ledger/src/**.rs'
+  ':metrics/src/**.rs'
+  ':net-utils/src/**.rs'
+  ':runtime/src/**.rs'
+  ':sdk/bpf/rust/rust-utils/**.rs'
+  ':sdk/**.rs'
+  ':programs/**.rs'
+  ':^**bin**.rs'
+  ':^**bench**.rs'
+  ':^**test**.rs'
+  ':^**/build.rs'
 )
 
-if _ git --no-pager grep -n --max-depth=0 "${prints[@]/#/-e }" -- "${print_free_tree[@]}"; then
+if _ git --no-pager grep -n "${prints[@]/#/-e}" -- "${print_free_tree[@]}"; then
     exit 1
 fi
 

ci/publish-book.sh

@@ -1,99 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-cd "$(dirname "$0")/.."
-
-me=$(basename "$0")
-
-BOOK="book"
-
-source ci/rust-version.sh stable
-eval "$(ci/channel-info.sh)"
-
-if [[ -n $PUBLISH_BOOK_TAG ]]; then
-  CURRENT_TAG="$(git describe --tags)"
-  COMMIT_TO_PUBLISH="$(git rev-list -n 1 "${PUBLISH_BOOK_TAG}")"
-
-  # book is manually published at a specified release tag
-  if [[ $PUBLISH_BOOK_TAG != "$CURRENT_TAG" ]]; then
-    (
-      cat <<EOF
-steps:
-  - trigger: "$BUILDKITE_PIPELINE_SLUG"
-    async: true
-    build:
-      message: "$BUILDKITE_MESSAGE"
-      commit: "$COMMIT_TO_PUBLISH"
-      env:
-        PUBLISH_BOOK_TAG: "$PUBLISH_BOOK_TAG"
-EOF
-    ) | buildkite-agent pipeline upload
-    exit 0
-  fi
-  repo=git@github.com:solana-labs/book.git
-else
-  # book-edge and book-beta are published automatically on the tip of the branch
-  case $CHANNEL in
-  edge)
-    repo=git@github.com:solana-labs/book-edge.git
-    ;;
-  beta)
-    repo=git@github.com:solana-labs/book-beta.git
-    ;;
-  *)
-   echo "--- publish skipped"
-   exit 0
-   ;;
-  esac
-  BOOK=$CHANNEL
-fi
-
-ci/docker-run.sh "$rust_stable_docker_image" bash -exc "book/build.sh"
-
-echo --- create book repo
-(
-  set -x
-  cd book/html/
-  git init .
-  git add ./* ./.nojekyll
-  git config user.email maintainers@solana.com
-  git config user.name "$me"
-  git commit -m "${CI_COMMIT:-local}"
-)
-
-echo "--- publish $BOOK"
-(
-  cd book/html/
-  git remote add origin $repo
-  git fetch origin master
-  if ! git diff HEAD origin/master --quiet; then
-    git push -f origin HEAD:master
-  else
-    echo "Content unchanged, publish skipped"
-  fi
-)
-
-echo --- update gitbook-cage
-(
-  if [[ -z $CI_BRANCH ]]; then
-    exit 0
-  fi
-
-  set -x
-  (
-    . ci/rust-version.sh
-    ci/docker-run.sh $rust_stable_docker_image make -Cbook -B svg
-  )
-  # make a local commit for the svgs
-  git add -A -f book/src/.gitbook/assets/.
-  if ! git diff-index --quiet HEAD; then
-    git config user.email maintainers@solana.com
-    git config user.name "$me"
-    git commit -m "gitbook-cage update $(date -Is)"
-    git push -f git@github.com:solana-labs/solana-gitbook-cage.git HEAD:refs/heads/"$CI_BRANCH"
-    # pop off the local commit
-    git reset --hard HEAD~
-  fi
-)
-
-exit 0

ci/publish-bpf-sdk.sh

@@ -4,7 +4,12 @@ set -e
 cd "$(dirname "$0")/.."
 eval "$(ci/channel-info.sh)"
 
-echo --- Creating tarball
+if [[ -n "$CI_TAG" ]]; then
+  CHANNEL_OR_TAG=$CI_TAG
+else
+  CHANNEL_OR_TAG=$CHANNEL
+fi
+
 (
   set -x
   sdk/bpf/scripts/package.sh
@@ -12,7 +17,7 @@ echo --- Creating tarball
 )
 
 echo --- AWS S3 Store
-if [[ -z $CHANNEL ]]; then
+if [[ -z $CHANNEL_OR_TAG ]]; then
   echo Skipped
 else
   (
@@ -24,7 +29,7 @@ else
       --volume "$PWD:/solana" \
       eremite/aws-cli:2018.12.18 \
       /usr/bin/s3cmd --acl-public put /solana/bpf-sdk.tar.bz2 \
-      s3://solana-sdk/"$CHANNEL"/bpf-sdk.tar.bz2
+      s3://solana-sdk/"$CHANNEL_OR_TAG"/bpf-sdk.tar.bz2
   )
 fi
 

ci/publish-docs.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$(dirname "$0")/.."
+
+me=$(basename "$0")
+
+echo --- update gitbook-cage
+if [[ -n $CI_BRANCH ]]; then
+  (
+    set -x
+    (
+      . ci/rust-version.sh stable
+      ci/docker-run.sh "$rust_stable_docker_image" make -C docs
+    )
+    # make a local commit for the svgs and generated/updated markdown
+    git add -f docs/src
+    if ! git diff-index --quiet HEAD; then
+      git config user.email maintainers@solana.com
+      git config user.name "$me"
+      git commit -m "gitbook-cage update $(date -Is)"
+      git push -f git@github.com:solana-labs/solana-gitbook-cage.git HEAD:refs/heads/"$CI_BRANCH"
+      # pop off the local commit
+      git reset --hard HEAD~
+    fi
+  )
+else
+  echo CI_BRANCH not set
+fi
+
+exit 0

ci/publish-tarball.sh

@@ -53,7 +53,7 @@ windows)
   ;;
 esac
 
-echo --- Creating tarball
+echo --- Creating release tarball
 (
   set -x
   rm -rf solana-release/
@@ -89,15 +89,21 @@ echo --- Creating tarball
 )
 
 # Metrics tarball is platform agnostic, only publish it from Linux
-MAYBE_METRICS_TARBALL=
+MAYBE_TARBALLS=
 if [[ "$CI_OS_NAME" = linux ]]; then
   metrics/create-metrics-tarball.sh
-  MAYBE_METRICS_TARBALL=solana-metrics.tar.bz2
+  (
+    set -x
+    sdk/bpf/scripts/package.sh
+    [[ -f bpf-sdk.tar.bz2 ]]
+
+  )
+  MAYBE_TARBALLS="bpf-sdk.tar.bz2 solana-metrics.tar.bz2"
 fi
 
 source ci/upload-ci-artifact.sh
 
-for file in solana-release-$TARGET.tar.bz2 solana-release-$TARGET.yml solana-install-init-"$TARGET"* $MAYBE_METRICS_TARBALL; do
+for file in solana-release-$TARGET.tar.bz2 solana-release-$TARGET.yml solana-install-init-"$TARGET"* $MAYBE_TARBALLS; do
   upload-ci-artifact "$file"
 
   if [[ -n $DO_NOT_PUBLISH_TAR ]]; then

ci/rust-version.sh

@@ -16,13 +16,13 @@
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.40.0
+  stable_version=1.42.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2019-12-19
+  nightly_version=2020-03-12
 fi
 
 

ci/test-checks.sh

@@ -10,6 +10,9 @@ source ci/rust-version.sh nightly
 export RUST_BACKTRACE=1
 export RUSTFLAGS="-D warnings"
 
+# Look for failed mergify.io backports
+_ git show HEAD --check --oneline
+
 _ cargo +"$rust_stable" fmt --all -- --check
 
 # Clippy gets stuck for unknown reasons if sdk-c is included in the build, so check it separately.
@@ -19,15 +22,15 @@ _ cargo +"$rust_stable" clippy --all --exclude solana-sdk-c -- --deny=warnings
 _ cargo +"$rust_stable" clippy --manifest-path sdk-c/Cargo.toml -- --deny=warnings
 
 _ cargo +"$rust_stable" audit --version
-_ cargo +"$rust_stable" audit --ignore RUSTSEC-2019-0013 --ignore RUSTSEC-2018-0015 --ignore RUSTSEC-2019-0031
+_ cargo +"$rust_stable" audit --ignore RUSTSEC-2020-0002
 _ ci/nits.sh
 _ ci/order-crates-for-publishing.py
-_ book/build.sh
+_ docs/build.sh
 _ ci/check-ssh-keys.sh
 
 {
   cd programs/bpf
-  _ cargo +"$rust_stable" audit --ignore RUSTSEC-2019-0031
+  _ cargo +"$rust_stable" audit
   for project in rust/*/ ; do
     echo "+++ do_bpf_checks $project"
     (

ci/test-coverage.sh

@@ -41,7 +41,10 @@ if [[ -z "$CODECOV_TOKEN" ]]; then
   echo "^^^ +++"
   echo CODECOV_TOKEN undefined, codecov.io upload skipped
 else
-  bash <(curl -s https://codecov.io/bash) -X gcov -f target/cov/lcov.info
+  # We normalize CI to `1`; but codecov expects it to be `true` to detect Buildkite...
+  # Unfortunately, codecov.io fails sometimes:
+  #   curl: (7) Failed to connect to codecov.io port 443: Connection timed out
+  CI=true bash <(while ! curl -sS --retry 5 --retry-delay 2 --retry-connrefused https://codecov.io/bash; do sleep 10; done) -Z -X gcov -f target/cov/lcov.info
 
   annotate --style success --context codecov.io \
     "CodeCov report: https://codecov.io/github/solana-labs/solana/commit/${CI_COMMIT:0:9}"

ci/test-stable.sh

@@ -13,12 +13,12 @@ annotate() {
 # Run the appropriate test based on entrypoint
 testName=$(basename "$0" .sh)
 
-# Skip if only the book has been modified
+# Skip if only the docs have been modified
 ci/affects-files.sh \
-  \!^book/ \
+  \!^docs/ \
 || {
   annotate --style info \
-    "Skipped $testName as only book files were modified"
+    "Skipped $testName as only docs/ files were modified"
   exit 0
 }
 
@@ -86,7 +86,7 @@ test-stable-perf)
   fi
 
   _ cargo +"$rust_stable" build --bins ${V:+--verbose}
-  _ cargo +"$rust_stable" test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
+  _ cargo +"$rust_stable" test --package solana-chacha-cuda --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
   ;;
 test-move)
   ci/affects-files.sh \
@@ -111,7 +111,7 @@ test-move)
   ;;
 test-local-cluster)
   _ cargo +"$rust_stable" build --release --bins ${V:+--verbose}
-  _ cargo +"$rust_stable" test --release --package solana-local-cluster ${V:+--verbose} -- --nocapture
+  _ cargo +"$rust_stable" test --release --package solana-local-cluster ${V:+--verbose} -- --nocapture --test-threads=1
   exit 0
   ;;
 *)

ci/testnet-manager.sh

@@ -32,20 +32,10 @@ steps:
         options:
           - label: "testnet"
             value: "testnet"
-          - label: "testnet-perf"
-            value: "testnet-perf"
           - label: "testnet-edge"
             value: "testnet-edge"
-          - label: "testnet-edge-perf"
-            value: "testnet-edge-perf"
           - label: "testnet-beta"
             value: "testnet-beta"
-          - label: "testnet-beta-perf"
-            value: "testnet-beta-perf"
-          - label: "testnet-demo"
-            value: "testnet-demo"
-          - label: "tds"
-            value: "tds"
       - select: "Operation"
         key: "testnet-operation"
         default: "sanity-or-restart"
@@ -131,32 +121,18 @@ GCE_LOW_QUOTA_ZONES=(
 )
 
 case $TESTNET in
-testnet-edge|testnet-edge-perf)
+testnet-edge)
   CHANNEL_OR_TAG=edge
   CHANNEL_BRANCH=$EDGE_CHANNEL
   ;;
-testnet-beta|testnet-beta-perf)
+testnet-beta)
   CHANNEL_OR_TAG=beta
   CHANNEL_BRANCH=$BETA_CHANNEL
   ;;
 testnet)
   CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
   CHANNEL_BRANCH=$STABLE_CHANNEL
-  ;;
-testnet-perf)
-  CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
-  CHANNEL_BRANCH=$STABLE_CHANNEL
-  ;;
-testnet-demo)
-  CHANNEL_OR_TAG=beta
-  CHANNEL_BRANCH=$BETA_CHANNEL
-  : "${GCE_NODE_COUNT:=150}"
-  : "${GCE_LOW_QUOTA_NODE_COUNT:=70}"
-  ;;
-tds)
-  : "${TDS_CHANNEL_OR_TAG:=edge}"
-  CHANNEL_OR_TAG="$TDS_CHANNEL_OR_TAG"
-  CHANNEL_BRANCH="$CI_BRANCH"
+  export CLOUDSDK_CORE_PROJECT=testnet-solana-com
   ;;
 *)
   echo "Error: Invalid TESTNET=$TESTNET"
@@ -204,6 +180,7 @@ steps:
         TESTNET_DB_HOST: "$TESTNET_DB_HOST"
         GCE_NODE_COUNT: "$GCE_NODE_COUNT"
         GCE_LOW_QUOTA_NODE_COUNT: "$GCE_LOW_QUOTA_NODE_COUNT"
+        RUST_LOG: "$RUST_LOG"
 EOF
     ) | buildkite-agent pipeline upload
     exit 0
@@ -231,64 +208,22 @@ sanity() {
     (
       set -x
       NO_INSTALL_CHECK=1 \
-        ci/testnet-sanity.sh edge-testnet-solana-com gce -P us-west1-b
+        ci/testnet-sanity.sh edge-devnet-solana-com gce -P us-west1-b
       maybe_deploy_software
     )
     ;;
-  testnet-edge-perf)
-    (
-      set -x
-      REJECT_EXTRA_NODES=1 \
-        ci/testnet-sanity.sh edge-perf-testnet-solana-com ec2 us-west-2b
-    )
-    ;;
   testnet-beta)
     (
       set -x
       NO_INSTALL_CHECK=1 \
-        ci/testnet-sanity.sh beta-testnet-solana-com gce -P us-west1-b
+        ci/testnet-sanity.sh beta-devnet-solana-com gce -P us-west1-b
       maybe_deploy_software --deploy-if-newer
     )
     ;;
-  testnet-beta-perf)
-    (
-      set -x
-      REJECT_EXTRA_NODES=1 \
-        ci/testnet-sanity.sh beta-perf-testnet-solana-com ec2 us-west-2b
-    )
-    ;;
   testnet)
     (
       set -x
-      ci/testnet-sanity.sh testnet-solana-com gce -P us-west1-b
-    )
-    ;;
-  testnet-perf)
-    (
-      set -x
-      REJECT_EXTRA_NODES=1 \
-        ci/testnet-sanity.sh perf-testnet-solana-com gce us-west1-b
-      #ci/testnet-sanity.sh perf-testnet-solana-com ec2 us-east-1a
-    )
-    ;;
-  testnet-demo)
-    (
-      set -x
-
-      ok=true
-      if [[ -n $GCE_NODE_COUNT ]]; then
-        ci/testnet-sanity.sh demo-testnet-solana-com gce "${GCE_ZONES[0]}" -f || ok=false
-      else
-        echo "Error: no GCE nodes"
-        ok=false
-      fi
-      $ok
-    )
-    ;;
-  tds)
-    (
-      set -x
-      ci/testnet-sanity.sh tds-solana-com gce "${GCE_ZONES[0]}" -f
+      ci/testnet-sanity.sh devnet-solana-com gce -P us-west1-b
     )
     ;;
   *)
@@ -324,9 +259,9 @@ deploy() {
   testnet-edge)
     (
       set -x
-      ci/testnet-deploy.sh -p edge-testnet-solana-com -C gce -z us-west1-b \
+      ci/testnet-deploy.sh -p edge-devnet-solana-com -C gce -z us-west1-b \
         -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P \
-        -a edge-testnet-solana-com --letsencrypt edge.testnet.solana.com \
+        -a edge-devnet-solana-com --letsencrypt edge.devnet.solana.com \
         --limit-ledger-size \
         ${skipCreate:+-e} \
         ${skipStart:+-s} \
@@ -334,24 +269,12 @@ deploy() {
         ${maybeDelete:+-D}
     )
     ;;
-  testnet-edge-perf)
-    (
-      set -x
-      RUST_LOG=solana=warn \
-        ci/testnet-deploy.sh -p edge-perf-testnet-solana-com -C ec2 -z us-west-2b \
-          -g -t "$CHANNEL_OR_TAG" -c 2 \
-          ${skipCreate:+-e} \
-          ${skipStart:+-s} \
-          ${maybeStop:+-S} \
-          ${maybeDelete:+-D}
-    )
-    ;;
   testnet-beta)
     (
       set -x
-      ci/testnet-deploy.sh -p beta-testnet-solana-com -C gce -z us-west1-b \
+      ci/testnet-deploy.sh -p beta-devnet-solana-com -C gce -z us-west1-b \
         -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P \
-        -a beta-testnet-solana-com --letsencrypt beta.testnet.solana.com \
+        -a beta-devnet-solana-com --letsencrypt beta.devnet.solana.com \
         --limit-ledger-size \
         ${skipCreate:+-e} \
         ${skipStart:+-s} \
@@ -359,24 +282,12 @@ deploy() {
         ${maybeDelete:+-D}
     )
     ;;
-  testnet-beta-perf)
-    (
-      set -x
-      RUST_LOG=solana=warn \
-        ci/testnet-deploy.sh -p beta-perf-testnet-solana-com -C ec2 -z us-west-2b \
-          -g -t "$CHANNEL_OR_TAG" -c 2 \
-          ${skipCreate:+-e} \
-          ${skipStart:+-s} \
-          ${maybeStop:+-S} \
-          ${maybeDelete:+-D}
-    )
-    ;;
   testnet)
     (
       set -x
-      ci/testnet-deploy.sh -p testnet-solana-com -C gce -z us-west1-b \
-        -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P \
-        -a testnet-solana-com --letsencrypt testnet.solana.com \
+      ci/testnet-deploy.sh -p devnet-solana-com -C gce -z us-west1-b \
+        -t "$CHANNEL_OR_TAG" -n 0 -c 0 -u -P \
+        -a testnet-solana-com --letsencrypt devnet.solana.com \
         --limit-ledger-size \
         ${skipCreate:+-e} \
         ${skipStart:+-s} \
@@ -386,132 +297,7 @@ deploy() {
     (
       echo "--- net.sh update"
       set -x
-      time net/net.sh update -t "$CHANNEL_OR_TAG" --platform linux --platform osx --platform windows
-    )
-    ;;
-  testnet-perf)
-    (
-      set -x
-      RUST_LOG=solana=warn \
-        ci/testnet-deploy.sh -p perf-testnet-solana-com -C gce -z us-west1-b \
-          -G "--machine-type n1-standard-16 --accelerator count=2,type=nvidia-tesla-v100" \
-          -t "$CHANNEL_OR_TAG" -c 2 \
-          -d pd-ssd \
-          ${skipCreate:+-e} \
-          ${skipStart:+-s} \
-          ${maybeStop:+-S} \
-          ${maybeDelete:+-D}
-    )
-    ;;
-  testnet-demo)
-    (
-      set -x
-
-      if [[ -n $GCE_LOW_QUOTA_NODE_COUNT ]] || [[ -n $skipStart ]]; then
-        maybeSkipStart="skip"
-      fi
-
-      # shellcheck disable=SC2068
-      ci/testnet-deploy.sh -p demo-testnet-solana-com -C gce ${GCE_ZONE_ARGS[@]} \
-        -t "$CHANNEL_OR_TAG" -n "$GCE_NODE_COUNT" -c 0 -P -u --allow-boot-failures \
-        --skip-remote-log-retrieval \
-        -a demo-testnet-solana-com \
-        ${skipCreate:+-e} \
-        ${maybeSkipStart:+-s} \
-        ${maybeStop:+-S} \
-        ${maybeDelete:+-D}
-
-      if [[ -n $GCE_LOW_QUOTA_NODE_COUNT ]]; then
-        # shellcheck disable=SC2068
-        ci/testnet-deploy.sh -p demo-testnet-solana-com2 -C gce ${GCE_LOW_QUOTA_ZONE_ARGS[@]} \
-          -t "$CHANNEL_OR_TAG" -n "$GCE_LOW_QUOTA_NODE_COUNT" -c 0 -P --allow-boot-failures -x \
-          --skip-remote-log-retrieval \
-          ${skipCreate:+-e} \
-          ${skipStart:+-s} \
-          ${maybeStop:+-S} \
-          ${maybeDelete:+-D}
-      fi
-    )
-    ;;
-  tds)
-    (
-      set -x
-
-      # Allow cluster configuration to be overridden from env vars
-
-      if [[ -z $TDS_ZONES ]]; then
-        TDS_ZONES="us-west1-a,us-central1-a,europe-west4-a"
-      fi
-      GCE_CLOUD_ZONES=(); while read -r -d, ; do GCE_CLOUD_ZONES+=( "$REPLY" ); done <<< "${TDS_ZONES},"
-
-      if [[ -z $TDS_NODE_COUNT ]]; then
-        TDS_NODE_COUNT="3"
-      fi
-
-      if [[ -z $TDS_CLIENT_COUNT ]]; then
-        TDS_CLIENT_COUNT="1"
-      fi
-
-      if [[ -z $ENABLE_GPU ]]; then
-        maybeGpu=(-G "--machine-type n1-standard-16 --accelerator count=2,type=nvidia-tesla-v100")
-      elif [[ $ENABLE_GPU == skip ]]; then
-        maybeGpu=()
-      else
-        maybeGpu=(-G "${ENABLE_GPU}")
-      fi
-
-      if [[ -z $DISABLE_AIRDROPS ]]; then
-        DISABLE_AIRDROPS="true"
-      fi
-
-      if [[ $DISABLE_AIRDROPS == true ]] ; then
-        maybeDisableAirdrops="--no-airdrop"
-      else
-        maybeDisableAirdrops=""
-      fi
-
-      EXTERNAL_ACCOUNTS_FILE=/tmp/validator.yml
-      if [[ -z $EXTERNAL_ACCOUNTS_FILE_URL ]]; then
-        EXTERNAL_ACCOUNTS_FILE_URL=https://raw.githubusercontent.com/solana-labs/tour-de-sol/master/validators/all.yml
-        wget ${EXTERNAL_ACCOUNTS_FILE_URL} -O ${EXTERNAL_ACCOUNTS_FILE}
-        maybeExternalAccountsFile="--external-accounts-file ${EXTERNAL_ACCOUNTS_FILE}"
-      elif [[ $EXTERNAL_ACCOUNTS_FILE_URL == skip ]]; then
-        maybeExternalAccountsFile=""
-      else
-        wget ${EXTERNAL_ACCOUNTS_FILE_URL} -O ${EXTERNAL_ACCOUNTS_FILE}
-        maybeExternalAccountsFile="--external-accounts-file ${EXTERNAL_ACCOUNTS_FILE}"
-      fi
-
-      if [[ -z $ADDITIONAL_DISK_SIZE_GB ]]; then
-        maybeAdditionalDisk="--validator-additional-disk-size-gb 32000"
-      elif [[ $ADDITIONAL_DISK_SIZE_GB == skip ]]; then
-        maybeAdditionalDisk=""
-      else
-        maybeAdditionalDisk="--validator-additional-disk-size-gb ${ADDITIONAL_DISK_SIZE_GB}"
-      fi
-
-      # Multiple V100 GPUs are available in us-west1, us-central1 and europe-west4
-      # shellcheck disable=SC2068
-      # shellcheck disable=SC2086
-      ci/testnet-deploy.sh -p tds-solana-com -C gce \
-        "${maybeGpu[@]}" \
-        -d pd-ssd \
-        ${GCE_CLOUD_ZONES[@]/#/-z } \
-        -t "$CHANNEL_OR_TAG" \
-        -n ${TDS_NODE_COUNT} \
-        -c ${TDS_CLIENT_COUNT} \
-        --idle-clients \
-        -P -u \
-        -a tds-solana-com --letsencrypt tds.solana.com \
-        ${skipCreate:+-e} \
-        ${skipStart:+-s} \
-        ${maybeStop:+-S} \
-        ${maybeDelete:+-D} \
-        ${maybeDisableAirdrops} \
-        ${maybeExternalAccountsFile} \
-        --target-lamports-per-signature 0 \
-        --slots-per-epoch 4096 \
-        ${maybeAdditionalDisk}
+      time net/net.sh update -t "$CHANNEL_OR_TAG" --platform linux --platform osx #--platform windows
     )
     ;;
   *)

ci/testnet-sanity.sh

@@ -60,7 +60,7 @@ trap shutdown EXIT INT
 set -x
 for zone in "$@"; do
   echo "--- $cloudProvider config [$zone]"
-  timeout 5m net/"$cloudProvider".sh config $maybePublicNetwork -p "$netName" -z "$zone"
+  timeout 5m net/"$cloudProvider".sh config $maybePublicNetwork -n 1 -p "$netName" -z "$zone"
   net/init-metrics.sh -e
   echo "+++ $cloudProvider.sh info"
   net/"$cloudProvider".sh info

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "0.22.0"
+version = "1.0.7"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,9 +11,10 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-semver = "0.9.0"
-solana-sdk = { path = "../sdk", version = "0.22.0" }
-tiny-bip39 = "0.6.2"
+solana-remote-wallet = { path = "../remote-wallet", version = "1.0.7" }
+solana-sdk = { path = "../sdk", version = "1.0.7" }
+thiserror = "1.0.11"
+tiny-bip39 = "0.7.0"
 url = "2.1.0"
 chrono = "0.4"
 

clap-utils/src/input_parsers.rs

@@ -1,13 +1,17 @@
-use crate::keypair::{keypair_from_seed_phrase, ASK_KEYWORD, SKIP_SEED_PHRASE_VALIDATION_ARG};
+use crate::keypair::{
+    keypair_from_seed_phrase, pubkey_from_path, resolve_signer_from_path, signer_from_path,
+    ASK_KEYWORD, SKIP_SEED_PHRASE_VALIDATION_ARG,
+};
 use chrono::DateTime;
 use clap::ArgMatches;
+use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     clock::UnixTimestamp,
     native_token::sol_to_lamports,
     pubkey::Pubkey,
-    signature::{read_keypair_file, Keypair, KeypairUtil, Signature},
+    signature::{read_keypair_file, Keypair, Signature, Signer},
 };
-use std::str::FromStr;
+use std::{str::FromStr, sync::Arc};
 
 // Return parsed values from matches at `name`
 pub fn values_of<T>(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<T>>
@@ -33,7 +37,10 @@ where
     }
 }
 
-pub fn unix_timestamp_of(matches: &ArgMatches<'_>, name: &str) -> Option<UnixTimestamp> {
+pub fn unix_timestamp_from_rfc3339_datetime(
+    matches: &ArgMatches<'_>,
+    name: &str,
+) -> Option<UnixTimestamp> {
     matches.value_of(name).and_then(|value| {
         DateTime::parse_from_rfc3339(value)
             .ok()
@@ -61,6 +68,20 @@ pub fn pubkey_of(matches: &ArgMatches<'_>, name: &str) -> Option<Pubkey> {
     value_of(matches, name).or_else(|| keypair_of(matches, name).map(|keypair| keypair.pubkey()))
 }
 
+pub fn pubkeys_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<Pubkey>> {
+    matches.values_of(name).map(|values| {
+        values
+            .map(|value| {
+                value.parse::<Pubkey>().unwrap_or_else(|_| {
+                    read_keypair_file(value)
+                        .expect("read_keypair_file failed")
+                        .pubkey()
+                })
+            })
+            .collect()
+    })
+}
+
 // Return pubkey/signature pairs for a string of the form pubkey=signature
 pub fn pubkeys_sigs_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<(Pubkey, Signature)>> {
     matches.values_of(name).map(|values| {
@@ -75,14 +96,56 @@ pub fn pubkeys_sigs_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<(Pubk
     })
 }
 
-pub fn amount_of(matches: &ArgMatches<'_>, name: &str, unit: &str) -> Option<u64> {
-    if matches.value_of(unit) == Some("lamports") {
-        value_of(matches, name)
+// Return a signer from matches at `name`
+#[allow(clippy::type_complexity)]
+pub fn signer_of(
+    matches: &ArgMatches<'_>,
+    name: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<(Option<Box<dyn Signer>>, Option<Pubkey>), Box<dyn std::error::Error>> {
+    if let Some(location) = matches.value_of(name) {
+        let signer = signer_from_path(matches, location, name, wallet_manager)?;
+        let signer_pubkey = signer.pubkey();
+        Ok((Some(signer), Some(signer_pubkey)))
     } else {
-        value_of(matches, name).map(sol_to_lamports)
+        Ok((None, None))
     }
 }
 
+pub fn pubkey_of_signer(
+    matches: &ArgMatches<'_>,
+    name: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<Option<Pubkey>, Box<dyn std::error::Error>> {
+    if let Some(location) = matches.value_of(name) {
+        Ok(Some(pubkey_from_path(
+            matches,
+            location,
+            name,
+            wallet_manager,
+        )?))
+    } else {
+        Ok(None)
+    }
+}
+
+pub fn resolve_signer(
+    matches: &ArgMatches<'_>,
+    name: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<Option<String>, Box<dyn std::error::Error>> {
+    Ok(resolve_signer_from_path(
+        matches,
+        matches.value_of(name).unwrap(),
+        name,
+        wallet_manager,
+    )?)
+}
+
+pub fn lamports_of_sol(matches: &ArgMatches<'_>, name: &str) -> Option<u64> {
+    value_of(matches, name).map(sol_to_lamports)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -151,7 +214,7 @@ mod tests {
     #[test]
     fn test_keypair_of() {
         let keypair = Keypair::new();
-        let outfile = tmp_file_path("test_gen_keypair_file.json", &keypair.pubkey());
+        let outfile = tmp_file_path("test_keypair_of.json", &keypair.pubkey());
         let _ = write_keypair_file(&keypair, &outfile).unwrap();
 
         let matches = app()
@@ -175,7 +238,7 @@ mod tests {
     #[test]
     fn test_pubkey_of() {
         let keypair = Keypair::new();
-        let outfile = tmp_file_path("test_gen_keypair_file.json", &keypair.pubkey());
+        let outfile = tmp_file_path("test_pubkey_of.json", &keypair.pubkey());
         let _ = write_keypair_file(&keypair, &outfile).unwrap();
 
         let matches = app()
@@ -199,6 +262,26 @@ mod tests {
         fs::remove_file(&outfile).unwrap();
     }
 
+    #[test]
+    fn test_pubkeys_of() {
+        let keypair = Keypair::new();
+        let outfile = tmp_file_path("test_pubkeys_of.json", &keypair.pubkey());
+        let _ = write_keypair_file(&keypair, &outfile).unwrap();
+
+        let matches = app().clone().get_matches_from(vec![
+            "test",
+            "--multiple",
+            &keypair.pubkey().to_string(),
+            "--multiple",
+            &outfile,
+        ]);
+        assert_eq!(
+            pubkeys_of(&matches, "multiple"),
+            Some(vec![keypair.pubkey(), keypair.pubkey()])
+        );
+        fs::remove_file(&outfile).unwrap();
+    }
+
     #[test]
     fn test_pubkeys_sigs_of() {
         let key1 = Pubkey::new_rand();
@@ -221,23 +304,20 @@ mod tests {
     }
 
     #[test]
-    fn test_amount_of() {
+    fn test_lamports_of_sol() {
         let matches = app()
             .clone()
-            .get_matches_from(vec!["test", "--single", "50", "--unit", "lamports"]);
-        assert_eq!(amount_of(&matches, "single", "unit"), Some(50));
-        assert_eq!(amount_of(&matches, "multiple", "unit"), None);
+            .get_matches_from(vec!["test", "--single", "50"]);
+        assert_eq!(lamports_of_sol(&matches, "single"), Some(50000000000));
+        assert_eq!(lamports_of_sol(&matches, "multiple"), None);
         let matches = app()
             .clone()
-            .get_matches_from(vec!["test", "--single", "50", "--unit", "SOL"]);
-        assert_eq!(amount_of(&matches, "single", "unit"), Some(50000000000));
+            .get_matches_from(vec!["test", "--single", "1.5"]);
+        assert_eq!(lamports_of_sol(&matches, "single"), Some(1500000000));
+        assert_eq!(lamports_of_sol(&matches, "multiple"), None);
         let matches = app()
             .clone()
-            .get_matches_from(vec!["test", "--single", "1.5", "--unit", "SOL"]);
-        assert_eq!(amount_of(&matches, "single", "unit"), Some(1500000000));
-        let matches = app()
-            .clone()
-            .get_matches_from(vec!["test", "--single", "1.5", "--unit", "lamports"]);
-        assert_eq!(amount_of(&matches, "single", "unit"), None);
+            .get_matches_from(vec!["test", "--single", "0.03"]);
+        assert_eq!(lamports_of_sol(&matches, "single"), Some(30000000));
     }
 }

clap-utils/src/input_validators.rs

@@ -1,14 +1,18 @@
-use crate::keypair::ASK_KEYWORD;
-use solana_sdk::hash::Hash;
-use solana_sdk::pubkey::Pubkey;
-use solana_sdk::signature::{read_keypair_file, Signature};
+use crate::keypair::{parse_keypair_path, KeypairUrl, ASK_KEYWORD};
+use chrono::DateTime;
+use solana_sdk::{
+    clock::Slot,
+    hash::Hash,
+    pubkey::Pubkey,
+    signature::{read_keypair_file, Signature},
+};
 use std::str::FromStr;
 
 // Return an error if a pubkey cannot be parsed.
 pub fn is_pubkey(string: String) -> Result<(), String> {
     match string.parse::<Pubkey>() {
         Ok(_) => Ok(()),
-        Err(err) => Err(format!("{:?}", err)),
+        Err(err) => Err(format!("{}", err)),
     }
 }
 
@@ -16,7 +20,7 @@ pub fn is_pubkey(string: String) -> Result<(), String> {
 pub fn is_hash(string: String) -> Result<(), String> {
     match string.parse::<Hash>() {
         Ok(_) => Ok(()),
-        Err(err) => Err(format!("{:?}", err)),
+        Err(err) => Err(format!("{}", err)),
     }
 }
 
@@ -24,7 +28,7 @@ pub fn is_hash(string: String) -> Result<(), String> {
 pub fn is_keypair(string: String) -> Result<(), String> {
     read_keypair_file(&string)
         .map(|_| ())
-        .map_err(|err| format!("{:?}", err))
+        .map_err(|err| format!("{}", err))
 }
 
 // Return an error if a keypair file cannot be parsed
@@ -34,7 +38,7 @@ pub fn is_keypair_or_ask_keyword(string: String) -> Result<(), String> {
     }
     read_keypair_file(&string)
         .map(|_| ())
-        .map_err(|err| format!("{:?}", err))
+        .map_err(|err| format!("{}", err))
 }
 
 // Return an error if string cannot be parsed as pubkey string or keypair file location
@@ -42,6 +46,18 @@ pub fn is_pubkey_or_keypair(string: String) -> Result<(), String> {
     is_pubkey(string.clone()).or_else(|_| is_keypair(string))
 }
 
+// Return an error if string cannot be parsed as pubkey or keypair file or keypair ask keyword
+pub fn is_pubkey_or_keypair_or_ask_keyword(string: String) -> Result<(), String> {
+    is_pubkey(string.clone()).or_else(|_| is_keypair_or_ask_keyword(string))
+}
+
+pub fn is_valid_signer(string: String) -> Result<(), String> {
+    match parse_keypair_path(&string) {
+        KeypairUrl::Filepath(path) => is_keypair(path),
+        _ => Ok(()),
+    }
+}
+
 // Return an error if string cannot be parsed as pubkey=signature string
 pub fn is_pubkey_sig(string: String) -> Result<(), String> {
     let mut signer = string.split('=');
@@ -57,10 +73,10 @@ pub fn is_pubkey_sig(string: String) -> Result<(), String> {
                     .ok_or_else(|| "Malformed signer string".to_string())?,
             ) {
                 Ok(_) => Ok(()),
-                Err(err) => Err(format!("{:?}", err)),
+                Err(err) => Err(format!("{}", err)),
             }
         }
-        Err(err) => Err(format!("{:?}", err)),
+        Err(err) => Err(format!("{}", err)),
     }
 }
 
@@ -74,28 +90,20 @@ pub fn is_url(string: String) -> Result<(), String> {
                 Err("no host provided".to_string())
             }
         }
-        Err(err) => Err(format!("{:?}", err)),
+        Err(err) => Err(format!("{}", err)),
     }
 }
 
-pub fn is_semver(semver: &str) -> Result<(), String> {
-    match semver::Version::parse(&semver) {
-        Ok(_) => Ok(()),
-        Err(err) => Err(format!("{:?}", err)),
-    }
-}
-
-pub fn is_release_channel(channel: &str) -> Result<(), String> {
-    match channel {
-        "edge" | "beta" | "stable" => Ok(()),
-        _ => Err(format!("Invalid release channel {}", channel)),
-    }
+pub fn is_slot(slot: String) -> Result<(), String> {
+    slot.parse::<Slot>()
+        .map(|_| ())
+        .map_err(|e| format!("{}", e))
 }
 
 pub fn is_port(port: String) -> Result<(), String> {
     port.parse::<u16>()
         .map(|_| ())
-        .map_err(|e| format!("{:?}", e))
+        .map_err(|e| format!("{}", e))
 }
 
 pub fn is_valid_percentage(percentage: String) -> Result<(), String> {
@@ -103,7 +111,7 @@ pub fn is_valid_percentage(percentage: String) -> Result<(), String> {
         .parse::<u8>()
         .map_err(|e| {
             format!(
-                "Unable to parse input percentage, provided: {}, err: {:?}",
+                "Unable to parse input percentage, provided: {}, err: {}",
                 percentage, e
             )
         })
@@ -129,3 +137,54 @@ pub fn is_amount(amount: String) -> Result<(), String> {
         ))
     }
 }
+
+pub fn is_rfc3339_datetime(value: String) -> Result<(), String> {
+    DateTime::parse_from_rfc3339(&value)
+        .map(|_| ())
+        .map_err(|e| format!("{}", e))
+}
+
+pub fn is_derivation(value: String) -> Result<(), String> {
+    let value = value.replace("'", "");
+    let mut parts = value.split('/');
+    let account = parts.next().unwrap();
+    account
+        .parse::<u32>()
+        .map_err(|e| {
+            format!(
+                "Unable to parse derivation, provided: {}, err: {}",
+                account, e
+            )
+        })
+        .and_then(|_| {
+            if let Some(change) = parts.next() {
+                change.parse::<u32>().map_err(|e| {
+                    format!(
+                        "Unable to parse derivation, provided: {}, err: {}",
+                        change, e
+                    )
+                })
+            } else {
+                Ok(0)
+            }
+        })
+        .map(|_| ())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_is_derivation() {
+        assert_eq!(is_derivation("2".to_string()), Ok(()));
+        assert_eq!(is_derivation("0".to_string()), Ok(()));
+        assert_eq!(is_derivation("65537".to_string()), Ok(()));
+        assert_eq!(is_derivation("0/2".to_string()), Ok(()));
+        assert_eq!(is_derivation("0'/2'".to_string()), Ok(()));
+        assert!(is_derivation("a".to_string()).is_err());
+        assert!(is_derivation("4294967296".to_string()).is_err());
+        assert!(is_derivation("a/b".to_string()).is_err());
+        assert!(is_derivation("0/4294967296".to_string()).is_err());
+    }
+}

clap-utils/src/keypair.rs

@@ -1,53 +1,183 @@
-use crate::ArgConstant;
+use crate::{input_parsers::pubkeys_sigs_of, offline::SIGNER_ARG, ArgConstant};
 use bip39::{Language, Mnemonic, Seed};
-use clap::values_t;
+use clap::ArgMatches;
 use rpassword::prompt_password_stderr;
+use solana_remote_wallet::{
+    remote_keypair::generate_remote_keypair,
+    remote_wallet::{RemoteWalletError, RemoteWalletManager},
+};
 use solana_sdk::{
     pubkey::Pubkey,
     signature::{
-        keypair_from_seed, keypair_from_seed_phrase_and_passphrase, read_keypair_file, Keypair,
-        KeypairUtil,
+        keypair_from_seed, keypair_from_seed_phrase_and_passphrase, read_keypair,
+        read_keypair_file, Keypair, Presigner, Signature, Signer,
     },
 };
 use std::{
     error,
     io::{stdin, stdout, Write},
     process::exit,
+    str::FromStr,
+    sync::Arc,
 };
 
+pub enum KeypairUrl {
+    Ask,
+    Filepath(String),
+    Usb(String),
+    Stdin,
+    Pubkey(Pubkey),
+}
+
+pub fn parse_keypair_path(path: &str) -> KeypairUrl {
+    if path == "-" {
+        KeypairUrl::Stdin
+    } else if path == ASK_KEYWORD {
+        KeypairUrl::Ask
+    } else if path.starts_with("usb://") {
+        KeypairUrl::Usb(path.to_string())
+    } else if let Ok(pubkey) = Pubkey::from_str(path) {
+        KeypairUrl::Pubkey(pubkey)
+    } else {
+        KeypairUrl::Filepath(path.to_string())
+    }
+}
+
+pub fn presigner_from_pubkey_sigs(
+    pubkey: &Pubkey,
+    signers: &[(Pubkey, Signature)],
+) -> Option<Presigner> {
+    signers.iter().find_map(|(signer, sig)| {
+        if *signer == *pubkey {
+            Some(Presigner::new(signer, sig))
+        } else {
+            None
+        }
+    })
+}
+
+pub fn signer_from_path(
+    matches: &ArgMatches,
+    path: &str,
+    keypair_name: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<Box<dyn Signer>, Box<dyn error::Error>> {
+    match parse_keypair_path(path) {
+        KeypairUrl::Ask => {
+            let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
+            Ok(Box::new(keypair_from_seed_phrase(
+                keypair_name,
+                skip_validation,
+                false,
+            )?))
+        }
+        KeypairUrl::Filepath(path) => match read_keypair_file(&path) {
+            Err(e) => Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                format!("could not find keypair file: {} error: {}", path, e),
+            )
+            .into()),
+            Ok(file) => Ok(Box::new(file)),
+        },
+        KeypairUrl::Stdin => {
+            let mut stdin = std::io::stdin();
+            Ok(Box::new(read_keypair(&mut stdin)?))
+        }
+        KeypairUrl::Usb(path) => {
+            if let Some(wallet_manager) = wallet_manager {
+                Ok(Box::new(generate_remote_keypair(
+                    path,
+                    wallet_manager,
+                    matches.is_present("confirm_key"),
+                    keypair_name,
+                )?))
+            } else {
+                Err(RemoteWalletError::NoDeviceFound.into())
+            }
+        }
+        KeypairUrl::Pubkey(pubkey) => {
+            let presigner = pubkeys_sigs_of(matches, SIGNER_ARG.name)
+                .as_ref()
+                .and_then(|presigners| presigner_from_pubkey_sigs(&pubkey, presigners));
+            if let Some(presigner) = presigner {
+                Ok(Box::new(presigner))
+            } else {
+                Err(std::io::Error::new(
+                    std::io::ErrorKind::Other,
+                    format!("missing signature for supplied pubkey: {}", pubkey),
+                )
+                .into())
+            }
+        }
+    }
+}
+
+pub fn pubkey_from_path(
+    matches: &ArgMatches,
+    path: &str,
+    keypair_name: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<Pubkey, Box<dyn error::Error>> {
+    match parse_keypair_path(path) {
+        KeypairUrl::Pubkey(pubkey) => Ok(pubkey),
+        _ => Ok(signer_from_path(matches, path, keypair_name, wallet_manager)?.pubkey()),
+    }
+}
+
+pub fn resolve_signer_from_path(
+    matches: &ArgMatches,
+    path: &str,
+    keypair_name: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<Option<String>, Box<dyn error::Error>> {
+    match parse_keypair_path(path) {
+        KeypairUrl::Ask => {
+            let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
+            // This method validates the seed phrase, but returns `None` because there is no path
+            // on disk or to a device
+            keypair_from_seed_phrase(keypair_name, skip_validation, false).map(|_| None)
+        }
+        KeypairUrl::Filepath(path) => match read_keypair_file(&path) {
+            Err(e) => Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                format!("could not find keypair file: {} error: {}", path, e),
+            )
+            .into()),
+            Ok(_) => Ok(Some(path.to_string())),
+        },
+        KeypairUrl::Stdin => {
+            let mut stdin = std::io::stdin();
+            // This method validates the keypair from stdin, but returns `None` because there is no
+            // path on disk or to a device
+            read_keypair(&mut stdin).map(|_| None)
+        }
+        KeypairUrl::Usb(path) => {
+            if let Some(wallet_manager) = wallet_manager {
+                let path = generate_remote_keypair(
+                    path,
+                    wallet_manager,
+                    matches.is_present("confirm_key"),
+                    keypair_name,
+                )
+                .map(|keypair| keypair.path)?;
+                Ok(Some(path))
+            } else {
+                Err(RemoteWalletError::NoDeviceFound.into())
+            }
+        }
+        _ => Ok(Some(path.to_string())),
+    }
+}
+
 // Keyword used to indicate that the user should be asked for a keypair seed phrase
 pub const ASK_KEYWORD: &str = "ASK";
 
-pub const ASK_SEED_PHRASE_ARG: ArgConstant<'static> = ArgConstant {
-    long: "ask-seed-phrase",
-    name: "ask_seed_phrase",
-    help: "Securely recover a keypair using a seed phrase and optional passphrase",
-};
-
 pub const SKIP_SEED_PHRASE_VALIDATION_ARG: ArgConstant<'static> = ArgConstant {
     long: "skip-seed-phrase-validation",
     name: "skip_seed_phrase_validation",
     help: "Skip validation of seed phrases. Use this if your phrase does not use the BIP39 official English word list",
 };
 
-#[derive(Debug, PartialEq)]
-pub enum Source {
-    File,
-    Generated,
-    SeedPhrase,
-}
-
-pub struct KeypairWithSource {
-    pub keypair: Keypair,
-    pub source: Source,
-}
-
-impl KeypairWithSource {
-    fn new(keypair: Keypair, source: Source) -> Self {
-        Self { keypair, source }
-    }
-}
-
 /// Prompts user for a passphrase and then asks for confirmirmation to check for mistakes
 pub fn prompt_passphrase(prompt: &str) -> Result<String, Box<dyn error::Error>> {
     let passphrase = prompt_password_stderr(&prompt)?;
@@ -80,14 +210,14 @@ pub fn keypair_from_seed_phrase(
         keypair_from_seed_phrase_and_passphrase(&seed_phrase, &passphrase)?
     } else {
         let sanitized = sanitize_seed_phrase(seed_phrase);
-        let mnemonic = Mnemonic::from_phrase(sanitized, Language::English)?;
+        let mnemonic = Mnemonic::from_phrase(&sanitized, Language::English)?;
         let passphrase = prompt_passphrase(&passphrase_prompt)?;
         let seed = Seed::new(&mnemonic, &passphrase);
         keypair_from_seed(seed.as_bytes())?
     };
 
     if confirm_pubkey {
-        let pubkey = Pubkey::new(keypair.public.as_ref());
+        let pubkey = keypair.pubkey();
         print!("Recovered pubkey `{:?}`. Continue? (y/n): ", pubkey);
         let _ignored = stdout().flush();
         let mut input = String::new();
@@ -101,42 +231,6 @@ pub fn keypair_from_seed_phrase(
     Ok(keypair)
 }
 
-/// Checks CLI arguments to determine whether a keypair should be:
-///   - inputted securely via stdin,
-///   - read in from a file,
-///   - or newly generated
-pub fn keypair_input(
-    matches: &clap::ArgMatches,
-    keypair_name: &str,
-) -> Result<KeypairWithSource, Box<dyn error::Error>> {
-    let ask_seed_phrase_matches =
-        values_t!(matches.values_of(ASK_SEED_PHRASE_ARG.name), String).unwrap_or_default();
-    let keypair_match_name = keypair_name.replace('-', "_");
-    if ask_seed_phrase_matches
-        .iter()
-        .any(|s| s.as_str() == keypair_name)
-    {
-        if matches.value_of(keypair_match_name).is_some() {
-            clap::Error::with_description(
-                &format!(
-                    "`--{} {}` cannot be used with `{} <PATH>`",
-                    ASK_SEED_PHRASE_ARG.long, keypair_name, keypair_name
-                ),
-                clap::ErrorKind::ArgumentConflict,
-            )
-            .exit();
-        }
-
-        let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
-        keypair_from_seed_phrase(keypair_name, skip_validation, true)
-            .map(|keypair| KeypairWithSource::new(keypair, Source::SeedPhrase))
-    } else if let Some(keypair_file) = matches.value_of(keypair_match_name) {
-        read_keypair_file(keypair_file).map(|keypair| KeypairWithSource::new(keypair, Source::File))
-    } else {
-        Ok(KeypairWithSource::new(Keypair::new(), Source::Generated))
-    }
-}
-
 fn sanitize_seed_phrase(seed_phrase: &str) -> String {
     seed_phrase
         .split_whitespace()
@@ -147,14 +241,6 @@ fn sanitize_seed_phrase(seed_phrase: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use clap::ArgMatches;
-
-    #[test]
-    fn test_keypair_input() {
-        let arg_matches = ArgMatches::default();
-        let KeypairWithSource { source, .. } = keypair_input(&arg_matches, "").unwrap();
-        assert_eq!(source, Source::Generated);
-    }
 
     #[test]
     fn test_sanitize_seed_phrase() {

clap-utils/src/lib.rs

@@ -1,3 +1,5 @@
+use thiserror::Error;
+
 #[macro_export]
 macro_rules! version {
     () => {
@@ -23,6 +25,24 @@ pub struct ArgConstant<'a> {
     pub help: &'a str,
 }
 
+/// Error type for forwarding Errors out of `main()` of a `clap` app
+/// and still using the `Display` formatter
+#[derive(Error)]
+#[error("{0}")]
+pub struct DisplayError(Box<dyn std::error::Error>);
+impl DisplayError {
+    pub fn new_as_boxed(inner: Box<dyn std::error::Error>) -> Box<Self> {
+        DisplayError(inner).into()
+    }
+}
+
+impl std::fmt::Debug for DisplayError {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(fmt, "{}", self.0)
+    }
+}
+
 pub mod input_parsers;
 pub mod input_validators;
 pub mod keypair;
+pub mod offline;

clap-utils/src/offline.rs

@@ -0,0 +1,19 @@
+use crate::ArgConstant;
+
+pub const BLOCKHASH_ARG: ArgConstant<'static> = ArgConstant {
+    name: "blockhash",
+    long: "blockhash",
+    help: "Use the supplied blockhash",
+};
+
+pub const SIGN_ONLY_ARG: ArgConstant<'static> = ArgConstant {
+    name: "sign_only",
+    long: "sign-only",
+    help: "Sign the transaction offline",
+};
+
+pub const SIGNER_ARG: ArgConstant<'static> = ArgConstant {
+    name: "signer",
+    long: "signer",
+    help: "Provide a public-key/signature pair for the transaction",
+};

cli-config/Cargo.toml

@@ -1,19 +1,17 @@
 [package]
-name = "solana-fixed-buf"
-version = "0.22.0"
-description = "A fixed-size byte array that supports bincode serde"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
+edition = "2018"
+name = "solana-cli-config"
+description = "Blockchain, Rebuilt for Scale"
+version = "1.0.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
-edition = "2018"
 
 [dependencies]
-bincode = "1.2.1"
+dirs = "2.0.2"
+lazy_static = "1.4.0"
 serde = "1.0.104"
-
-[lib]
-name = "solana_fixed_buf"
-
-[dev-dependencies]
 serde_derive = "1.0.103"
+serde_yaml = "0.8.11"
+url = "2.1.1"

cli-config/src/config.rs

@@ -0,0 +1,92 @@
+// Wallet settings that can be configured for long-term use
+use serde_derive::{Deserialize, Serialize};
+use std::{
+    fs::{create_dir_all, File},
+    io::{self, Write},
+    path::Path,
+};
+use url::Url;
+
+lazy_static! {
+    pub static ref CONFIG_FILE: Option<String> = {
+        dirs::home_dir().map(|mut path| {
+            path.extend(&[".config", "solana", "cli", "config.yml"]);
+            path.to_str().unwrap().to_string()
+        })
+    };
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq)]
+pub struct Config {
+    pub json_rpc_url: String,
+    pub websocket_url: String,
+    pub keypair_path: String,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        let keypair_path = {
+            let mut keypair_path = dirs::home_dir().expect("home directory");
+            keypair_path.extend(&[".config", "solana", "id.json"]);
+            keypair_path.to_str().unwrap().to_string()
+        };
+        let json_rpc_url = "http://127.0.0.1:8899".to_string();
+
+        // Empty websocket_url string indicates the client should
+        // `Config::compute_websocket_url(&json_rpc_url)`
+        let websocket_url = "".to_string();
+
+        Self {
+            json_rpc_url,
+            websocket_url,
+            keypair_path,
+        }
+    }
+}
+
+impl Config {
+    pub fn load(config_file: &str) -> Result<Self, io::Error> {
+        let file = File::open(config_file.to_string())?;
+        let config = serde_yaml::from_reader(file)
+            .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
+        Ok(config)
+    }
+
+    pub fn save(&self, config_file: &str) -> Result<(), io::Error> {
+        let serialized = serde_yaml::to_string(self)
+            .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
+
+        if let Some(outdir) = Path::new(&config_file).parent() {
+            create_dir_all(outdir)?;
+        }
+        let mut file = File::create(config_file)?;
+        file.write_all(&serialized.into_bytes())?;
+
+        Ok(())
+    }
+
+    pub fn compute_websocket_url(json_rpc_url: &str) -> String {
+        let json_rpc_url: Option<Url> = json_rpc_url.parse().ok();
+        if json_rpc_url.is_none() {
+            return "".to_string();
+        }
+        let json_rpc_url = json_rpc_url.unwrap();
+        let is_secure = json_rpc_url.scheme().to_ascii_lowercase() == "https";
+        let mut ws_url = json_rpc_url.clone();
+        ws_url
+            .set_scheme(if is_secure { "wss" } else { "ws" })
+            .expect("unable to set scheme");
+        let ws_port = match json_rpc_url.port() {
+            Some(port) => port + 1,
+            None => {
+                if is_secure {
+                    8901
+                } else {
+                    8900
+                }
+            }
+        };
+        ws_url.set_port(Some(ws_port)).expect("unable to set port");
+        ws_url.to_string()
+    }
+}

cli-config/src/lib.rs

@@ -0,0 +1,5 @@
+#[macro_use]
+extern crate lazy_static;
+
+mod config;
+pub use config::{Config, CONFIG_FILE};

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "0.22.0"
+version = "1.0.7"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,38 +14,40 @@ bs58 = "0.3.0"
 chrono = { version = "0.4.10", features = ["serde"] }
 clap = "2.33.0"
 criterion-stats = "0.3.0"
-ctrlc = { version = "3.1.3", features = ["termination"] }
-console = "0.9.1"
+ctrlc = { version = "3.1.4", features = ["termination"] }
+console = "0.9.2"
 dirs = "2.0.2"
-lazy_static = "1.4.0"
 log = "0.4.8"
-indicatif = "0.13.0"
-humantime = "1.3.0"
+indicatif = "0.14.0"
+humantime = "2.0.0"
 num-traits = "0.2"
 pretty-hex = "0.1.1"
-reqwest = { version = "0.9.24", default-features = false, features = ["rustls-tls"] }
+reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
 serde = "1.0.104"
 serde_derive = "1.0.103"
-serde_json = "1.0.44"
-serde_yaml = "0.8.11"
-solana-budget-program = { path = "../programs/budget", version = "0.22.0" }
-solana-clap-utils = { path = "../clap-utils", version = "0.22.0" }
-solana-client = { path = "../client", version = "0.22.0" }
-solana-config-program = { path = "../programs/config", version = "0.22.0" }
-solana-faucet = { path = "../faucet", version = "0.22.0" }
-solana-logger = { path = "../logger", version = "0.22.0" }
-solana-net-utils = { path = "../net-utils", version = "0.22.0" }
-solana-runtime = { path = "../runtime", version = "0.22.0" }
-solana-sdk = { path = "../sdk", version = "0.22.0" }
-solana-stake-program = { path = "../programs/stake", version = "0.22.0" }
-solana-storage-program = { path = "../programs/storage", version = "0.22.0" }
-solana-vote-program = { path = "../programs/vote", version = "0.22.0" }
-solana-vote-signer = { path = "../vote-signer", version = "0.22.0" }
-url = "2.1.0"
+serde_json = "1.0.46"
+solana-budget-program = { path = "../programs/budget", version = "1.0.7" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.7" }
+solana-cli-config = { path = "../cli-config", version = "1.0.7" }
+solana-client = { path = "../client", version = "1.0.7" }
+solana-config-program = { path = "../programs/config", version = "1.0.7" }
+solana-faucet = { path = "../faucet", version = "1.0.7" }
+solana-logger = { path = "../logger", version = "1.0.7" }
+solana-net-utils = { path = "../net-utils", version = "1.0.7" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.0.7" }
+solana-runtime = { path = "../runtime", version = "1.0.7" }
+solana-sdk = { path = "../sdk", version = "1.0.7" }
+solana-stake-program = { path = "../programs/stake", version = "1.0.7" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.7" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.7" }
+solana-vote-signer = { path = "../vote-signer", version = "1.0.7" }
+titlecase = "1.1.0"
+thiserror = "1.0.11"
+url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "0.22.0" }
-solana-budget-program = { path = "../programs/budget", version = "0.22.0" }
+solana-core = { path = "../core", version = "1.0.7" }
+solana-budget-program = { path = "../programs/budget", version = "1.0.7" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cluster_query.rs

@@ -8,20 +8,32 @@ use crate::{
 use clap::{value_t, value_t_or_exit, App, Arg, ArgMatches, SubCommand};
 use console::{style, Emoji};
 use indicatif::{ProgressBar, ProgressStyle};
-use solana_clap_utils::{input_parsers::*, input_validators::*};
-use solana_client::{rpc_client::RpcClient, rpc_request::RpcVoteAccountInfo};
+use solana_clap_utils::{input_parsers::*, input_validators::*, keypair::signer_from_path};
+use solana_client::{
+    pubsub_client::{PubsubClient, SlotInfoMessage},
+    rpc_client::RpcClient,
+    rpc_response::RpcVoteAccountInfo,
+};
+use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
+    account_utils::StateMut,
     clock::{self, Slot},
     commitment_config::CommitmentConfig,
-    epoch_schedule::{Epoch, EpochSchedule},
+    epoch_schedule::Epoch,
     hash::Hash,
+    message::Message,
     pubkey::Pubkey,
-    signature::{Keypair, KeypairUtil},
-    system_transaction,
+    signature::{Keypair, Signer},
+    system_instruction,
+    transaction::Transaction,
 };
 use std::{
     collections::{HashMap, VecDeque},
     net::SocketAddr,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
     thread::sleep,
     time::{Duration, Instant},
 };
@@ -47,6 +59,14 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .validator(is_pubkey_or_keypair)
                         .required(true)
                         .help("Identity pubkey of the validator"),
+                )
+                .arg(
+                    Arg::with_name("node_json_rpc_url")
+                        .index(2)
+                        .value_name("URL")
+                        .takes_value(true)
+                        .validator(is_url)
+                        .help("JSON RPC URL for validator, which is useful for validators with a private RPC service")
                 ),
         )
         .subcommand(
@@ -54,8 +74,9 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                 .about("Get the version of the cluster entrypoint"),
         )
         .subcommand(SubCommand::with_name("fees").about("Display current cluster fees"))
-        .subcommand(SubCommand::with_name("get-block-time")
+        .subcommand(SubCommand::with_name("block-time")
             .about("Get estimated production time of a block")
+            .alias("get-block-time")
             .arg(
                 Arg::with_name("slot")
                     .index(1)
@@ -65,9 +86,11 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                     .help("Slot number of the block to query")
             )
         )
+        .subcommand(SubCommand::with_name("leader-schedule").about("Display leader schedule"))
         .subcommand(
-            SubCommand::with_name("get-epoch-info")
+            SubCommand::with_name("epoch-info")
             .about("Get information about the current epoch")
+            .alias("get-epoch-info")
             .arg(
                 Arg::with_name("confirmed")
                     .long("confirmed")
@@ -78,10 +101,13 @@ impl ClusterQuerySubCommands for App<'_, '_> {
             ),
         )
         .subcommand(
-            SubCommand::with_name("get-genesis-hash").about("Get the genesis hash"),
+            SubCommand::with_name("genesis-hash")
+            .about("Get the genesis hash")
+            .alias("get-genesis-hash")
         )
         .subcommand(
-            SubCommand::with_name("get-slot").about("Get current slot")
+            SubCommand::with_name("slot").about("Get current slot")
+            .alias("get-slot")
             .arg(
                 Arg::with_name("confirmed")
                     .long("confirmed")
@@ -92,7 +118,8 @@ impl ClusterQuerySubCommands for App<'_, '_> {
             ),
         )
         .subcommand(
-            SubCommand::with_name("get-transaction-count").about("Get current transaction count")
+            SubCommand::with_name("transaction-count").about("Get current transaction count")
+            .alias("get-transaction-count")
             .arg(
                 Arg::with_name("confirmed")
                     .long("confirmed")
@@ -150,8 +177,13 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                 ),
         )
         .subcommand(
-            SubCommand::with_name("show-block-production")
+            SubCommand::with_name("live-slots")
+                .about("Show information about the current slot progression"),
+        )
+        .subcommand(
+            SubCommand::with_name("block-production")
                 .about("Show information about block production")
+                .alias("show-block-production")
                 .arg(
                     Arg::with_name("epoch")
                         .long("epoch")
@@ -166,12 +198,41 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                 ),
         )
         .subcommand(
-            SubCommand::with_name("show-gossip")
-                .about("Show the current gossip network nodes"),
+            SubCommand::with_name("gossip")
+                .about("Show the current gossip network nodes")
+                .alias("show-gossip")
         )
         .subcommand(
-            SubCommand::with_name("show-validators")
-                .about("Show information about the current validators")
+            SubCommand::with_name("stakes")
+                .about("Show stake account information")
+                .arg(
+                    Arg::with_name("vote_account_pubkeys")
+                        .index(1)
+                        .value_name("VOTE ACCOUNT PUBKEYS")
+                        .takes_value(true)
+                        .multiple(true)
+                        .validator(is_pubkey_or_keypair)
+                        .help("Only show stake accounts delegated to the provided vote accounts"),
+                )
+                .arg(
+                    Arg::with_name("lamports")
+                        .long("lamports")
+                        .takes_value(false)
+                        .help("Display balance in lamports instead of SOL"),
+                ),
+        )
+        .subcommand(
+            SubCommand::with_name("validators")
+                .about("Show summary information about the current validators")
+                .alias("show-validators")
+                .arg(
+                    Arg::with_name("confirmed")
+                        .long("confirmed")
+                        .takes_value(false)
+                        .help(
+                            "Return information at maximum-lockout commitment level",
+                        ),
+                )
                 .arg(
                     Arg::with_name("lamports")
                         .long("lamports")
@@ -184,13 +245,21 @@ impl ClusterQuerySubCommands for App<'_, '_> {
 
 pub fn parse_catchup(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let node_pubkey = pubkey_of(matches, "node_pubkey").unwrap();
+    let node_json_rpc_url = value_t!(matches, "node_json_rpc_url", String).ok();
     Ok(CliCommandInfo {
-        command: CliCommand::Catchup { node_pubkey },
-        require_keypair: false,
+        command: CliCommand::Catchup {
+            node_pubkey,
+            node_json_rpc_url,
+        },
+        signers: vec![],
     })
 }
 
-pub fn parse_cluster_ping(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+pub fn parse_cluster_ping(
+    matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
     let lamports = value_t_or_exit!(matches, "lamports", u64);
     let interval = Duration::from_secs(value_t_or_exit!(matches, "interval", u64));
     let count = if matches.is_present("count") {
@@ -212,7 +281,12 @@ pub fn parse_cluster_ping(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, Cl
             timeout,
             commitment_config,
         },
-        require_keypair: true,
+        signers: vec![signer_from_path(
+            matches,
+            default_signer_path,
+            "keypair",
+            wallet_manager,
+        )?],
     })
 }
 
@@ -220,7 +294,7 @@ pub fn parse_get_block_time(matches: &ArgMatches<'_>) -> Result<CliCommandInfo,
     let slot = value_t_or_exit!(matches, "slot", u64);
     Ok(CliCommandInfo {
         command: CliCommand::GetBlockTime { slot },
-        require_keypair: false,
+        signers: vec![],
     })
 }
 
@@ -232,7 +306,7 @@ pub fn parse_get_epoch_info(matches: &ArgMatches<'_>) -> Result<CliCommandInfo,
     };
     Ok(CliCommandInfo {
         command: CliCommand::GetEpochInfo { commitment_config },
-        require_keypair: false,
+        signers: vec![],
     })
 }
 
@@ -244,7 +318,7 @@ pub fn parse_get_slot(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliErr
     };
     Ok(CliCommandInfo {
         command: CliCommand::GetSlot { commitment_config },
-        require_keypair: false,
+        signers: vec![],
     })
 }
 
@@ -256,16 +330,37 @@ pub fn parse_get_transaction_count(matches: &ArgMatches<'_>) -> Result<CliComman
     };
     Ok(CliCommandInfo {
         command: CliCommand::GetTransactionCount { commitment_config },
-        require_keypair: false,
+        signers: vec![],
+    })
+}
+
+pub fn parse_show_stakes(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+    let use_lamports_unit = matches.is_present("lamports");
+    let vote_account_pubkeys = pubkeys_of(matches, "vote_account_pubkeys");
+
+    Ok(CliCommandInfo {
+        command: CliCommand::ShowStakes {
+            use_lamports_unit,
+            vote_account_pubkeys,
+        },
+        signers: vec![],
     })
 }
 
 pub fn parse_show_validators(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let use_lamports_unit = matches.is_present("lamports");
+    let commitment_config = if matches.is_present("confirmed") {
+        CommitmentConfig::default()
+    } else {
+        CommitmentConfig::recent()
+    };
 
     Ok(CliCommandInfo {
-        command: CliCommand::ShowValidators { use_lamports_unit },
-        require_keypair: false,
+        command: CliCommand::ShowValidators {
+            use_lamports_unit,
+            commitment_config,
+        },
+        signers: vec![],
     })
 }
 
@@ -278,34 +373,42 @@ fn new_spinner_progress_bar() -> ProgressBar {
     progress_bar
 }
 
-/// Aggregate epoch credit stats and return (total credits, total slots, total epochs)
-pub fn aggregate_epoch_credits(
-    epoch_credits: &[(Epoch, u64, u64)],
-    epoch_schedule: &EpochSchedule,
-) -> (u64, u64, u64) {
-    epoch_credits
-        .iter()
-        .fold((0, 0, 0), |acc, (epoch, credits, prev_credits)| {
-            let credits_earned = credits - prev_credits;
-            let slots_in_epoch = epoch_schedule.get_slots_in_epoch(*epoch);
-            (acc.0 + credits_earned, acc.1 + slots_in_epoch, acc.2 + 1)
-        })
-}
-
-pub fn process_catchup(rpc_client: &RpcClient, node_pubkey: &Pubkey) -> ProcessResult {
+pub fn process_catchup(
+    rpc_client: &RpcClient,
+    node_pubkey: &Pubkey,
+    node_json_rpc_url: &Option<String>,
+) -> ProcessResult {
     let cluster_nodes = rpc_client.get_cluster_nodes()?;
 
-    let rpc_addr = cluster_nodes
-        .iter()
-        .find(|contact_info| contact_info.pubkey == node_pubkey.to_string())
-        .ok_or_else(|| format!("Contact information not found for {}", node_pubkey))?
-        .rpc
-        .ok_or_else(|| format!("RPC service not found for {}", node_pubkey))?;
+    let node_client = if let Some(node_json_rpc_url) = node_json_rpc_url {
+        RpcClient::new(node_json_rpc_url.to_string())
+    } else {
+        RpcClient::new_socket(
+            cluster_nodes
+                .iter()
+                .find(|contact_info| contact_info.pubkey == node_pubkey.to_string())
+                .ok_or_else(|| format!("Contact information not found for {}", node_pubkey))?
+                .rpc
+                .ok_or_else(|| format!("RPC service not found for {}", node_pubkey))?,
+        )
+    };
+
+    let reported_node_pubkey = node_client.get_identity()?;
+    if reported_node_pubkey != *node_pubkey {
+        return Err(format!(
+            "The identity reported by node RPC URL does not match.  Expected: {:?}.  Reported: {:?}",
+            node_pubkey, reported_node_pubkey
+        )
+        .into());
+    }
+
+    if rpc_client.get_identity()? == *node_pubkey {
+        return Err("Both RPC URLs reference the same node, unable to monitor for catchup.  Try a different --url".into());
+    }
 
     let progress_bar = new_spinner_progress_bar();
     progress_bar.set_message("Connecting...");
 
-    let node_client = RpcClient::new_socket(rpc_addr);
     let mut previous_rpc_slot = std::u64::MAX;
     let mut previous_slot_distance = 0;
     let sleep_interval = 5;
@@ -364,35 +467,90 @@ pub fn process_fees(rpc_client: &RpcClient) -> ProcessResult {
     ))
 }
 
+pub fn process_leader_schedule(rpc_client: &RpcClient) -> ProcessResult {
+    let epoch_info = rpc_client.get_epoch_info()?;
+    let first_slot_in_epoch = epoch_info.absolute_slot - epoch_info.slot_index;
+
+    let leader_schedule = rpc_client.get_leader_schedule(Some(first_slot_in_epoch))?;
+    if leader_schedule.is_none() {
+        return Err(format!(
+            "Unable to fetch leader schedule for slot {}",
+            first_slot_in_epoch
+        )
+        .into());
+    }
+    let leader_schedule = leader_schedule.unwrap();
+
+    let mut leader_per_slot_index = Vec::new();
+    for (pubkey, leader_slots) in leader_schedule.iter() {
+        for slot_index in leader_slots.iter() {
+            if *slot_index >= leader_per_slot_index.len() {
+                leader_per_slot_index.resize(*slot_index + 1, "?");
+            }
+            leader_per_slot_index[*slot_index] = pubkey;
+        }
+    }
+
+    for (slot_index, leader) in leader_per_slot_index.iter().enumerate() {
+        println!(
+            "  {:<15} {:<44}",
+            first_slot_in_epoch + slot_index as u64,
+            leader
+        );
+    }
+
+    Ok("".to_string())
+}
+
 pub fn process_get_block_time(rpc_client: &RpcClient, slot: Slot) -> ProcessResult {
     let timestamp = rpc_client.get_block_time(slot)?;
     Ok(timestamp.to_string())
 }
 
+fn slot_to_human_time(slot: Slot) -> String {
+    humantime::format_duration(Duration::from_secs(
+        slot * clock::DEFAULT_TICKS_PER_SLOT / clock::DEFAULT_TICKS_PER_SECOND,
+    ))
+    .to_string()
+}
+
 pub fn process_get_epoch_info(
     rpc_client: &RpcClient,
-    commitment_config: &CommitmentConfig,
+    commitment_config: CommitmentConfig,
 ) -> ProcessResult {
     let epoch_info = rpc_client.get_epoch_info_with_commitment(commitment_config.clone())?;
     println!();
-    println_name_value("Current epoch:", &epoch_info.epoch.to_string());
-    println_name_value("Current slot:", &epoch_info.absolute_slot.to_string());
+    println_name_value("Slot:", &epoch_info.absolute_slot.to_string());
+    println_name_value("Epoch:", &epoch_info.epoch.to_string());
+    let start_slot = epoch_info.absolute_slot - epoch_info.slot_index;
+    let end_slot = start_slot + epoch_info.slots_in_epoch;
     println_name_value(
-        "Total slots in current epoch:",
-        &epoch_info.slots_in_epoch.to_string(),
+        "Epoch Slot Range:",
+        &format!("[{}..{})", start_slot, end_slot),
+    );
+    println_name_value(
+        "Epoch Completed Percent:",
+        &format!(
+            "{:>3.3}%",
+            epoch_info.slot_index as f64 / epoch_info.slots_in_epoch as f64 * 100_f64
+        ),
     );
     let remaining_slots_in_epoch = epoch_info.slots_in_epoch - epoch_info.slot_index;
     println_name_value(
-        "Remaining slots in current epoch:",
-        &remaining_slots_in_epoch.to_string(),
-    );
-
-    let remaining_time_in_epoch = Duration::from_secs(
-        remaining_slots_in_epoch * clock::DEFAULT_TICKS_PER_SLOT / clock::DEFAULT_TICKS_PER_SECOND,
+        "Epoch Completed Slots:",
+        &format!(
+            "{}/{} ({} remaining)",
+            epoch_info.slot_index, epoch_info.slots_in_epoch, remaining_slots_in_epoch
+        ),
     );
     println_name_value(
-        "Time remaining in current epoch:",
-        &humantime::format_duration(remaining_time_in_epoch).to_string(),
+        "Epoch Completed Time:",
+        &format!(
+            "{}/{} ({} remaining)",
+            slot_to_human_time(epoch_info.slot_index),
+            slot_to_human_time(epoch_info.slots_in_epoch),
+            slot_to_human_time(remaining_slots_in_epoch)
+        ),
     );
     Ok("".to_string())
 }
@@ -404,7 +562,7 @@ pub fn process_get_genesis_hash(rpc_client: &RpcClient) -> ProcessResult {
 
 pub fn process_get_slot(
     rpc_client: &RpcClient,
-    commitment_config: &CommitmentConfig,
+    commitment_config: CommitmentConfig,
 ) -> ProcessResult {
     let slot = rpc_client.get_slot_with_commitment(commitment_config.clone())?;
     Ok(slot.to_string())
@@ -416,7 +574,7 @@ pub fn parse_show_block_production(matches: &ArgMatches<'_>) -> Result<CliComman
 
     Ok(CliCommandInfo {
         command: CliCommand::ShowBlockProduction { epoch, slot_limit },
-        require_keypair: false,
+        signers: vec![],
     })
 }
 
@@ -434,19 +592,39 @@ pub fn process_show_block_production(
         return Err(format!("Epoch {} is in the future", epoch).into());
     }
 
+    let minimum_ledger_slot = rpc_client.minimum_ledger_slot()?;
+
     let first_slot_in_epoch = epoch_schedule.get_first_slot_in_epoch(epoch);
     let end_slot = std::cmp::min(
         epoch_info.absolute_slot,
         epoch_schedule.get_last_slot_in_epoch(epoch),
     );
 
-    let start_slot = if let Some(slot_limit) = slot_limit {
+    let mut start_slot = if let Some(slot_limit) = slot_limit {
         std::cmp::max(end_slot.saturating_sub(slot_limit), first_slot_in_epoch)
     } else {
         first_slot_in_epoch
     };
-    let start_slot_index = (start_slot - first_slot_in_epoch) as usize;
-    let end_slot_index = (end_slot - start_slot) as usize;
+
+    if minimum_ledger_slot > end_slot {
+        return Err(format!(
+            "Ledger data not available for slots {} to {} (minimum ledger slot is {})",
+            start_slot, end_slot, minimum_ledger_slot
+        )
+        .into());
+    }
+
+    if minimum_ledger_slot > start_slot {
+        println!(
+            "\n{}",
+            style(format!(
+                "Note: Requested start slot was {} but minimum ledger slot is {}",
+                start_slot, minimum_ledger_slot
+            ))
+            .italic(),
+        );
+        start_slot = minimum_ledger_slot;
+    }
 
     let progress_bar = new_spinner_progress_bar();
     progress_bar.set_message(&format!(
@@ -455,12 +633,14 @@ pub fn process_show_block_production(
     ));
     let confirmed_blocks = rpc_client.get_confirmed_blocks(start_slot, Some(end_slot))?;
 
+    let start_slot_index = (start_slot - first_slot_in_epoch) as usize;
+    let end_slot_index = (end_slot - first_slot_in_epoch) as usize;
     let total_slots = end_slot_index - start_slot_index + 1;
     let total_blocks = confirmed_blocks.len();
     assert!(total_blocks <= total_slots);
-    let total_slots_missed = total_slots - total_blocks;
+    let total_slots_skipped = total_slots - total_blocks;
     let mut leader_slot_count = HashMap::new();
-    let mut leader_missed_slots = HashMap::new();
+    let mut leader_skipped_slots = HashMap::new();
 
     progress_bar.set_message(&format!("Fetching leader schedule for epoch {}...", epoch));
     let leader_schedule = rpc_client
@@ -482,7 +662,7 @@ pub fn process_show_block_production(
 
     progress_bar.set_message(&format!(
         "Processing {} slots containing {} blocks and {} empty slots...",
-        total_slots, total_blocks, total_slots_missed
+        total_slots, total_blocks, total_slots_skipped
     ));
 
     let mut confirmed_blocks_index = 0;
@@ -491,10 +671,10 @@ pub fn process_show_block_production(
         let slot = start_slot + slot_index as u64;
         let slot_count = leader_slot_count.entry(leader).or_insert(0);
         *slot_count += 1;
-        let missed_slots = leader_missed_slots.entry(leader).or_insert(0);
+        let skipped_slots = leader_skipped_slots.entry(leader).or_insert(0);
 
         loop {
-            if !confirmed_blocks.is_empty() {
+            if confirmed_blocks_index < confirmed_blocks.len() {
                 let slot_of_next_confirmed_block = confirmed_blocks[confirmed_blocks_index];
                 if slot_of_next_confirmed_block < slot {
                     confirmed_blocks_index += 1;
@@ -506,9 +686,9 @@ pub fn process_show_block_production(
                     break;
                 }
             }
-            *missed_slots += 1;
+            *skipped_slots += 1;
             individual_slot_status.push(
-                style(format!("  {:<15} {:<44} MISSED", slot, leader))
+                style(format!("  {:<15} {:<44} SKIPPED", slot, leader))
                     .red()
                     .to_string(),
             );
@@ -524,23 +704,23 @@ pub fn process_show_block_production(
             "Identity Pubkey",
             "Leader Slots",
             "Blocks Produced",
-            "Missed Slots",
-            "Missed Block Percentage",
+            "Skipped Slots",
+            "Skipped Slot Percentage",
         ))
         .bold()
     );
 
     let mut table = vec![];
     for (leader, leader_slots) in leader_slot_count.iter() {
-        let missed_slots = leader_missed_slots.get(leader).unwrap();
-        let blocks_produced = leader_slots - missed_slots;
+        let skipped_slots = leader_skipped_slots.get(leader).unwrap();
+        let blocks_produced = leader_slots - skipped_slots;
         table.push(format!(
             "  {:<44}  {:>15}  {:>15}  {:>15}  {:>22.2}%",
             leader,
             leader_slots,
             blocks_produced,
-            missed_slots,
-            *missed_slots as f64 / *leader_slots as f64 * 100.
+            skipped_slots,
+            *skipped_slots as f64 / *leader_slots as f64 * 100.
         ));
     }
     table.sort();
@@ -551,8 +731,8 @@ pub fn process_show_block_production(
         format!("Epoch {} total:", epoch),
         total_slots,
         total_blocks,
-        total_slots_missed,
-        total_slots_missed as f64 / total_slots as f64 * 100.
+        total_slots_skipped,
+        total_slots_skipped as f64 / total_slots as f64 * 100.
     );
     println!(
         "  (using data from {} slots: {} to {})",
@@ -571,7 +751,7 @@ pub fn process_show_block_production(
 
 pub fn process_get_transaction_count(
     rpc_client: &RpcClient,
-    commitment_config: &CommitmentConfig,
+    commitment_config: CommitmentConfig,
 ) -> ProcessResult {
     let transaction_count =
         rpc_client.get_transaction_count_with_commitment(commitment_config.clone())?;
@@ -585,12 +765,12 @@ pub fn process_ping(
     interval: &Duration,
     count: &Option<u64>,
     timeout: &Duration,
-    commitment_config: &CommitmentConfig,
+    commitment_config: CommitmentConfig,
 ) -> ProcessResult {
     let to = Keypair::new().pubkey();
 
-    println_name_value("Source account:", &config.keypair.pubkey().to_string());
-    println_name_value("Destination account:", &to.to_string());
+    println_name_value("Source Account:", &config.signers[0].pubkey().to_string());
+    println_name_value("Destination Account:", &to.to_string());
     println!();
 
     let (signal_sender, signal_receiver) = std::sync::mpsc::channel();
@@ -608,11 +788,13 @@ pub fn process_ping(
         let (recent_blockhash, fee_calculator) = rpc_client.get_new_blockhash(&last_blockhash)?;
         last_blockhash = recent_blockhash;
 
-        let transaction =
-            system_transaction::transfer(&config.keypair, &to, lamports, recent_blockhash);
+        let ix = system_instruction::transfer(&config.signers[0].pubkey(), &to, lamports);
+        let message = Message::new(&[ix]);
+        let mut transaction = Transaction::new_unsigned(message);
+        transaction.try_sign(&config.signers, recent_blockhash)?;
         check_account_for_fee(
             rpc_client,
-            &config.keypair.pubkey(),
+            &config.signers[0].pubkey(),
             &fee_calculator,
             &transaction.message,
         )?;
@@ -704,6 +886,101 @@ pub fn process_ping(
     Ok("".to_string())
 }
 
+pub fn process_live_slots(url: &str) -> ProcessResult {
+    let exit = Arc::new(AtomicBool::new(false));
+
+    // Disable Ctrl+C handler as sometimes the PubsubClient shutdown can stall.  Also it doesn't
+    // really matter that the shutdown is clean because the process is terminating.
+    /*
+    let exit_clone = exit.clone();
+    ctrlc::set_handler(move || {
+        exit_clone.store(true, Ordering::Relaxed);
+    })?;
+    */
+
+    let mut current: Option<SlotInfoMessage> = None;
+    let mut message = "".to_string();
+
+    let slot_progress = new_spinner_progress_bar();
+    slot_progress.set_message("Connecting...");
+    let (mut client, receiver) = PubsubClient::slot_subscribe(url)?;
+    slot_progress.set_message("Connected.");
+
+    let spacer = "|";
+    slot_progress.println(spacer);
+
+    let mut last_root = std::u64::MAX;
+    let mut last_root_update = Instant::now();
+    let mut slots_per_second = std::f64::NAN;
+    loop {
+        if exit.load(Ordering::Relaxed) {
+            eprintln!("{}", message);
+            client.shutdown().unwrap();
+            break;
+        }
+
+        match receiver.recv() {
+            Ok(new_info) => {
+                if last_root == std::u64::MAX {
+                    last_root = new_info.root;
+                    last_root_update = Instant::now();
+                }
+                if last_root_update.elapsed().as_secs() >= 5 {
+                    let root = new_info.root;
+                    slots_per_second =
+                        (root - last_root) as f64 / last_root_update.elapsed().as_secs() as f64;
+                    last_root_update = Instant::now();
+                    last_root = root;
+                }
+
+                message = if slots_per_second.is_nan() {
+                    format!("{:?}", new_info)
+                } else {
+                    format!(
+                        "{:?} | root slot advancing at {:.2} slots/second",
+                        new_info, slots_per_second
+                    )
+                }
+                .to_owned();
+                slot_progress.set_message(&message);
+
+                if let Some(previous) = current {
+                    let slot_delta: i64 = new_info.slot as i64 - previous.slot as i64;
+                    let root_delta: i64 = new_info.root as i64 - previous.root as i64;
+
+                    //
+                    // if slot has advanced out of step with the root, we detect
+                    // a mismatch and output the slot information
+                    //
+                    if slot_delta != root_delta {
+                        let prev_root = format!(
+                            "|<--- {} <- … <- {} <- {}   (prev)",
+                            previous.root, previous.parent, previous.slot
+                        );
+                        slot_progress.println(&prev_root);
+
+                        let new_root = format!(
+                            "|  '- {} <- … <- {} <- {}   (next)",
+                            new_info.root, new_info.parent, new_info.slot
+                        );
+
+                        slot_progress.println(prev_root);
+                        slot_progress.println(new_root);
+                        slot_progress.println(spacer);
+                    }
+                }
+                current = Some(new_info);
+            }
+            Err(err) => {
+                eprintln!("disconnected: {}", err);
+                break;
+            }
+        }
+    }
+
+    Ok("".to_string())
+}
+
 pub fn process_show_gossip(rpc_client: &RpcClient) -> ProcessResult {
     let cluster_nodes = rpc_client.get_cluster_nodes()?;
 
@@ -740,9 +1017,52 @@ pub fn process_show_gossip(rpc_client: &RpcClient) -> ProcessResult {
     ))
 }
 
-pub fn process_show_validators(rpc_client: &RpcClient, use_lamports_unit: bool) -> ProcessResult {
-    let epoch_schedule = rpc_client.get_epoch_schedule()?;
-    let vote_accounts = rpc_client.get_vote_accounts()?;
+pub fn process_show_stakes(
+    rpc_client: &RpcClient,
+    use_lamports_unit: bool,
+    vote_account_pubkeys: Option<&[Pubkey]>,
+) -> ProcessResult {
+    use crate::stake::print_stake_state;
+    use solana_stake_program::stake_state::StakeState;
+
+    let progress_bar = new_spinner_progress_bar();
+    progress_bar.set_message("Fetching stake accounts...");
+    let all_stake_accounts = rpc_client.get_program_accounts(&solana_stake_program::id())?;
+    progress_bar.finish_and_clear();
+
+    for (stake_pubkey, stake_account) in all_stake_accounts {
+        if let Ok(stake_state) = stake_account.state() {
+            match stake_state {
+                StakeState::Initialized(_) => {
+                    if vote_account_pubkeys.is_none() {
+                        println!("\nstake pubkey: {}", stake_pubkey);
+                        print_stake_state(stake_account.lamports, &stake_state, use_lamports_unit);
+                    }
+                }
+                StakeState::Stake(_, stake) => {
+                    if vote_account_pubkeys.is_none()
+                        || vote_account_pubkeys
+                            .unwrap()
+                            .contains(&stake.delegation.voter_pubkey)
+                    {
+                        println!("\nstake pubkey: {}", stake_pubkey);
+                        print_stake_state(stake_account.lamports, &stake_state, use_lamports_unit);
+                    }
+                }
+                _ => {}
+            }
+        }
+    }
+    Ok("".to_string())
+}
+
+pub fn process_show_validators(
+    rpc_client: &RpcClient,
+    use_lamports_unit: bool,
+    commitment_config: CommitmentConfig,
+) -> ProcessResult {
+    let epoch_info = rpc_client.get_epoch_info_with_commitment(commitment_config)?;
+    let vote_accounts = rpc_client.get_vote_accounts_with_commitment(commitment_config)?;
     let total_active_stake = vote_accounts
         .current
         .iter()
@@ -790,7 +1110,7 @@ pub fn process_show_validators(rpc_client: &RpcClient, use_lamports_unit: bool)
             "Commission",
             "Last Vote",
             "Root Block",
-            "Uptime",
+            "Credits",
             "Active Stake",
         ))
         .bold()
@@ -798,7 +1118,7 @@ pub fn process_show_validators(rpc_client: &RpcClient, use_lamports_unit: bool)
 
     fn print_vote_account(
         vote_account: RpcVoteAccountInfo,
-        epoch_schedule: &EpochSchedule,
+        current_epoch: Epoch,
         total_active_stake: f64,
         use_lamports_unit: bool,
         delinquent: bool,
@@ -811,17 +1131,6 @@ pub fn process_show_validators(rpc_client: &RpcClient, use_lamports_unit: bool)
             }
         }
 
-        fn uptime(epoch_credits: Vec<(Epoch, u64, u64)>, epoch_schedule: &EpochSchedule) -> String {
-            let (total_credits, total_slots, _) =
-                aggregate_epoch_credits(&epoch_credits, &epoch_schedule);
-            if total_slots > 0 {
-                let total_uptime = 100_f64 * total_credits as f64 / total_slots as f64;
-                format!("{:.2}%", total_uptime)
-            } else {
-                "-".into()
-            }
-        }
-
         println!(
             "{} {:<44}  {:<44}  {:>9}%   {:>8}  {:>10}  {:>7}  {}",
             if delinquent {
@@ -834,7 +1143,15 @@ pub fn process_show_validators(rpc_client: &RpcClient, use_lamports_unit: bool)
             vote_account.commission,
             non_zero_or_dash(vote_account.last_vote),
             non_zero_or_dash(vote_account.root_slot),
-            uptime(vote_account.epoch_credits, epoch_schedule),
+            vote_account
+                .epoch_credits
+                .iter()
+                .find_map(|(epoch, credits, _)| if *epoch == current_epoch {
+                    Some(*credits)
+                } else {
+                    None
+                })
+                .unwrap_or(0),
             if vote_account.activated_stake > 0 {
                 format!(
                     "{} ({:.2}%)",
@@ -850,7 +1167,7 @@ pub fn process_show_validators(rpc_client: &RpcClient, use_lamports_unit: bool)
     for vote_account in vote_accounts.current.into_iter() {
         print_vote_account(
             vote_account,
-            &epoch_schedule,
+            epoch_info.epoch,
             total_active_stake,
             use_lamports_unit,
             false,
@@ -859,7 +1176,7 @@ pub fn process_show_validators(rpc_client: &RpcClient, use_lamports_unit: bool)
     for vote_account in vote_accounts.delinquent.into_iter() {
         print_vote_account(
             vote_account,
-            &epoch_schedule,
+            epoch_info.epoch,
             total_active_stake,
             use_lamports_unit,
             true,
@@ -873,92 +1190,99 @@ pub fn process_show_validators(rpc_client: &RpcClient, use_lamports_unit: bool)
 mod tests {
     use super::*;
     use crate::cli::{app, parse_command};
+    use solana_sdk::signature::{write_keypair, Keypair};
+    use tempfile::NamedTempFile;
+
+    fn make_tmp_file() -> (String, NamedTempFile) {
+        let tmp_file = NamedTempFile::new().unwrap();
+        (String::from(tmp_file.path().to_str().unwrap()), tmp_file)
+    }
 
     #[test]
     fn test_parse_command() {
         let test_commands = app("test", "desc", "version");
+        let default_keypair = Keypair::new();
+        let (default_keypair_file, mut tmp_file) = make_tmp_file();
+        write_keypair(&default_keypair, tmp_file.as_file_mut()).unwrap();
 
         let test_cluster_version = test_commands
             .clone()
             .get_matches_from(vec!["test", "cluster-version"]);
         assert_eq!(
-            parse_command(&test_cluster_version).unwrap(),
+            parse_command(&test_cluster_version, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::ClusterVersion,
-                require_keypair: false
+                signers: vec![],
             }
         );
 
         let test_fees = test_commands.clone().get_matches_from(vec!["test", "fees"]);
         assert_eq!(
-            parse_command(&test_fees).unwrap(),
+            parse_command(&test_fees, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::Fees,
-                require_keypair: false
+                signers: vec![],
             }
         );
 
         let slot = 100;
-        let test_get_block_time = test_commands.clone().get_matches_from(vec![
-            "test",
-            "get-block-time",
-            &slot.to_string(),
-        ]);
+        let test_get_block_time =
+            test_commands
+                .clone()
+                .get_matches_from(vec!["test", "block-time", &slot.to_string()]);
         assert_eq!(
-            parse_command(&test_get_block_time).unwrap(),
+            parse_command(&test_get_block_time, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::GetBlockTime { slot },
-                require_keypair: false
+                signers: vec![],
             }
         );
 
         let test_get_epoch_info = test_commands
             .clone()
-            .get_matches_from(vec!["test", "get-epoch-info"]);
+            .get_matches_from(vec!["test", "epoch-info"]);
         assert_eq!(
-            parse_command(&test_get_epoch_info).unwrap(),
+            parse_command(&test_get_epoch_info, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::GetEpochInfo {
                     commitment_config: CommitmentConfig::recent(),
                 },
-                require_keypair: false
+                signers: vec![],
             }
         );
 
         let test_get_genesis_hash = test_commands
             .clone()
-            .get_matches_from(vec!["test", "get-genesis-hash"]);
+            .get_matches_from(vec!["test", "genesis-hash"]);
         assert_eq!(
-            parse_command(&test_get_genesis_hash).unwrap(),
+            parse_command(&test_get_genesis_hash, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::GetGenesisHash,
-                require_keypair: false
+                signers: vec![],
             }
         );
 
-        let test_get_slot = test_commands
-            .clone()
-            .get_matches_from(vec!["test", "get-slot"]);
+        let test_get_slot = test_commands.clone().get_matches_from(vec!["test", "slot"]);
         assert_eq!(
-            parse_command(&test_get_slot).unwrap(),
+            parse_command(&test_get_slot, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::GetSlot {
                     commitment_config: CommitmentConfig::recent(),
                 },
-                require_keypair: false
+                signers: vec![],
             }
         );
 
         let test_transaction_count = test_commands
             .clone()
-            .get_matches_from(vec!["test", "get-transaction-count"]);
+            .get_matches_from(vec!["test", "transaction-count"]);
         assert_eq!(
-            parse_command(&test_transaction_count).unwrap(),
+            parse_command(&test_transaction_count, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::GetTransactionCount {
                     commitment_config: CommitmentConfig::recent(),
                 },
-                require_keypair: false
+                signers: vec![],
             }
         );
 
@@ -974,7 +1298,7 @@ mod tests {
             "--confirmed",
         ]);
         assert_eq!(
-            parse_command(&test_ping).unwrap(),
+            parse_command(&test_ping, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::Ping {
                     lamports: 1,
@@ -983,7 +1307,7 @@ mod tests {
                     timeout: Duration::from_secs(3),
                     commitment_config: CommitmentConfig::default(),
                 },
-                require_keypair: true
+                signers: vec![default_keypair.into()],
             }
         );
     }

cli/src/config.rs

@@ -1,49 +0,0 @@
-// Wallet settings that can be configured for long-term use
-use serde_derive::{Deserialize, Serialize};
-use std::fs::{create_dir_all, File};
-use std::io::{self, Write};
-use std::path::Path;
-
-lazy_static! {
-    pub static ref CONFIG_FILE: Option<String> = {
-        dirs::home_dir().map(|mut path| {
-            path.extend(&[".config", "solana", "cli", "config.yml"]);
-            path.to_str().unwrap().to_string()
-        })
-    };
-}
-
-#[derive(Serialize, Deserialize, Default, Debug, PartialEq)]
-pub struct Config {
-    pub url: String,
-    pub keypair_path: String,
-}
-
-impl Config {
-    pub fn new(url: &str, keypair_path: &str) -> Self {
-        Self {
-            url: url.to_string(),
-            keypair_path: keypair_path.to_string(),
-        }
-    }
-
-    pub fn load(config_file: &str) -> Result<Self, io::Error> {
-        let file = File::open(config_file.to_string())?;
-        let config = serde_yaml::from_reader(file)
-            .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
-        Ok(config)
-    }
-
-    pub fn save(&self, config_file: &str) -> Result<(), io::Error> {
-        let serialized = serde_yaml::to_string(self)
-            .map_err(|err| io::Error::new(io::ErrorKind::Other, format!("{:?}", err)))?;
-
-        if let Some(outdir) = Path::new(&config_file).parent() {
-            create_dir_all(outdir)?;
-        }
-        let mut file = File::create(config_file)?;
-        file.write_all(&serialized.into_bytes())?;
-
-        Ok(())
-    }
-}

cli/src/display.rs

@@ -1,3 +1,4 @@
+use crate::cli::SettingType;
 use console::style;
 use solana_sdk::transaction::Transaction;
 
@@ -11,17 +12,19 @@ pub fn println_name_value(name: &str, value: &str) {
     println!("{} {}", style(name).bold(), styled_value);
 }
 
-pub fn println_name_value_or(name: &str, value: &str, default_value: &str) {
-    if value == "" {
-        println!(
-            "{} {} {}",
-            style(name).bold(),
-            style(default_value),
-            style("(default)").italic()
-        );
-    } else {
-        println!("{} {}", style(name).bold(), style(value));
+pub fn println_name_value_or(name: &str, value: &str, setting_type: SettingType) {
+    let description = match setting_type {
+        SettingType::Explicit => "",
+        SettingType::Computed => "(computed)",
+        SettingType::SystemDefault => "(default)",
     };
+
+    println!(
+        "{} {} {}",
+        style(name).bold(),
+        style(value),
+        style(description).italic(),
+    );
 }
 
 pub fn println_signers(tx: &Transaction) {

cli/src/lib.rs

@@ -1,11 +1,8 @@
-#[macro_use]
-extern crate lazy_static;
-
 pub mod cli;
 pub mod cluster_query;
-pub mod config;
 pub mod display;
 pub mod nonce;
+pub mod offline;
 pub mod stake;
 pub mod storage;
 pub mod validator_info;

cli/src/main.rs

@@ -1,146 +1,143 @@
-use clap::{crate_description, crate_name, Arg, ArgGroup, ArgMatches, SubCommand};
+use clap::{crate_description, crate_name, AppSettings, Arg, ArgGroup, ArgMatches, SubCommand};
 use console::style;
 
 use solana_clap_utils::{
-    input_validators::is_url,
-    keypair::{
-        self, keypair_input, KeypairWithSource, ASK_SEED_PHRASE_ARG,
-        SKIP_SEED_PHRASE_VALIDATION_ARG,
-    },
+    input_validators::is_url, keypair::SKIP_SEED_PHRASE_VALIDATION_ARG, DisplayError,
 };
 use solana_cli::{
-    cli::{app, parse_command, process_command, CliCommandInfo, CliConfig, CliError},
-    config::{self, Config},
+    cli::{app, parse_command, process_command, CliCommandInfo, CliConfig, CliSigners},
     display::{println_name_value, println_name_value_or},
 };
-use solana_sdk::signature::read_keypair_file;
-
-use std::error;
+use solana_cli_config::{Config, CONFIG_FILE};
+use solana_remote_wallet::remote_wallet::{maybe_wallet_manager, RemoteWalletManager};
+use std::{error, sync::Arc};
 
 fn parse_settings(matches: &ArgMatches<'_>) -> Result<bool, Box<dyn error::Error>> {
     let parse_args = match matches.subcommand() {
-        ("get", Some(subcommand_matches)) => {
-            if let Some(config_file) = matches.value_of("config_file") {
-                let config = Config::load(config_file).unwrap_or_default();
-                if let Some(field) = subcommand_matches.value_of("specific_setting") {
-                    let (value, default_value) = match field {
-                        "url" => (config.url, CliConfig::default_json_rpc_url()),
-                        "keypair" => (config.keypair_path, CliConfig::default_keypair_path()),
-                        _ => unreachable!(),
-                    };
-                    println_name_value_or(&format!("* {}:", field), &value, &default_value);
+        ("config", Some(matches)) => match matches.subcommand() {
+            ("get", Some(subcommand_matches)) => {
+                if let Some(config_file) = matches.value_of("config_file") {
+                    let config = Config::load(config_file).unwrap_or_default();
+
+                    let (url_setting_type, json_rpc_url) =
+                        CliConfig::compute_json_rpc_url_setting("", &config.json_rpc_url);
+                    let (ws_setting_type, websocket_url) = CliConfig::compute_websocket_url_setting(
+                        "",
+                        &config.websocket_url,
+                        "",
+                        &config.json_rpc_url,
+                    );
+                    let (keypair_setting_type, keypair_path) =
+                        CliConfig::compute_keypair_path_setting("", &config.keypair_path);
+
+                    if let Some(field) = subcommand_matches.value_of("specific_setting") {
+                        let (field_name, value, setting_type) = match field {
+                            "json_rpc_url" => ("RPC URL", json_rpc_url, url_setting_type),
+                            "websocket_url" => ("WS URL", websocket_url, ws_setting_type),
+                            "keypair" => ("Key Path", keypair_path, keypair_setting_type),
+                            _ => unreachable!(),
+                        };
+                        println_name_value_or(&format!("{}:", field_name), &value, setting_type);
+                    } else {
+                        println_name_value("Config File:", config_file);
+                        println_name_value_or("RPC URL:", &json_rpc_url, url_setting_type);
+                        println_name_value_or("WS URL:", &websocket_url, ws_setting_type);
+                        println_name_value_or("Keypair Path:", &keypair_path, keypair_setting_type);
+                    }
                 } else {
-                    println_name_value("Wallet Config:", config_file);
-                    println_name_value_or(
-                        "* url:",
-                        &config.url,
-                        &CliConfig::default_json_rpc_url(),
-                    );
-                    println_name_value_or(
-                        "* keypair:",
-                        &config.keypair_path,
-                        &CliConfig::default_keypair_path(),
+                    println!(
+                        "{} Either provide the `--config` arg or ensure home directory exists to use the default config location",
+                        style("No config file found.").bold()
                     );
                 }
-            } else {
-                println!(
-                    "{} Either provide the `--config` arg or ensure home directory exists to use the default config location",
-                    style("No config file found.").bold()
-                );
+                false
             }
-            false
-        }
-        ("set", Some(subcommand_matches)) => {
-            if let Some(config_file) = matches.value_of("config_file") {
-                let mut config = Config::load(config_file).unwrap_or_default();
-                if let Some(url) = subcommand_matches.value_of("json_rpc_url") {
-                    config.url = url.to_string();
+            ("set", Some(subcommand_matches)) => {
+                if let Some(config_file) = matches.value_of("config_file") {
+                    let mut config = Config::load(config_file).unwrap_or_default();
+                    if let Some(url) = subcommand_matches.value_of("json_rpc_url") {
+                        config.json_rpc_url = url.to_string();
+                        // Revert to a computed `websocket_url` value when `json_rpc_url` is
+                        // changed
+                        config.websocket_url = "".to_string();
+                    }
+                    if let Some(url) = subcommand_matches.value_of("websocket_url") {
+                        config.websocket_url = url.to_string();
+                    }
+                    if let Some(keypair) = subcommand_matches.value_of("keypair") {
+                        config.keypair_path = keypair.to_string();
+                    }
+                    config.save(config_file)?;
+
+                    let (url_setting_type, json_rpc_url) =
+                        CliConfig::compute_json_rpc_url_setting("", &config.json_rpc_url);
+                    let (ws_setting_type, websocket_url) = CliConfig::compute_websocket_url_setting(
+                        "",
+                        &config.websocket_url,
+                        "",
+                        &config.json_rpc_url,
+                    );
+                    let (keypair_setting_type, keypair_path) =
+                        CliConfig::compute_keypair_path_setting("", &config.keypair_path);
+
+                    println_name_value("Config File:", config_file);
+                    println_name_value_or("RPC URL:", &json_rpc_url, url_setting_type);
+                    println_name_value_or("WS URL:", &websocket_url, ws_setting_type);
+                    println_name_value_or("Keypair Path:", &keypair_path, keypair_setting_type);
+                } else {
+                    println!(
+                        "{} Either provide the `--config` arg or ensure home directory exists to use the default config location",
+                        style("No config file found.").bold()
+                    );
                 }
-                if let Some(keypair) = subcommand_matches.value_of("keypair") {
-                    config.keypair_path = keypair.to_string();
-                }
-                config.save(config_file)?;
-                println_name_value("Wallet Config Updated:", config_file);
-                println_name_value("* url:", &config.url);
-                println_name_value("* keypair:", &config.keypair_path);
-            } else {
-                println!(
-                    "{} Either provide the `--config` arg or ensure home directory exists to use the default config location",
-                    style("No config file found.").bold()
-                );
+                false
             }
-            false
-        }
+            _ => unreachable!(),
+        },
         _ => true,
     };
     Ok(parse_args)
 }
 
-pub fn parse_args(matches: &ArgMatches<'_>) -> Result<CliConfig, Box<dyn error::Error>> {
+pub fn parse_args<'a>(
+    matches: &ArgMatches<'_>,
+    wallet_manager: Option<Arc<RemoteWalletManager>>,
+) -> Result<(CliConfig<'a>, CliSigners), Box<dyn error::Error>> {
     let config = if let Some(config_file) = matches.value_of("config_file") {
         Config::load(config_file).unwrap_or_default()
     } else {
         Config::default()
     };
-    let json_rpc_url = if let Some(url) = matches.value_of("json_rpc_url") {
-        url.to_string()
-    } else if config.url != "" {
-        config.url
-    } else {
-        let default = CliConfig::default();
-        default.json_rpc_url
-    };
+    let (_, json_rpc_url) = CliConfig::compute_json_rpc_url_setting(
+        matches.value_of("json_rpc_url").unwrap_or(""),
+        &config.json_rpc_url,
+    );
+    let (_, websocket_url) = CliConfig::compute_websocket_url_setting(
+        matches.value_of("websocket_url").unwrap_or(""),
+        &config.websocket_url,
+        matches.value_of("json_rpc_url").unwrap_or(""),
+        &config.json_rpc_url,
+    );
+    let (_, default_signer_path) = CliConfig::compute_keypair_path_setting(
+        matches.value_of("keypair").unwrap_or(""),
+        &config.keypair_path,
+    );
 
-    let CliCommandInfo {
-        command,
-        require_keypair,
-    } = parse_command(&matches)?;
+    let CliCommandInfo { command, signers } =
+        parse_command(&matches, &default_signer_path, wallet_manager.as_ref())?;
 
-    let (keypair, keypair_path) = if require_keypair {
-        let KeypairWithSource { keypair, source } = keypair_input(&matches, "keypair")?;
-        match source {
-            keypair::Source::File => (
-                keypair,
-                Some(matches.value_of("keypair").unwrap().to_string()),
-            ),
-            keypair::Source::SeedPhrase => (keypair, None),
-            keypair::Source::Generated => {
-                let keypair_path = if config.keypair_path != "" {
-                    config.keypair_path
-                } else {
-                    let default_keypair_path = CliConfig::default_keypair_path();
-                    if !std::path::Path::new(&default_keypair_path).exists() {
-                        return Err(CliError::KeypairFileNotFound(
-                            "Generate a new keypair with `solana-keygen new`".to_string(),
-                        )
-                        .into());
-                    }
-                    default_keypair_path
-                };
-
-                let keypair = read_keypair_file(&keypair_path).or_else(|err| {
-                    Err(CliError::BadParameter(format!(
-                        "{}: Unable to open keypair file: {}",
-                        err, keypair_path
-                    )))
-                })?;
-
-                (keypair, Some(keypair_path))
-            }
-        }
-    } else {
-        let default = CliConfig::default();
-        (default.keypair, None)
-    };
-
-    Ok(CliConfig {
-        command,
-        json_rpc_url,
-        keypair,
-        keypair_path,
-        rpc_client: None,
-        verbose: matches.is_present("verbose"),
-    })
+    Ok((
+        CliConfig {
+            command,
+            json_rpc_url,
+            websocket_url,
+            signers: vec![],
+            keypair_path: default_signer_path,
+            rpc_client: None,
+            verbose: matches.is_present("verbose"),
+        },
+        signers,
+    ))
 }
 
 fn main() -> Result<(), Box<dyn error::Error>> {
@@ -158,7 +155,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
             .takes_value(true)
             .global(true)
             .help("Configuration file to use");
-        if let Some(ref config_file) = *config::CONFIG_FILE {
+        if let Some(ref config_file) = *CONFIG_FILE {
             arg.default_value(&config_file)
         } else {
             arg
@@ -174,6 +171,15 @@ fn main() -> Result<(), Box<dyn error::Error>> {
             .validator(is_url)
             .help("JSON RPC URL for the solana cluster"),
     )
+    .arg(
+        Arg::with_name("websocket_url")
+            .long("ws")
+            .value_name("URL")
+            .takes_value(true)
+            .global(true)
+            .validator(is_url)
+            .help("WebSocket URL for the solana cluster"),
+    )
     .arg(
         Arg::with_name("keypair")
             .short("k")
@@ -181,7 +187,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
             .value_name("PATH")
             .global(true)
             .takes_value(true)
-            .help("/path/to/id.json"),
+            .help("/path/to/id.json or usb://remote/wallet/path"),
     )
     .arg(
         Arg::with_name("verbose")
@@ -190,15 +196,6 @@ fn main() -> Result<(), Box<dyn error::Error>> {
             .global(true)
             .help("Show extra information header"),
     )
-    .arg(
-        Arg::with_name(ASK_SEED_PHRASE_ARG.name)
-            .long(ASK_SEED_PHRASE_ARG.long)
-            .value_name("KEYPAIR NAME")
-            .global(true)
-            .takes_value(true)
-            .possible_values(&["keypair"])
-            .help(ASK_SEED_PHRASE_ARG.help),
-    )
     .arg(
         Arg::with_name(SKIP_SEED_PHRASE_VALIDATION_ARG.name)
             .long(SKIP_SEED_PHRASE_VALIDATION_ARG.long)
@@ -206,33 +203,46 @@ fn main() -> Result<(), Box<dyn error::Error>> {
             .help(SKIP_SEED_PHRASE_VALIDATION_ARG.help),
     )
     .subcommand(
-        SubCommand::with_name("get")
-            .about("Get cli config settings")
-            .arg(
-                Arg::with_name("specific_setting")
-                    .index(1)
-                    .value_name("CONFIG_FIELD")
-                    .takes_value(true)
-                    .possible_values(&["url", "keypair"])
-                    .help("Return a specific config setting"),
-            ),
-    )
-    .subcommand(
-        SubCommand::with_name("set")
-            .about("Set a cli config setting")
-            .group(
-                ArgGroup::with_name("config_settings")
-                    .args(&["json_rpc_url", "keypair"])
-                    .multiple(true)
-                    .required(true),
+        SubCommand::with_name("config")
+            .about("Solana command-line tool configuration settings")
+            .aliases(&["get", "set"])
+            .setting(AppSettings::SubcommandRequiredElseHelp)
+            .subcommand(
+                SubCommand::with_name("get")
+                    .about("Get current config settings")
+                    .arg(
+                        Arg::with_name("specific_setting")
+                            .index(1)
+                            .value_name("CONFIG_FIELD")
+                            .takes_value(true)
+                            .possible_values(&["json_rpc_url", "websocket_url", "keypair"])
+                            .help("Return a specific config setting"),
+                    ),
+            )
+            .subcommand(
+                SubCommand::with_name("set")
+                    .about("Set a config setting")
+                    .group(
+                        ArgGroup::with_name("config_settings")
+                            .args(&["json_rpc_url", "websocket_url", "keypair"])
+                            .multiple(true)
+                            .required(true),
+                    ),
             ),
     )
     .get_matches();
 
+    do_main(&matches).map_err(|err| DisplayError::new_as_boxed(err).into())
+}
+
+fn do_main(matches: &ArgMatches<'_>) -> Result<(), Box<dyn error::Error>> {
     if parse_settings(&matches)? {
-        let config = parse_args(&matches)?;
+        let wallet_manager = maybe_wallet_manager()?;
+
+        let (mut config, signers) = parse_args(&matches, wallet_manager)?;
+        config.signers = signers.iter().map(|s| s.as_ref()).collect();
         let result = process_command(&config)?;
         println!("{}", result);
-    }
+    };
     Ok(())
 }

cli/src/offline/blockhash_query.rs

@@ -0,0 +1,394 @@
+use super::*;
+
+#[derive(Debug, PartialEq)]
+pub enum Source {
+    Cluster,
+    NonceAccount(Pubkey),
+}
+
+impl Source {
+    pub fn get_blockhash_and_fee_calculator(
+        &self,
+        rpc_client: &RpcClient,
+    ) -> Result<(Hash, FeeCalculator), Box<dyn std::error::Error>> {
+        match self {
+            Self::Cluster => {
+                let res = rpc_client.get_recent_blockhash()?;
+                Ok(res)
+            }
+            Self::NonceAccount(ref pubkey) => {
+                let data = nonce::get_account(rpc_client, pubkey)
+                    .and_then(|ref a| nonce::data_from_account(a))?;
+                Ok((data.blockhash, data.fee_calculator))
+            }
+        }
+    }
+
+    pub fn get_fee_calculator(
+        &self,
+        rpc_client: &RpcClient,
+        blockhash: &Hash,
+    ) -> Result<Option<FeeCalculator>, Box<dyn std::error::Error>> {
+        match self {
+            Self::Cluster => {
+                let res = rpc_client.get_fee_calculator_for_blockhash(blockhash)?;
+                Ok(res)
+            }
+            Self::NonceAccount(ref pubkey) => {
+                let res = nonce::get_account(rpc_client, pubkey)
+                    .and_then(|ref a| nonce::data_from_account(a))
+                    .and_then(|d| {
+                        if d.blockhash == *blockhash {
+                            Ok(Some(d.fee_calculator))
+                        } else {
+                            Ok(None)
+                        }
+                    })?;
+                Ok(res)
+            }
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum BlockhashQuery {
+    None(Hash),
+    FeeCalculator(Source, Hash),
+    All(Source),
+}
+
+impl BlockhashQuery {
+    pub fn new(blockhash: Option<Hash>, sign_only: bool, nonce_account: Option<Pubkey>) -> Self {
+        let source = nonce_account
+            .map(Source::NonceAccount)
+            .unwrap_or(Source::Cluster);
+        match blockhash {
+            Some(hash) if sign_only => Self::None(hash),
+            Some(hash) if !sign_only => Self::FeeCalculator(source, hash),
+            None if !sign_only => Self::All(source),
+            _ => panic!("Cannot resolve blockhash"),
+        }
+    }
+
+    pub fn new_from_matches(matches: &ArgMatches<'_>) -> Self {
+        let blockhash = value_of(matches, BLOCKHASH_ARG.name);
+        let sign_only = matches.is_present(SIGN_ONLY_ARG.name);
+        let nonce_account = pubkey_of(matches, nonce::NONCE_ARG.name);
+        BlockhashQuery::new(blockhash, sign_only, nonce_account)
+    }
+
+    pub fn get_blockhash_and_fee_calculator(
+        &self,
+        rpc_client: &RpcClient,
+    ) -> Result<(Hash, FeeCalculator), Box<dyn std::error::Error>> {
+        match self {
+            BlockhashQuery::None(hash) => Ok((*hash, FeeCalculator::default())),
+            BlockhashQuery::FeeCalculator(source, hash) => {
+                let fee_calculator = source
+                    .get_fee_calculator(rpc_client, hash)?
+                    .ok_or(format!("Hash has expired {:?}", hash))?;
+                Ok((*hash, fee_calculator))
+            }
+            BlockhashQuery::All(source) => source.get_blockhash_and_fee_calculator(rpc_client),
+        }
+    }
+}
+
+impl Default for BlockhashQuery {
+    fn default() -> Self {
+        BlockhashQuery::All(Source::Cluster)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{nonce::nonce_arg, offline::blockhash_query::BlockhashQuery};
+    use clap::App;
+    use serde_json::{self, json, Value};
+    use solana_client::{
+        rpc_request::RpcRequest,
+        rpc_response::{Response, RpcAccount, RpcFeeCalculator, RpcResponseContext},
+    };
+    use solana_sdk::{
+        account::Account, fee_calculator::FeeCalculator, hash::hash, nonce, system_program,
+    };
+    use std::collections::HashMap;
+
+    #[test]
+    fn test_blockhash_query_new_ok() {
+        let blockhash = hash(&[1u8]);
+        let nonce_pubkey = Pubkey::new(&[1u8; 32]);
+
+        assert_eq!(
+            BlockhashQuery::new(Some(blockhash), true, None),
+            BlockhashQuery::None(blockhash),
+        );
+        assert_eq!(
+            BlockhashQuery::new(Some(blockhash), false, None),
+            BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
+        );
+        assert_eq!(
+            BlockhashQuery::new(None, false, None),
+            BlockhashQuery::All(blockhash_query::Source::Cluster)
+        );
+
+        assert_eq!(
+            BlockhashQuery::new(Some(blockhash), true, Some(nonce_pubkey)),
+            BlockhashQuery::None(blockhash),
+        );
+        assert_eq!(
+            BlockhashQuery::new(Some(blockhash), false, Some(nonce_pubkey)),
+            BlockhashQuery::FeeCalculator(
+                blockhash_query::Source::NonceAccount(nonce_pubkey),
+                blockhash
+            ),
+        );
+        assert_eq!(
+            BlockhashQuery::new(None, false, Some(nonce_pubkey)),
+            BlockhashQuery::All(blockhash_query::Source::NonceAccount(nonce_pubkey)),
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_blockhash_query_new_no_nonce_fail() {
+        BlockhashQuery::new(None, true, None);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_blockhash_query_new_nonce_fail() {
+        let nonce_pubkey = Pubkey::new(&[1u8; 32]);
+        BlockhashQuery::new(None, true, Some(nonce_pubkey));
+    }
+
+    #[test]
+    fn test_blockhash_query_new_from_matches_ok() {
+        let test_commands = App::new("blockhash_query_test")
+            .arg(nonce_arg())
+            .offline_args();
+        let blockhash = hash(&[1u8]);
+        let blockhash_string = blockhash.to_string();
+
+        let matches = test_commands.clone().get_matches_from(vec![
+            "blockhash_query_test",
+            "--blockhash",
+            &blockhash_string,
+            "--sign-only",
+        ]);
+        assert_eq!(
+            BlockhashQuery::new_from_matches(&matches),
+            BlockhashQuery::None(blockhash),
+        );
+
+        let matches = test_commands.clone().get_matches_from(vec![
+            "blockhash_query_test",
+            "--blockhash",
+            &blockhash_string,
+        ]);
+        assert_eq!(
+            BlockhashQuery::new_from_matches(&matches),
+            BlockhashQuery::FeeCalculator(blockhash_query::Source::Cluster, blockhash),
+        );
+
+        let matches = test_commands
+            .clone()
+            .get_matches_from(vec!["blockhash_query_test"]);
+        assert_eq!(
+            BlockhashQuery::new_from_matches(&matches),
+            BlockhashQuery::All(blockhash_query::Source::Cluster),
+        );
+
+        let nonce_pubkey = Pubkey::new(&[1u8; 32]);
+        let nonce_string = nonce_pubkey.to_string();
+        let matches = test_commands.clone().get_matches_from(vec![
+            "blockhash_query_test",
+            "--blockhash",
+            &blockhash_string,
+            "--sign-only",
+            "--nonce",
+            &nonce_string,
+        ]);
+        assert_eq!(
+            BlockhashQuery::new_from_matches(&matches),
+            BlockhashQuery::None(blockhash),
+        );
+
+        let matches = test_commands.clone().get_matches_from(vec![
+            "blockhash_query_test",
+            "--blockhash",
+            &blockhash_string,
+            "--nonce",
+            &nonce_string,
+        ]);
+        assert_eq!(
+            BlockhashQuery::new_from_matches(&matches),
+            BlockhashQuery::FeeCalculator(
+                blockhash_query::Source::NonceAccount(nonce_pubkey),
+                blockhash
+            ),
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_blockhash_query_new_from_matches_without_nonce_fail() {
+        let test_commands = App::new("blockhash_query_test")
+            .arg(blockhash_arg())
+            // We can really only hit this case if the arg requirements
+            // are broken, so unset the requires() to recreate that condition
+            .arg(sign_only_arg().requires(""));
+
+        let matches = test_commands
+            .clone()
+            .get_matches_from(vec!["blockhash_query_test", "--sign-only"]);
+        BlockhashQuery::new_from_matches(&matches);
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_blockhash_query_new_from_matches_with_nonce_fail() {
+        let test_commands = App::new("blockhash_query_test")
+            .arg(blockhash_arg())
+            // We can really only hit this case if the arg requirements
+            // are broken, so unset the requires() to recreate that condition
+            .arg(sign_only_arg().requires(""));
+        let nonce_pubkey = Pubkey::new(&[1u8; 32]);
+        let nonce_string = nonce_pubkey.to_string();
+
+        let matches = test_commands.clone().get_matches_from(vec![
+            "blockhash_query_test",
+            "--sign-only",
+            "--nonce",
+            &nonce_string,
+        ]);
+        BlockhashQuery::new_from_matches(&matches);
+    }
+
+    #[test]
+    fn test_blockhash_query_get_blockhash_fee_calc() {
+        let test_blockhash = hash(&[0u8]);
+        let rpc_blockhash = hash(&[1u8]);
+        let rpc_fee_calc = FeeCalculator::new(42);
+        let get_recent_blockhash_response = json!(Response {
+            context: RpcResponseContext { slot: 1 },
+            value: json!((
+                Value::String(rpc_blockhash.to_string()),
+                serde_json::to_value(rpc_fee_calc.clone()).unwrap()
+            )),
+        });
+        let get_fee_calculator_for_blockhash_response = json!(Response {
+            context: RpcResponseContext { slot: 1 },
+            value: json!(RpcFeeCalculator {
+                fee_calculator: rpc_fee_calc.clone()
+            }),
+        });
+        let mut mocks = HashMap::new();
+        mocks.insert(
+            RpcRequest::GetRecentBlockhash,
+            get_recent_blockhash_response.clone(),
+        );
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert_eq!(
+            BlockhashQuery::default()
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .unwrap(),
+            (rpc_blockhash, rpc_fee_calc.clone()),
+        );
+        let mut mocks = HashMap::new();
+        mocks.insert(
+            RpcRequest::GetRecentBlockhash,
+            get_recent_blockhash_response.clone(),
+        );
+        mocks.insert(
+            RpcRequest::GetFeeCalculatorForBlockhash,
+            get_fee_calculator_for_blockhash_response.clone(),
+        );
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert_eq!(
+            BlockhashQuery::FeeCalculator(Source::Cluster, test_blockhash)
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .unwrap(),
+            (test_blockhash, rpc_fee_calc.clone()),
+        );
+        let mut mocks = HashMap::new();
+        mocks.insert(
+            RpcRequest::GetRecentBlockhash,
+            get_recent_blockhash_response.clone(),
+        );
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert_eq!(
+            BlockhashQuery::None(test_blockhash)
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .unwrap(),
+            (test_blockhash, FeeCalculator::default()),
+        );
+        let rpc_client = RpcClient::new_mock("fails".to_string());
+        assert!(BlockhashQuery::default()
+            .get_blockhash_and_fee_calculator(&rpc_client)
+            .is_err());
+
+        let nonce_blockhash = Hash::new(&[2u8; 32]);
+        let nonce_fee_calc = FeeCalculator::new(4242);
+        let data = nonce::state::Data {
+            authority: Pubkey::new(&[3u8; 32]),
+            blockhash: nonce_blockhash,
+            fee_calculator: nonce_fee_calc.clone(),
+        };
+        let nonce_account = Account::new_data_with_space(
+            42,
+            &nonce::state::Versions::new_current(nonce::State::Initialized(data)),
+            nonce::State::size(),
+            &system_program::id(),
+        )
+        .unwrap();
+        let nonce_pubkey = Pubkey::new(&[4u8; 32]);
+        let rpc_nonce_account = RpcAccount::encode(nonce_account);
+        let get_account_response = json!(Response {
+            context: RpcResponseContext { slot: 1 },
+            value: json!(Some(rpc_nonce_account.clone())),
+        });
+
+        let mut mocks = HashMap::new();
+        mocks.insert(RpcRequest::GetAccountInfo, get_account_response.clone());
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert_eq!(
+            BlockhashQuery::All(Source::NonceAccount(nonce_pubkey))
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .unwrap(),
+            (nonce_blockhash, nonce_fee_calc.clone()),
+        );
+        let mut mocks = HashMap::new();
+        mocks.insert(RpcRequest::GetAccountInfo, get_account_response.clone());
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert_eq!(
+            BlockhashQuery::FeeCalculator(Source::NonceAccount(nonce_pubkey), nonce_blockhash)
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .unwrap(),
+            (nonce_blockhash, nonce_fee_calc.clone()),
+        );
+        let mut mocks = HashMap::new();
+        mocks.insert(RpcRequest::GetAccountInfo, get_account_response.clone());
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert!(
+            BlockhashQuery::FeeCalculator(Source::NonceAccount(nonce_pubkey), test_blockhash)
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .is_err()
+        );
+        let mut mocks = HashMap::new();
+        mocks.insert(RpcRequest::GetAccountInfo, get_account_response.clone());
+        let rpc_client = RpcClient::new_mock_with_mocks("".to_string(), mocks);
+        assert_eq!(
+            BlockhashQuery::None(nonce_blockhash)
+                .get_blockhash_and_fee_calculator(&rpc_client)
+                .unwrap(),
+            (nonce_blockhash, FeeCalculator::default()),
+        );
+
+        let rpc_client = RpcClient::new_mock("fails".to_string());
+        assert!(BlockhashQuery::All(Source::NonceAccount(nonce_pubkey))
+            .get_blockhash_and_fee_calculator(&rpc_client)
+            .is_err());
+    }
+}

cli/src/offline/mod.rs

@@ -0,0 +1,70 @@
+pub mod blockhash_query;
+
+use crate::nonce;
+use clap::{App, Arg, ArgMatches};
+use serde_json::Value;
+use solana_clap_utils::{
+    input_parsers::{pubkey_of, value_of},
+    input_validators::{is_hash, is_pubkey_sig},
+    offline::{BLOCKHASH_ARG, SIGNER_ARG, SIGN_ONLY_ARG},
+};
+use solana_client::rpc_client::RpcClient;
+use solana_sdk::{fee_calculator::FeeCalculator, hash::Hash, pubkey::Pubkey, signature::Signature};
+use std::str::FromStr;
+
+fn blockhash_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(BLOCKHASH_ARG.name)
+        .long(BLOCKHASH_ARG.long)
+        .takes_value(true)
+        .value_name("BLOCKHASH")
+        .validator(is_hash)
+        .help(BLOCKHASH_ARG.help)
+}
+
+fn sign_only_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(SIGN_ONLY_ARG.name)
+        .long(SIGN_ONLY_ARG.long)
+        .takes_value(false)
+        .requires(BLOCKHASH_ARG.name)
+        .help(SIGN_ONLY_ARG.help)
+}
+
+fn signer_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(SIGNER_ARG.name)
+        .long(SIGNER_ARG.long)
+        .takes_value(true)
+        .value_name("BASE58_PUBKEY=BASE58_SIG")
+        .validator(is_pubkey_sig)
+        .requires(BLOCKHASH_ARG.name)
+        .multiple(true)
+        .help(SIGNER_ARG.help)
+}
+
+pub trait OfflineArgs {
+    fn offline_args(self) -> Self;
+}
+
+impl OfflineArgs for App<'_, '_> {
+    fn offline_args(self) -> Self {
+        self.arg(blockhash_arg())
+            .arg(sign_only_arg())
+            .arg(signer_arg())
+    }
+}
+
+pub fn parse_sign_only_reply_string(reply: &str) -> (Hash, Vec<(Pubkey, Signature)>) {
+    let object: Value = serde_json::from_str(&reply).unwrap();
+    let blockhash_str = object.get("blockhash").unwrap().as_str().unwrap();
+    let blockhash = blockhash_str.parse::<Hash>().unwrap();
+    let signer_strings = object.get("signers").unwrap().as_array().unwrap();
+    let signers = signer_strings
+        .iter()
+        .map(|signer_string| {
+            let mut signer = signer_string.as_str().unwrap().split('=');
+            let key = Pubkey::from_str(signer.next().unwrap()).unwrap();
+            let sig = Signature::from_str(signer.next().unwrap()).unwrap();
+            (key, sig)
+        })
+        .collect();
+    (blockhash, signers)
+}

cli/src/storage.rs

@@ -1,16 +1,18 @@
 use crate::cli::{
-    check_account_for_fee, check_unique_pubkeys, log_instruction_custom_error, CliCommand,
-    CliCommandInfo, CliConfig, CliError, ProcessResult,
+    check_account_for_fee, check_unique_pubkeys, generate_unique_signers,
+    log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError, ProcessResult,
+    SignerIndex,
 };
 use clap::{App, Arg, ArgMatches, SubCommand};
-use solana_clap_utils::{input_parsers::*, input_validators::*};
+use solana_clap_utils::{input_parsers::*, input_validators::*, keypair::signer_from_path};
 use solana_client::rpc_client::RpcClient;
-use solana_sdk::signature::Keypair;
+use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
-    account_utils::State, message::Message, pubkey::Pubkey, signature::KeypairUtil,
-    system_instruction::SystemError, transaction::Transaction,
+    account_utils::StateMut, message::Message, pubkey::Pubkey, system_instruction::SystemError,
+    transaction::Transaction,
 };
 use solana_storage_program::storage_instruction::{self, StorageAccountType};
+use std::sync::Arc;
 
 pub trait StorageSubCommands {
     fn storage_subcommands(self) -> Self;
@@ -35,7 +37,7 @@ impl StorageSubCommands for App<'_, '_> {
                         .value_name("STORAGE ACCOUNT")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_keypair_or_ask_keyword),
+                        .validator(is_valid_signer),
                 ),
         )
         .subcommand(
@@ -55,7 +57,7 @@ impl StorageSubCommands for App<'_, '_> {
                         .value_name("STORAGE ACCOUNT")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_keypair_or_ask_keyword),
+                        .validator(is_valid_signer),
                 ),
         )
         .subcommand(
@@ -81,8 +83,9 @@ impl StorageSubCommands for App<'_, '_> {
                 ),
         )
         .subcommand(
-            SubCommand::with_name("show-storage-account")
+            SubCommand::with_name("storage-account")
                 .about("Show the contents of a storage account")
+                .alias("show-storage-account")
                 .arg(
                     Arg::with_name("storage_account_pubkey")
                         .index(1)
@@ -98,35 +101,63 @@ impl StorageSubCommands for App<'_, '_> {
 
 pub fn parse_storage_create_archiver_account(
     matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
     let account_owner = pubkey_of(matches, "storage_account_owner").unwrap();
-    let storage_account = keypair_of(matches, "storage_account").unwrap();
+    let (storage_account, storage_account_pubkey) =
+        signer_of(matches, "storage_account", wallet_manager)?;
+
+    let payer_provided = None;
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, storage_account],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
+
     Ok(CliCommandInfo {
         command: CliCommand::CreateStorageAccount {
             account_owner,
-            storage_account: storage_account.into(),
+            storage_account: signer_info.index_of(storage_account_pubkey).unwrap(),
             account_type: StorageAccountType::Archiver,
         },
-        require_keypair: true,
+        signers: signer_info.signers,
     })
 }
 
 pub fn parse_storage_create_validator_account(
     matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
 ) -> Result<CliCommandInfo, CliError> {
     let account_owner = pubkey_of(matches, "storage_account_owner").unwrap();
-    let storage_account = keypair_of(matches, "storage_account").unwrap();
+    let (storage_account, storage_account_pubkey) =
+        signer_of(matches, "storage_account", wallet_manager)?;
+
+    let payer_provided = None;
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, storage_account],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
+
     Ok(CliCommandInfo {
         command: CliCommand::CreateStorageAccount {
             account_owner,
-            storage_account: storage_account.into(),
+            storage_account: signer_info.index_of(storage_account_pubkey).unwrap(),
             account_type: StorageAccountType::Validator,
         },
-        require_keypair: true,
+        signers: signer_info.signers,
     })
 }
 
-pub fn parse_storage_claim_reward(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+pub fn parse_storage_claim_reward(
+    matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
     let node_account_pubkey = pubkey_of(matches, "node_account_pubkey").unwrap();
     let storage_account_pubkey = pubkey_of(matches, "storage_account_pubkey").unwrap();
     Ok(CliCommandInfo {
@@ -134,7 +165,12 @@ pub fn parse_storage_claim_reward(matches: &ArgMatches<'_>) -> Result<CliCommand
             node_account_pubkey,
             storage_account_pubkey,
         },
-        require_keypair: true,
+        signers: vec![signer_from_path(
+            matches,
+            default_signer_path,
+            "keypair",
+            wallet_manager,
+        )?],
     })
 }
 
@@ -144,32 +180,46 @@ pub fn parse_storage_get_account_command(
     let storage_account_pubkey = pubkey_of(matches, "storage_account_pubkey").unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::ShowStorageAccount(storage_account_pubkey),
-        require_keypair: false,
+        signers: vec![],
     })
 }
 
 pub fn process_create_storage_account(
     rpc_client: &RpcClient,
     config: &CliConfig,
+    storage_account: SignerIndex,
     account_owner: &Pubkey,
-    storage_account: &Keypair,
     account_type: StorageAccountType,
 ) -> ProcessResult {
+    let storage_account = config.signers[storage_account];
     let storage_account_pubkey = storage_account.pubkey();
     check_unique_pubkeys(
-        (&config.keypair.pubkey(), "cli keypair".to_string()),
+        (&config.signers[0].pubkey(), "cli keypair".to_string()),
         (
             &storage_account_pubkey,
             "storage_account_pubkey".to_string(),
         ),
     )?;
+
+    if let Ok(storage_account) = rpc_client.get_account(&storage_account_pubkey) {
+        let err_msg = if storage_account.owner == solana_storage_program::id() {
+            format!("Storage account {} already exists", storage_account_pubkey)
+        } else {
+            format!(
+                "Account {} already exists and is not a storage account",
+                storage_account_pubkey
+            )
+        };
+        return Err(CliError::BadParameter(err_msg).into());
+    }
+
     use solana_storage_program::storage_contract::STORAGE_ACCOUNT_SPACE;
     let required_balance = rpc_client
         .get_minimum_balance_for_rent_exemption(STORAGE_ACCOUNT_SPACE as usize)?
         .max(1);
 
     let ixs = storage_instruction::create_storage_account(
-        &config.keypair.pubkey(),
+        &config.signers[0].pubkey(),
         &account_owner,
         &storage_account_pubkey,
         required_balance,
@@ -177,19 +227,16 @@ pub fn process_create_storage_account(
     );
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
 
-    let mut tx = Transaction::new_signed_instructions(
-        &[&config.keypair, &storage_account],
-        ixs,
-        recent_blockhash,
-    );
+    let message = Message::new(&ixs);
+    let mut tx = Transaction::new_unsigned(message);
+    tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
         rpc_client,
-        &config.keypair.pubkey(),
+        &config.signers[0].pubkey(),
         &fee_calculator,
         &tx.message,
     )?;
-    let result =
-        rpc_client.send_and_confirm_transaction(&mut tx, &[&config.keypair, &storage_account]);
+    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
     log_instruction_custom_error::<SystemError>(result)
 }
 
@@ -203,13 +250,13 @@ pub fn process_claim_storage_reward(
 
     let instruction =
         storage_instruction::claim_reward(node_account_pubkey, storage_account_pubkey);
-    let signers = [&config.keypair];
-    let message = Message::new_with_payer(vec![instruction], Some(&signers[0].pubkey()));
-
-    let mut tx = Transaction::new(&signers, message, recent_blockhash);
+    let signers = [config.signers[0]];
+    let message = Message::new_with_payer(&[instruction], Some(&signers[0].pubkey()));
+    let mut tx = Transaction::new_unsigned(message);
+    tx.try_sign(&signers, recent_blockhash)?;
     check_account_for_fee(
         rpc_client,
-        &config.keypair.pubkey(),
+        &config.signers[0].pubkey(),
         &fee_calculator,
         &tx.message,
     )?;
@@ -234,10 +281,10 @@ pub fn process_show_storage_account(
 
     use solana_storage_program::storage_contract::StorageContract;
     let storage_contract: StorageContract = account.state().map_err(|err| {
-        CliError::RpcRequestError(format!("Unable to deserialize storage account: {:?}", err))
+        CliError::RpcRequestError(format!("Unable to deserialize storage account: {}", err))
     })?;
     println!("{:#?}", storage_contract);
-    println!("account lamports: {}", account.lamports);
+    println!("Account Lamports: {}", account.lamports);
     Ok("".to_string())
 }
 
@@ -245,7 +292,7 @@ pub fn process_show_storage_account(
 mod tests {
     use super::*;
     use crate::cli::{app, parse_command};
-    use solana_sdk::signature::write_keypair;
+    use solana_sdk::signature::{read_keypair_file, write_keypair, Keypair, Signer};
     use tempfile::NamedTempFile;
 
     fn make_tmp_file() -> (String, NamedTempFile) {
@@ -259,6 +306,10 @@ mod tests {
         let pubkey = Pubkey::new_rand();
         let pubkey_string = pubkey.to_string();
 
+        let default_keypair = Keypair::new();
+        let (default_keypair_file, mut tmp_file) = make_tmp_file();
+        write_keypair(&default_keypair, tmp_file.as_file_mut()).unwrap();
+
         let (keypair_file, mut tmp_file) = make_tmp_file();
         let storage_account_keypair = Keypair::new();
         write_keypair(&storage_account_keypair, tmp_file.as_file_mut()).unwrap();
@@ -270,14 +321,22 @@ mod tests {
             &keypair_file,
         ]);
         assert_eq!(
-            parse_command(&test_create_archiver_storage_account).unwrap(),
+            parse_command(
+                &test_create_archiver_storage_account,
+                &default_keypair_file,
+                None
+            )
+            .unwrap(),
             CliCommandInfo {
                 command: CliCommand::CreateStorageAccount {
                     account_owner: pubkey,
-                    storage_account: storage_account_keypair.into(),
+                    storage_account: 1,
                     account_type: StorageAccountType::Archiver,
                 },
-                require_keypair: true
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    storage_account_keypair.into()
+                ],
             }
         );
 
@@ -294,14 +353,22 @@ mod tests {
             &keypair_file,
         ]);
         assert_eq!(
-            parse_command(&test_create_validator_storage_account).unwrap(),
+            parse_command(
+                &test_create_validator_storage_account,
+                &default_keypair_file,
+                None
+            )
+            .unwrap(),
             CliCommandInfo {
                 command: CliCommand::CreateStorageAccount {
                     account_owner: pubkey,
-                    storage_account: storage_account_keypair.into(),
+                    storage_account: 1,
                     account_type: StorageAccountType::Validator,
                 },
-                require_keypair: true
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    storage_account_keypair.into()
+                ],
             }
         );
 
@@ -312,13 +379,13 @@ mod tests {
             &storage_account_string,
         ]);
         assert_eq!(
-            parse_command(&test_claim_storage_reward).unwrap(),
+            parse_command(&test_claim_storage_reward, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::ClaimStorageReward {
                     node_account_pubkey: pubkey,
                     storage_account_pubkey,
                 },
-                require_keypair: true
+                signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
         );
     }

cli/src/validator_info.rs

@@ -3,27 +3,29 @@ use crate::{
     display::println_name_value,
 };
 use bincode::deserialize;
-use clap::{App, Arg, ArgMatches, SubCommand};
-use reqwest::Client;
+use clap::{App, AppSettings, Arg, ArgMatches, SubCommand};
+use reqwest::blocking::Client;
 use serde_derive::{Deserialize, Serialize};
 use serde_json::{Map, Value};
 
 use solana_clap_utils::{
     input_parsers::pubkey_of,
     input_validators::{is_pubkey, is_url},
+    keypair::signer_from_path,
 };
 use solana_client::rpc_client::RpcClient;
 use solana_config_program::{config_instruction, get_config_data, ConfigKeys, ConfigState};
+use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     account::Account,
     commitment_config::CommitmentConfig,
     message::Message,
     pubkey::Pubkey,
-    signature::{Keypair, KeypairUtil},
+    signature::{Keypair, Signer},
     transaction::Transaction,
 };
-
-use std::error;
+use std::{error, sync::Arc};
+use titlecase::titlecase;
 
 pub const MAX_SHORT_FIELD_LENGTH: usize = 70;
 pub const MAX_LONG_FIELD_LENGTH: usize = 300;
@@ -151,6 +153,7 @@ impl ValidatorInfoSubCommands for App<'_, '_> {
         self.subcommand(
             SubCommand::with_name("validator-info")
                 .about("Publish/get Validator info on Solana")
+                .setting(AppSettings::SubcommandRequiredElseHelp)
                 .subcommand(
                     SubCommand::with_name("publish")
                         .about("Publish Validator info on Solana")
@@ -223,17 +226,26 @@ impl ValidatorInfoSubCommands for App<'_, '_> {
     }
 }
 
-pub fn parse_validator_info_command(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+pub fn parse_validator_info_command(
+    matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
     let info_pubkey = pubkey_of(matches, "info_pubkey");
     // Prepare validator info
-    let validator_info = parse_args(&matches);
+    let validator_info = parse_args(matches);
     Ok(CliCommandInfo {
         command: CliCommand::SetValidatorInfo {
             validator_info,
             force_keybase: matches.is_present("force"),
             info_pubkey,
         },
-        require_keypair: true,
+        signers: vec![signer_from_path(
+            matches,
+            default_signer_path,
+            "keypair",
+            wallet_manager,
+        )?],
     })
 }
 
@@ -243,7 +255,7 @@ pub fn parse_get_validator_info_command(
     let info_pubkey = pubkey_of(matches, "info_pubkey");
     Ok(CliCommandInfo {
         command: CliCommand::GetValidatorInfo(info_pubkey),
-        require_keypair: false,
+        signers: vec![],
     })
 }
 
@@ -256,13 +268,13 @@ pub fn process_set_validator_info(
 ) -> ProcessResult {
     // Validate keybase username
     if let Some(string) = validator_info.get("keybaseUsername") {
-        let result = verify_keybase(&config.keypair.pubkey(), &string);
+        let result = verify_keybase(&config.signers[0].pubkey(), &string);
         if result.is_err() {
             if force_keybase {
                 println!("--force supplied, ignoring: {:?}", result);
             } else {
                 result.map_err(|err| {
-                    CliError::BadParameter(format!("Invalid validator keybase username: {:?}", err))
+                    CliError::BadParameter(format!("Invalid validator keybase username: {}", err))
                 })?;
             }
         }
@@ -281,7 +293,7 @@ pub fn process_set_validator_info(
         })
         .find(|(pubkey, account)| {
             let (validator_pubkey, _) = parse_validator_info(&pubkey, &account).unwrap();
-            validator_pubkey == config.keypair.pubkey()
+            validator_pubkey == config.signers[0].pubkey()
         });
 
     // Create validator-info keypair to use if info_pubkey not provided or does not exist
@@ -299,8 +311,8 @@ pub fn process_set_validator_info(
         .poll_get_balance_with_commitment(&info_pubkey, CommitmentConfig::default())
         .unwrap_or(0);
 
-    let keys = vec![(id(), false), (config.keypair.pubkey(), true)];
-    let (message, signers): (Message, Vec<&Keypair>) = if balance == 0 {
+    let keys = vec![(id(), false), (config.signers[0].pubkey(), true)];
+    let (message, signers): (Message, Vec<&dyn Signer>) = if balance == 0 {
         if info_pubkey != info_keypair.pubkey() {
             println!(
                 "Account {:?} does not exist. Generating new keypair...",
@@ -310,12 +322,14 @@ pub fn process_set_validator_info(
         }
         println!(
             "Publishing info for Validator {:?}",
-            config.keypair.pubkey()
+            config.signers[0].pubkey()
         );
+        let lamports = rpc_client
+            .get_minimum_balance_for_rent_exemption(ValidatorInfo::max_space() as usize)?;
         let mut instructions = config_instruction::create_account::<ValidatorInfo>(
-            &config.keypair.pubkey(),
+            &config.signers[0].pubkey(),
             &info_keypair.pubkey(),
-            1,
+            lamports,
             keys.clone(),
         );
         instructions.extend_from_slice(&[config_instruction::store(
@@ -324,13 +338,13 @@ pub fn process_set_validator_info(
             keys,
             &validator_info,
         )]);
-        let signers = vec![&config.keypair, &info_keypair];
-        let message = Message::new(instructions);
+        let signers = vec![config.signers[0], &info_keypair];
+        let message = Message::new(&instructions);
         (message, signers)
     } else {
         println!(
             "Updating Validator {:?} info at: {:?}",
-            config.keypair.pubkey(),
+            config.signers[0].pubkey(),
             info_pubkey
         );
         let instructions = vec![config_instruction::store(
@@ -339,17 +353,18 @@ pub fn process_set_validator_info(
             keys,
             &validator_info,
         )];
-        let message = Message::new_with_payer(instructions, Some(&config.keypair.pubkey()));
-        let signers = vec![&config.keypair];
+        let message = Message::new_with_payer(&instructions, Some(&config.signers[0].pubkey()));
+        let signers = vec![config.signers[0]];
         (message, signers)
     };
 
     // Submit transaction
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
-    let mut tx = Transaction::new(&signers, message, recent_blockhash);
+    let mut tx = Transaction::new_unsigned(message);
+    tx.try_sign(&signers, recent_blockhash)?;
     check_account_for_fee(
         rpc_client,
-        &config.keypair.pubkey(),
+        &config.signers[0].pubkey(),
         &fee_calculator,
         &tx.message,
     )?;
@@ -387,9 +402,12 @@ pub fn process_get_validator_info(rpc_client: &RpcClient, pubkey: Option<Pubkey>
             parse_validator_info(&validator_info_pubkey, &validator_info_account)?;
         println!();
         println_name_value("Validator Identity Pubkey:", &validator_pubkey.to_string());
-        println_name_value("  info pubkey:", &validator_info_pubkey.to_string());
+        println_name_value("  Info Pubkey:", &validator_info_pubkey.to_string());
         for (key, value) in validator_info.iter() {
-            println_name_value(&format!("  {}:", key), &value.as_str().unwrap_or("?"));
+            println_name_value(
+                &format!("  {}:", titlecase(key)),
+                &value.as_str().unwrap_or("?"),
+            );
         }
     }
 

cli/src/vote.rs

@@ -1,23 +1,25 @@
-use crate::{
-    cli::{
-        build_balance_message, check_account_for_fee, check_unique_pubkeys,
-        log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError,
-        ProcessResult,
-    },
-    cluster_query::aggregate_epoch_credits,
+use crate::cli::{
+    build_balance_message, check_account_for_fee, check_unique_pubkeys, generate_unique_signers,
+    log_instruction_custom_error, CliCommand, CliCommandInfo, CliConfig, CliError, CliSignerInfo,
+    ProcessResult, SignerIndex,
 };
 use clap::{value_t_or_exit, App, Arg, ArgMatches, SubCommand};
 use solana_clap_utils::{input_parsers::*, input_validators::*};
 use solana_client::rpc_client::RpcClient;
-use solana_sdk::signature::Keypair;
+use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
-    account::Account, pubkey::Pubkey, signature::KeypairUtil, system_instruction::SystemError,
+    account::Account,
+    commitment_config::CommitmentConfig,
+    message::Message,
+    pubkey::Pubkey,
+    system_instruction::{create_address_with_seed, SystemError},
     transaction::Transaction,
 };
 use solana_vote_program::{
-    vote_instruction::{self, VoteError},
+    vote_instruction::{self, withdraw, VoteError},
     vote_state::{VoteAuthorize, VoteInit, VoteState},
 };
+use std::sync::Arc;
 
 pub trait VoteSubCommands {
     fn vote_subcommands(self) -> Self;
@@ -34,7 +36,7 @@ impl VoteSubCommands for App<'_, '_> {
                         .value_name("VOTE ACCOUNT KEYPAIR")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_keypair_or_ask_keyword)
+                        .validator(is_valid_signer)
                         .help("Vote account keypair to fund"),
                 )
                 .arg(
@@ -51,7 +53,8 @@ impl VoteSubCommands for App<'_, '_> {
                         .long("commission")
                         .value_name("NUM")
                         .takes_value(true)
-                        .help("The commission taken on reward redemption (0-100), default: 0"),
+                        .default_value("100")
+                        .help("The commission taken on reward redemption (0-100)"),
                 )
                 .arg(
                     Arg::with_name("authorized_voter")
@@ -59,7 +62,7 @@ impl VoteSubCommands for App<'_, '_> {
                         .value_name("PUBKEY")
                         .takes_value(true)
                         .validator(is_pubkey_or_keypair)
-                        .help("Public key of the authorized voter (defaults to vote account)"),
+                        .help("Public key of the authorized voter [default: validator identity pubkey]"),
                 )
                 .arg(
                     Arg::with_name("authorized_withdrawer")
@@ -67,7 +70,14 @@ impl VoteSubCommands for App<'_, '_> {
                         .value_name("PUBKEY")
                         .takes_value(true)
                         .validator(is_pubkey_or_keypair)
-                        .help("Public key of the authorized withdrawer (defaults to cli config pubkey)"),
+                        .help("Public key of the authorized withdrawer [default: validator identity pubkey]"),
+                )
+                .arg(
+                    Arg::with_name("seed")
+                        .long("seed")
+                        .value_name("SEED STRING")
+                        .takes_value(true)
+                        .help("Seed for address generation; if specified, the resulting account will be at a derived address of the VOTE ACCOUNT pubkey")
                 ),
         )
         .subcommand(
@@ -97,7 +107,7 @@ impl VoteSubCommands for App<'_, '_> {
                         .value_name("AUTHORIZED VOTER KEYPAIR")
                         .takes_value(true)
                         .required(true)
-                        .validator(is_keypair)
+                        .validator(is_valid_signer)
                         .help("Authorized voter keypair"),
                 )
         )
@@ -146,8 +156,17 @@ impl VoteSubCommands for App<'_, '_> {
                 ),
         )
         .subcommand(
-            SubCommand::with_name("show-vote-account")
+            SubCommand::with_name("vote-account")
                 .about("Show the contents of a vote account")
+                .alias("show-vote-account")
+                .arg(
+                    Arg::with_name("confirmed")
+                        .long("confirmed")
+                        .takes_value(false)
+                        .help(
+                            "Return information at maximum-lockout commitment level",
+                        ),
+                )
                 .arg(
                     Arg::with_name("vote_account_pubkey")
                         .index(1)
@@ -165,8 +184,8 @@ impl VoteSubCommands for App<'_, '_> {
                 ),
         )
         .subcommand(
-            SubCommand::with_name("uptime")
-                .about("Show the uptime of a validator, based on epoch voting history")
+            SubCommand::with_name("withdraw-from-vote-account")
+                .about("Withdraw lamports from a vote account into a specified account")
                 .arg(
                     Arg::with_name("vote_account_pubkey")
                         .index(1)
@@ -174,72 +193,120 @@ impl VoteSubCommands for App<'_, '_> {
                         .takes_value(true)
                         .required(true)
                         .validator(is_pubkey_or_keypair)
-                        .help("Vote account pubkey"),
+                        .help("Vote account from which to withdraw"),
                 )
                 .arg(
-                    Arg::with_name("span")
-                        .long("span")
-                        .value_name("NUM OF EPOCHS")
+                    Arg::with_name("destination_account_pubkey")
+                        .index(2)
+                        .value_name("DESTINATION ACCOUNT")
                         .takes_value(true)
-                        .help("Number of recent epochs to examine"),
+                        .required(true)
+                        .validator(is_pubkey_or_keypair)
+                        .help("The account to which the SOL should be transferred"),
                 )
                 .arg(
-                    Arg::with_name("aggregate")
-                        .long("aggregate")
-                        .help("Aggregate uptime data across span"),
-                ),
+                    Arg::with_name("amount")
+                        .index(3)
+                        .value_name("AMOUNT")
+                        .takes_value(true)
+                        .required(true)
+                        .validator(is_amount)
+                        .help("The amount to withdraw, in SOL"),
+                )
+                .arg(
+                    Arg::with_name("authorized_withdrawer")
+                        .long("authorized-withdrawer")
+                        .value_name("KEYPAIR or PUBKEY or REMOTE WALLET PATH")
+                        .takes_value(true)
+                        .validator(is_valid_signer)
+                        .help("Authorized withdrawer [default: cli config keypair]"),
+                )
         )
     }
 }
 
-pub fn parse_vote_create_account(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let vote_account = keypair_of(matches, "vote_account").unwrap();
+pub fn parse_create_vote_account(
+    matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
+    let (vote_account, _) = signer_of(matches, "vote_account", wallet_manager)?;
+    let seed = matches.value_of("seed").map(|s| s.to_string());
     let identity_pubkey = pubkey_of(matches, "identity_pubkey").unwrap();
-    let commission = value_of(&matches, "commission").unwrap_or(0);
+    let commission = value_t_or_exit!(matches, "commission", u8);
     let authorized_voter = pubkey_of(matches, "authorized_voter");
     let authorized_withdrawer = pubkey_of(matches, "authorized_withdrawer");
 
+    let payer_provided = None;
+    let CliSignerInfo { signers } = generate_unique_signers(
+        vec![payer_provided, vote_account],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
+
     Ok(CliCommandInfo {
         command: CliCommand::CreateVoteAccount {
-            vote_account: vote_account.into(),
+            seed,
             node_pubkey: identity_pubkey,
             authorized_voter,
             authorized_withdrawer,
             commission,
         },
-        require_keypair: true,
+        signers,
     })
 }
 
 pub fn parse_vote_authorize(
     matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
     vote_authorize: VoteAuthorize,
 ) -> Result<CliCommandInfo, CliError> {
     let vote_account_pubkey = pubkey_of(matches, "vote_account_pubkey").unwrap();
     let new_authorized_pubkey = pubkey_of(matches, "new_authorized_pubkey").unwrap();
 
+    let authorized_voter_provided = None;
+    let CliSignerInfo { signers } = generate_unique_signers(
+        vec![authorized_voter_provided],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
+
     Ok(CliCommandInfo {
         command: CliCommand::VoteAuthorize {
             vote_account_pubkey,
             new_authorized_pubkey,
             vote_authorize,
         },
-        require_keypair: true,
+        signers,
     })
 }
 
-pub fn parse_vote_update_validator(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+pub fn parse_vote_update_validator(
+    matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
     let vote_account_pubkey = pubkey_of(matches, "vote_account_pubkey").unwrap();
     let new_identity_pubkey = pubkey_of(matches, "new_identity_pubkey").unwrap();
-    let authorized_voter = keypair_of(matches, "authorized_voter").unwrap();
+    let (authorized_voter, _) = signer_of(matches, "authorized_voter", wallet_manager)?;
+
+    let payer_provided = None;
+    let CliSignerInfo { signers } = generate_unique_signers(
+        vec![payer_provided, authorized_voter],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
 
     Ok(CliCommandInfo {
         command: CliCommand::VoteUpdateValidator {
             vote_account_pubkey,
             new_identity_pubkey,
-            authorized_voter: authorized_voter.into(),
         },
-        require_keypair: true,
+        signers,
     })
 }
 
@@ -248,51 +315,88 @@ pub fn parse_vote_get_account_command(
 ) -> Result<CliCommandInfo, CliError> {
     let vote_account_pubkey = pubkey_of(matches, "vote_account_pubkey").unwrap();
     let use_lamports_unit = matches.is_present("lamports");
+    let commitment_config = if matches.is_present("confirmed") {
+        CommitmentConfig::default()
+    } else {
+        CommitmentConfig::recent()
+    };
     Ok(CliCommandInfo {
         command: CliCommand::ShowVoteAccount {
             pubkey: vote_account_pubkey,
             use_lamports_unit,
+            commitment_config,
         },
-        require_keypair: false,
+        signers: vec![],
     })
 }
 
-pub fn parse_vote_uptime_command(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
+pub fn parse_withdraw_from_vote_account(
+    matches: &ArgMatches<'_>,
+    default_signer_path: &str,
+    wallet_manager: Option<&Arc<RemoteWalletManager>>,
+) -> Result<CliCommandInfo, CliError> {
     let vote_account_pubkey = pubkey_of(matches, "vote_account_pubkey").unwrap();
-    let aggregate = matches.is_present("aggregate");
-    let span = if matches.is_present("span") {
-        Some(value_t_or_exit!(matches, "span", u64))
-    } else {
-        None
-    };
+    let destination_account_pubkey = pubkey_of(matches, "destination_account_pubkey").unwrap();
+    let lamports = lamports_of_sol(matches, "amount").unwrap();
+    let (withdraw_authority, withdraw_authority_pubkey) =
+        signer_of(matches, "authorized_withdrawer", wallet_manager)?;
+
+    let payer_provided = None;
+    let signer_info = generate_unique_signers(
+        vec![payer_provided, withdraw_authority],
+        matches,
+        default_signer_path,
+        wallet_manager,
+    )?;
+
     Ok(CliCommandInfo {
-        command: CliCommand::Uptime {
-            pubkey: vote_account_pubkey,
-            aggregate,
-            span,
+        command: CliCommand::WithdrawFromVoteAccount {
+            vote_account_pubkey,
+            destination_account_pubkey,
+            withdraw_authority: signer_info.index_of(withdraw_authority_pubkey).unwrap(),
+            lamports,
         },
-        require_keypair: false,
+        signers: signer_info.signers,
     })
 }
 
 pub fn process_create_vote_account(
     rpc_client: &RpcClient,
     config: &CliConfig,
-    vote_account: &Keypair,
+    seed: &Option<String>,
     identity_pubkey: &Pubkey,
     authorized_voter: &Option<Pubkey>,
     authorized_withdrawer: &Option<Pubkey>,
     commission: u8,
 ) -> ProcessResult {
+    let vote_account = config.signers[1];
     let vote_account_pubkey = vote_account.pubkey();
+    let vote_account_address = if let Some(seed) = seed {
+        create_address_with_seed(&vote_account_pubkey, &seed, &solana_vote_program::id())?
+    } else {
+        vote_account_pubkey
+    };
     check_unique_pubkeys(
-        (&vote_account_pubkey, "vote_account_pubkey".to_string()),
+        (&config.signers[0].pubkey(), "cli keypair".to_string()),
+        (&vote_account_address, "vote_account".to_string()),
+    )?;
+
+    check_unique_pubkeys(
+        (&vote_account_address, "vote_account".to_string()),
         (&identity_pubkey, "identity_pubkey".to_string()),
     )?;
-    check_unique_pubkeys(
-        (&config.keypair.pubkey(), "cli keypair".to_string()),
-        (&vote_account_pubkey, "vote_account_pubkey".to_string()),
-    )?;
+
+    if let Ok(vote_account) = rpc_client.get_account(&vote_account_address) {
+        let err_msg = if vote_account.owner == solana_vote_program::id() {
+            format!("Vote account {} already exists", vote_account_address)
+        } else {
+            format!(
+                "Account {} already exists and is not a vote account",
+                vote_account_address
+            )
+        };
+        return Err(CliError::BadParameter(err_msg).into());
+    }
 
     let required_balance = rpc_client
         .get_minimum_balance_for_rent_exemption(VoteState::size_of())?
@@ -300,29 +404,40 @@ pub fn process_create_vote_account(
 
     let vote_init = VoteInit {
         node_pubkey: *identity_pubkey,
-        authorized_voter: authorized_voter.unwrap_or(vote_account_pubkey),
-        authorized_withdrawer: authorized_withdrawer.unwrap_or(config.keypair.pubkey()),
+        authorized_voter: authorized_voter.unwrap_or(*identity_pubkey),
+        authorized_withdrawer: authorized_withdrawer.unwrap_or(*identity_pubkey),
         commission,
     };
-    let ixs = vote_instruction::create_account(
-        &config.keypair.pubkey(),
-        &vote_account_pubkey,
-        &vote_init,
-        required_balance,
-    );
+
+    let ixs = if let Some(seed) = seed {
+        vote_instruction::create_account_with_seed(
+            &config.signers[0].pubkey(), // from
+            &vote_account_address,       // to
+            &vote_account_pubkey,        // base
+            seed,                        // seed
+            &vote_init,
+            required_balance,
+        )
+    } else {
+        vote_instruction::create_account(
+            &config.signers[0].pubkey(),
+            &vote_account_pubkey,
+            &vote_init,
+            required_balance,
+        )
+    };
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
-    let mut tx = Transaction::new_signed_instructions(
-        &[&config.keypair, vote_account],
-        ixs,
-        recent_blockhash,
-    );
+
+    let message = Message::new(&ixs);
+    let mut tx = Transaction::new_unsigned(message);
+    tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
         rpc_client,
-        &config.keypair.pubkey(),
+        &config.signers[0].pubkey(),
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &[&config.keypair, vote_account]);
+    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
     log_instruction_custom_error::<SystemError>(result)
 }
 
@@ -339,25 +454,22 @@ pub fn process_vote_authorize(
     )?;
     let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
     let ixs = vec![vote_instruction::authorize(
-        vote_account_pubkey,      // vote account to update
-        &config.keypair.pubkey(), // current authorized voter
-        new_authorized_pubkey,    // new vote signer/withdrawer
-        vote_authorize,           // vote or withdraw
+        vote_account_pubkey,         // vote account to update
+        &config.signers[0].pubkey(), // current authorized voter
+        new_authorized_pubkey,       // new vote signer/withdrawer
+        vote_authorize,              // vote or withdraw
     )];
 
-    let mut tx = Transaction::new_signed_with_payer(
-        ixs,
-        Some(&config.keypair.pubkey()),
-        &[&config.keypair],
-        recent_blockhash,
-    );
+    let message = Message::new_with_payer(&ixs, Some(&config.signers[0].pubkey()));
+    let mut tx = Transaction::new_unsigned(message);
+    tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
         rpc_client,
-        &config.keypair.pubkey(),
+        &config.signers[0].pubkey(),
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &[&config.keypair]);
+    let result = rpc_client.send_and_confirm_transaction(&mut tx, &[config.signers[0]]);
     log_instruction_custom_error::<VoteError>(result)
 }
 
@@ -366,8 +478,8 @@ pub fn process_vote_update_validator(
     config: &CliConfig,
     vote_account_pubkey: &Pubkey,
     new_identity_pubkey: &Pubkey,
-    authorized_voter: &Keypair,
 ) -> ProcessResult {
+    let authorized_voter = config.signers[1];
     check_unique_pubkeys(
         (vote_account_pubkey, "vote_account_pubkey".to_string()),
         (new_identity_pubkey, "new_identity_pubkey".to_string()),
@@ -379,27 +491,30 @@ pub fn process_vote_update_validator(
         new_identity_pubkey,
     )];
 
-    let mut tx = Transaction::new_signed_with_payer(
-        ixs,
-        Some(&config.keypair.pubkey()),
-        &[&config.keypair, authorized_voter],
-        recent_blockhash,
-    );
+    let message = Message::new_with_payer(&ixs, Some(&config.signers[0].pubkey()));
+    let mut tx = Transaction::new_unsigned(message);
+    tx.try_sign(&config.signers, recent_blockhash)?;
     check_account_for_fee(
         rpc_client,
-        &config.keypair.pubkey(),
+        &config.signers[0].pubkey(),
         &fee_calculator,
         &tx.message,
     )?;
-    let result = rpc_client.send_and_confirm_transaction(&mut tx, &[&config.keypair]);
+    let result = rpc_client.send_and_confirm_transaction(&mut tx, &config.signers);
     log_instruction_custom_error::<VoteError>(result)
 }
 
 fn get_vote_account(
     rpc_client: &RpcClient,
     vote_account_pubkey: &Pubkey,
+    commitment_config: CommitmentConfig,
 ) -> Result<(Account, VoteState), Box<dyn std::error::Error>> {
-    let vote_account = rpc_client.get_account(vote_account_pubkey)?;
+    let vote_account = rpc_client
+        .get_account_with_commitment(vote_account_pubkey, commitment_config)?
+        .value
+        .ok_or_else(|| {
+            CliError::RpcRequestError(format!("{:?} account does not exist", vote_account_pubkey))
+        })?;
 
     if vote_account.owner != solana_vote_program::id() {
         return Err(CliError::RpcRequestError(format!(
@@ -422,31 +537,33 @@ pub fn process_show_vote_account(
     _config: &CliConfig,
     vote_account_pubkey: &Pubkey,
     use_lamports_unit: bool,
+    commitment_config: CommitmentConfig,
 ) -> ProcessResult {
-    let (vote_account, vote_state) = get_vote_account(rpc_client, vote_account_pubkey)?;
+    let (vote_account, vote_state) =
+        get_vote_account(rpc_client, vote_account_pubkey, commitment_config)?;
 
     let epoch_schedule = rpc_client.get_epoch_schedule()?;
 
     println!(
-        "account balance: {}",
+        "Account Balance: {}",
         build_balance_message(vote_account.lamports, use_lamports_unit, true)
     );
-    println!("validator identity: {}", vote_state.node_pubkey);
-    println!("authorized voter: {}", vote_state.authorized_voter);
+    println!("Validator Identity: {}", vote_state.node_pubkey);
+    println!("Authorized Voter: {:?}", vote_state.authorized_voters());
     println!(
-        "authorized withdrawer: {}",
+        "Authorized Withdrawer: {}",
         vote_state.authorized_withdrawer
     );
-    println!("credits: {}", vote_state.credits());
-    println!("commission: {}%", vote_state.commission);
+    println!("Credits: {}", vote_state.credits());
+    println!("Commission: {}%", vote_state.commission);
     println!(
-        "root slot: {}",
+        "Root Slot: {}",
         match vote_state.root_slot {
             Some(slot) => slot.to_string(),
             None => "~".to_string(),
         }
     );
-    println!("recent timestamp: {:?}", vote_state.last_timestamp);
+    println!("Recent Timestamp: {:?}", vote_state.last_timestamp);
     if !vote_state.votes.is_empty() {
         println!("recent votes:");
         for vote in &vote_state.votes {
@@ -456,7 +573,7 @@ pub fn process_show_vote_account(
             );
         }
 
-        println!("epoch voting history:");
+        println!("Epoch Voting History:");
         for (epoch, credits, prev_credits) in vote_state.epoch_credits() {
             let credits_earned = credits - prev_credits;
             let slots_in_epoch = epoch_schedule.get_slots_in_epoch(*epoch);
@@ -469,65 +586,42 @@ pub fn process_show_vote_account(
     Ok("".to_string())
 }
 
-pub fn process_uptime(
+pub fn process_withdraw_from_vote_account(
     rpc_client: &RpcClient,
-    _config: &CliConfig,
+    config: &CliConfig,
     vote_account_pubkey: &Pubkey,
-    aggregate: bool,
-    span: Option<u64>,
+    withdraw_authority: SignerIndex,
+    lamports: u64,
+    destination_account_pubkey: &Pubkey,
 ) -> ProcessResult {
-    let (_vote_account, vote_state) = get_vote_account(rpc_client, vote_account_pubkey)?;
+    let (recent_blockhash, fee_calculator) = rpc_client.get_recent_blockhash()?;
+    let withdraw_authority = config.signers[withdraw_authority];
 
-    let epoch_schedule = rpc_client.get_epoch_schedule()?;
+    let ix = withdraw(
+        vote_account_pubkey,
+        &withdraw_authority.pubkey(),
+        lamports,
+        destination_account_pubkey,
+    );
 
-    println!("validator identity: {}", vote_state.node_pubkey);
-    println!("authorized voter: {}", vote_state.authorized_voter);
-    if !vote_state.votes.is_empty() {
-        println!("uptime:");
-
-        let epoch_credits: Vec<(u64, u64, u64)> = if let Some(x) = span {
-            vote_state
-                .epoch_credits()
-                .iter()
-                .rev()
-                .take(x as usize)
-                .cloned()
-                .collect()
-        } else {
-            vote_state.epoch_credits().iter().rev().cloned().collect()
-        };
-
-        if aggregate {
-            let (total_credits, total_slots, epochs) =
-                aggregate_epoch_credits(&epoch_credits, &epoch_schedule);
-            if total_slots > 0 {
-                let total_uptime = 100_f64 * total_credits as f64 / total_slots as f64;
-                println!("{:.2}% over {} epochs", total_uptime, epochs);
-            } else {
-                println!("Insufficient voting history available");
-            }
-        } else {
-            for (epoch, credits, prev_credits) in epoch_credits {
-                let credits_earned = credits - prev_credits;
-                let slots_in_epoch = epoch_schedule.get_slots_in_epoch(epoch);
-                let uptime = credits_earned as f64 / slots_in_epoch as f64;
-                println!("- epoch: {} {:.2}% uptime", epoch, uptime * 100_f64,);
-            }
-        }
-        if let Some(x) = span {
-            if x > vote_state.epoch_credits().len() as u64 {
-                println!("(span longer than available epochs)");
-            }
-        }
-    }
-    Ok("".to_string())
+    let message = Message::new_with_payer(&[ix], Some(&config.signers[0].pubkey()));
+    let mut transaction = Transaction::new_unsigned(message);
+    transaction.try_sign(&config.signers, recent_blockhash)?;
+    check_account_for_fee(
+        rpc_client,
+        &config.signers[0].pubkey(),
+        &fee_calculator,
+        &transaction.message,
+    )?;
+    let result = rpc_client.send_and_confirm_transaction(&mut transaction, &config.signers);
+    log_instruction_custom_error::<VoteError>(result)
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::cli::{app, parse_command};
-    use solana_sdk::signature::write_keypair;
+    use solana_sdk::signature::{read_keypair_file, write_keypair, Keypair, Signer};
     use tempfile::NamedTempFile;
 
     fn make_tmp_file() -> (String, NamedTempFile) {
@@ -545,6 +639,10 @@ mod tests {
         let pubkey2 = keypair2.pubkey();
         let pubkey2_string = pubkey2.to_string();
 
+        let default_keypair = Keypair::new();
+        let (default_keypair_file, mut tmp_file) = make_tmp_file();
+        write_keypair(&default_keypair, tmp_file.as_file_mut()).unwrap();
+
         let test_authorize_voter = test_commands.clone().get_matches_from(vec![
             "test",
             "vote-authorize-voter",
@@ -552,14 +650,14 @@ mod tests {
             &pubkey2_string,
         ]);
         assert_eq!(
-            parse_command(&test_authorize_voter).unwrap(),
+            parse_command(&test_authorize_voter, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::VoteAuthorize {
                     vote_account_pubkey: pubkey,
                     new_authorized_pubkey: pubkey2,
                     vote_authorize: VoteAuthorize::Voter
                 },
-                require_keypair: true
+                signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
             }
         );
 
@@ -578,16 +676,19 @@ mod tests {
             "10",
         ]);
         assert_eq!(
-            parse_command(&test_create_vote_account).unwrap(),
+            parse_command(&test_create_vote_account, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::CreateVoteAccount {
-                    vote_account: keypair.into(),
+                    seed: None,
                     node_pubkey,
                     authorized_voter: None,
                     authorized_withdrawer: None,
                     commission: 10,
                 },
-                require_keypair: true
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    Box::new(keypair)
+                ],
             }
         );
 
@@ -602,16 +703,19 @@ mod tests {
             &node_pubkey_string,
         ]);
         assert_eq!(
-            parse_command(&test_create_vote_account2).unwrap(),
+            parse_command(&test_create_vote_account2, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::CreateVoteAccount {
-                    vote_account: keypair.into(),
+                    seed: None,
                     node_pubkey,
                     authorized_voter: None,
                     authorized_withdrawer: None,
-                    commission: 0,
+                    commission: 100,
                 },
-                require_keypair: true
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    Box::new(keypair)
+                ],
             }
         );
 
@@ -630,16 +734,19 @@ mod tests {
             &authed.to_string(),
         ]);
         assert_eq!(
-            parse_command(&test_create_vote_account3).unwrap(),
+            parse_command(&test_create_vote_account3, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::CreateVoteAccount {
-                    vote_account: keypair.into(),
+                    seed: None,
                     node_pubkey,
                     authorized_voter: Some(authed),
                     authorized_withdrawer: None,
-                    commission: 0
+                    commission: 100
                 },
-                require_keypair: true
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    Box::new(keypair)
+                ],
             }
         );
 
@@ -656,16 +763,19 @@ mod tests {
             &authed.to_string(),
         ]);
         assert_eq!(
-            parse_command(&test_create_vote_account4).unwrap(),
+            parse_command(&test_create_vote_account4, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::CreateVoteAccount {
-                    vote_account: keypair.into(),
+                    seed: None,
                     node_pubkey,
                     authorized_voter: None,
                     authorized_withdrawer: Some(authed),
-                    commission: 0
+                    commission: 100
                 },
-                require_keypair: true
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    Box::new(keypair)
+                ],
             }
         );
 
@@ -677,38 +787,76 @@ mod tests {
             &keypair_file,
         ]);
         assert_eq!(
-            parse_command(&test_update_validator).unwrap(),
+            parse_command(&test_update_validator, &default_keypair_file, None).unwrap(),
             CliCommandInfo {
                 command: CliCommand::VoteUpdateValidator {
                     vote_account_pubkey: pubkey,
                     new_identity_pubkey: pubkey2,
-                    authorized_voter: solana_sdk::signature::read_keypair_file(&keypair_file)
-                        .unwrap()
-                        .into(),
                 },
-                require_keypair: true
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    Box::new(read_keypair_file(&keypair_file).unwrap())
+                ],
             }
         );
 
-        // Test Uptime Subcommand
-        let pubkey = Pubkey::new_rand();
-        let matches = test_commands.clone().get_matches_from(vec![
+        // Test WithdrawFromVoteAccount subcommand
+        let test_withdraw_from_vote_account = test_commands.clone().get_matches_from(vec![
             "test",
-            "uptime",
-            &pubkey.to_string(),
-            "--span",
-            "4",
-            "--aggregate",
+            "withdraw-from-vote-account",
+            &keypair_file,
+            &pubkey_string,
+            "42",
         ]);
         assert_eq!(
-            parse_command(&matches).unwrap(),
+            parse_command(
+                &test_withdraw_from_vote_account,
+                &default_keypair_file,
+                None
+            )
+            .unwrap(),
             CliCommandInfo {
-                command: CliCommand::Uptime {
-                    pubkey,
-                    aggregate: true,
-                    span: Some(4)
+                command: CliCommand::WithdrawFromVoteAccount {
+                    vote_account_pubkey: read_keypair_file(&keypair_file).unwrap().pubkey(),
+                    destination_account_pubkey: pubkey,
+                    withdraw_authority: 0,
+                    lamports: 42_000_000_000
                 },
-                require_keypair: false
+                signers: vec![read_keypair_file(&default_keypair_file).unwrap().into()],
+            }
+        );
+
+        // Test WithdrawFromVoteAccount subcommand with authority
+        let withdraw_authority = Keypair::new();
+        let (withdraw_authority_file, mut tmp_file) = make_tmp_file();
+        write_keypair(&withdraw_authority, tmp_file.as_file_mut()).unwrap();
+        let test_withdraw_from_vote_account = test_commands.clone().get_matches_from(vec![
+            "test",
+            "withdraw-from-vote-account",
+            &keypair_file,
+            &pubkey_string,
+            "42",
+            "--authorized-withdrawer",
+            &withdraw_authority_file,
+        ]);
+        assert_eq!(
+            parse_command(
+                &test_withdraw_from_vote_account,
+                &default_keypair_file,
+                None
+            )
+            .unwrap(),
+            CliCommandInfo {
+                command: CliCommand::WithdrawFromVoteAccount {
+                    vote_account_pubkey: read_keypair_file(&keypair_file).unwrap().pubkey(),
+                    destination_account_pubkey: pubkey,
+                    withdraw_authority: 1,
+                    lamports: 42_000_000_000
+                },
+                signers: vec![
+                    read_keypair_file(&default_keypair_file).unwrap().into(),
+                    read_keypair_file(&withdraw_authority_file).unwrap().into()
+                ],
             }
         );
     }