Backport Accounts Fixes #16838 and the test #17038 (#19412)

* reclaims unref accounts from index (#16838)

* Test account index and store alignment (#17038)

* Use ReclaimResult::Default() instead of building subtypes

* Add test to ensure account_db store and index are aligned

Co-authored-by: Jeff Washington (jwash) <75863576+jeffwashington@users.noreply.github.com>
Co-authored-by: steviez <steven@solana.com>

.buildkite/env/secrets.ejson

@@ -1,6 +1,7 @@
 {
     "_public_key": "ae29f4f7ad2fc92de70d470e411c8426d5d48db8817c9e3dae574b122192335f",
+    "_comment": "These credentials are encrypted and pose no risk",
     "environment": {
-      "CODECOV_TOKEN": "EJ[1:Z7OneT3RdJJ0DipCHQ7rC84snQ+FPbgHwZADQiz54wk=:3K68mE38LJ2RB98VWmjuNLFBNn1XTGR4:cR4r05/TOZQKmEZp1v4CSgUJtC6QJiOaL85QjXW0qZ061fMnsBA8AtAPMDoDq4WCGOZM1A==]"
+      "CODECOV_TOKEN": "EJ[1:KToenD1Sr3w82lHGxz1n+j3hwNlLk/1pYrjZHlvY6kE=:hN1Q25omtJ+4yYVn+qzIsPLKT3O6J9XN:DMLNLXi/pkWgvwF6gNIcNF222sgsRR9LnwLZYj0P0wGj7q6w8YQnd1Rskj+sRroI/z5pQg==]"
     }
 }

.buildkite/hooks/pre-command

@@ -31,4 +31,12 @@ export CARGO_TARGET_CACHE=$HOME/cargo-target-cache/"$CHANNEL"-"$BUILDKITE_LABEL"
 
   mkdir -p "$CARGO_TARGET_CACHE"/target
   rsync -a --delete --link-dest="$CARGO_TARGET_CACHE" "$CARGO_TARGET_CACHE"/target .
+
+  # Don't reuse BPF target build artifacts due to incremental build issues with
+  # `std:
+  #    "found possibly newer version of crate `std` which `xyz` depends on
+  rm -rf target/bpfel-unknown-unknown
+  if [[ $BUILDKITE_LABEL = "stable-perf" ]]; then
+    rm -rf target/release
+  fi
 )

.github/stale.yml

@@ -1,11 +1,3 @@
-only: pulls
-
-# Number of days of inactivity before a pull request becomes stale
-daysUntilStale: 7
-
-# Number of days of inactivity before a stale pull request is closed
-daysUntilClose: 7
-
 # Issues with these labels will never be considered stale
 exemptLabels:
   - security
@@ -14,12 +6,34 @@ exemptLabels:
 # Label to use when marking a pull request as stale
 staleLabel: stale
 
-# Comment to post when marking a pull request as stale. Set to `false` to disable
-markComment: >
-  This pull request has been automatically marked as stale because it has not had
-  recent activity. It will be closed if no further activity occurs.
+pulls:
+  # Number of days of inactivity before a pull request becomes stale
+  daysUntilStale: 7
 
-# Comment to post when closing a stale pull request. Set to `false` to disable
-closeComment: >
-  This stale pull request has been automatically closed.
-  Thank you for your contributions.
+  # Number of days of inactivity before a stale pull request is closed
+  daysUntilClose: 7
+  # Comment to post when marking a pull request as stale. Set to `false` to disable
+  markComment: >
+    This pull request has been automatically marked as stale because it has not had
+    recent activity. It will be closed if no further activity occurs.
+
+  # Comment to post when closing a stale pull request. Set to `false` to disable
+  closeComment: >
+    This stale pull request has been automatically closed.
+    Thank you for your contributions.
+
+issues:
+  # Number of days of inactivity before a issue becomes stale
+  daysUntilStale: 365
+
+  # Number of days of inactivity before a stale issue is closed
+  daysUntilClose: 7
+  # Comment to post when marking a issue as stale. Set to `false` to disable
+  markComment: >
+    This issue has been automatically marked as stale because it has not had
+    recent activity. It will be closed if no further activity occurs.
+
+  # Comment to post when closing a stale issue. Set to `false` to disable
+  closeComment: >
+    This stale issue has been automatically closed.
+    Thank you for your contributions.

.gitignore

@@ -1,7 +1,3 @@
-/docs/html/
-/docs/src/tests.ok
-/docs/src/cli/usage.md
-/docs/src/.gitbook/assets/*.svg
 /farf/
 /solana-release/
 /solana-release.tar.bz2
@@ -14,6 +10,8 @@
 
 /config/
 
+.cache
+
 # log files
 *.log
 log-*.txt

.mergify.yml

@@ -50,27 +50,35 @@ pull_request_rules:
       label:
         add:
           - automerge
-  - name: v1.1 backport
+  - name: v1.4 backport
     conditions:
-      - label=v1.1
+      - label=v1.4
     actions:
       backport:
         ignore_conflicts: true
         branches:
-          - v1.1
-  - name: v1.2 backport
+          - v1.4
+  - name: v1.5 backport
     conditions:
-      - label=v1.2
+      - label=v1.5
     actions:
       backport:
         ignore_conflicts: true
         branches:
-          - v1.2
-  - name: v1.3 backport
+          - v1.5
+  - name: v1.6 backport
     conditions:
-      - label=v1.3
+      - label=v1.6
     actions:
       backport:
         ignore_conflicts: true
         branches:
-          - v1.3
+          - v1.6
+  - name: v1.7 backport
+    conditions:
+      - label=v1.7
+    actions:
+      backport:
+        ignore_conflicts: true
+        branches:
+          - v1.7

.travis.yml

@@ -34,8 +34,11 @@ jobs:
         - stable
       install:
         - source ci/rust-version.sh
+        - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
+        - readlink -f .
       script:
         - source ci/env.sh
+        - rustup set profile default
         - ci/publish-tarball.sh
       deploy:
         - provider: s3
@@ -124,6 +127,8 @@ jobs:
           - ~/.npm
 
       before_install:
+        - source ci/env.sh
+        - .travis/channel_restriction.sh edge beta || travis_terminate 0
         - .travis/affects.sh docs/ .travis || travis_terminate 0
         - cd docs/
         - source .travis/before_install.sh

.travis/channel_restriction.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+#
+# Only proceed if we are on one of the channels passed in, or a tag build
+#
+
+set -ex
+
+[[ -n $CI_TAG ]] && exit 0
+
+eval "$(ci/channel-info.sh)"
+
+for acceptable_channel in "$@"; do
+  if [[ "$CHANNEL" == "$acceptable_channel" ]]; then
+    exit 0
+  fi
+done
+
+echo "Not running from one of the following channels: $*"
+exit 1

CONTRIBUTING.md

@@ -232,7 +232,7 @@ confused with 3-letter acronyms.
 Solana's architecture is described by docs generated from markdown files in
 the `docs/src/` directory, maintained by an *editor* (currently @garious). To
 add a design proposal, you'll need to include it in the
-[Accepted Design Proposals](https://docs.solana.com/proposals)
+[Accepted Design Proposals](https://docs.solana.com/proposals/accepted-design-proposals)
 section of the Solana docs.  Here's the full process:
 
 1. Propose a design by creating a PR that adds a markdown document to the

Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = [
+    "accounts-cluster-bench",
     "bench-exchange",
     "bench-streamer",
     "bench-tps",
@@ -16,10 +17,10 @@ members = [
     "dos",
     "download-utils",
     "faucet",
+    "frozen-abi",
     "perf",
     "validator",
     "genesis",
-    "genesis-programs",
     "gossip",
     "install",
     "keygen",
@@ -28,15 +29,18 @@ members = [
     "local-cluster",
     "logger",
     "log-analyzer",
+    "merkle-root-bench",
     "merkle-tree",
     "stake-o-matic",
     "storage-bigtable",
+    "storage-proto",
     "streamer",
     "measure",
     "metrics",
     "net-shaper",
     "notifier",
     "poh-bench",
+    "program-test",
     "programs/secp256k1",
     "programs/bpf_loader",
     "programs/budget",
@@ -51,10 +55,12 @@ members = [
     "remote-wallet",
     "ramp-tps",
     "runtime",
+    "runtime/store-tool",
     "sdk",
+    "sdk/cargo-build-bpf",
+    "sdk/cargo-test-bpf",
     "scripts",
     "stake-accounts",
-    "stake-monitor",
     "sys-tuner",
     "tokens",
     "transaction-status",
@@ -62,7 +68,6 @@ members = [
     "upload-perf",
     "net-utils",
     "version",
-    "vote-signer",
     "cli",
     "rayon-threadlimit",
     "watchtower",
@@ -71,3 +76,6 @@ members = [
 exclude = [
     "programs/bpf",
 ]
+
+[profile.dev]
+split-debuginfo = "unpacked"

README.md

@@ -19,7 +19,7 @@ $ source $HOME/.cargo/env
 $ rustup component add rustfmt
 ```
 
-If your rustc version is lower than 1.39.0, please update it:
+Please make sure you are always using the latest stable rust version by running:
 
 ```bash
 $ rustup update
@@ -29,7 +29,7 @@ On Linux systems you may need to install libssl-dev, pkg-config, zlib1g-dev, etc
 
 ```bash
 $ sudo apt-get update
-$ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang
+$ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang make
 ```
 
 ## **2. Download the source code.**
@@ -59,10 +59,11 @@ $ cargo test
 ```
 
 ### Starting a local testnet
-Start your own testnet locally, instructions are in the [online docs](https://docs.solana.com/bench-tps).
+Start your own testnet locally, instructions are in the [online docs](https://docs.solana.com/cluster/bench-tps).
 
-### Accessing the remote testnet
-* `testnet` - public stable testnet accessible via devnet.solana.com. Runs 24/7
+### Accessing the remote development cluster
+* `devnet` - stable public cluster for development accessible via
+devnet.solana.com. Runs 24/7. Learn more about the [public clusters](https://docs.solana.com/clusters)
 
 # Benchmarking
 
@@ -106,4 +107,41 @@ send us that patch!
 
 # Disclaimer
 
-All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort.  It is up to the reader to check and validate their accuracy and truthfulness.  Furthermore nothing in this project constitutes a solicitation for investment.
+All claims, content, designs, algorithms, estimates, roadmaps,
+specifications, and performance measurements described in this project
+are done with the Solana Foundation's ("SF") best efforts. It is up to
+the reader to check and validate their accuracy and truthfulness.
+Furthermore nothing in this project constitutes a solicitation for
+investment.
+
+Any content produced by SF or developer resources that SF provides, are
+for educational and inspiration purposes only. SF does not encourage,
+induce or sanction the deployment, integration or use of any such
+applications (including the code comprising the Solana blockchain
+protocol) in violation of applicable laws or regulations and hereby
+prohibits any such deployment, integration or use. This includes use of
+any such applications by the reader (a) in violation of export control
+or sanctions laws of the United States or any other applicable
+jurisdiction, (b) if the reader is located in or ordinarily resident in
+a country or territory subject to comprehensive sanctions administered
+by the U.S. Office of Foreign Assets Control (OFAC), or (c) if the
+reader is or is working on behalf of a Specially Designated National
+(SDN) or a person subject to similar blocking or denied party
+prohibitions.
+
+The reader should be aware that U.S. export control and sanctions laws
+prohibit U.S. persons (and other persons that are subject to such laws)
+from transacting with persons in certain countries and territories or
+that are on the SDN list. As a project based primarily on open-source
+software, it is possible that such sanctioned persons may nevertheless
+bypass prohibitions, obtain the code comprising the Solana blockchain
+protocol (or other project code or applications) and deploy, integrate,
+or otherwise use it. Accordingly, there is a risk to individuals that
+other persons using the Solana blockchain protocol may be sanctioned
+persons and that transactions with such persons would be a violation of
+U.S. export controls and sanctions law. This risk applies to
+individuals, organizations, and other ecosystem participants that
+deploy, integrate, or use the Solana blockchain protocol code directly
+(e.g., as a node operator), and individuals that transact on the Solana
+blockchain through light clients, third party interfaces, and/or wallet
+software.

RELEASE.md

@@ -107,11 +107,15 @@ Alternatively use the Github UI.
    1.  If the Cargo.toml version field is **0.12.3**, then the release tag must be **v0.12.3**
 1. Make sure the Target Branch field matches the branch you want to make a release on.
    1.  If you want to release v0.12.0, the target branch must be v0.12
-1. If this is the first release on the branch (e.g. v0.13.**0**), paste in [this
-   template](https://raw.githubusercontent.com/solana-labs/solana/master/.github/RELEASE_TEMPLATE.md).  Engineering Lead can provide summary contents for release notes if needed.  If this is a patch release, review all the commits since the previous release on this branch and add details as needed.
+1. Fill the release notes.
+   1.  If this is the first release on the branch (e.g. v0.13.**0**), paste in [this
+   template](https://raw.githubusercontent.com/solana-labs/solana/master/.github/RELEASE_TEMPLATE.md).  Engineering Lead can provide summary contents for release notes if needed.
+   1. If this is a patch release, review all the commits since the previous release on this branch and add details as needed.
 1. Click "Save Draft", then confirm the release notes look good and the tag name and branch are correct.
-1. Ensure the release is marked **"This is a pre-release"**.  This flag will then need to be be removed once the the Linux binary artifacts appear later.
-1. Go back into edit the release and click "Publish release" when ready.
+1. Ensure all desired commits (usually backports) are landed on the branch by now.
+1. Ensure the release is marked **"This is a pre-release"**.  This flag will need to be be removed manually after confirming the the Linux binary artifacts appear at a later step.
+1. Go back into edit the release and click "Publish release" while being marked as a pre-release.
+1. Confirm there is new git tag with intended version number at the intended revision after running `git fetch` locally.
 
 
 ### Update release branch with the next patch version
@@ -131,7 +135,8 @@ Alternatively use the Github UI.
 1. Open a PR against origin/vX.Y and then merge the PR after passing CI.
 
 ### Prepare for the next release
-1.  Go to [GitHub Releases](https://github.com/solana-labs/solana/releases) and create a new draft release for `X.Y.Z+1` with empty release nodes.  This allows people to incrementally add new release notes until it's time for the next release
+1.  Go to [GitHub Releases](https://github.com/solana-labs/solana/releases) and create a new draft release for `X.Y.Z+1` with empty release notes.  This allows people to incrementally add new release notes until it's time for the next release
+    1. Also, point the branch field to the same branch and mark the relese as **"This is a pre-release"**.
 1.  Go to the [Github Milestones](https://github.com/solana-labs/solana/milestones).  Create a new milestone for the `X.Y.Z+1`, move over
 unresolved issues still in the `X.Y.Z` milestone, then close the `X.Y.Z` milestone.
 
@@ -147,5 +152,5 @@ appearing.  To check for progress:
 [Crates.io](https://crates.io/crates/solana) should have an updated Solana version.  This can take 2-3 hours, and sometimes fails in the `solana-secondary` job.
 If this happens and the error is non-fatal, click "Retry" on the "publish crate" job
 
-### Update software on devnet.solana.com/testnet.solama.com/mainnet-beta.solana.com
+### Update software on devnet.solana.com/testnet.solana.com/mainnet-beta.solana.com
 See the documentation at https://github.com/solana-labs/cluster-ops/

SECURITY.md

@@ -0,0 +1,155 @@
+# Security Policy
+
+1. [Reporting security problems](#reporting)
+4. [Security Bug Bounties](#bounty)
+2. [Incident Response Process](#process)
+
+<a name="reporting"></a>
+## Reporting security problems to Solana
+
+**DO NOT CREATE AN ISSUE** to report a security problem. Instead, please send an
+email to security@solana.com and provide your github username so we can add you
+to a new draft security advisory for further discussion.
+
+Expect a response as fast as possible, within one business day at the latest.
+
+<a name="bounty"></a>
+## Security Bug Bounties
+We offer bounties for critical security issues. Please see below for more details.
+
+Loss of Funds:
+$500,000 USD in locked SOL tokens (locked for 12 months)
+* Theft of funds without users signature from any account
+* Theft of funds without users interaction in system, token, stake, vote programs
+* Theft of funds that requires users signature - creating a vote program that drains the delegated stakes.
+
+Consensus/Safety Violations:
+$250,000 USD in locked SOL tokens (locked for 12 months)
+* Consensus safety violation
+* Tricking a validator to accept an optimistic confirmation or rooted slot without a double vote, etc..
+
+Other Attacks:
+ $100,000 USD in locked SOL tokens (locked for 12 months)
+* Protocol liveness attacks,
+* Eclipse attacks,
+* Remote attacks that partition the network,
+
+DoS Attacks:
+$25,000 USD in locked SOL tokens (locked for 12 months)
+* Remote resource exaustion via Non-RPC protocols
+
+RPC DoS/Crashes:
+$5,000 USD in locked SOL tokens (locked for 12 months)
+* RPC attacks
+
+Out of Scope:
+The following components are out of scope for the bounty program
+* Metrics: `/metrics` in the monorepo as well as https://metrics.solana.com
+* Explorer: `/explorer` in the monorepo as well as https://explorer.solana.com
+* Any encrypted credentials, auth tokens, etc. checked into the repo
+* Bugs in dependencies. Please take them upstream!
+* Attacks that require social engineering
+
+Eligibility:
+* The participant submitting the bug bounty shall follow the process outlined within this document
+* Valid exploits can be eligible even if they are not successfully executed on the cluster
+* Multiple submissions for the same class of exploit are still eligible for compensation, though may be compensated at a lower rate, however these will be assessed on a case-by-case basis
+* Participants must complete KYC and sign the participation agreement here when the registrations are open https://solana.com/validator-registration. Security exploits will still be assessed and open for submission at all times. This needs only be done prior to distribution of tokens.
+
+Notes:
+* All locked tokens can be staked during the lockup period
+
+<a name="process"></a>
+## Incident Response Process
+
+In case an incident is discovered or reported, the following process will be
+followed to contain, respond and remediate:
+
+### 1. Establish a new draft security advisory
+In response to an email to security@solana.com, a member of the `solana-labs/admins` group will
+1. Create a new draft security advisory for the incident at https://github.com/solana-labs/solana/security/advisories
+1. Add the reporter's github user and the `solana-labs/security-incident-response` group to the draft security advisory
+1. Create a private fork of the repository (grey button towards the bottom of the page)
+1. Respond to the reporter by email, sharing a link to the draft security advisory
+
+### 2. Triage
+Within the draft security advisory, discuss and determine the severity of the
+issue. If necessary, members of the `solana-labs/security-incident-response`
+group may add other github users to the advisory to assist.
+
+If it is determined that this not a critical network issue then the advisory
+should be closed and if more follow-up is required a normal Solana public github
+issue should be created.
+
+### 3. Prepare Fixes
+For the affected branches, typically all three (edge, beta and stable), prepare
+a fix for the issue and push them to the corresponding branch in the private
+repository associated with the draft security advisory.
+
+There is no CI available in the private repository so you must build from source
+and manually verify fixes.
+
+Code review from the reporter is ideal, as well as from multiple members of the
+core development team.
+
+### 4. Notify Security Group Validators
+Once an ETA is available for the fix, a member of the
+`solana-labs/security-incident-response` group should notify the validators so
+they can prepare for an update using the "Solana Red Alert" notification system.
+
+The teams are all over the world and it's critical to provide actionable
+information at the right time. Don't be the person that wakes everybody up at
+2am when a fix won't be available for hours.
+
+### 5. Ship the patch
+Once the fix is accepted, a member of the
+`solana-labs/security-incident-response` group should prepare a single patch
+file for each affected branch. The commit title for the patch should only
+contain the advisory id, and not disclose any further details about the
+incident.
+
+Copy the patches to https://release.solana.com/ under a subdirectory named after
+the advisory id (example:
+https://release.solana.com/GHSA-hx59-f5g4-jghh/v1.4.patch). Contact a member of
+the `solana-labs/admins` group if you require access to release.solana.com
+
+Using the "Solana Red Alert" channel:
+1. Notify validators that there's an issue and a patch will be provided in X minutes
+2. If X minutes expires and there's no patch, notify of the delay and provide a
+   new ETA
+3. Provide links to patches of https://release.solana.com/ for each affected branch
+
+Validators can be expected to build the patch from source against the latest
+release for the affected branch.
+
+Since the software version will not change after the patch is applied, request
+that each validator notify in the existing channel once they've updated. Manually
+monitor the roll out until a sufficient amount of stake has updated - typically
+at least 33.3% or 66.6% depending on the issue.
+
+### 6. Public Disclosure and Release
+Once the fix has been deployed to the security group validators, the patches from the security
+advisory may be merged into the main source repository. A new official release
+for each affected branch should be shipped and all validators requested to
+upgrade as quickly as possible.
+
+### 7. Security Advisory Bounty Accounting and Cleanup
+
+If this issue is eligible for a bounty, prefix the title of the security
+advisory with one of the following, depending on the severity:
+* `[Bounty Category: Critical: Loss of Funds]`
+* `[Bounty Category: Critical: Loss of Availability]`
+* `[Bounty Category: Critical: DoS]`
+* `[Bounty Category: Critical: Other]`
+* `[Bounty Category: Non-critical]`
+* `[Bounty Category: RPC]`
+
+Confirm with the reporter that they agree with the severity assessment, and
+discuss as required to reach a conclusion.
+
+We currently do not use the Github workflow to publish security advisories.
+Once the issue and fix have been disclosed, and a bounty category is assessed if
+appropriate, the GitHub security advisory is no longer needed and can be closed.
+
+Bounties are currently awarded once a quarter (TODO: link to this process, or
+inline the workflow)

account-decoder/Cargo.toml

@@ -1,10 +1,11 @@
 [package]
 name = "solana-account-decoder"
-version = "1.3.13"
+version = "1.6.22"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-account-decoder"
 license = "Apache-2.0"
 edition = "2018"
 
@@ -15,15 +16,16 @@ bs58 = "0.3.1"
 bv = "0.11.1"
 Inflector = "0.11.4"
 lazy_static = "1.4.0"
-serde = "1.0.112"
+serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "1.3.13" }
-solana-sdk = { path = "../sdk", version = "1.3.13" }
-solana-stake-program = { path = "../programs/stake", version = "1.3.13" }
-solana-vote-program = { path = "../programs/vote", version = "1.3.13" }
-spl-token-v2-0 = { package = "spl-token", version = "2.0.6", features = ["skip-no-mangle"] }
+solana-config-program = { path = "../programs/config", version = "=1.6.22" }
+solana-sdk = { path = "../sdk", version = "=1.6.22" }
+solana-stake-program = { path = "../programs/stake", version = "=1.6.22" }
+solana-vote-program = { path = "../programs/vote", version = "=1.6.22" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.1.1", features = ["no-entrypoint"] }
 thiserror = "1.0"
+zstd = "0.5.1"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

account-decoder/src/lib.rs

@@ -1,9 +1,11 @@
+#![allow(clippy::integer_arithmetic)]
 #[macro_use]
 extern crate lazy_static;
 #[macro_use]
 extern crate serde_derive;
 
 pub mod parse_account_data;
+pub mod parse_bpf_loader;
 pub mod parse_config;
 pub mod parse_nonce;
 pub mod parse_stake;
@@ -12,11 +14,20 @@ pub mod parse_token;
 pub mod parse_vote;
 pub mod validator_info;
 
-use crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount};
-use solana_sdk::{account::Account, clock::Epoch, fee_calculator::FeeCalculator, pubkey::Pubkey};
-use std::str::FromStr;
+use {
+    crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount},
+    solana_sdk::{
+        account::ReadableAccount, account::WritableAccount, clock::Epoch,
+        fee_calculator::FeeCalculator, pubkey::Pubkey,
+    },
+    std::{
+        io::{Read, Write},
+        str::FromStr,
+    },
+};
 
 pub type StringAmount = String;
+pub type StringDecimals = String;
 
 /// A duplicate representation of an Account for pretty JSON serialization
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -37,71 +48,99 @@ pub enum UiAccountData {
     Binary(String, UiAccountEncoding),
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub enum UiAccountEncoding {
     Binary, // Legacy. Retained for RPC backwards compatibility
     Base58,
     Base64,
     JsonParsed,
+    #[serde(rename = "base64+zstd")]
+    Base64Zstd,
 }
 
 impl UiAccount {
-    pub fn encode(
+    pub fn encode<T: ReadableAccount>(
         pubkey: &Pubkey,
-        account: Account,
+        account: &T,
         encoding: UiAccountEncoding,
         additional_data: Option<AccountAdditionalData>,
         data_slice_config: Option<UiDataSliceConfig>,
     ) -> Self {
         let data = match encoding {
             UiAccountEncoding::Binary => UiAccountData::LegacyBinary(
-                bs58::encode(slice_data(&account.data, data_slice_config)).into_string(),
+                bs58::encode(slice_data(&account.data(), data_slice_config)).into_string(),
             ),
             UiAccountEncoding::Base58 => UiAccountData::Binary(
-                bs58::encode(slice_data(&account.data, data_slice_config)).into_string(),
+                bs58::encode(slice_data(&account.data(), data_slice_config)).into_string(),
                 encoding,
             ),
             UiAccountEncoding::Base64 => UiAccountData::Binary(
-                base64::encode(slice_data(&account.data, data_slice_config)),
+                base64::encode(slice_data(&account.data(), data_slice_config)),
                 encoding,
             ),
+            UiAccountEncoding::Base64Zstd => {
+                let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 0).unwrap();
+                match encoder
+                    .write_all(slice_data(&account.data(), data_slice_config))
+                    .and_then(|()| encoder.finish())
+                {
+                    Ok(zstd_data) => UiAccountData::Binary(base64::encode(zstd_data), encoding),
+                    Err(_) => UiAccountData::Binary(
+                        base64::encode(slice_data(&account.data(), data_slice_config)),
+                        UiAccountEncoding::Base64,
+                    ),
+                }
+            }
             UiAccountEncoding::JsonParsed => {
                 if let Ok(parsed_data) =
-                    parse_account_data(pubkey, &account.owner, &account.data, additional_data)
+                    parse_account_data(pubkey, &account.owner(), &account.data(), additional_data)
                 {
                     UiAccountData::Json(parsed_data)
                 } else {
-                    UiAccountData::Binary(base64::encode(&account.data), UiAccountEncoding::Base64)
+                    UiAccountData::Binary(
+                        base64::encode(&account.data()),
+                        UiAccountEncoding::Base64,
+                    )
                 }
             }
         };
         UiAccount {
-            lamports: account.lamports,
+            lamports: account.lamports(),
             data,
-            owner: account.owner.to_string(),
-            executable: account.executable,
-            rent_epoch: account.rent_epoch,
+            owner: account.owner().to_string(),
+            executable: account.executable(),
+            rent_epoch: account.rent_epoch(),
         }
     }
 
-    pub fn decode(&self) -> Option<Account> {
+    pub fn decode<T: WritableAccount>(&self) -> Option<T> {
         let data = match &self.data {
             UiAccountData::Json(_) => None,
             UiAccountData::LegacyBinary(blob) => bs58::decode(blob).into_vec().ok(),
             UiAccountData::Binary(blob, encoding) => match encoding {
                 UiAccountEncoding::Base58 => bs58::decode(blob).into_vec().ok(),
                 UiAccountEncoding::Base64 => base64::decode(blob).ok(),
+                UiAccountEncoding::Base64Zstd => base64::decode(blob)
+                    .ok()
+                    .map(|zstd_data| {
+                        let mut data = vec![];
+                        zstd::stream::read::Decoder::new(zstd_data.as_slice())
+                            .and_then(|mut reader| reader.read_to_end(&mut data))
+                            .map(|_| data)
+                            .ok()
+                    })
+                    .flatten(),
                 UiAccountEncoding::Binary | UiAccountEncoding::JsonParsed => None,
             },
         }?;
-        Some(Account {
-            lamports: self.lamports,
+        Some(T::create(
+            self.lamports,
             data,
-            owner: Pubkey::from_str(&self.owner).ok()?,
-            executable: self.executable,
-            rent_epoch: self.rent_epoch,
-        })
+            Pubkey::from_str(&self.owner).ok()?,
+            self.executable,
+            self.rent_epoch,
+        ))
     }
 }
 
@@ -151,6 +190,7 @@ fn slice_data(data: &[u8], data_slice_config: Option<UiDataSliceConfig>) -> &[u8
 #[cfg(test)]
 mod test {
     use super::*;
+    use solana_sdk::account::{Account, AccountSharedData};
 
     #[test]
     fn test_slice_data() {
@@ -179,4 +219,27 @@ mod test {
         });
         assert_eq!(slice_data(&data, slice_config), &[] as &[u8]);
     }
+
+    #[test]
+    fn test_base64_zstd() {
+        let encoded_account = UiAccount::encode(
+            &Pubkey::default(),
+            &AccountSharedData::from(Account {
+                data: vec![0; 1024],
+                ..Account::default()
+            }),
+            UiAccountEncoding::Base64Zstd,
+            None,
+            None,
+        );
+        assert!(matches!(
+            encoded_account.data,
+            UiAccountData::Binary(_, UiAccountEncoding::Base64Zstd)
+        ));
+
+        let decoded_account = encoded_account.decode::<Account>().unwrap();
+        assert_eq!(decoded_account.data(), &vec![0; 1024]);
+        let decoded_account = encoded_account.decode::<AccountSharedData>().unwrap();
+        assert_eq!(decoded_account.data(), &vec![0; 1024]);
+    }
 }

account-decoder/src/parse_account_data.rs

@@ -1,4 +1,5 @@
 use crate::{
+    parse_bpf_loader::parse_bpf_upgradeable_loader,
     parse_config::parse_config,
     parse_nonce::parse_nonce,
     parse_stake::parse_stake,
@@ -13,6 +14,7 @@ use std::collections::HashMap;
 use thiserror::Error;
 
 lazy_static! {
+    static ref BPF_UPGRADEABLE_LOADER_PROGRAM_ID: Pubkey = solana_sdk::bpf_loader_upgradeable::id();
     static ref CONFIG_PROGRAM_ID: Pubkey = solana_config_program::id();
     static ref STAKE_PROGRAM_ID: Pubkey = solana_stake_program::id();
     static ref SYSTEM_PROGRAM_ID: Pubkey = system_program::id();
@@ -21,6 +23,10 @@ lazy_static! {
     static ref VOTE_PROGRAM_ID: Pubkey = solana_vote_program::id();
     pub static ref PARSABLE_PROGRAM_IDS: HashMap<Pubkey, ParsableAccount> = {
         let mut m = HashMap::new();
+        m.insert(
+            *BPF_UPGRADEABLE_LOADER_PROGRAM_ID,
+            ParsableAccount::BpfUpgradeableLoader,
+        );
         m.insert(*CONFIG_PROGRAM_ID, ParsableAccount::Config);
         m.insert(*SYSTEM_PROGRAM_ID, ParsableAccount::Nonce);
         m.insert(*TOKEN_PROGRAM_ID, ParsableAccount::SplToken);
@@ -60,6 +66,7 @@ pub struct ParsedAccount {
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum ParsableAccount {
+    BpfUpgradeableLoader,
     Config,
     Nonce,
     SplToken,
@@ -81,9 +88,12 @@ pub fn parse_account_data(
 ) -> Result<ParsedAccount, ParseAccountError> {
     let program_name = PARSABLE_PROGRAM_IDS
         .get(program_id)
-        .ok_or_else(|| ParseAccountError::ProgramNotParsable)?;
+        .ok_or(ParseAccountError::ProgramNotParsable)?;
     let additional_data = additional_data.unwrap_or_default();
     let parsed_json = match program_name {
+        ParsableAccount::BpfUpgradeableLoader => {
+            serde_json::to_value(parse_bpf_upgradeable_loader(data)?)?
+        }
         ParsableAccount::Config => serde_json::to_value(parse_config(data, pubkey)?)?,
         ParsableAccount::Nonce => serde_json::to_value(parse_nonce(data)?)?,
         ParsableAccount::SplToken => {
@@ -111,14 +121,14 @@ mod test {
 
     #[test]
     fn test_parse_account_data() {
-        let account_pubkey = Pubkey::new_rand();
-        let other_program = Pubkey::new_rand();
+        let account_pubkey = solana_sdk::pubkey::new_rand();
+        let other_program = solana_sdk::pubkey::new_rand();
         let data = vec![0; 4];
         assert!(parse_account_data(&account_pubkey, &other_program, &data, None).is_err());
 
         let vote_state = VoteState::default();
         let mut vote_account_data: Vec<u8> = vec![0; VoteState::size_of()];
-        let versioned = VoteStateVersions::Current(Box::new(vote_state));
+        let versioned = VoteStateVersions::new_current(vote_state);
         VoteState::serialize(&versioned, &mut vote_account_data).unwrap();
         let parsed = parse_account_data(
             &account_pubkey,

account-decoder/src/parse_bpf_loader.rs

@@ -0,0 +1,181 @@
+use crate::{
+    parse_account_data::{ParsableAccount, ParseAccountError},
+    UiAccountData, UiAccountEncoding,
+};
+use bincode::{deserialize, serialized_size};
+use solana_sdk::{bpf_loader_upgradeable::UpgradeableLoaderState, pubkey::Pubkey};
+
+pub fn parse_bpf_upgradeable_loader(
+    data: &[u8],
+) -> Result<BpfUpgradeableLoaderAccountType, ParseAccountError> {
+    let account_state: UpgradeableLoaderState = deserialize(data).map_err(|_| {
+        ParseAccountError::AccountNotParsable(ParsableAccount::BpfUpgradeableLoader)
+    })?;
+    let parsed_account = match account_state {
+        UpgradeableLoaderState::Uninitialized => BpfUpgradeableLoaderAccountType::Uninitialized,
+        UpgradeableLoaderState::Buffer { authority_address } => {
+            let offset = if authority_address.is_some() {
+                UpgradeableLoaderState::buffer_data_offset().unwrap()
+            } else {
+                // This case included for code completeness; in practice, a Buffer account will
+                // always have authority_address.is_some()
+                UpgradeableLoaderState::buffer_data_offset().unwrap()
+                    - serialized_size(&Pubkey::default()).unwrap() as usize
+            };
+            BpfUpgradeableLoaderAccountType::Buffer(UiBuffer {
+                authority: authority_address.map(|pubkey| pubkey.to_string()),
+                data: UiAccountData::Binary(
+                    base64::encode(&data[offset as usize..]),
+                    UiAccountEncoding::Base64,
+                ),
+            })
+        }
+        UpgradeableLoaderState::Program {
+            programdata_address,
+        } => BpfUpgradeableLoaderAccountType::Program(UiProgram {
+            program_data: programdata_address.to_string(),
+        }),
+        UpgradeableLoaderState::ProgramData {
+            slot,
+            upgrade_authority_address,
+        } => {
+            let offset = if upgrade_authority_address.is_some() {
+                UpgradeableLoaderState::programdata_data_offset().unwrap()
+            } else {
+                UpgradeableLoaderState::programdata_data_offset().unwrap()
+                    - serialized_size(&Pubkey::default()).unwrap() as usize
+            };
+            BpfUpgradeableLoaderAccountType::ProgramData(UiProgramData {
+                slot,
+                authority: upgrade_authority_address.map(|pubkey| pubkey.to_string()),
+                data: UiAccountData::Binary(
+                    base64::encode(&data[offset as usize..]),
+                    UiAccountEncoding::Base64,
+                ),
+            })
+        }
+    };
+    Ok(parsed_account)
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase", tag = "type", content = "info")]
+pub enum BpfUpgradeableLoaderAccountType {
+    Uninitialized,
+    Buffer(UiBuffer),
+    Program(UiProgram),
+    ProgramData(UiProgramData),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiBuffer {
+    pub authority: Option<String>,
+    pub data: UiAccountData,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiProgram {
+    pub program_data: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiProgramData {
+    pub slot: u64,
+    pub authority: Option<String>,
+    pub data: UiAccountData,
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use bincode::serialize;
+    use solana_sdk::pubkey::Pubkey;
+
+    #[test]
+    fn test_parse_bpf_upgradeable_loader_accounts() {
+        let bpf_loader_state = UpgradeableLoaderState::Uninitialized;
+        let account_data = serialize(&bpf_loader_state).unwrap();
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Uninitialized
+        );
+
+        let program = vec![7u8; 64]; // Arbitrary program data
+
+        let authority = Pubkey::new_unique();
+        let bpf_loader_state = UpgradeableLoaderState::Buffer {
+            authority_address: Some(authority),
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Buffer(UiBuffer {
+                authority: Some(authority.to_string()),
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+
+        // This case included for code completeness; in practice, a Buffer account will always have
+        // authority_address.is_some()
+        let bpf_loader_state = UpgradeableLoaderState::Buffer {
+            authority_address: None,
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Buffer(UiBuffer {
+                authority: None,
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+
+        let programdata_address = Pubkey::new_unique();
+        let bpf_loader_state = UpgradeableLoaderState::Program {
+            programdata_address,
+        };
+        let account_data = serialize(&bpf_loader_state).unwrap();
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Program(UiProgram {
+                program_data: programdata_address.to_string(),
+            })
+        );
+
+        let authority = Pubkey::new_unique();
+        let slot = 42;
+        let bpf_loader_state = UpgradeableLoaderState::ProgramData {
+            slot,
+            upgrade_authority_address: Some(authority),
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::ProgramData(UiProgramData {
+                slot,
+                authority: Some(authority.to_string()),
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+
+        let bpf_loader_state = UpgradeableLoaderState::ProgramData {
+            slot,
+            upgrade_authority_address: None,
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::ProgramData(UiProgramData {
+                slot,
+                authority: None,
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+    }
+}

account-decoder/src/parse_config.rs

@@ -91,6 +91,7 @@ mod test {
     use crate::validator_info::ValidatorInfo;
     use serde_json::json;
     use solana_config_program::create_config_account;
+    use solana_sdk::account::ReadableAccount;
 
     #[test]
     fn test_parse_config() {
@@ -101,7 +102,7 @@ mod test {
         let stake_config_account = create_config_account(vec![], &stake_config, 10);
         assert_eq!(
             parse_config(
-                &stake_config_account.data,
+                &stake_config_account.data(),
                 &solana_stake_program::config::id()
             )
             .unwrap(),
@@ -117,14 +118,14 @@ mod test {
             }))
             .unwrap(),
         };
-        let info_pubkey = Pubkey::new_rand();
+        let info_pubkey = solana_sdk::pubkey::new_rand();
         let validator_info_config_account = create_config_account(
             vec![(validator_info::id(), false), (info_pubkey, true)],
             &validator_info,
             10,
         );
         assert_eq!(
-            parse_config(&validator_info_config_account.data, &info_pubkey).unwrap(),
+            parse_config(&validator_info_config_account.data(), &info_pubkey).unwrap(),
             ConfigAccountType::ValidatorInfo(UiConfig {
                 keys: vec![
                     UiConfigKey {

account-decoder/src/parse_nonce.rs

@@ -9,7 +9,13 @@ pub fn parse_nonce(data: &[u8]) -> Result<UiNonceState, ParseAccountError> {
         .map_err(|_| ParseAccountError::from(InstructionError::InvalidAccountData))?;
     let nonce_state = nonce_state.convert_to_current();
     match nonce_state {
-        State::Uninitialized => Ok(UiNonceState::Uninitialized),
+        // This prevents parsing an allocated System-owned account with empty data of any non-zero
+        // length as `uninitialized` nonce. An empty account of the wrong length can never be
+        // initialized as a nonce account, and an empty account of the correct length may not be an
+        // uninitialized nonce account, since it can be assigned to another program.
+        State::Uninitialized => Err(ParseAccountError::from(
+            InstructionError::InvalidAccountData,
+        )),
         State::Initialized(data) => Ok(UiNonceState::Initialized(UiNonceData {
             authority: data.authority.to_string(),
             blockhash: data.blockhash.to_string(),

account-decoder/src/parse_stake.rs

@@ -134,7 +134,6 @@ impl From<Delegation> for UiDelegation {
 mod test {
     use super::*;
     use bincode::serialize;
-    use solana_sdk::pubkey::Pubkey;
 
     #[test]
     fn test_parse_stake() {
@@ -145,8 +144,8 @@ mod test {
             StakeAccountType::Uninitialized
         );
 
-        let pubkey = Pubkey::new_rand();
-        let custodian = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
+        let custodian = solana_sdk::pubkey::new_rand();
         let authorized = Authorized::auto(&pubkey);
         let lockup = Lockup {
             unix_timestamp: 0,
@@ -180,7 +179,7 @@ mod test {
             })
         );
 
-        let voter_pubkey = Pubkey::new_rand();
+        let voter_pubkey = solana_sdk::pubkey::new_rand();
         let stake = Stake {
             delegation: Delegation {
                 voter_pubkey,

account-decoder/src/parse_sysvar.rs

@@ -105,6 +105,7 @@ pub enum SysvarAccountType {
 pub struct UiClock {
     pub slot: Slot,
     pub epoch: Epoch,
+    pub epoch_start_timestamp: UnixTimestamp,
     pub leader_schedule_epoch: Epoch,
     pub unix_timestamp: UnixTimestamp,
 }
@@ -114,6 +115,7 @@ impl From<Clock> for UiClock {
         Self {
             slot: clock.slot,
             epoch: clock.epoch,
+            epoch_start_timestamp: clock.epoch_start_timestamp,
             leader_schedule_epoch: clock.leader_schedule_epoch,
             unix_timestamp: clock.unix_timestamp,
         }
@@ -212,15 +214,13 @@ pub struct UiStakeHistoryEntry {
 mod test {
     use super::*;
     use solana_sdk::{
-        fee_calculator::FeeCalculator,
-        hash::Hash,
-        sysvar::{recent_blockhashes::IterItem, Sysvar},
+        account::create_account_for_test, fee_calculator::FeeCalculator, hash::Hash,
+        sysvar::recent_blockhashes::IterItem,
     };
-    use std::iter::FromIterator;
 
     #[test]
     fn test_parse_sysvars() {
-        let clock_sysvar = Clock::default().create_account(1);
+        let clock_sysvar = create_account_for_test(&Clock::default());
         assert_eq!(
             parse_sysvar(&clock_sysvar.data, &sysvar::clock::id()).unwrap(),
             SysvarAccountType::Clock(UiClock::default()),
@@ -233,13 +233,13 @@ mod test {
             first_normal_epoch: 1,
             first_normal_slot: 12,
         };
-        let epoch_schedule_sysvar = epoch_schedule.create_account(1);
+        let epoch_schedule_sysvar = create_account_for_test(&epoch_schedule);
         assert_eq!(
             parse_sysvar(&epoch_schedule_sysvar.data, &sysvar::epoch_schedule::id()).unwrap(),
             SysvarAccountType::EpochSchedule(epoch_schedule),
         );
 
-        let fees_sysvar = Fees::default().create_account(1);
+        let fees_sysvar = create_account_for_test(&Fees::default());
         assert_eq!(
             parse_sysvar(&fees_sysvar.data, &sysvar::fees::id()).unwrap(),
             SysvarAccountType::Fees(UiFees::default()),
@@ -249,9 +249,10 @@ mod test {
         let fee_calculator = FeeCalculator {
             lamports_per_signature: 10,
         };
-        let recent_blockhashes =
-            RecentBlockhashes::from_iter(vec![IterItem(0, &hash, &fee_calculator)].into_iter());
-        let recent_blockhashes_sysvar = recent_blockhashes.create_account(1);
+        let recent_blockhashes: RecentBlockhashes = vec![IterItem(0, &hash, &fee_calculator)]
+            .into_iter()
+            .collect();
+        let recent_blockhashes_sysvar = create_account_for_test(&recent_blockhashes);
         assert_eq!(
             parse_sysvar(
                 &recent_blockhashes_sysvar.data,
@@ -269,13 +270,13 @@ mod test {
             exemption_threshold: 2.0,
             burn_percent: 5,
         };
-        let rent_sysvar = rent.create_account(1);
+        let rent_sysvar = create_account_for_test(&rent);
         assert_eq!(
             parse_sysvar(&rent_sysvar.data, &sysvar::rent::id()).unwrap(),
             SysvarAccountType::Rent(rent.into()),
         );
 
-        let rewards_sysvar = Rewards::default().create_account(1);
+        let rewards_sysvar = create_account_for_test(&Rewards::default());
         assert_eq!(
             parse_sysvar(&rewards_sysvar.data, &sysvar::rewards::id()).unwrap(),
             SysvarAccountType::Rewards(UiRewards::default()),
@@ -283,7 +284,7 @@ mod test {
 
         let mut slot_hashes = SlotHashes::default();
         slot_hashes.add(1, hash);
-        let slot_hashes_sysvar = slot_hashes.create_account(1);
+        let slot_hashes_sysvar = create_account_for_test(&slot_hashes);
         assert_eq!(
             parse_sysvar(&slot_hashes_sysvar.data, &sysvar::slot_hashes::id()).unwrap(),
             SysvarAccountType::SlotHashes(vec![UiSlotHashEntry {
@@ -294,7 +295,7 @@ mod test {
 
         let mut slot_history = SlotHistory::default();
         slot_history.add(42);
-        let slot_history_sysvar = slot_history.create_account(1);
+        let slot_history_sysvar = create_account_for_test(&slot_history);
         assert_eq!(
             parse_sysvar(&slot_history_sysvar.data, &sysvar::slot_history::id()).unwrap(),
             SysvarAccountType::SlotHistory(UiSlotHistory {
@@ -310,7 +311,7 @@ mod test {
             deactivating: 3,
         };
         stake_history.add(1, stake_history_entry.clone());
-        let stake_history_sysvar = stake_history.create_account(1);
+        let stake_history_sysvar = create_account_for_test(&stake_history);
         assert_eq!(
             parse_sysvar(&stake_history_sysvar.data, &sysvar::stake_history::id()).unwrap(),
             SysvarAccountType::StakeHistory(vec![UiStakeHistoryEntry {
@@ -319,7 +320,7 @@ mod test {
             }]),
         );
 
-        let bad_pubkey = Pubkey::new_rand();
+        let bad_pubkey = solana_sdk::pubkey::new_rand();
         assert!(parse_sysvar(&stake_history_sysvar.data, &bad_pubkey).is_err());
 
         let bad_data = vec![0; 4];

account-decoder/src/parse_token.rs

@@ -1,10 +1,12 @@
 use crate::{
     parse_account_data::{ParsableAccount, ParseAccountError},
-    StringAmount,
+    StringAmount, StringDecimals,
 };
 use solana_sdk::pubkey::Pubkey;
 use spl_token_v2_0::{
-    solana_sdk::{program_option::COption, program_pack::Pack, pubkey::Pubkey as SplTokenPubkey},
+    solana_program::{
+        program_option::COption, program_pack::Pack, pubkey::Pubkey as SplTokenPubkey,
+    },
     state::{Account, AccountState, Mint, Multisig},
 };
 use std::str::FromStr;
@@ -12,13 +14,23 @@ use std::str::FromStr;
 // A helper function to convert spl_token_v2_0::id() as spl_sdk::pubkey::Pubkey to
 // solana_sdk::pubkey::Pubkey
 pub fn spl_token_id_v2_0() -> Pubkey {
-    Pubkey::from_str(&spl_token_v2_0::id().to_string()).unwrap()
+    Pubkey::new_from_array(spl_token_v2_0::id().to_bytes())
 }
 
 // A helper function to convert spl_token_v2_0::native_mint::id() as spl_sdk::pubkey::Pubkey to
 // solana_sdk::pubkey::Pubkey
 pub fn spl_token_v2_0_native_mint() -> Pubkey {
-    Pubkey::from_str(&spl_token_v2_0::native_mint::id().to_string()).unwrap()
+    Pubkey::new_from_array(spl_token_v2_0::native_mint::id().to_bytes())
+}
+
+// A helper function to convert a solana_sdk::pubkey::Pubkey to spl_sdk::pubkey::Pubkey
+pub fn spl_token_v2_0_pubkey(pubkey: &Pubkey) -> SplTokenPubkey {
+    SplTokenPubkey::new_from_array(pubkey.to_bytes())
+}
+
+// A helper function to convert a spl_sdk::pubkey::Pubkey to solana_sdk::pubkey::Pubkey
+pub fn pubkey_from_spl_token_v2_0(pubkey: &SplTokenPubkey) -> Pubkey {
+    Pubkey::new_from_array(pubkey.to_bytes())
 }
 
 pub fn parse_token(
@@ -146,21 +158,66 @@ impl From<AccountState> for UiAccountState {
     }
 }
 
+pub fn real_number_string(amount: u64, decimals: u8) -> StringDecimals {
+    let decimals = decimals as usize;
+    if decimals > 0 {
+        // Left-pad zeros to decimals + 1, so we at least have an integer zero
+        let mut s = format!("{:01$}", amount, decimals + 1);
+        // Add the decimal point (Sorry, "," locales!)
+        s.insert(s.len() - decimals, '.');
+        s
+    } else {
+        amount.to_string()
+    }
+}
+
+pub fn real_number_string_trimmed(amount: u64, decimals: u8) -> StringDecimals {
+    let mut s = real_number_string(amount, decimals);
+    if decimals > 0 {
+        let zeros_trimmed = s.trim_end_matches('0');
+        s = zeros_trimmed.trim_end_matches('.').to_string();
+    }
+    s
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct UiTokenAmount {
-    pub ui_amount: f64,
+    pub ui_amount: Option<f64>,
     pub decimals: u8,
     pub amount: StringAmount,
+    pub ui_amount_string: StringDecimals,
+}
+
+impl UiTokenAmount {
+    pub fn real_number_string(&self) -> String {
+        real_number_string(
+            u64::from_str(&self.amount).unwrap_or_default(),
+            self.decimals as u8,
+        )
+    }
+
+    pub fn real_number_string_trimmed(&self) -> String {
+        if !self.ui_amount_string.is_empty() {
+            self.ui_amount_string.clone()
+        } else {
+            real_number_string_trimmed(
+                u64::from_str(&self.amount).unwrap_or_default(),
+                self.decimals as u8,
+            )
+        }
+    }
 }
 
 pub fn token_amount_to_ui_amount(amount: u64, decimals: u8) -> UiTokenAmount {
-    // Use `amount_to_ui_amount()` once spl_token is bumped to a version that supports it: https://github.com/solana-labs/solana-program-library/pull/211
-    let amount_decimals = amount as f64 / 10_usize.pow(decimals as u32) as f64;
+    let amount_decimals = 10_usize
+        .checked_pow(decimals as u32)
+        .map(|dividend| amount as f64 / dividend as f64);
     UiTokenAmount {
         ui_amount: amount_decimals,
         decimals,
         amount: amount.to_string(),
+        ui_amount_string: real_number_string_trimmed(amount, decimals),
     }
 }
 
@@ -216,9 +273,10 @@ mod test {
                 mint: mint_pubkey.to_string(),
                 owner: owner_pubkey.to_string(),
                 token_amount: UiTokenAmount {
-                    ui_amount: 0.42,
+                    ui_amount: Some(0.42),
                     decimals: 2,
-                    amount: "42".to_string()
+                    amount: "42".to_string(),
+                    ui_amount_string: "0.42".to_string()
                 },
                 delegate: None,
                 state: UiAccountState::Initialized,
@@ -296,4 +354,90 @@ mod test {
             Some(expected_mint_pubkey)
         );
     }
+
+    #[test]
+    fn test_ui_token_amount_real_string() {
+        assert_eq!(&real_number_string(1, 0), "1");
+        assert_eq!(&real_number_string_trimmed(1, 0), "1");
+        let token_amount = token_amount_to_ui_amount(1, 0);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(1, 0)
+        );
+        assert_eq!(token_amount.ui_amount, Some(1.0));
+        assert_eq!(&real_number_string(10, 0), "10");
+        assert_eq!(&real_number_string_trimmed(10, 0), "10");
+        let token_amount = token_amount_to_ui_amount(10, 0);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(10, 0)
+        );
+        assert_eq!(token_amount.ui_amount, Some(10.0));
+        assert_eq!(&real_number_string(1, 9), "0.000000001");
+        assert_eq!(&real_number_string_trimmed(1, 9), "0.000000001");
+        let token_amount = token_amount_to_ui_amount(1, 9);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(1, 9)
+        );
+        assert_eq!(token_amount.ui_amount, Some(0.000000001));
+        assert_eq!(&real_number_string(1_000_000_000, 9), "1.000000000");
+        assert_eq!(&real_number_string_trimmed(1_000_000_000, 9), "1");
+        let token_amount = token_amount_to_ui_amount(1_000_000_000, 9);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(1_000_000_000, 9)
+        );
+        assert_eq!(token_amount.ui_amount, Some(1.0));
+        assert_eq!(&real_number_string(1_234_567_890, 3), "1234567.890");
+        assert_eq!(&real_number_string_trimmed(1_234_567_890, 3), "1234567.89");
+        let token_amount = token_amount_to_ui_amount(1_234_567_890, 3);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(1_234_567_890, 3)
+        );
+        assert_eq!(token_amount.ui_amount, Some(1234567.89));
+        assert_eq!(
+            &real_number_string(1_234_567_890, 25),
+            "0.0000000000000001234567890"
+        );
+        assert_eq!(
+            &real_number_string_trimmed(1_234_567_890, 25),
+            "0.000000000000000123456789"
+        );
+        let token_amount = token_amount_to_ui_amount(1_234_567_890, 20);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(1_234_567_890, 20)
+        );
+        assert_eq!(token_amount.ui_amount, None);
+    }
+
+    #[test]
+    fn test_ui_token_amount_real_string_zero() {
+        assert_eq!(&real_number_string(0, 0), "0");
+        assert_eq!(&real_number_string_trimmed(0, 0), "0");
+        let token_amount = token_amount_to_ui_amount(0, 0);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(0, 0)
+        );
+        assert_eq!(token_amount.ui_amount, Some(0.0));
+        assert_eq!(&real_number_string(0, 9), "0.000000000");
+        assert_eq!(&real_number_string_trimmed(0, 9), "0");
+        let token_amount = token_amount_to_ui_amount(0, 9);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(0, 9)
+        );
+        assert_eq!(token_amount.ui_amount, Some(0.0));
+        assert_eq!(&real_number_string(0, 25), "0.0000000000000000000000000");
+        assert_eq!(&real_number_string_trimmed(0, 25), "0");
+        let token_amount = token_amount_to_ui_amount(0, 20);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(0, 20)
+        );
+        assert_eq!(token_amount.ui_amount, None);
+    }
 }

account-decoder/src/parse_vote.rs

@@ -128,11 +128,13 @@ mod test {
     fn test_parse_vote() {
         let vote_state = VoteState::default();
         let mut vote_account_data: Vec<u8> = vec![0; VoteState::size_of()];
-        let versioned = VoteStateVersions::Current(Box::new(vote_state));
+        let versioned = VoteStateVersions::new_current(vote_state);
         VoteState::serialize(&versioned, &mut vote_account_data).unwrap();
-        let mut expected_vote_state = UiVoteState::default();
-        expected_vote_state.node_pubkey = Pubkey::default().to_string();
-        expected_vote_state.authorized_withdrawer = Pubkey::default().to_string();
+        let expected_vote_state = UiVoteState {
+            node_pubkey: Pubkey::default().to_string(),
+            authorized_withdrawer: Pubkey::default().to_string(),
+            ..UiVoteState::default()
+        };
         assert_eq!(
             parse_vote(&vote_account_data).unwrap(),
             VoteAccountType::Vote(expected_vote_state)

accounts-bench/Cargo.toml

@@ -2,19 +2,20 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-accounts-bench"
-version = "1.3.13"
+version = "1.6.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
+publish = false
 
 [dependencies]
-log = "0.4.6"
-rayon = "1.4.0"
-solana-logger = { path = "../logger", version = "1.3.13" }
-solana-runtime = { path = "../runtime", version = "1.3.13" }
-solana-measure = { path = "../measure", version = "1.3.13" }
-solana-sdk = { path = "../sdk", version = "1.3.13" }
-solana-version = { path = "../version", version = "1.3.13" }
+log = "0.4.11"
+rayon = "1.5.0"
+solana-logger = { path = "../logger", version = "=1.6.22" }
+solana-runtime = { path = "../runtime", version = "=1.6.22" }
+solana-measure = { path = "../measure", version = "=1.6.22" }
+solana-sdk = { path = "../sdk", version = "=1.6.22" }
+solana-version = { path = "../version", version = "=1.6.22" }
 rand = "0.7.0"
 clap = "2.33.1"
 crossbeam-channel = "0.4"

accounts-bench/src/main.rs

@@ -1,14 +1,15 @@
+#![allow(clippy::integer_arithmetic)]
+#[macro_use]
+extern crate log;
 use clap::{crate_description, crate_name, value_t, App, Arg};
 use rayon::prelude::*;
 use solana_measure::measure::Measure;
 use solana_runtime::{
-    accounts::{create_test_accounts, update_accounts, Accounts},
-    accounts_index::Ancestors,
+    accounts::{create_test_accounts, update_accounts_bench, Accounts},
+    accounts_index::{AccountSecondaryIndexes, Ancestors},
 };
 use solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey};
-use std::env;
-use std::fs;
-use std::path::PathBuf;
+use std::{env, fs, path::PathBuf};
 
 fn main() {
     solana_logger::setup();
@@ -53,10 +54,16 @@ fn main() {
 
     let path = PathBuf::from(env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_owned()))
         .join("accounts-bench");
+    println!("cleaning file system: {:?}", path);
     if fs::remove_dir_all(path.clone()).is_err() {
         println!("Warning: Couldn't remove {:?}", path);
     }
-    let accounts = Accounts::new(vec![path], &ClusterType::Testnet);
+    let accounts = Accounts::new_with_config(
+        vec![path],
+        &ClusterType::Testnet,
+        AccountSecondaryIndexes::default(),
+        false,
+    );
     println!("Creating {} accounts", num_accounts);
     let mut create_time = Measure::start("create accounts");
     let pubkeys: Vec<_> = (0..num_slots)
@@ -85,23 +92,52 @@ fn main() {
         ancestors.insert(i as u64, i - 1);
         accounts.add_root(i as u64);
     }
+    let mut elapsed = vec![0; iterations];
+    let mut elapsed_store = vec![0; iterations];
     for x in 0..iterations {
         if clean {
             let mut time = Measure::start("clean");
-            accounts.accounts_db.clean_accounts();
+            accounts.accounts_db.clean_accounts(None);
             time.stop();
             println!("{}", time);
             for slot in 0..num_slots {
-                update_accounts(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64);
+                update_accounts_bench(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64);
                 accounts.add_root((x * num_slots + slot) as u64);
             }
         } else {
             let mut pubkeys: Vec<Pubkey> = vec![];
             let mut time = Measure::start("hash");
-            let hash = accounts.accounts_db.update_accounts_hash(0, &ancestors).0;
+            let results = accounts.accounts_db.update_accounts_hash(0, &ancestors);
             time.stop();
-            println!("hash: {} {}", hash, time);
+            let mut time_store = Measure::start("hash using store");
+            let results_store = accounts.accounts_db.update_accounts_hash_with_index_option(
+                false,
+                false,
+                solana_sdk::clock::Slot::default(),
+                &ancestors,
+                None,
+            );
+            time_store.stop();
+            if results != results_store {
+                error!("results different: \n{:?}\n{:?}", results, results_store);
+            }
+            println!(
+                "hash,{},{},{},{}%",
+                results.0,
+                time,
+                time_store,
+                (time_store.as_us() as f64 / time.as_us() as f64 * 100.0f64) as u32
+            );
             create_test_accounts(&accounts, &mut pubkeys, 1, 0);
+            elapsed[x] = time.as_us();
+            elapsed_store[x] = time_store.as_us();
         }
     }
+
+    for x in elapsed {
+        info!("update_accounts_hash(us),{}", x);
+    }
+    for x in elapsed_store {
+        info!("calculate_accounts_hash_without_index(us),{}", x);
+    }
 }

accounts-cluster-bench/.gitignore

@@ -0,0 +1 @@
+/farf/

accounts-cluster-bench/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2018"
+name = "solana-accounts-cluster-bench"
+version = "1.6.22"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+publish = false
+
+[dependencies]
+clap = "2.33.1"
+log = "0.4.11"
+rand = "0.7.0"
+rayon = "1.4.1"
+solana-account-decoder = { path = "../account-decoder", version = "=1.6.22" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.22" }
+solana-client = { path = "../client", version = "=1.6.22" }
+solana-core = { path = "../core", version = "=1.6.22" }
+solana-measure = { path = "../measure", version = "=1.6.22" }
+solana-logger = { path = "../logger", version = "=1.6.22" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.22" }
+solana-faucet = { path = "../faucet", version = "=1.6.22" }
+solana-runtime = { path = "../runtime", version = "=1.6.22" }
+solana-sdk = { path = "../sdk", version = "=1.6.22" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.6.22" }
+solana-version = { path = "../version", version = "=1.6.22" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.1.1", features = ["no-entrypoint"] }
+
+[dev-dependencies]
+solana-local-cluster = { path = "../local-cluster", version = "=1.6.22" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accounts-cluster-bench/src/main.rs

@@ -0,0 +1,742 @@
+#![allow(clippy::integer_arithmetic)]
+use clap::{crate_description, crate_name, value_t, values_t_or_exit, App, Arg};
+use log::*;
+use rand::{thread_rng, Rng};
+use rayon::prelude::*;
+use solana_account_decoder::parse_token::spl_token_v2_0_pubkey;
+use solana_clap_utils::input_parsers::pubkey_of;
+use solana_client::rpc_client::RpcClient;
+use solana_core::gossip_service::discover;
+use solana_faucet::faucet::{request_airdrop_transaction, FAUCET_PORT};
+use solana_measure::measure::Measure;
+use solana_runtime::inline_spl_token_v2_0;
+use solana_sdk::{
+    commitment_config::CommitmentConfig,
+    message::Message,
+    pubkey::Pubkey,
+    rpc_port::DEFAULT_RPC_PORT,
+    signature::{read_keypair_file, Keypair, Signature, Signer},
+    system_instruction, system_program,
+    timing::timestamp,
+    transaction::Transaction,
+};
+use solana_transaction_status::parse_token::spl_token_v2_0_instruction;
+use std::{
+    net::SocketAddr,
+    process::exit,
+    sync::{
+        atomic::{AtomicBool, AtomicU64, Ordering},
+        Arc, RwLock,
+    },
+    thread::{sleep, Builder, JoinHandle},
+    time::{Duration, Instant},
+};
+
+// Create and close messages both require 2 signatures; if transaction construction changes, update
+// this magic number
+const NUM_SIGNATURES: u64 = 2;
+
+pub fn airdrop_lamports(
+    client: &RpcClient,
+    faucet_addr: &SocketAddr,
+    id: &Keypair,
+    desired_balance: u64,
+) -> bool {
+    let starting_balance = client.get_balance(&id.pubkey()).unwrap_or(0);
+    info!("starting balance {}", starting_balance);
+
+    if starting_balance < desired_balance {
+        let airdrop_amount = desired_balance - starting_balance;
+        info!(
+            "Airdropping {:?} lamports from {} for {}",
+            airdrop_amount,
+            faucet_addr,
+            id.pubkey(),
+        );
+
+        let (blockhash, _fee_calculator) = client.get_recent_blockhash().unwrap();
+        match request_airdrop_transaction(&faucet_addr, &id.pubkey(), airdrop_amount, blockhash) {
+            Ok(transaction) => {
+                let mut tries = 0;
+                loop {
+                    tries += 1;
+                    let result = client.send_and_confirm_transaction(&transaction);
+
+                    if result.is_ok() {
+                        break;
+                    }
+                    if tries >= 5 {
+                        panic!(
+                            "Error requesting airdrop: to addr: {:?} amount: {} {:?}",
+                            faucet_addr, airdrop_amount, result
+                        )
+                    }
+                }
+            }
+            Err(err) => {
+                panic!(
+                    "Error requesting airdrop: {:?} to addr: {:?} amount: {}",
+                    err, faucet_addr, airdrop_amount
+                );
+            }
+        };
+
+        let current_balance = client.get_balance(&id.pubkey()).unwrap_or_else(|e| {
+            panic!("airdrop error {}", e);
+        });
+        info!("current balance {}...", current_balance);
+
+        if current_balance - starting_balance != airdrop_amount {
+            info!(
+                "Airdrop failed? {} {} {} {}",
+                id.pubkey(),
+                current_balance,
+                starting_balance,
+                airdrop_amount,
+            );
+        }
+    }
+    true
+}
+
+// signature, timestamp, id
+type PendingQueue = Vec<(Signature, u64, u64)>;
+
+struct TransactionExecutor {
+    sig_clear_t: JoinHandle<()>,
+    sigs: Arc<RwLock<PendingQueue>>,
+    cleared: Arc<RwLock<Vec<u64>>>,
+    exit: Arc<AtomicBool>,
+    counter: AtomicU64,
+    client: RpcClient,
+}
+
+impl TransactionExecutor {
+    fn new(entrypoint_addr: SocketAddr) -> Self {
+        let sigs = Arc::new(RwLock::new(Vec::new()));
+        let cleared = Arc::new(RwLock::new(Vec::new()));
+        let exit = Arc::new(AtomicBool::new(false));
+        let sig_clear_t = Self::start_sig_clear_thread(&exit, &sigs, &cleared, entrypoint_addr);
+        let client =
+            RpcClient::new_socket_with_commitment(entrypoint_addr, CommitmentConfig::confirmed());
+        Self {
+            sigs,
+            cleared,
+            sig_clear_t,
+            exit,
+            counter: AtomicU64::new(0),
+            client,
+        }
+    }
+
+    fn num_outstanding(&self) -> usize {
+        self.sigs.read().unwrap().len()
+    }
+
+    fn push_transactions(&self, txs: Vec<Transaction>) -> Vec<u64> {
+        let mut ids = vec![];
+        let new_sigs = txs.into_iter().filter_map(|tx| {
+            let id = self.counter.fetch_add(1, Ordering::Relaxed);
+            ids.push(id);
+            match self.client.send_transaction(&tx) {
+                Ok(sig) => {
+                    return Some((sig, timestamp(), id));
+                }
+                Err(e) => {
+                    info!("error: {:#?}", e);
+                }
+            }
+            None
+        });
+        let mut sigs_w = self.sigs.write().unwrap();
+        sigs_w.extend(new_sigs);
+        ids
+    }
+
+    fn drain_cleared(&self) -> Vec<u64> {
+        std::mem::take(&mut *self.cleared.write().unwrap())
+    }
+
+    fn close(self) {
+        self.exit.store(true, Ordering::Relaxed);
+        self.sig_clear_t.join().unwrap();
+    }
+
+    fn start_sig_clear_thread(
+        exit: &Arc<AtomicBool>,
+        sigs: &Arc<RwLock<PendingQueue>>,
+        cleared: &Arc<RwLock<Vec<u64>>>,
+        entrypoint_addr: SocketAddr,
+    ) -> JoinHandle<()> {
+        let sigs = sigs.clone();
+        let exit = exit.clone();
+        let cleared = cleared.clone();
+        Builder::new()
+            .name("sig_clear".to_string())
+            .spawn(move || {
+                let client = RpcClient::new_socket_with_commitment(
+                    entrypoint_addr,
+                    CommitmentConfig::confirmed(),
+                );
+                let mut success = 0;
+                let mut error_count = 0;
+                let mut timed_out = 0;
+                let mut last_log = Instant::now();
+                while !exit.load(Ordering::Relaxed) {
+                    let sigs_len = sigs.read().unwrap().len();
+                    if sigs_len > 0 {
+                        let mut sigs_w = sigs.write().unwrap();
+                        let mut start = Measure::start("sig_status");
+                        let statuses: Vec<_> = sigs_w
+                            .chunks(200)
+                            .flat_map(|sig_chunk| {
+                                let only_sigs: Vec<_> = sig_chunk.iter().map(|s| s.0).collect();
+                                client
+                                    .get_signature_statuses(&only_sigs)
+                                    .expect("status fail")
+                                    .value
+                            })
+                            .collect();
+                        let mut num_cleared = 0;
+                        let start_len = sigs_w.len();
+                        let now = timestamp();
+                        let mut new_ids = vec![];
+                        let mut i = 0;
+                        let mut j = 0;
+                        while i != sigs_w.len() {
+                            let mut retain = true;
+                            let sent_ts = sigs_w[i].1;
+                            if let Some(e) = &statuses[j] {
+                                debug!("error: {:?}", e);
+                                if e.status.is_ok() {
+                                    success += 1;
+                                } else {
+                                    error_count += 1;
+                                }
+                                num_cleared += 1;
+                                retain = false;
+                            } else if now - sent_ts > 30_000 {
+                                retain = false;
+                                timed_out += 1;
+                            }
+                            if !retain {
+                                new_ids.push(sigs_w.remove(i).2);
+                            } else {
+                                i += 1;
+                            }
+                            j += 1;
+                        }
+                        let final_sigs_len = sigs_w.len();
+                        drop(sigs_w);
+                        cleared.write().unwrap().extend(new_ids);
+                        start.stop();
+                        debug!(
+                            "sigs len: {:?} success: {} took: {}ms cleared: {}/{}",
+                            final_sigs_len,
+                            success,
+                            start.as_ms(),
+                            num_cleared,
+                            start_len,
+                        );
+                        if last_log.elapsed().as_millis() > 5000 {
+                            info!(
+                                "success: {} error: {} timed_out: {}",
+                                success, error_count, timed_out,
+                            );
+                            last_log = Instant::now();
+                        }
+                    }
+                    sleep(Duration::from_millis(200));
+                }
+            })
+            .unwrap()
+    }
+}
+
+struct SeedTracker {
+    max_created: Arc<AtomicU64>,
+    max_closed: Arc<AtomicU64>,
+}
+
+fn make_create_message(
+    keypair: &Keypair,
+    base_keypair: &Keypair,
+    max_created_seed: Arc<AtomicU64>,
+    num_instructions: usize,
+    balance: u64,
+    maybe_space: Option<u64>,
+    mint: Option<Pubkey>,
+) -> Message {
+    let space = maybe_space.unwrap_or_else(|| thread_rng().gen_range(0, 1000));
+
+    let instructions: Vec<_> = (0..num_instructions)
+        .into_iter()
+        .map(|_| {
+            let program_id = if mint.is_some() {
+                inline_spl_token_v2_0::id()
+            } else {
+                system_program::id()
+            };
+            let seed = max_created_seed.fetch_add(1, Ordering::Relaxed).to_string();
+            let to_pubkey =
+                Pubkey::create_with_seed(&base_keypair.pubkey(), &seed, &program_id).unwrap();
+            let mut instructions = vec![system_instruction::create_account_with_seed(
+                &keypair.pubkey(),
+                &to_pubkey,
+                &base_keypair.pubkey(),
+                &seed,
+                balance,
+                space,
+                &program_id,
+            )];
+            if let Some(mint_address) = mint {
+                instructions.push(spl_token_v2_0_instruction(
+                    spl_token_v2_0::instruction::initialize_account(
+                        &spl_token_v2_0::id(),
+                        &spl_token_v2_0_pubkey(&to_pubkey),
+                        &spl_token_v2_0_pubkey(&mint_address),
+                        &spl_token_v2_0_pubkey(&base_keypair.pubkey()),
+                    )
+                    .unwrap(),
+                ));
+            }
+
+            instructions
+        })
+        .collect();
+    let instructions: Vec<_> = instructions.into_iter().flatten().collect();
+
+    Message::new(&instructions, Some(&keypair.pubkey()))
+}
+
+fn make_close_message(
+    keypair: &Keypair,
+    base_keypair: &Keypair,
+    max_closed_seed: Arc<AtomicU64>,
+    num_instructions: usize,
+    balance: u64,
+    spl_token: bool,
+) -> Message {
+    let instructions: Vec<_> = (0..num_instructions)
+        .into_iter()
+        .map(|_| {
+            let program_id = if spl_token {
+                inline_spl_token_v2_0::id()
+            } else {
+                system_program::id()
+            };
+            let seed = max_closed_seed.fetch_add(1, Ordering::Relaxed).to_string();
+            let address =
+                Pubkey::create_with_seed(&base_keypair.pubkey(), &seed, &program_id).unwrap();
+            if spl_token {
+                spl_token_v2_0_instruction(
+                    spl_token_v2_0::instruction::close_account(
+                        &spl_token_v2_0::id(),
+                        &spl_token_v2_0_pubkey(&address),
+                        &spl_token_v2_0_pubkey(&keypair.pubkey()),
+                        &spl_token_v2_0_pubkey(&base_keypair.pubkey()),
+                        &[],
+                    )
+                    .unwrap(),
+                )
+            } else {
+                system_instruction::transfer_with_seed(
+                    &address,
+                    &base_keypair.pubkey(),
+                    seed,
+                    &program_id,
+                    &keypair.pubkey(),
+                    balance,
+                )
+            }
+        })
+        .collect();
+
+    Message::new(&instructions, Some(&keypair.pubkey()))
+}
+
+#[allow(clippy::too_many_arguments)]
+fn run_accounts_bench(
+    entrypoint_addr: SocketAddr,
+    faucet_addr: SocketAddr,
+    payer_keypairs: &[&Keypair],
+    iterations: usize,
+    maybe_space: Option<u64>,
+    batch_size: usize,
+    close_nth: u64,
+    maybe_lamports: Option<u64>,
+    num_instructions: usize,
+    mint: Option<Pubkey>,
+) {
+    assert!(num_instructions > 0);
+    let client =
+        RpcClient::new_socket_with_commitment(entrypoint_addr, CommitmentConfig::confirmed());
+
+    info!("Targeting {}", entrypoint_addr);
+
+    let mut last_blockhash = Instant::now();
+    let mut last_log = Instant::now();
+    let mut count = 0;
+    let mut recent_blockhash = client.get_recent_blockhash().expect("blockhash");
+    let mut tx_sent_count = 0;
+    let mut total_accounts_created = 0;
+    let mut total_accounts_closed = 0;
+    let mut balances: Vec<_> = payer_keypairs
+        .iter()
+        .map(|keypair| client.get_balance(&keypair.pubkey()).unwrap_or(0))
+        .collect();
+    let mut last_balance = Instant::now();
+
+    let default_max_lamports = 1000;
+    let min_balance = maybe_lamports.unwrap_or_else(|| {
+        let space = maybe_space.unwrap_or(default_max_lamports);
+        client
+            .get_minimum_balance_for_rent_exemption(space as usize)
+            .expect("min balance")
+    });
+
+    let base_keypair = Keypair::new();
+    let seed_tracker = SeedTracker {
+        max_created: Arc::new(AtomicU64::default()),
+        max_closed: Arc::new(AtomicU64::default()),
+    };
+
+    info!("Starting balance(s): {:?}", balances);
+
+    let executor = TransactionExecutor::new(entrypoint_addr);
+
+    loop {
+        if last_blockhash.elapsed().as_millis() > 10_000 {
+            recent_blockhash = client.get_recent_blockhash().expect("blockhash");
+            last_blockhash = Instant::now();
+        }
+
+        let fee = recent_blockhash
+            .1
+            .lamports_per_signature
+            .saturating_mul(NUM_SIGNATURES);
+        let lamports = min_balance + fee;
+
+        for (i, balance) in balances.iter_mut().enumerate() {
+            if *balance < lamports || last_balance.elapsed().as_millis() > 2000 {
+                if let Ok(b) = client.get_balance(&payer_keypairs[i].pubkey()) {
+                    *balance = b;
+                }
+                last_balance = Instant::now();
+                if *balance < lamports * 2 {
+                    info!(
+                        "Balance {} is less than needed: {}, doing aidrop...",
+                        balance, lamports
+                    );
+                    if !airdrop_lamports(
+                        &client,
+                        &faucet_addr,
+                        &payer_keypairs[i],
+                        lamports * 100_000,
+                    ) {
+                        warn!("failed airdrop, exiting");
+                        return;
+                    }
+                }
+            }
+        }
+
+        let sigs_len = executor.num_outstanding();
+        if sigs_len < batch_size {
+            let num_to_create = batch_size - sigs_len;
+            if num_to_create >= payer_keypairs.len() {
+                info!("creating {} new", num_to_create);
+                let chunk_size = num_to_create / payer_keypairs.len();
+                if chunk_size > 0 {
+                    for (i, keypair) in payer_keypairs.iter().enumerate() {
+                        let txs: Vec<_> = (0..chunk_size)
+                            .into_par_iter()
+                            .map(|_| {
+                                let message = make_create_message(
+                                    keypair,
+                                    &base_keypair,
+                                    seed_tracker.max_created.clone(),
+                                    num_instructions,
+                                    min_balance,
+                                    maybe_space,
+                                    mint,
+                                );
+                                let signers: Vec<&Keypair> = vec![keypair, &base_keypair];
+                                Transaction::new(&signers, message, recent_blockhash.0)
+                            })
+                            .collect();
+                        balances[i] = balances[i].saturating_sub(lamports * txs.len() as u64);
+                        info!("txs: {}", txs.len());
+                        let new_ids = executor.push_transactions(txs);
+                        info!("ids: {}", new_ids.len());
+                        tx_sent_count += new_ids.len();
+                        total_accounts_created += num_instructions * new_ids.len();
+                    }
+                }
+            }
+
+            if close_nth > 0 {
+                let expected_closed = total_accounts_created as u64 / close_nth;
+                if expected_closed > total_accounts_closed {
+                    let txs: Vec<_> = (0..expected_closed - total_accounts_closed)
+                        .into_par_iter()
+                        .map(|_| {
+                            let message = make_close_message(
+                                &payer_keypairs[0],
+                                &base_keypair,
+                                seed_tracker.max_closed.clone(),
+                                1,
+                                min_balance,
+                                mint.is_some(),
+                            );
+                            let signers: Vec<&Keypair> = vec![&payer_keypairs[0], &base_keypair];
+                            Transaction::new(&signers, message, recent_blockhash.0)
+                        })
+                        .collect();
+                    balances[0] = balances[0].saturating_sub(fee * txs.len() as u64);
+                    info!("close txs: {}", txs.len());
+                    let new_ids = executor.push_transactions(txs);
+                    info!("close ids: {}", new_ids.len());
+                    tx_sent_count += new_ids.len();
+                    total_accounts_closed += new_ids.len() as u64;
+                }
+            }
+        } else {
+            let _ = executor.drain_cleared();
+        }
+
+        count += 1;
+        if last_log.elapsed().as_millis() > 3000 {
+            info!(
+                "total_accounts_created: {} total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance(s): {:?}",
+                total_accounts_created, total_accounts_closed, tx_sent_count, count, balances
+            );
+            last_log = Instant::now();
+        }
+        if iterations != 0 && count >= iterations {
+            break;
+        }
+        if executor.num_outstanding() >= batch_size {
+            sleep(Duration::from_millis(500));
+        }
+    }
+    executor.close();
+}
+
+fn main() {
+    solana_logger::setup_with_default("solana=info");
+    let matches = App::new(crate_name!())
+        .about(crate_description!())
+        .version(solana_version::version!())
+        .arg(
+            Arg::with_name("entrypoint")
+                .long("entrypoint")
+                .takes_value(true)
+                .value_name("HOST:PORT")
+                .help("RPC entrypoint address. Usually <ip>:8899"),
+        )
+        .arg(
+            Arg::with_name("faucet_addr")
+                .long("faucet")
+                .takes_value(true)
+                .value_name("HOST:PORT")
+                .help("Faucet entrypoint address. Usually <ip>:9900"),
+        )
+        .arg(
+            Arg::with_name("space")
+                .long("space")
+                .takes_value(true)
+                .value_name("BYTES")
+                .help("Size of accounts to create"),
+        )
+        .arg(
+            Arg::with_name("lamports")
+                .long("lamports")
+                .takes_value(true)
+                .value_name("LAMPORTS")
+                .help("How many lamports to fund each account"),
+        )
+        .arg(
+            Arg::with_name("identity")
+                .long("identity")
+                .takes_value(true)
+                .multiple(true)
+                .value_name("FILE")
+                .help("keypair file"),
+        )
+        .arg(
+            Arg::with_name("batch_size")
+                .long("batch-size")
+                .takes_value(true)
+                .value_name("BYTES")
+                .help("Number of transactions to send per batch"),
+        )
+        .arg(
+            Arg::with_name("close_nth")
+                .long("close-frequency")
+                .takes_value(true)
+                .value_name("BYTES")
+                .help(
+                    "Send close transactions after this many accounts created. \
+                    Note: a `close-frequency` value near or below `batch-size` \
+                    may result in transaction-simulation errors, as the close \
+                    transactions will be submitted before the corresponding \
+                    create transactions have been confirmed",
+                ),
+        )
+        .arg(
+            Arg::with_name("num_instructions")
+                .long("num-instructions")
+                .takes_value(true)
+                .value_name("NUM")
+                .help("Number of accounts to create on each transaction"),
+        )
+        .arg(
+            Arg::with_name("iterations")
+                .long("iterations")
+                .takes_value(true)
+                .value_name("NUM")
+                .help("Number of iterations to make"),
+        )
+        .arg(
+            Arg::with_name("check_gossip")
+                .long("check-gossip")
+                .help("Just use entrypoint address directly"),
+        )
+        .arg(
+            Arg::with_name("mint")
+                .long("mint")
+                .takes_value(true)
+                .help("Mint address to initialize account"),
+        )
+        .get_matches();
+
+    let skip_gossip = !matches.is_present("check_gossip");
+
+    let port = if skip_gossip { DEFAULT_RPC_PORT } else { 8001 };
+    let mut entrypoint_addr = SocketAddr::from(([127, 0, 0, 1], port));
+    if let Some(addr) = matches.value_of("entrypoint") {
+        entrypoint_addr = solana_net_utils::parse_host_port(addr).unwrap_or_else(|e| {
+            eprintln!("failed to parse entrypoint address: {}", e);
+            exit(1)
+        });
+    }
+    let mut faucet_addr = SocketAddr::from(([127, 0, 0, 1], FAUCET_PORT));
+    if let Some(addr) = matches.value_of("faucet_addr") {
+        faucet_addr = solana_net_utils::parse_host_port(addr).unwrap_or_else(|e| {
+            eprintln!("failed to parse entrypoint address: {}", e);
+            exit(1)
+        });
+    }
+
+    let space = value_t!(matches, "space", u64).ok();
+    let lamports = value_t!(matches, "lamports", u64).ok();
+    let batch_size = value_t!(matches, "batch_size", usize).unwrap_or(4);
+    let close_nth = value_t!(matches, "close_nth", u64).unwrap_or(0);
+    let iterations = value_t!(matches, "iterations", usize).unwrap_or(10);
+    let num_instructions = value_t!(matches, "num_instructions", usize).unwrap_or(1);
+    if num_instructions == 0 || num_instructions > 500 {
+        eprintln!("bad num_instructions: {}", num_instructions);
+        exit(1);
+    }
+
+    let mint = pubkey_of(&matches, "mint");
+
+    let payer_keypairs: Vec<_> = values_t_or_exit!(matches, "identity", String)
+        .iter()
+        .map(|keypair_string| {
+            read_keypair_file(keypair_string)
+                .unwrap_or_else(|_| panic!("bad keypair {:?}", keypair_string))
+        })
+        .collect();
+    let mut payer_keypair_refs: Vec<&Keypair> = vec![];
+    for keypair in payer_keypairs.iter() {
+        payer_keypair_refs.push(keypair);
+    }
+
+    let rpc_addr = if !skip_gossip {
+        info!("Finding cluster entry: {:?}", entrypoint_addr);
+        let (gossip_nodes, _validators) = discover(
+            None, // keypair
+            Some(&entrypoint_addr),
+            None,                    // num_nodes
+            Duration::from_secs(60), // timeout
+            None,                    // find_node_by_pubkey
+            Some(&entrypoint_addr),  // find_node_by_gossip_addr
+            None,                    // my_gossip_addr
+            0,                       // my_shred_version
+        )
+        .unwrap_or_else(|err| {
+            eprintln!("Failed to discover {} node: {:?}", entrypoint_addr, err);
+            exit(1);
+        });
+
+        info!("done found {} nodes", gossip_nodes.len());
+        gossip_nodes[0].rpc
+    } else {
+        info!("Using {:?} as the RPC address", entrypoint_addr);
+        entrypoint_addr
+    };
+
+    run_accounts_bench(
+        rpc_addr,
+        faucet_addr,
+        &payer_keypair_refs,
+        iterations,
+        space,
+        batch_size,
+        close_nth,
+        lamports,
+        num_instructions,
+        mint,
+    );
+}
+
+#[cfg(test)]
+pub mod test {
+    use super::*;
+    use solana_core::validator::ValidatorConfig;
+    use solana_local_cluster::{
+        local_cluster::{ClusterConfig, LocalCluster},
+        validator_configs::make_identical_validator_configs,
+    };
+    use solana_sdk::poh_config::PohConfig;
+
+    #[test]
+    fn test_accounts_cluster_bench() {
+        solana_logger::setup();
+        let validator_config = ValidatorConfig::default();
+        let num_nodes = 1;
+        let mut config = ClusterConfig {
+            cluster_lamports: 10_000_000,
+            poh_config: PohConfig::new_sleep(Duration::from_millis(50)),
+            node_stakes: vec![100; num_nodes],
+            validator_configs: make_identical_validator_configs(&validator_config, num_nodes),
+            ..ClusterConfig::default()
+        };
+
+        let faucet_addr = SocketAddr::from(([127, 0, 0, 1], 9900));
+        let cluster = LocalCluster::new(&mut config);
+        let iterations = 10;
+        let maybe_space = None;
+        let batch_size = 100;
+        let close_nth = 100;
+        let maybe_lamports = None;
+        let num_instructions = 2;
+        let mut start = Measure::start("total accounts run");
+        run_accounts_bench(
+            cluster.entry_point_info.rpc,
+            faucet_addr,
+            &[&cluster.funding_keypair],
+            iterations,
+            maybe_space,
+            batch_size,
+            close_nth,
+            maybe_lamports,
+            num_instructions,
+            None,
+        );
+        start.stop();
+        info!("{}", start);
+    }
+}

banking-bench/Cargo.toml

@@ -2,27 +2,28 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.3.13"
+version = "1.6.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
+publish = false
 
 [dependencies]
 clap = "2.33.1"
 crossbeam-channel = "0.4"
-log = "0.4.6"
+log = "0.4.11"
 rand = "0.7.0"
-rayon = "1.4.0"
-solana-core = { path = "../core", version = "1.3.13" }
-solana-clap-utils = { path = "../clap-utils", version = "1.3.13" }
-solana-streamer = { path = "../streamer", version = "1.3.13" }
-solana-perf = { path = "../perf", version = "1.3.13" }
-solana-ledger = { path = "../ledger", version = "1.3.13" }
-solana-logger = { path = "../logger", version = "1.3.13" }
-solana-runtime = { path = "../runtime", version = "1.3.13" }
-solana-measure = { path = "../measure", version = "1.3.13" }
-solana-sdk = { path = "../sdk", version = "1.3.13" }
-solana-version = { path = "../version", version = "1.3.13" }
+rayon = "1.5.0"
+solana-core = { path = "../core", version = "=1.6.22" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.22" }
+solana-streamer = { path = "../streamer", version = "=1.6.22" }
+solana-perf = { path = "../perf", version = "=1.6.22" }
+solana-ledger = { path = "../ledger", version = "=1.6.22" }
+solana-logger = { path = "../logger", version = "=1.6.22" }
+solana-runtime = { path = "../runtime", version = "=1.6.22" }
+solana-measure = { path = "../measure", version = "=1.6.22" }
+solana-sdk = { path = "../sdk", version = "=1.6.22" }
+solana-version = { path = "../version", version = "=1.6.22" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 use clap::{crate_description, crate_name, value_t, App, Arg};
 use crossbeam_channel::unbounded;
 use log::*;
@@ -17,10 +18,11 @@ use solana_ledger::{
 };
 use solana_measure::measure::Measure;
 use solana_perf::packet::to_packets_chunked;
-use solana_runtime::{bank::Bank, bank_forks::BankForks};
+use solana_runtime::{
+    accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks,
+};
 use solana_sdk::{
     hash::Hash,
-    pubkey::Pubkey,
     signature::Keypair,
     signature::Signature,
     system_transaction,
@@ -69,7 +71,7 @@ fn make_accounts_txs(
     hash: Hash,
     same_payer: bool,
 ) -> Vec<Transaction> {
-    let to_pubkey = Pubkey::new_rand();
+    let to_pubkey = solana_sdk::pubkey::new_rand();
     let payer_key = Keypair::new();
     let dummy = system_transaction::transfer(&payer_key, &to_pubkey, 1, hash);
     (0..total_num_transactions)
@@ -78,9 +80,9 @@ fn make_accounts_txs(
             let mut new = dummy.clone();
             let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
             if !same_payer {
-                new.message.account_keys[0] = Pubkey::new_rand();
+                new.message.account_keys[0] = solana_sdk::pubkey::new_rand();
             }
-            new.message.account_keys[1] = Pubkey::new_rand();
+            new.message.account_keys[1] = solana_sdk::pubkey::new_rand();
             new.signatures = vec![Signature::new(&sig[0..64])];
             new
         })
@@ -241,7 +243,7 @@ fn main() {
         let base_tx_count = bank.transaction_count();
         let mut txs_processed = 0;
         let mut root = 1;
-        let collector = Pubkey::new_rand();
+        let collector = solana_sdk::pubkey::new_rand();
         let config = Config {
             packets_per_batch: packets_per_chunk,
             chunk_len,
@@ -324,7 +326,7 @@ fn main() {
                 poh_recorder.lock().unwrap().set_bank(&bank);
                 assert!(poh_recorder.lock().unwrap().bank().is_some());
                 if bank.slot() > 32 {
-                    bank_forks.set_root(root, &None, None);
+                    bank_forks.set_root(root, &AbsRequestSender::default(), None);
                     root += 1;
                 }
                 debug!(

banks-client/Cargo.toml

@@ -1,26 +1,30 @@
 [package]
 name = "solana-banks-client"
-version = "1.3.13"
+version = "1.6.22"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-banks-client"
 edition = "2018"
 
 [dependencies]
-async-trait = "0.1.36"
 bincode = "1.3.1"
+borsh = "0.9.0"
+borsh-derive = "0.9.0"
 futures = "0.3"
-solana-banks-interface = { path = "../banks-interface", version = "1.3.13" }
-solana-sdk = { path = "../sdk", version = "1.3.13" }
-tarpc = { version = "0.21.0", features = ["full"] }
-tokio = "0.2"
-tokio-serde = { version = "0.6", features = ["bincode"] }
+mio = "0.7.6"
+solana-banks-interface = { path = "../banks-interface", version = "=1.6.22" }
+solana-program = { path = "../sdk/program", version = "=1.6.22" }
+solana-sdk = { path = "../sdk", version = "=1.6.22" }
+tarpc = { version = "0.24.1", features = ["full"] }
+tokio = { version = "1", features = ["full"] }
+tokio-serde = { version = "0.8", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "1.3.13" }
-solana-banks-server = { path = "../banks-server", version = "1.3.13" }
+solana-runtime = { path = "../runtime", version = "=1.6.22" }
+solana-banks-server = { path = "../banks-server", version = "=1.6.22" }
 
 [lib]
 crate-type = ["lib"]

banks-client/src/lib.rs

@@ -5,165 +5,276 @@
 //! but they are undocumented, may change over time, and are generally more
 //! cumbersome to use.
 
-use async_trait::async_trait;
-use futures::future::join_all;
-pub use solana_banks_interface::{BanksClient, TransactionStatus};
+use borsh::BorshDeserialize;
+use futures::{future::join_all, Future, FutureExt};
+pub use solana_banks_interface::{BanksClient as TarpcClient, TransactionStatus};
 use solana_banks_interface::{BanksRequest, BanksResponse};
+use solana_program::{
+    clock::Slot, fee_calculator::FeeCalculator, hash::Hash, program_pack::Pack, pubkey::Pubkey,
+    rent::Rent, sysvar,
+};
 use solana_sdk::{
-    account::Account, clock::Slot, commitment_config::CommitmentLevel,
-    fee_calculator::FeeCalculator, hash::Hash, pubkey::Pubkey, signature::Signature,
-    transaction::Transaction, transport,
+    account::{from_account, Account},
+    commitment_config::CommitmentLevel,
+    signature::Signature,
+    transaction::{self, Transaction},
+    transport,
 };
 use std::io::{self, Error, ErrorKind};
 use tarpc::{
-    client, context,
-    rpc::{transport::channel::UnboundedChannel, ClientMessage, Response},
+    client::{self, channel::RequestDispatch, NewClient},
+    context::{self, Context},
+    rpc::{ClientMessage, Response},
     serde_transport::tcp,
+    Transport,
 };
 use tokio::{net::ToSocketAddrs, time::Duration};
 use tokio_serde::formats::Bincode;
 
-#[async_trait]
-pub trait BanksClientExt {
+// This exists only for backward compatibility
+pub trait BanksClientExt {}
+
+#[derive(Clone)]
+pub struct BanksClient {
+    inner: TarpcClient,
+}
+
+impl BanksClient {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new<C>(
+        config: client::Config,
+        transport: C,
+    ) -> NewClient<TarpcClient, RequestDispatch<BanksRequest, BanksResponse, C>>
+    where
+        C: Transport<ClientMessage<BanksRequest>, Response<BanksResponse>>,
+    {
+        TarpcClient::new(config, transport)
+    }
+
+    pub fn send_transaction_with_context(
+        &mut self,
+        ctx: Context,
+        transaction: Transaction,
+    ) -> impl Future<Output = io::Result<()>> + '_ {
+        self.inner.send_transaction_with_context(ctx, transaction)
+    }
+
+    pub fn get_fees_with_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, Slot)>> + '_ {
+        self.inner
+            .get_fees_with_commitment_and_context(ctx, commitment)
+    }
+
+    pub fn get_transaction_status_with_context(
+        &mut self,
+        ctx: Context,
+        signature: Signature,
+    ) -> impl Future<Output = io::Result<Option<TransactionStatus>>> + '_ {
+        self.inner
+            .get_transaction_status_with_context(ctx, signature)
+    }
+
+    pub fn get_slot_with_context(
+        &mut self,
+        ctx: Context,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Slot>> + '_ {
+        self.inner.get_slot_with_context(ctx, commitment)
+    }
+
+    pub fn process_transaction_with_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Option<transaction::Result<()>>>> + '_ {
+        self.inner
+            .process_transaction_with_commitment_and_context(ctx, transaction, commitment)
+    }
+
+    pub fn get_account_with_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+        self.inner
+            .get_account_with_commitment_and_context(ctx, address, commitment)
+    }
+
     /// Send a transaction and return immediately. The server will resend the
     /// transaction until either it is accepted by the cluster or the transaction's
     /// blockhash expires.
-    async fn send_transaction(&mut self, transaction: Transaction) -> io::Result<()>;
-
-    /// Return a recent, rooted blockhash from the server. The cluster will only accept
-    /// transactions with a blockhash that has not yet expired. Use the `get_fees`
-    /// method to get both a blockhash and the blockhash's last valid slot.
-    async fn get_recent_blockhash(&mut self) -> io::Result<Hash>;
+    pub fn send_transaction(
+        &mut self,
+        transaction: Transaction,
+    ) -> impl Future<Output = io::Result<()>> + '_ {
+        self.send_transaction_with_context(context::current(), transaction)
+    }
 
     /// Return the fee parameters associated with a recent, rooted blockhash. The cluster
     /// will use the transaction's blockhash to look up these same fee parameters and
     /// use them to calculate the transaction fee.
-    async fn get_fees(&mut self) -> io::Result<(FeeCalculator, Hash, Slot)>;
+    pub fn get_fees(
+        &mut self,
+    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, Slot)>> + '_ {
+        self.get_fees_with_commitment_and_context(context::current(), CommitmentLevel::default())
+    }
+
+    /// Return the cluster rent
+    pub fn get_rent(&mut self) -> impl Future<Output = io::Result<Rent>> + '_ {
+        self.get_account(sysvar::rent::id()).map(|result| {
+            let rent_sysvar = result?
+                .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Rent sysvar not present"))?;
+            from_account::<Rent, _>(&rent_sysvar).ok_or_else(|| {
+                io::Error::new(io::ErrorKind::Other, "Failed to deserialize Rent sysvar")
+            })
+        })
+    }
+
+    /// Return a recent, rooted blockhash from the server. The cluster will only accept
+    /// transactions with a blockhash that has not yet expired. Use the `get_fees`
+    /// method to get both a blockhash and the blockhash's last valid slot.
+    pub fn get_recent_blockhash(&mut self) -> impl Future<Output = io::Result<Hash>> + '_ {
+        self.get_fees().map(|result| Ok(result?.1))
+    }
 
     /// Send a transaction and return after the transaction has been rejected or
     /// reached the given level of commitment.
-    async fn process_transaction_with_commitment(
+    pub fn process_transaction_with_commitment(
         &mut self,
         transaction: Transaction,
         commitment: CommitmentLevel,
-    ) -> transport::Result<()>;
+    ) -> impl Future<Output = transport::Result<()>> + '_ {
+        let mut ctx = context::current();
+        ctx.deadline += Duration::from_secs(50);
+        self.process_transaction_with_commitment_and_context(ctx, transaction, commitment)
+            .map(|result| match result? {
+                None => {
+                    Err(Error::new(ErrorKind::TimedOut, "invalid blockhash or fee-payer").into())
+                }
+                Some(transaction_result) => Ok(transaction_result?),
+            })
+    }
 
-    /// Send a transaction and return after the transaction has been finalized or rejected.
-    async fn process_transaction(&mut self, transaction: Transaction) -> transport::Result<()>;
+    /// Send a transaction and return until the transaction has been finalized or rejected.
+    pub fn process_transaction(
+        &mut self,
+        transaction: Transaction,
+    ) -> impl Future<Output = transport::Result<()>> + '_ {
+        self.process_transaction_with_commitment(transaction, CommitmentLevel::default())
+    }
+
+    pub async fn process_transactions_with_commitment(
+        &mut self,
+        transactions: Vec<Transaction>,
+        commitment: CommitmentLevel,
+    ) -> transport::Result<()> {
+        let mut clients: Vec<_> = transactions.iter().map(|_| self.clone()).collect();
+        let futures = clients
+            .iter_mut()
+            .zip(transactions)
+            .map(|(client, transaction)| {
+                client.process_transaction_with_commitment(transaction, commitment)
+            });
+        let statuses = join_all(futures).await;
+        statuses.into_iter().collect() // Convert Vec<Result<_, _>> to Result<Vec<_>>
+    }
+
+    /// Send transactions and return until the transaction has been finalized or rejected.
+    pub fn process_transactions(
+        &mut self,
+        transactions: Vec<Transaction>,
+    ) -> impl Future<Output = transport::Result<()>> + '_ {
+        self.process_transactions_with_commitment(transactions, CommitmentLevel::default())
+    }
+
+    /// Return the most recent rooted slot height. All transactions at or below this height
+    /// are said to be finalized. The cluster will not fork to a higher slot height.
+    pub fn get_root_slot(&mut self) -> impl Future<Output = io::Result<Slot>> + '_ {
+        self.get_slot_with_context(context::current(), CommitmentLevel::default())
+    }
+
+    /// Return the account at the given address at the slot corresponding to the given
+    /// commitment level. If the account is not found, None is returned.
+    pub fn get_account_with_commitment(
+        &mut self,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+        self.get_account_with_commitment_and_context(context::current(), address, commitment)
+    }
+
+    /// Return the account at the given address at the time of the most recent root slot.
+    /// If the account is not found, None is returned.
+    pub fn get_account(
+        &mut self,
+        address: Pubkey,
+    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+        self.get_account_with_commitment(address, CommitmentLevel::default())
+    }
+
+    /// Return the unpacked account data at the given address
+    /// If the account is not found, an error is returned
+    pub fn get_packed_account_data<T: Pack>(
+        &mut self,
+        address: Pubkey,
+    ) -> impl Future<Output = io::Result<T>> + '_ {
+        self.get_account(address).map(|result| {
+            let account =
+                result?.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Account not found"))?;
+            T::unpack_from_slice(&account.data)
+                .map_err(|_| io::Error::new(io::ErrorKind::Other, "Failed to deserialize account"))
+        })
+    }
+
+    /// Return the unpacked account data at the given address
+    /// If the account is not found, an error is returned
+    pub fn get_account_data_with_borsh<T: BorshDeserialize>(
+        &mut self,
+        address: Pubkey,
+    ) -> impl Future<Output = io::Result<T>> + '_ {
+        self.get_account(address).map(|result| {
+            let account =
+                result?.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "account not found"))?;
+            T::try_from_slice(&account.data)
+        })
+    }
+
+    /// Return the balance in lamports of an account at the given address at the slot
+    /// corresponding to the given commitment level.
+    pub fn get_balance_with_commitment(
+        &mut self,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<u64>> + '_ {
+        self.get_account_with_commitment_and_context(context::current(), address, commitment)
+            .map(|result| Ok(result?.map(|x| x.lamports).unwrap_or(0)))
+    }
+
+    /// Return the balance in lamports of an account at the given address at the time
+    /// of the most recent root slot.
+    pub fn get_balance(&mut self, address: Pubkey) -> impl Future<Output = io::Result<u64>> + '_ {
+        self.get_balance_with_commitment(address, CommitmentLevel::default())
+    }
 
     /// Return the status of a transaction with a signature matching the transaction's first
     /// signature. Return None if the transaction is not found, which may be because the
     /// blockhash was expired or the fee-paying account had insufficient funds to pay the
     /// transaction fee. Note that servers rarely store the full transaction history. This
     /// method may return None if the transaction status has been discarded.
-    async fn get_transaction_status(
+    pub fn get_transaction_status(
         &mut self,
         signature: Signature,
-    ) -> io::Result<Option<TransactionStatus>>;
+    ) -> impl Future<Output = io::Result<Option<TransactionStatus>>> + '_ {
+        self.get_transaction_status_with_context(context::current(), signature)
+    }
 
     /// Same as get_transaction_status, but for multiple transactions.
-    async fn get_transaction_statuses(
-        &mut self,
-        signatures: Vec<Signature>,
-    ) -> io::Result<Vec<Option<TransactionStatus>>>;
-
-    /// Return the most recent rooted slot height. All transactions at or below this height
-    /// are said to be finalized. The cluster will not fork to a higher slot height.
-    async fn get_root_slot(&mut self) -> io::Result<Slot>;
-
-    /// Return the account at the given address at the time of the most recent root slot.
-    /// If the account is not found, None is returned.
-    async fn get_account(&mut self, address: Pubkey) -> io::Result<Option<Account>>;
-
-    /// Return the balance in lamports of an account at the given address at the slot
-    /// corresponding to the given commitment level.
-    async fn get_balance_with_commitment(
-        &mut self,
-        address: Pubkey,
-        commitment: CommitmentLevel,
-    ) -> io::Result<u64>;
-
-    /// Return the balance in lamports of an account at the given address at the time
-    /// of the most recent root slot.
-    async fn get_balance(&mut self, address: Pubkey) -> io::Result<u64>;
-}
-
-#[async_trait]
-impl BanksClientExt for BanksClient {
-    async fn send_transaction(&mut self, transaction: Transaction) -> io::Result<()> {
-        self.send_transaction_with_context(context::current(), transaction)
-            .await
-    }
-
-    async fn get_fees(&mut self) -> io::Result<(FeeCalculator, Hash, Slot)> {
-        self.get_fees_with_commitment_and_context(context::current(), CommitmentLevel::Root)
-            .await
-    }
-
-    async fn get_recent_blockhash(&mut self) -> io::Result<Hash> {
-        Ok(self.get_fees().await?.1)
-    }
-
-    async fn process_transaction_with_commitment(
-        &mut self,
-        transaction: Transaction,
-        commitment: CommitmentLevel,
-    ) -> transport::Result<()> {
-        let mut ctx = context::current();
-        ctx.deadline += Duration::from_secs(50);
-        let result = self
-            .process_transaction_with_commitment_and_context(ctx, transaction, commitment)
-            .await?;
-        match result {
-            None => Err(Error::new(ErrorKind::TimedOut, "invalid blockhash or fee-payer").into()),
-            Some(transaction_result) => Ok(transaction_result?),
-        }
-    }
-
-    async fn process_transaction(&mut self, transaction: Transaction) -> transport::Result<()> {
-        self.process_transaction_with_commitment(transaction, CommitmentLevel::default())
-            .await
-    }
-
-    async fn get_root_slot(&mut self) -> io::Result<Slot> {
-        self.get_slot_with_context(context::current(), CommitmentLevel::Root)
-            .await
-    }
-
-    async fn get_account(&mut self, address: Pubkey) -> io::Result<Option<Account>> {
-        self.get_account_with_commitment_and_context(
-            context::current(),
-            address,
-            CommitmentLevel::default(),
-        )
-        .await
-    }
-
-    async fn get_balance_with_commitment(
-        &mut self,
-        address: Pubkey,
-        commitment: CommitmentLevel,
-    ) -> io::Result<u64> {
-        let account = self
-            .get_account_with_commitment_and_context(context::current(), address, commitment)
-            .await?;
-        Ok(account.map(|x| x.lamports).unwrap_or(0))
-    }
-
-    async fn get_balance(&mut self, address: Pubkey) -> io::Result<u64> {
-        self.get_balance_with_commitment(address, CommitmentLevel::default())
-            .await
-    }
-
-    async fn get_transaction_status(
-        &mut self,
-        signature: Signature,
-    ) -> io::Result<Option<TransactionStatus>> {
-        self.get_transaction_status_with_context(context::current(), signature)
-            .await
-    }
-
-    async fn get_transaction_statuses(
+    pub async fn get_transaction_statuses(
         &mut self,
         signatures: Vec<Signature>,
     ) -> io::Result<Vec<Option<TransactionStatus>>> {
@@ -184,26 +295,34 @@ impl BanksClientExt for BanksClient {
     }
 }
 
-pub async fn start_client(
-    transport: UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>>,
-) -> io::Result<BanksClient> {
-    BanksClient::new(client::Config::default(), transport).spawn()
+pub async fn start_client<C>(transport: C) -> io::Result<BanksClient>
+where
+    C: Transport<ClientMessage<BanksRequest>, Response<BanksResponse>> + Send + 'static,
+{
+    Ok(BanksClient {
+        inner: TarpcClient::new(client::Config::default(), transport).spawn()?,
+    })
 }
 
 pub async fn start_tcp_client<T: ToSocketAddrs>(addr: T) -> io::Result<BanksClient> {
-    let transport = tcp::connect(addr, Bincode::default()).await?;
-    BanksClient::new(client::Config::default(), transport).spawn()
+    let transport = tcp::connect(addr, Bincode::default).await?;
+    Ok(BanksClient {
+        inner: TarpcClient::new(client::Config::default(), transport).spawn()?,
+    })
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use solana_banks_server::banks_server::start_local_server;
-    use solana_runtime::{bank::Bank, bank_forks::BankForks, genesis_utils::create_genesis_config};
-    use solana_sdk::{message::Message, pubkey::Pubkey, signature::Signer, system_instruction};
+    use solana_runtime::{
+        bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache,
+        genesis_utils::create_genesis_config,
+    };
+    use solana_sdk::{message::Message, signature::Signer, system_instruction};
     use std::sync::{Arc, RwLock};
     use tarpc::transport;
-    use tokio::{runtime::Runtime, time::delay_for};
+    use tokio::{runtime::Runtime, time::sleep};
 
     #[test]
     fn test_banks_client_new() {
@@ -218,19 +337,21 @@ mod tests {
         // `runtime.block_on()` just once, to run all the async code.
 
         let genesis = create_genesis_config(10);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(
-            &genesis.genesis_config,
-        ))));
+        let bank = Bank::new(&genesis.genesis_config);
+        let slot = bank.slot();
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
+        ));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
 
-        let bob_pubkey = Pubkey::new_rand();
+        let bob_pubkey = solana_sdk::pubkey::new_rand();
         let mint_pubkey = genesis.mint_keypair.pubkey();
         let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(&bank_forks).await;
-            let mut banks_client =
-                BanksClient::new(client::Config::default(), client_transport).spawn()?;
+            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
+            let mut banks_client = start_client(client_transport).await?;
 
             let recent_blockhash = banks_client.get_recent_blockhash().await?;
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
@@ -247,19 +368,21 @@ mod tests {
         // server-side functionality is available to the client.
 
         let genesis = create_genesis_config(10);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(
-            &genesis.genesis_config,
-        ))));
+        let bank = Bank::new(&genesis.genesis_config);
+        let slot = bank.slot();
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
+        ));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
 
         let mint_pubkey = &genesis.mint_keypair.pubkey();
-        let bob_pubkey = Pubkey::new_rand();
+        let bob_pubkey = solana_sdk::pubkey::new_rand();
         let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(&bank_forks).await;
-            let mut banks_client =
-                BanksClient::new(client::Config::default(), client_transport).spawn()?;
+            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
+            let mut banks_client = start_client(client_transport).await?;
             let (_, recent_blockhash, last_valid_slot) = banks_client.get_fees().await?;
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
             let signature = transaction.signatures[0];
@@ -272,7 +395,7 @@ mod tests {
                 if root_slot > last_valid_slot {
                     break;
                 }
-                delay_for(Duration::from_millis(100)).await;
+                sleep(Duration::from_millis(100)).await;
                 status = banks_client.get_transaction_status(signature).await?;
             }
             assert!(status.unwrap().err.is_none());

banks-interface/Cargo.toml

@@ -1,17 +1,22 @@
 [package]
 name = "solana-banks-interface"
-version = "1.3.13"
+version = "1.6.22"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-banks-interface"
 edition = "2018"
 
 [dependencies]
-serde = { version = "1.0.112", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "1.3.13" }
-tarpc = { version = "0.21.0", features = ["full"] }
+mio = "0.7.6"
+serde = { version = "1.0.122", features = ["derive"] }
+solana-sdk = { path = "../sdk", version = "=1.6.22" }
+tarpc = { version = "0.24.1", features = ["full"] }
+
+[dev-dependencies]
+tokio = { version = "1", features = ["full"] }
 
 [lib]
 crate-type = ["lib"]

banks-interface/src/lib.rs

@@ -10,11 +10,19 @@ use solana_sdk::{
     transaction::{self, Transaction, TransactionError},
 };
 
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub enum TransactionConfirmationStatus {
+    Processed,
+    Confirmed,
+    Finalized,
+}
+
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct TransactionStatus {
     pub slot: Slot,
     pub confirmations: Option<usize>, // None = rooted
     pub err: Option<TransactionError>,
+    pub confirmation_status: Option<TransactionConfirmationStatus>,
 }
 
 #[tarpc::service]

banks-server/Cargo.toml

@@ -1,24 +1,27 @@
 [package]
 name = "solana-banks-server"
-version = "1.3.13"
+version = "1.6.22"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-banks-server"
 edition = "2018"
 
 [dependencies]
 bincode = "1.3.1"
 futures = "0.3"
-log = "0.4.8"
-solana-banks-interface = { path = "../banks-interface", version = "1.3.13" }
-solana-runtime = { path = "../runtime", version = "1.3.13" }
-solana-sdk = { path = "../sdk", version = "1.3.13" }
-solana-metrics = { path = "../metrics", version = "1.3.13" }
-tarpc = { version = "0.21.0", features = ["full"] }
-tokio = "0.2"
-tokio-serde = { version = "0.6", features = ["bincode"] }
+log = "0.4.11"
+mio = "0.7.6"
+solana-banks-interface = { path = "../banks-interface", version = "=1.6.22" }
+solana-runtime = { path = "../runtime", version = "=1.6.22" }
+solana-sdk = { path = "../sdk", version = "=1.6.22" }
+solana-metrics = { path = "../metrics", version = "=1.6.22" }
+tarpc = { version = "0.24.1", features = ["full"] }
+tokio = { version = "1", features = ["full"] }
+tokio-serde = { version = "0.8", features = ["bincode"] }
+tokio-stream = "0.1"
 
 [lib]
 crate-type = ["lib"]

banks-server/src/banks_server.rs

@@ -4,12 +4,10 @@ use futures::{
     future,
     prelude::stream::{self, StreamExt},
 };
-use solana_banks_interface::{Banks, BanksRequest, BanksResponse, TransactionStatus};
-use solana_runtime::{
-    bank::Bank,
-    bank_forks::BankForks,
-    commitment::{BlockCommitmentCache, CommitmentSlots},
+use solana_banks_interface::{
+    Banks, BanksRequest, BanksResponse, TransactionConfirmationStatus, TransactionStatus,
 };
+use solana_runtime::{bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache};
 use solana_sdk::{
     account::Account,
     clock::Slot,
@@ -21,7 +19,6 @@ use solana_sdk::{
     transaction::{self, Transaction},
 };
 use std::{
-    collections::HashMap,
     io,
     net::{Ipv4Addr, SocketAddr},
     sync::{
@@ -38,7 +35,7 @@ use tarpc::{
     server::{self, Channel, Handler},
     transport,
 };
-use tokio::time::delay_for;
+use tokio::time::sleep;
 use tokio_serde::formats::Bincode;
 
 #[derive(Clone)]
@@ -65,7 +62,7 @@ impl BanksServer {
         }
     }
 
-    fn run(bank: &Bank, transaction_receiver: Receiver<TransactionInfo>) {
+    fn run(bank_forks: Arc<RwLock<BankForks>>, transaction_receiver: Receiver<TransactionInfo>) {
         while let Ok(info) = transaction_receiver.recv() {
             let mut transaction_infos = vec![info];
             while let Ok(info) = transaction_receiver.try_recv() {
@@ -75,28 +72,28 @@ impl BanksServer {
                 .into_iter()
                 .map(|info| deserialize(&info.wire_transaction).unwrap())
                 .collect();
+            let bank = bank_forks.read().unwrap().working_bank();
             let _ = bank.process_transactions(&transactions);
         }
     }
 
     /// Useful for unit-testing
-    fn new_loopback(bank_forks: Arc<RwLock<BankForks>>) -> Self {
+    fn new_loopback(
+        bank_forks: Arc<RwLock<BankForks>>,
+        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+    ) -> Self {
         let (transaction_sender, transaction_receiver) = channel();
         let bank = bank_forks.read().unwrap().working_bank();
         let slot = bank.slot();
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new(
-            HashMap::default(),
-            0,
-            CommitmentSlots {
-                slot,
-                root: 0,
-                highest_confirmed_slot: 0,
-                highest_confirmed_root: 0,
-            },
-        )));
+        {
+            // ensure that the commitment cache and bank are synced
+            let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
+            w_block_commitment_cache.set_all_slots(slot, slot);
+        }
+        let server_bank_forks = bank_forks.clone();
         Builder::new()
             .name("solana-bank-forks-client".to_string())
-            .spawn(move || Self::run(&bank, transaction_receiver))
+            .spawn(move || Self::run(server_bank_forks, transaction_receiver))
             .unwrap();
         Self::new(bank_forks, block_commitment_cache, transaction_sender)
     }
@@ -114,23 +111,39 @@ impl BanksServer {
 
     async fn poll_signature_status(
         self,
-        signature: Signature,
+        signature: &Signature,
+        blockhash: &Hash,
         last_valid_slot: Slot,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
-        let mut status = self.bank(commitment).get_signature_status(&signature);
+        let mut status = self
+            .bank(commitment)
+            .get_signature_status_with_blockhash(signature, blockhash);
         while status.is_none() {
-            delay_for(Duration::from_millis(200)).await;
+            sleep(Duration::from_millis(200)).await;
             let bank = self.bank(commitment);
             if bank.slot() > last_valid_slot {
                 break;
             }
-            status = bank.get_signature_status(&signature);
+            status = bank.get_signature_status_with_blockhash(signature, blockhash);
         }
         status
     }
 }
 
+fn verify_transaction(
+    transaction: &Transaction,
+    libsecp256k1_0_5_upgrade_enabled: bool,
+) -> transaction::Result<()> {
+    if let Err(err) = transaction.verify() {
+        Err(err)
+    } else if let Err(err) = transaction.verify_precompiles(libsecp256k1_0_5_upgrade_enabled) {
+        Err(err)
+    } else {
+        Ok(())
+    }
+}
+
 #[tarpc::server]
 impl Banks for BanksServer {
     async fn send_transaction_with_context(self, _: Context, transaction: Transaction) {
@@ -164,11 +177,17 @@ impl Banks for BanksServer {
         _: Context,
         signature: Signature,
     ) -> Option<TransactionStatus> {
-        let bank = self.bank(CommitmentLevel::Recent);
+        let bank = self.bank(CommitmentLevel::Processed);
         let (slot, status) = bank.get_signature_status_slot(&signature)?;
         let r_block_commitment_cache = self.block_commitment_cache.read().unwrap();
 
-        let confirmations = if r_block_commitment_cache.root() >= slot {
+        let optimistically_confirmed_bank = self.bank(CommitmentLevel::Confirmed);
+        let optimistically_confirmed =
+            optimistically_confirmed_bank.get_signature_status_slot(&signature);
+
+        let confirmations = if r_block_commitment_cache.root() >= slot
+            && r_block_commitment_cache.highest_confirmed_root() >= slot
+        {
             None
         } else {
             r_block_commitment_cache
@@ -179,6 +198,13 @@ impl Banks for BanksServer {
             slot,
             confirmations,
             err: status.err(),
+            confirmation_status: if confirmations.is_none() {
+                Some(TransactionConfirmationStatus::Finalized)
+            } else if optimistically_confirmed.is_some() {
+                Some(TransactionConfirmationStatus::Confirmed)
+            } else {
+                Some(TransactionConfirmationStatus::Processed)
+            },
         })
     }
 
@@ -192,19 +218,26 @@ impl Banks for BanksServer {
         transaction: Transaction,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
+        if let Err(err) = verify_transaction(
+            &transaction,
+            self.bank(commitment).libsecp256k1_0_5_upgrade_enabled(),
+        ) {
+            return Some(Err(err));
+        }
+
         let blockhash = &transaction.message.recent_blockhash;
         let last_valid_slot = self
             .bank_forks
             .read()
             .unwrap()
             .root_bank()
-            .get_blockhash_last_valid_slot(&blockhash)
+            .get_blockhash_last_valid_slot(blockhash)
             .unwrap();
         let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
         let info =
             TransactionInfo::new(signature, serialize(&transaction).unwrap(), last_valid_slot);
         self.transaction_sender.send(info).unwrap();
-        self.poll_signature_status(signature, last_valid_slot, commitment)
+        self.poll_signature_status(&signature, blockhash, last_valid_slot, commitment)
             .await
     }
 
@@ -215,14 +248,15 @@ impl Banks for BanksServer {
         commitment: CommitmentLevel,
     ) -> Option<Account> {
         let bank = self.bank(commitment);
-        bank.get_account(&address)
+        bank.get_account(&address).map(Account::from)
     }
 }
 
 pub async fn start_local_server(
-    bank_forks: &Arc<RwLock<BankForks>>,
+    bank_forks: Arc<RwLock<BankForks>>,
+    block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
 ) -> UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>> {
-    let banks_server = BanksServer::new_loopback(bank_forks.clone());
+    let banks_server = BanksServer::new_loopback(bank_forks, block_commitment_cache);
     let (client_transport, server_transport) = transport::channel::unbounded();
     let server = server::new(server::Config::default())
         .incoming(stream::once(future::ready(server_transport)))

banks-server/src/lib.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 pub mod banks_server;
 pub mod rpc_banks_service;
 pub mod send_transaction_service;

banks-server/src/rpc_banks_service.rs

@@ -15,6 +15,7 @@ use tokio::{
     runtime::Runtime,
     time::{self, Duration},
 };
+use tokio_stream::wrappers::IntervalStream;
 
 pub struct RpcBanksService {
     thread_hdl: JoinHandle<()>,
@@ -35,7 +36,7 @@ async fn start_abortable_tcp_server(
         block_commitment_cache.clone(),
     )
     .fuse();
-    let interval = time::interval(Duration::from_millis(100)).fuse();
+    let interval = IntervalStream::new(time::interval(Duration::from_millis(100))).fuse();
     pin_mut!(server, interval);
     loop {
         select! {

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.3.13"
+version = "1.6.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,28 +11,28 @@ publish = false
 [dependencies]
 clap = "2.33.1"
 itertools = "0.9.0"
-log = "0.4.8"
+log = "0.4.11"
 num-derive = "0.3"
 num-traits = "0.2"
 rand = "0.7.0"
-rayon = "1.4.0"
+rayon = "1.5.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.13" }
-solana-core = { path = "../core", version = "1.3.13" }
-solana-genesis = { path = "../genesis", version = "1.3.13" }
-solana-client = { path = "../client", version = "1.3.13" }
-solana-faucet = { path = "../faucet", version = "1.3.13" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.3.13" }
-solana-logger = { path = "../logger", version = "1.3.13" }
-solana-metrics = { path = "../metrics", version = "1.3.13" }
-solana-net-utils = { path = "../net-utils", version = "1.3.13" }
-solana-runtime = { path = "../runtime", version = "1.3.13" }
-solana-sdk = { path = "../sdk", version = "1.3.13" }
-solana-version = { path = "../version", version = "1.3.13" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.22" }
+solana-core = { path = "../core", version = "=1.6.22" }
+solana-genesis = { path = "../genesis", version = "=1.6.22" }
+solana-client = { path = "../client", version = "=1.6.22" }
+solana-faucet = { path = "../faucet", version = "=1.6.22" }
+solana-exchange-program = { path = "../programs/exchange", version = "=1.6.22" }
+solana-logger = { path = "../logger", version = "=1.6.22" }
+solana-metrics = { path = "../metrics", version = "=1.6.22" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.22" }
+solana-runtime = { path = "../runtime", version = "=1.6.22" }
+solana-sdk = { path = "../sdk", version = "=1.6.22" }
+solana-version = { path = "../version", version = "=1.6.22" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.3.13" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.6.22" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/src/bench.rs

@@ -1,4 +1,5 @@
 #![allow(clippy::useless_attribute)]
+#![allow(clippy::integer_arithmetic)]
 
 use crate::order_book::*;
 use itertools::izip;
@@ -390,7 +391,7 @@ fn swapper<T>(
             while client
                 .get_balance_with_commitment(
                     &trade_infos[trade_index].trade_account,
-                    CommitmentConfig::recent(),
+                    CommitmentConfig::processed(),
                 )
                 .unwrap_or(0)
                 == 0
@@ -445,7 +446,7 @@ fn swapper<T>(
             account_group = (account_group + 1) % account_groups as usize;
 
             let (blockhash, _fee_calculator, _last_valid_slot) = client
-                .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
+                .get_recent_blockhash_with_commitment(CommitmentConfig::processed())
                 .expect("Failed to get blockhash");
             let to_swap_txs: Vec<_> = to_swap
                 .par_iter()
@@ -571,7 +572,7 @@ fn trader<T>(
         account_group = (account_group + 1) % account_groups as usize;
 
         let (blockhash, _fee_calculator, _last_valid_slot) = client
-            .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
+            .get_recent_blockhash_with_commitment(CommitmentConfig::processed())
             .expect("Failed to get blockhash");
 
         trades.chunks(chunk_size).for_each(|chunk| {
@@ -658,7 +659,7 @@ where
 {
     for s in &tx.signatures {
         if let Ok(Some(r)) =
-            sync_client.get_signature_status_with_commitment(s, CommitmentConfig::recent())
+            sync_client.get_signature_status_with_commitment(s, CommitmentConfig::processed())
         {
             match r {
                 Ok(_) => {
@@ -681,7 +682,7 @@ fn verify_funding_transfer<T: SyncClient + ?Sized>(
     if verify_transaction(client, tx) {
         for a in &tx.message().account_keys[1..] {
             if client
-                .get_balance_with_commitment(a, CommitmentConfig::recent())
+                .get_balance_with_commitment(a, CommitmentConfig::processed())
                 .unwrap_or(0)
                 >= amount
             {
@@ -764,7 +765,7 @@ pub fn fund_keys<T: Client>(client: &T, source: &Keypair, dests: &[Arc<Keypair>]
                 );
 
                 let (blockhash, _fee_calculator, _last_valid_slot) = client
-                    .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
+                    .get_recent_blockhash_with_commitment(CommitmentConfig::processed())
                     .expect("blockhash");
                 to_fund_txs.par_iter_mut().for_each(|(k, tx)| {
                     tx.sign(&[*k], blockhash);
@@ -803,7 +804,7 @@ pub fn fund_keys<T: Client>(client: &T, source: &Keypair, dests: &[Arc<Keypair>]
         funded.append(&mut new_funded);
         funded.retain(|(k, b)| {
             client
-                .get_balance_with_commitment(&k.pubkey(), CommitmentConfig::recent())
+                .get_balance_with_commitment(&k.pubkey(), CommitmentConfig::processed())
                 .unwrap_or(0)
                 > lamports
                 && *b > lamports
@@ -857,7 +858,7 @@ pub fn create_token_accounts<T: Client>(
             let mut retries = 0;
             while !to_create_txs.is_empty() {
                 let (blockhash, _fee_calculator, _last_valid_slot) = client
-                    .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
+                    .get_recent_blockhash_with_commitment(CommitmentConfig::processed())
                     .expect("Failed to get blockhash");
                 to_create_txs
                     .par_iter_mut()
@@ -903,7 +904,7 @@ pub fn create_token_accounts<T: Client>(
         let mut new_notfunded: Vec<(&Arc<Keypair>, &Keypair)> = vec![];
         for f in &notfunded {
             if client
-                .get_balance_with_commitment(&f.1.pubkey(), CommitmentConfig::recent())
+                .get_balance_with_commitment(&f.1.pubkey(), CommitmentConfig::processed())
                 .unwrap_or(0)
                 == 0
             {
@@ -968,7 +969,7 @@ pub fn airdrop_lamports<T: Client>(
     id: &Keypair,
     amount: u64,
 ) {
-    let balance = client.get_balance_with_commitment(&id.pubkey(), CommitmentConfig::recent());
+    let balance = client.get_balance_with_commitment(&id.pubkey(), CommitmentConfig::processed());
     let balance = balance.unwrap_or(0);
     if balance >= amount {
         return;
@@ -986,7 +987,7 @@ pub fn airdrop_lamports<T: Client>(
     let mut tries = 0;
     loop {
         let (blockhash, _fee_calculator, _last_valid_slot) = client
-            .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
+            .get_recent_blockhash_with_commitment(CommitmentConfig::processed())
             .expect("Failed to get blockhash");
         match request_airdrop_transaction(&faucet_addr, &id.pubkey(), amount_to_drop, blockhash) {
             Ok(transaction) => {
@@ -995,14 +996,14 @@ pub fn airdrop_lamports<T: Client>(
                 for _ in 0..30 {
                     if let Ok(Some(_)) = client.get_signature_status_with_commitment(
                         &signature,
-                        CommitmentConfig::recent(),
+                        CommitmentConfig::processed(),
                     ) {
                         break;
                     }
                     sleep(Duration::from_millis(100));
                 }
                 if client
-                    .get_balance_with_commitment(&id.pubkey(), CommitmentConfig::recent())
+                    .get_balance_with_commitment(&id.pubkey(), CommitmentConfig::processed())
                     .unwrap_or(0)
                     >= amount
                 {

bench-exchange/src/cli.rs

@@ -163,7 +163,8 @@ pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
         )
 }
 
-pub fn extract_args<'a>(matches: &ArgMatches<'a>) -> Config {
+#[allow(clippy::field_reassign_with_default)]
+pub fn extract_args(matches: &ArgMatches) -> Config {
     let mut args = Config::default();
 
     args.entrypoint_addr = solana_net_utils::parse_host_port(

bench-exchange/src/main.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 pub mod bench;
 mod cli;
 pub mod order_book;

bench-exchange/tests/bench_exchange.rs

@@ -1,19 +1,23 @@
 use log::*;
 use solana_bench_exchange::bench::{airdrop_lamports, do_bench_exchange, Config};
-use solana_core::gossip_service::{discover_cluster, get_multi_client};
-use solana_core::validator::ValidatorConfig;
-use solana_exchange_program::exchange_processor::process_instruction;
-use solana_exchange_program::id;
-use solana_exchange_program::solana_exchange_program;
-use solana_faucet::faucet::run_local_faucet;
-use solana_local_cluster::local_cluster::{ClusterConfig, LocalCluster};
-use solana_runtime::bank::Bank;
-use solana_runtime::bank_client::BankClient;
-use solana_sdk::genesis_config::create_genesis_config;
-use solana_sdk::signature::{Keypair, Signer};
-use std::process::exit;
-use std::sync::mpsc::channel;
-use std::time::Duration;
+use solana_core::{
+    gossip_service::{discover_cluster, get_multi_client},
+    validator::ValidatorConfig,
+};
+use solana_exchange_program::{
+    exchange_processor::process_instruction, id, solana_exchange_program,
+};
+use solana_faucet::faucet::run_local_faucet_with_port;
+use solana_local_cluster::{
+    local_cluster::{ClusterConfig, LocalCluster},
+    validator_configs::make_identical_validator_configs,
+};
+use solana_runtime::{bank::Bank, bank_client::BankClient};
+use solana_sdk::{
+    genesis_config::create_genesis_config,
+    signature::{Keypair, Signer},
+};
+use std::{process::exit, sync::mpsc::channel, time::Duration};
 
 #[test]
 #[ignore]
@@ -22,15 +26,17 @@ fn test_exchange_local_cluster() {
 
     const NUM_NODES: usize = 1;
 
-    let mut config = Config::default();
-    config.identity = Keypair::new();
-    config.duration = Duration::from_secs(1);
-    config.fund_amount = 100_000;
-    config.threads = 1;
-    config.transfer_delay = 20; // 15
-    config.batch_size = 100; // 1000;
-    config.chunk_size = 10; // 200;
-    config.account_groups = 1; // 10;
+    let config = Config {
+        identity: Keypair::new(),
+        duration: Duration::from_secs(1),
+        fund_amount: 100_000,
+        threads: 1,
+        transfer_delay: 20, // 15
+        batch_size: 100,    // 1000
+        chunk_size: 10,     // 200
+        account_groups: 1,  // 10
+        ..Config::default()
+    };
     let Config {
         fund_amount,
         batch_size,
@@ -39,10 +45,10 @@ fn test_exchange_local_cluster() {
     } = config;
     let accounts_in_groups = batch_size * account_groups;
 
-    let cluster = LocalCluster::new(&ClusterConfig {
+    let cluster = LocalCluster::new(&mut ClusterConfig {
         node_stakes: vec![100_000; NUM_NODES],
         cluster_lamports: 100_000_000_000_000,
-        validator_configs: vec![ValidatorConfig::default(); NUM_NODES],
+        validator_configs: make_identical_validator_configs(&ValidatorConfig::default(), NUM_NODES),
         native_instruction_processors: [solana_exchange_program!()].to_vec(),
         ..ClusterConfig::default()
     });
@@ -55,8 +61,11 @@ fn test_exchange_local_cluster() {
     );
 
     let (addr_sender, addr_receiver) = channel();
-    run_local_faucet(faucet_keypair, addr_sender, Some(1_000_000_000_000));
-    let faucet_addr = addr_receiver.recv_timeout(Duration::from_secs(2)).unwrap();
+    run_local_faucet_with_port(faucet_keypair, addr_sender, Some(1_000_000_000_000), 0);
+    let faucet_addr = addr_receiver
+        .recv_timeout(Duration::from_secs(2))
+        .expect("run_local_faucet")
+        .expect("faucet_addr");
 
     info!("Connecting to the cluster");
     let nodes =
@@ -86,18 +95,21 @@ fn test_exchange_bank_client() {
     solana_logger::setup();
     let (genesis_config, identity) = create_genesis_config(100_000_000_000_000);
     let mut bank = Bank::new(&genesis_config);
-    bank.add_builtin_program("exchange_program", id(), process_instruction);
+    bank.add_builtin("exchange_program", id(), process_instruction);
     let clients = vec![BankClient::new(bank)];
 
-    let mut config = Config::default();
-    config.identity = identity;
-    config.duration = Duration::from_secs(1);
-    config.fund_amount = 100_000;
-    config.threads = 1;
-    config.transfer_delay = 20; // 0;
-    config.batch_size = 100; // 1500;
-    config.chunk_size = 10; // 1500;
-    config.account_groups = 1; // 50;
-
-    do_bench_exchange(clients, config);
+    do_bench_exchange(
+        clients,
+        Config {
+            identity,
+            duration: Duration::from_secs(1),
+            fund_amount: 100_000,
+            threads: 1,
+            transfer_delay: 20, // 0;
+            batch_size: 100,    // 1500;
+            chunk_size: 10,     // 1500;
+            account_groups: 1,  // 50;
+            ..Config::default()
+        },
+    );
 }

bench-streamer/Cargo.toml

@@ -2,18 +2,19 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.3.13"
+version = "1.6.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
+publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.13" }
-solana-streamer = { path = "../streamer", version = "1.3.13" }
-solana-logger = { path = "../logger", version = "1.3.13" }
-solana-net-utils = { path = "../net-utils", version = "1.3.13" }
-solana-version = { path = "../version", version = "1.3.13" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.22" }
+solana-streamer = { path = "../streamer", version = "=1.6.22" }
+solana-logger = { path = "../logger", version = "=1.6.22" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.22" }
+solana-version = { path = "../version", version = "=1.6.22" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/src/main.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 use clap::{crate_description, crate_name, App, Arg};
 use solana_streamer::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
 use solana_streamer::streamer::{receiver, PacketReceiver};
@@ -74,7 +75,7 @@ fn main() -> Result<()> {
 
     let mut read_channels = Vec::new();
     let mut read_threads = Vec::new();
-    let recycler = PacketsRecycler::default();
+    let recycler = PacketsRecycler::new_without_limit("bench-streamer-recycler-shrink-stats");
     for _ in 0..num_sockets {
         let read = solana_net_utils::bind_to(ip_addr, port, false).unwrap();
         read.set_read_timeout(Some(Duration::new(1, 0))).unwrap();
@@ -90,6 +91,7 @@ fn main() -> Result<()> {
             s_reader,
             recycler.clone(),
             "bench-streamer-test",
+            1,
         ));
     }
 

bench-tps/Cargo.toml

@@ -2,35 +2,35 @@
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.3.13"
+version = "1.6.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
+publish = false
 
 [dependencies]
 bincode = "1.3.1"
 clap = "2.33.1"
-log = "0.4.8"
-rayon = "1.4.0"
+log = "0.4.11"
+rayon = "1.5.0"
 serde_json = "1.0.56"
 serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.13" }
-solana-core = { path = "../core", version = "1.3.13" }
-solana-genesis = { path = "../genesis", version = "1.3.13" }
-solana-client = { path = "../client", version = "1.3.13" }
-solana-faucet = { path = "../faucet", version = "1.3.13" }
-solana-logger = { path = "../logger", version = "1.3.13" }
-solana-metrics = { path = "../metrics", version = "1.3.13" }
-solana-measure = { path = "../measure", version = "1.3.13" }
-solana-net-utils = { path = "../net-utils", version = "1.3.13" }
-solana-runtime = { path = "../runtime", version = "1.3.13" }
-solana-sdk = { path = "../sdk", version = "1.3.13" }
-solana-version = { path = "../version", version = "1.3.13" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.22" }
+solana-core = { path = "../core", version = "=1.6.22" }
+solana-genesis = { path = "../genesis", version = "=1.6.22" }
+solana-client = { path = "../client", version = "=1.6.22" }
+solana-faucet = { path = "../faucet", version = "=1.6.22" }
+solana-logger = { path = "../logger", version = "=1.6.22" }
+solana-metrics = { path = "../metrics", version = "=1.6.22" }
+solana-measure = { path = "../measure", version = "=1.6.22" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.22" }
+solana-runtime = { path = "../runtime", version = "=1.6.22" }
+solana-sdk = { path = "../sdk", version = "=1.6.22" }
+solana-version = { path = "../version", version = "=1.6.22" }
 
 [dev-dependencies]
 serial_test = "0.4.0"
-serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.3.13" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.6.22" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/src/bench.rs

@@ -8,7 +8,7 @@ use solana_measure::measure::Measure;
 use solana_metrics::{self, datapoint_info};
 use solana_sdk::{
     client::Client,
-    clock::{DEFAULT_TICKS_PER_SECOND, DEFAULT_TICKS_PER_SLOT, MAX_PROCESSING_AGE},
+    clock::{DEFAULT_S_PER_SLOT, MAX_PROCESSING_AGE},
     commitment_config::CommitmentConfig,
     fee_calculator::FeeCalculator,
     hash::Hash,
@@ -32,8 +32,7 @@ use std::{
 };
 
 // The point at which transactions become "too old", in seconds.
-const MAX_TX_QUEUE_AGE: u64 =
-    MAX_PROCESSING_AGE as u64 * DEFAULT_TICKS_PER_SLOT / DEFAULT_TICKS_PER_SECOND;
+const MAX_TX_QUEUE_AGE: u64 = (MAX_PROCESSING_AGE as f64 * DEFAULT_S_PER_SLOT) as u64;
 
 pub const MAX_SPENDS_PER_TX: u64 = 4;
 
@@ -48,7 +47,7 @@ pub type SharedTransactions = Arc<RwLock<VecDeque<Vec<(Transaction, u64)>>>>;
 
 fn get_recent_blockhash<T: Client>(client: &T) -> (Hash, FeeCalculator) {
     loop {
-        match client.get_recent_blockhash_with_commitment(CommitmentConfig::recent()) {
+        match client.get_recent_blockhash_with_commitment(CommitmentConfig::processed()) {
             Ok((blockhash, fee_calculator, _last_valid_slot)) => {
                 return (blockhash, fee_calculator)
             }
@@ -497,7 +496,7 @@ fn do_tx_transfers<T: Client>(
 
 fn verify_funding_transfer<T: Client>(client: &Arc<T>, tx: &Transaction, amount: u64) -> bool {
     for a in &tx.message().account_keys[1..] {
-        match client.get_balance_with_commitment(a, CommitmentConfig::recent()) {
+        match client.get_balance_with_commitment(a, CommitmentConfig::processed()) {
             Ok(balance) => return balance >= amount,
             Err(err) => error!("failed to get balance {:?}", err),
         }
@@ -762,7 +761,7 @@ pub fn airdrop_lamports<T: Client>(
         };
 
         let current_balance = client
-            .get_balance_with_commitment(&id.pubkey(), CommitmentConfig::recent())
+            .get_balance_with_commitment(&id.pubkey(), CommitmentConfig::processed())
             .unwrap_or_else(|e| {
                 info!("airdrop error {}", e);
                 starting_balance
@@ -938,10 +937,12 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
 
-        let mut config = Config::default();
-        config.id = id;
-        config.tx_count = 10;
-        config.duration = Duration::from_secs(5);
+        let config = Config {
+            id,
+            tx_count: 10,
+            duration: Duration::from_secs(5),
+            ..Config::default()
+        };
 
         let keypair_count = config.tx_count * config.keypair_multiplier;
         let keypairs =
@@ -965,7 +966,7 @@ mod tests {
         for kp in &keypairs {
             assert_eq!(
                 client
-                    .get_balance_with_commitment(&kp.pubkey(), CommitmentConfig::recent())
+                    .get_balance_with_commitment(&kp.pubkey(), CommitmentConfig::processed())
                     .unwrap(),
                 lamports
             );

bench-tps/src/cli.rs

@@ -196,7 +196,7 @@ pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
 /// * `matches` - command line arguments parsed by clap
 /// # Panics
 /// Panics if there is trouble parsing any of the arguments
-pub fn extract_args<'a>(matches: &ArgMatches<'a>) -> Config {
+pub fn extract_args(matches: &ArgMatches) -> Config {
     let mut args = Config::default();
 
     if let Some(addr) = matches.value_of("entrypoint") {

bench-tps/src/lib.rs

@@ -1,2 +1,3 @@
+#![allow(clippy::integer_arithmetic)]
 pub mod bench;
 pub mod cli;

bench-tps/src/main.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::integer_arithmetic)]
 use log::*;
 use solana_bench_tps::bench::{do_bench_tps, generate_and_fund_keypairs, generate_keypairs};
 use solana_bench_tps::cli;

bench-tps/tests/bench_tps.rs

@@ -1,24 +1,31 @@
-use serial_test_derive::serial;
-use solana_bench_tps::bench::{do_bench_tps, generate_and_fund_keypairs};
-use solana_bench_tps::cli::Config;
+#![allow(clippy::integer_arithmetic)]
+use serial_test::serial;
+use solana_bench_tps::{
+    bench::{do_bench_tps, generate_and_fund_keypairs},
+    cli::Config,
+};
 use solana_client::thin_client::create_client;
-use solana_core::cluster_info::VALIDATOR_PORT_RANGE;
-use solana_core::validator::ValidatorConfig;
-use solana_faucet::faucet::run_local_faucet;
-use solana_local_cluster::local_cluster::{ClusterConfig, LocalCluster};
+use solana_core::{cluster_info::VALIDATOR_PORT_RANGE, validator::ValidatorConfig};
+use solana_faucet::faucet::run_local_faucet_with_port;
+use solana_local_cluster::{
+    local_cluster::{ClusterConfig, LocalCluster},
+    validator_configs::make_identical_validator_configs,
+};
 use solana_sdk::signature::{Keypair, Signer};
-use std::sync::{mpsc::channel, Arc};
-use std::time::Duration;
+use std::{
+    sync::{mpsc::channel, Arc},
+    time::Duration,
+};
 
 fn test_bench_tps_local_cluster(config: Config) {
     let native_instruction_processors = vec![];
 
     solana_logger::setup();
     const NUM_NODES: usize = 1;
-    let cluster = LocalCluster::new(&ClusterConfig {
+    let cluster = LocalCluster::new(&mut ClusterConfig {
         node_stakes: vec![999_990; NUM_NODES],
         cluster_lamports: 200_000_000,
-        validator_configs: vec![ValidatorConfig::default(); NUM_NODES],
+        validator_configs: make_identical_validator_configs(&ValidatorConfig::default(), NUM_NODES),
         native_instruction_processors,
         ..ClusterConfig::default()
     });
@@ -36,8 +43,11 @@ fn test_bench_tps_local_cluster(config: Config) {
     ));
 
     let (addr_sender, addr_receiver) = channel();
-    run_local_faucet(faucet_keypair, addr_sender, None);
-    let faucet_addr = addr_receiver.recv_timeout(Duration::from_secs(2)).unwrap();
+    run_local_faucet_with_port(faucet_keypair, addr_sender, None, 0);
+    let faucet_addr = addr_receiver
+        .recv_timeout(Duration::from_secs(2))
+        .expect("run_local_faucet")
+        .expect("faucet_addr");
 
     let lamports_per_account = 100;
 
@@ -60,9 +70,9 @@ fn test_bench_tps_local_cluster(config: Config) {
 #[test]
 #[serial]
 fn test_bench_tps_local_cluster_solana() {
-    let mut config = Config::default();
-    config.tx_count = 100;
-    config.duration = Duration::from_secs(10);
-
-    test_bench_tps_local_cluster(config);
+    test_bench_tps_local_cluster(Config {
+        tx_count: 100,
+        duration: Duration::from_secs(10),
+        ..Config::default()
+    });
 }

cargo

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# shellcheck source=ci/rust-version.sh
+here=$(dirname "$0")
+
+toolchain=
+case "$1" in
+  stable)
+    source "${here}"/ci/rust-version.sh stable
+    # shellcheck disable=SC2054 # rust_stable is sourced from rust-version.sh
+    toolchain="$rust_stable"
+    shift
+    ;;
+  nightly)
+    source "${here}"/ci/rust-version.sh nightly
+    # shellcheck disable=SC2054 # rust_nightly is sourced from rust-version.sh
+    toolchain="$rust_nightly"
+    shift
+    ;;
+  *)
+    source "${here}"/ci/rust-version.sh stable
+    # shellcheck disable=SC2054 # rust_stable is sourced from rust-version.sh
+    toolchain="$rust_stable"
+    ;;
+esac
+
+set -x
+exec cargo "+${toolchain}" "${@}"

cargo-build-bpf

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+here=$(dirname "$0")
+
+maybe_bpf_sdk="--bpf-sdk $here/sdk/bpf"
+for a in "$@"; do
+  if [[ $a = --bpf-sdk ]]; then
+    maybe_bpf_sdk=
+  fi
+done
+
+set -x
+exec "$here"/cargo run --manifest-path "$here"/sdk/cargo-build-bpf/Cargo.toml -- $maybe_bpf_sdk "$@"

cargo-test-bpf

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+here=$(dirname "$0")
+
+maybe_bpf_sdk="--bpf-sdk $here/sdk/bpf"
+for a in "$@"; do
+  if [[ $a = --bpf-sdk ]]; then
+    maybe_bpf_sdk=
+  fi
+done
+
+export CARGO_BUILD_BPF="$here"/cargo-build-bpf
+set -x
+exec "$here"/cargo run --manifest-path "$here"/sdk/cargo-test-bpf/Cargo.toml -- $maybe_bpf_sdk "$@"

ci/README.md

@@ -47,6 +47,8 @@ sudo ./setup-new-buildkite-agent/setup-buildkite.sh
 ```
 - Copy the pubkey contents from `~buildkite-agent/.ssh/id_ecdsa.pub` and
 add the pubkey as an authorized SSH key on github.
+  - In net/scripts/solana-user-authorized_keys.sh
+  - Bug mvines to add it to the "solana-grimes" github user
 - Edit `/etc/buildkite-agent/buildkite-agent.cfg` and/or `/etc/systemd/system/buildkite-agent@*` to the desired configuration of the agent(s)
 - Copy `ejson` keys from another CI node at `/opt/ejson/keys/`
 to the same location on the new node.

ci/buildkite-pipeline.sh

@@ -148,6 +148,33 @@ all_test_steps() {
   command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 60
   wait_step
 
+  # BPF test suite
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-stable-bpf.sh \
+             ^ci/test-stable.sh \
+             ^ci/test-local-cluster.sh \
+             ^core/build.rs \
+             ^fetch-perf-libs.sh \
+             ^programs/ \
+             ^sdk/ \
+      ; then
+    cat >> "$output_file" <<"EOF"
+  - command: "ci/test-stable-bpf.sh"
+    name: "stable-bpf"
+    timeout_in_minutes: 20
+    artifact_paths: "bpf-dumps.tar.bz2"
+    agents:
+      - "queue=default"
+EOF
+  else
+    annotate --style info \
+      "Stable-BPF skipped as no relevant files were modified"
+  fi
+
   # Perf test suite
   if affects \
              .rs$ \
@@ -165,7 +192,7 @@ all_test_steps() {
     cat >> "$output_file" <<"EOF"
   - command: "ci/test-stable-perf.sh"
     name: "stable-perf"
-    timeout_in_minutes: 40
+    timeout_in_minutes: 20
     artifact_paths: "log-*.txt"
     agents:
       - "queue=cuda"
@@ -175,6 +202,30 @@ EOF
       "Stable-perf skipped as no relevant files were modified"
   fi
 
+  # Downstream backwards compatibility
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-stable-perf.sh \
+             ^ci/test-stable.sh \
+             ^ci/test-local-cluster.sh \
+             ^core/build.rs \
+             ^fetch-perf-libs.sh \
+             ^programs/ \
+             ^sdk/ \
+             ^scripts/build-downstream-projects.sh \
+      ; then
+    cat >> "$output_file" <<"EOF"
+  - command: "scripts/build-downstream-projects.sh"
+    name: "downstream-projects"
+    timeout_in_minutes: 30
+EOF
+  else
+    annotate --style info \
+      "downstream-projects skipped as no relevant files were modified"
+  fi
   # Benches...
   if affects \
              .rs$ \
@@ -239,7 +290,7 @@ if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
   annotate --style info --context pr-backlink \
     "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
 
-  if [[ $GITHUB_USER = "dependabot-preview[bot]" ]]; then
+  if [[ $GITHUB_USER = "dependabot[bot]" ]]; then
     command_step dependabot "ci/dependabot-pr.sh" 5
     wait_step
   fi

ci/buildkite-secondary.yml

@@ -3,16 +3,24 @@
 # Pull requests to not run these steps.
 steps:
   - command: "ci/publish-tarball.sh"
+    agents:
+      - "queue=release-build"
     timeout_in_minutes: 60
     name: "publish tarball"
   - command: "ci/publish-bpf-sdk.sh"
+    agents:
+      - "queue=release-build"
     timeout_in_minutes: 5
     name: "publish bpf sdk"
   - wait
   - command: "sdk/docker-solana/build.sh"
+    agents:
+      - "queue=release-build"
     timeout_in_minutes: 60
     name: "publish docker"
   - command: "ci/publish-crate.sh"
+    agents:
+      - "queue=release-build"
     timeout_in_minutes: 240
     name: "publish crate"
     branches: "!master"

ci/channel-info.sh

@@ -95,12 +95,20 @@ elif [[ -n $CI_BRANCH ]]; then
   BRANCH="$CI_BRANCH"
 fi
 
-if [[ $BRANCH = "$STABLE_CHANNEL" ]]; then
-  CHANNEL=stable
-elif [[ $BRANCH = "$EDGE_CHANNEL" ]]; then
-  CHANNEL=edge
-elif [[ $BRANCH = "$BETA_CHANNEL" ]]; then
-  CHANNEL=beta
+if [[ -z "$CHANNEL" ]]; then
+  if [[ $BRANCH = "$STABLE_CHANNEL" ]]; then
+    CHANNEL=stable
+  elif [[ $BRANCH = "$EDGE_CHANNEL" ]]; then
+    CHANNEL=edge
+  elif [[ $BRANCH = "$BETA_CHANNEL" ]]; then
+    CHANNEL=beta
+  fi
+fi
+
+if [[ $CHANNEL = beta ]]; then
+  CHANNEL_LATEST_TAG="$BETA_CHANNEL_LATEST_TAG"
+elif [[ $CHANNEL = stable ]]; then
+  CHANNEL_LATEST_TAG="$STABLE_CHANNEL_LATEST_TAG"
 fi
 
 echo EDGE_CHANNEL="$EDGE_CHANNEL"
@@ -109,5 +117,6 @@ echo BETA_CHANNEL_LATEST_TAG="$BETA_CHANNEL_LATEST_TAG"
 echo STABLE_CHANNEL="$STABLE_CHANNEL"
 echo STABLE_CHANNEL_LATEST_TAG="$STABLE_CHANNEL_LATEST_TAG"
 echo CHANNEL="$CHANNEL"
+echo CHANNEL_LATEST_TAG="$CHANNEL_LATEST_TAG"
 
 exit 0

ci/dependabot-updater.sh

@@ -6,13 +6,13 @@ source ci/_
 
 commit_range="$(git merge-base HEAD origin/master)..HEAD"
 parsed_update_args="$(
-  git log "$commit_range" --author "dependabot-preview" --oneline -n1 |
+  git log "$commit_range" --author "dependabot\[bot\]" --oneline -n1 |
     grep -o '[Bb]ump.*$' |
     sed -r 's/[Bb]ump ([^ ]+) from ([^ ]+) to ([^ ]+)/-p \1:\2 --precise \3/'
 )"
 # relaxed_parsed_update_args is temporal measure...
 relaxed_parsed_update_args="$(
-  git log "$commit_range" --author "dependabot-preview" --oneline -n1 |
+  git log "$commit_range" --author "dependabot\[bot\]" --oneline -n1 |
     grep -o '[Bb]ump.*$' |
     sed -r 's/[Bb]ump ([^ ]+) from [^ ]+ to ([^ ]+)/-p \1 --precise \2/'
 )"

ci/do-audit.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -e
+
+here="$(dirname "$0")"
+src_root="$(readlink -f "${here}/..")"
+
+cd "${src_root}"
+
+cargo_audit_ignores=(
+  # failure is officially deprecated/unmaintained
+  #
+  # Blocked on multiple upstream crates removing their `failure` dependency.
+  --ignore RUSTSEC-2020-0036
+
+  # `net2` crate has been deprecated; use `socket2` instead
+  #
+  # Blocked on https://github.com/paritytech/jsonrpc/issues/575
+  --ignore RUSTSEC-2020-0016
+
+  # stdweb is unmaintained
+  #
+  # Blocked on multiple upstream crates removing their `stdweb` dependency.
+  --ignore RUSTSEC-2020-0056
+
+  # Potential segfault in the time crate
+  #
+  # Blocked on multiple crates updating `time` to >= 0.2.23
+  --ignore RUSTSEC-2020-0071
+
+  # generic-array: arr! macro erases lifetimes
+  #
+  # Blocked on libsecp256k1 releasing with upgraded dependencies
+  # https://github.com/paritytech/libsecp256k1/issues/66
+  --ignore RUSTSEC-2020-0146
+
+  # hyper: Lenient `hyper` header parsing of `Content-Length` could allow request smuggling
+  #
+  # Blocked on jsonrpc removing dependency on unmaintained `websocket`
+  # https://github.com/paritytech/jsonrpc/issues/605
+  --ignore RUSTSEC-2021-0078
+
+  # hyper: Integer overflow in `hyper`'s parsing of the `Transfer-Encoding` header leads to data loss
+  #
+  # Blocked on jsonrpc removing dependency on unmaintained `websocket`
+  # https://github.com/paritytech/jsonrpc/issues/605
+  --ignore RUSTSEC-2021-0079
+
+  # tar: Links in archive can create arbitrary directories
+  #
+  # Blocked on `tar` releasing safe upgrade
+  # https://github.com/alexcrichton/tar-rs/issues/238
+  --ignore RUSTSEC-2021-0080
+
+)
+scripts/cargo-for-all-lock-files.sh stable audit "${cargo_audit_ignores[@]}"

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.45.1
+FROM solanalabs/rust:1.51.0
 ARG date
 
 RUN set -x \

ci/docker-rust-nightly/README.md

@@ -20,7 +20,7 @@ To update the pinned version:
    specific YYYY-MM-DD that is desired (default is today's build).
    Check https://rust-lang.github.io/rustup-components-history/ for build
    status
-1. Update `ci/rust-version.sh` to reflect the new nightly `YYY-MM-DD`
+1. Update `ci/rust-version.sh` to reflect the new nightly `YYYY-MM-DD`
 1. Run `SOLANA_ALLOCATE_TTY=1 SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-checks.sh`
    and `SOLANA_ALLOCATE_TTY=1 SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-coverage.sh [args]...`
    to confirm the new nightly image builds.  Fix any issues as needed

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.45.1
+FROM rust:1.51.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0

ci/env.sh

@@ -74,10 +74,13 @@ else
   export CI_BUILD_ID=
   export CI_COMMIT=
   export CI_JOB_ID=
-  export CI_OS_NAME=
   export CI_PULL_REQUEST=
   export CI_REPO_SLUG=
   export CI_TAG=
+  # Don't override ci/run-local.sh
+  if [[ -z $CI_LOCAL_RUN ]]; then
+    export CI_OS_NAME=
+  fi
 fi
 
 cat <<EOF

ci/localnet-sanity.sh

@@ -70,7 +70,7 @@ done
 
 source ci/upload-ci-artifact.sh
 source scripts/configure-metrics.sh
-source multinode-demo/common.sh
+source multinode-demo/common.sh --prebuild
 
 nodes=(
   "multinode-demo/bootstrap-validator.sh \
@@ -78,7 +78,6 @@ nodes=(
     --init-complete-file init-complete-node0.log \
     --dynamic-port-range 8000-8050"
   "multinode-demo/validator.sh \
-    --enable-rpc-exit \
     --no-restart \
     --dynamic-port-range 8050-8100
     --init-complete-file init-complete-node1.log \
@@ -128,7 +127,7 @@ startNode() {
 waitForNodeToInit() {
   declare initCompleteFile=$1
   while [[ ! -r $initCompleteFile ]]; do
-    if [[ $SECONDS -ge 240 ]]; then
+    if [[ $SECONDS -ge 300 ]]; then
       echo "^^^ +++"
       echo "Error: $initCompleteFile not found in $SECONDS seconds"
       exit 1
@@ -201,17 +200,10 @@ killNodes() {
   [[ ${#pids[@]} -gt 0 ]] || return
 
   # Try to use the RPC exit API to cleanly exit the first two nodes
-  # (dynamic nodes, -x, are just killed since their RPC port is not known)
+  # (dynamic nodes, -x, are just killed)
   echo "--- RPC exit"
-  for port in 8899 18899; do
-    (
-      set -x
-      curl --retry 5 --retry-delay 2 --retry-connrefused \
-        -X POST -H 'Content-Type: application/json' \
-        -d '{"jsonrpc":"2.0","id":1, "method":"validatorExit"}' \
-        http://localhost:$port
-    )
-  done
+  $solana_validator --ledger "$SOLANA_CONFIG_DIR"/bootstrap-validator exit --force || true
+  $solana_validator --ledger "$SOLANA_CONFIG_DIR"/validator exit --force || true
 
   # Give the nodes a splash of time to cleanly exit before killing them
   sleep 2

ci/nits.sh

@@ -19,6 +19,7 @@ declare prints=(
 # Parts of the tree that are expected to be print free
 declare print_free_tree=(
   ':core/src/**.rs'
+  ':^core/src/validator.rs'
   ':faucet/src/**.rs'
   ':ledger/src/**.rs'
   ':metrics/src/**.rs'
@@ -26,7 +27,9 @@ declare print_free_tree=(
   ':runtime/src/**.rs'
   ':sdk/bpf/rust/rust-utils/**.rs'
   ':sdk/**.rs'
-  ':^sdk/src/program_option.rs'
+  ':^sdk/cargo-build-bpf/**.rs'
+  ':^sdk/program/src/program_option.rs'
+  ':^sdk/program/src/program_stubs.rs'
   ':programs/**.rs'
   ':^**bin**.rs'
   ':^**bench**.rs'

ci/order-crates-for-publishing.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python3
 #
 # This script figures the order in which workspace crates must be published to
 # crates.io.  Along the way it also ensures there are no circular dependencies
@@ -12,10 +12,14 @@ import json
 import subprocess
 import sys;
 
+real_file = os.path.realpath(__file__)
+ci_path = os.path.dirname(real_file)
+src_root = os.path.dirname(ci_path)
+
 def load_metadata():
+    cmd = f'{src_root}/cargo metadata --no-deps --format-version=1'
     return json.loads(subprocess.Popen(
-        'cargo metadata --no-deps --format-version=1',
-        shell=True, stdout=subprocess.PIPE).communicate()[0])
+        cmd, shell=True, stdout=subprocess.PIPE).communicate()[0])
 
 def get_packages():
     metadata = load_metadata()
@@ -45,21 +49,27 @@ def get_packages():
     sorted_dependency_graph = []
     max_iterations = pow(len(dependency_graph),2)
     while dependency_graph:
+        deleted_packages = []
         if max_iterations == 0:
             # One day be more helpful and find the actual cycle for the user...
             sys.exit('Error: Circular dependency suspected between these packages: \n {}\n'.format('\n '.join(dependency_graph.keys())))
 
         max_iterations -= 1
+
         for package, dependencies in dependency_graph.items():
+            if package in deleted_packages:
+                continue
             for dependency in dependencies:
                 if dependency in dependency_graph:
                     break
             else:
-                del dependency_graph[package]
+                deleted_packages.append(package)
                 sorted_dependency_graph.append((package, manifest_path[package]))
 
+        dependency_graph = {p: d for p, d in dependency_graph.items() if not p in deleted_packages }
+
 
     return sorted_dependency_graph
 
 for package, manifest in get_packages():
-    print os.path.relpath(manifest)
+    print(os.path.relpath(manifest))

ci/publish-bpf-sdk.sh

@@ -16,21 +16,12 @@ fi
   [[ -f bpf-sdk.tar.bz2 ]]
 )
 
+source ci/upload-ci-artifact.sh
 echo --- AWS S3 Store
 if [[ -z $CHANNEL_OR_TAG ]]; then
   echo Skipped
 else
-  (
-    set -x
-    docker run \
-      --rm \
-      --env AWS_ACCESS_KEY_ID \
-      --env AWS_SECRET_ACCESS_KEY \
-      --volume "$PWD:/solana" \
-      eremite/aws-cli:2018.12.18 \
-      /usr/bin/s3cmd --acl-public put /solana/bpf-sdk.tar.bz2 \
-      s3://solana-sdk/"$CHANNEL_OR_TAG"/bpf-sdk.tar.bz2
-  )
+  upload-s3-artifact "/solana/bpf-sdk.tar.bz2" "s3://solana-sdk/$CHANNEL_OR_TAG/bpf-sdk.tar.bz2"
 fi
 
 exit 0

ci/publish-crate.sh

@@ -4,6 +4,8 @@ cd "$(dirname "$0")/.."
 source ci/semver_bash/semver.sh
 source ci/rust-version.sh stable
 
+cargo="$(readlink -f ./cargo)"
+
 # shellcheck disable=SC2086
 is_crate_version_uploaded() {
   name=$1
@@ -38,7 +40,7 @@ for Cargo_toml in $Cargo_tomls; do
   crate_name=$(grep -m 1 '^name = ' "$Cargo_toml" | cut -f 3 -d ' ' | tr -d \")
 
   if grep -q "^publish = false" "$Cargo_toml"; then
-    echo "$crate_name is is marked as unpublishable"
+    echo "$crate_name is marked as unpublishable"
     continue
   fi
 
@@ -66,11 +68,11 @@ for Cargo_toml in $Cargo_tomls; do
       (
         set -x
         rm -rf crate-test
-        cargo +"$rust_stable" init crate-test
+        "$cargo" stable init crate-test
         cd crate-test/
         echo "${crate_name} = \"${expectedCrateVersion}\"" >> Cargo.toml
         echo "[workspace]" >> Cargo.toml
-        cargo +"$rust_stable" check
+        "$cargo" stable check
       ) && really_uploaded=1
       if ((really_uploaded)); then
         break;

ci/publish-tarball.sh

@@ -83,7 +83,7 @@ echo --- Creating release tarball
   export CHANNEL
 
   source ci/rust-version.sh stable
-  scripts/cargo-install-all.sh +"$rust_stable" "${RELEASE_BASENAME}"
+  scripts/cargo-install-all.sh stable "${RELEASE_BASENAME}"
 
   tar cvf "${TARBALL_BASENAME}"-$TARGET.tar "${RELEASE_BASENAME}"
   bzip2 "${TARBALL_BASENAME}"-$TARGET.tar
@@ -91,17 +91,15 @@ echo --- Creating release tarball
   cp "${RELEASE_BASENAME}"/version.yml "${TARBALL_BASENAME}"-$TARGET.yml
 )
 
-# Metrics tarball is platform agnostic, only publish it from Linux
+# Maybe tarballs are platform agnostic, only publish them from the Linux build
 MAYBE_TARBALLS=
 if [[ "$CI_OS_NAME" = linux ]]; then
-  metrics/create-metrics-tarball.sh
   (
     set -x
     sdk/bpf/scripts/package.sh
     [[ -f bpf-sdk.tar.bz2 ]]
-
   )
-  MAYBE_TARBALLS="bpf-sdk.tar.bz2 solana-metrics.tar.bz2"
+  MAYBE_TARBALLS="bpf-sdk.tar.bz2"
 fi
 
 source ci/upload-ci-artifact.sh
@@ -115,19 +113,10 @@ for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET.
 
   if [[ -n $BUILDKITE ]]; then
     echo --- AWS S3 Store: "$file"
-    (
-      set -x
-      $DRYRUN docker run \
-        --rm \
-        --env AWS_ACCESS_KEY_ID \
-        --env AWS_SECRET_ACCESS_KEY \
-        --volume "$PWD:/solana" \
-        eremite/aws-cli:2018.12.18 \
-        /usr/bin/s3cmd --acl-public put /solana/"$file" s3://release.solana.com/"$CHANNEL_OR_TAG"/"$file"
+    upload-s3-artifact "/solana/$file" s3://release.solana.com/"$CHANNEL_OR_TAG"/"$file"
 
-      echo Published to:
-      $DRYRUN ci/format-url.sh http://release.solana.com/"$CHANNEL_OR_TAG"/"$file"
-    )
+    echo Published to:
+    $DRYRUN ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/"$file"
 
     if [[ -n $TAG ]]; then
       ci/upload-github-release-asset.sh "$file"
@@ -149,4 +138,22 @@ for file in "${TARBALL_BASENAME}"-$TARGET.tar.bz2 "${TARBALL_BASENAME}"-$TARGET.
   fi
 done
 
+
+# Create install wrapper for release.solana.com
+if [[ -n $DO_NOT_PUBLISH_TAR ]]; then
+  echo "Skipping publishing install wrapper"
+elif [[ -n $BUILDKITE ]]; then
+  cat > release.solana.com-install <<EOF
+SOLANA_RELEASE=$CHANNEL_OR_TAG
+SOLANA_INSTALL_INIT_ARGS=$CHANNEL_OR_TAG
+SOLANA_DOWNLOAD_ROOT=http://release.solana.com
+EOF
+  cat install/solana-install-init.sh >> release.solana.com-install
+
+  echo --- AWS S3 Store: "install"
+  $DRYRUN upload-s3-artifact "/solana/release.solana.com-install" "s3://release.solana.com/$CHANNEL_OR_TAG/install"
+  echo Published to:
+  $DRYRUN ci/format-url.sh https://release.solana.com/"$CHANNEL_OR_TAG"/install
+fi
+
 echo --- ok

ci/run-local.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+cd "$(dirname "$0")/.."
+
+export CI_LOCAL_RUN=true
+
+set -e
+
+case $(uname -o) in
+  */Linux)
+    export CI_OS_NAME=linux
+    ;;
+  *)
+    echo "local CI runs are only supported on Linux" 1>&2
+    exit 1
+    ;;
+esac
+
+steps=()
+steps+=(test-sanity)
+steps+=(shellcheck)
+steps+=(test-checks)
+steps+=(test-coverage)
+steps+=(test-stable)
+steps+=(test-stable-bpf)
+steps+=(test-stable-perf)
+steps+=(test-downstream-builds)
+steps+=(test-bench)
+steps+=(test-local-cluster)
+
+step_index=0
+if [[ -n "$1" ]]; then
+  start_step="$1"
+  while [[ $step_index -lt ${#steps[@]} ]]; do
+    step="${steps[$step_index]}"
+    if [[ "$step" = "$start_step" ]]; then
+      break
+    fi
+    step_index=$((step_index + 1))
+  done
+  if [[ $step_index -eq ${#steps[@]} ]]; then
+    echo "unexpected start step: \"$start_step\"" 1>&2
+    exit 1
+  else
+    echo "** starting at step: \"$start_step\" **"
+    echo
+  fi
+fi
+
+while [[ $step_index -lt ${#steps[@]} ]]; do
+  step="${steps[$step_index]}"
+  cmd="ci/${step}.sh"
+  $cmd
+  step_index=$((step_index + 1))
+done

ci/run-sanity.sh

@@ -22,10 +22,11 @@ done
 snapshot_slot=1
 
 # wait a bit longer than snapshot_slot
-while [[ $($solana_cli --url http://localhost:8899 slot --commitment recent) -le $((snapshot_slot + 1)) ]]; do
+while [[ $($solana_cli --url http://localhost:8899 slot --commitment processed) -le $((snapshot_slot + 1)) ]]; do
   sleep 1
 done
-curl -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":1, "method":"validatorExit"}' http://localhost:8899
+
+$solana_validator --ledger config/ledger exit --force || true
 
 wait $pid
 

ci/rust-version.sh

@@ -18,13 +18,13 @@
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.45.1
+  stable_version=1.51.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2020-07-27
+  nightly_version=2021-04-18
 fi
 
 

ci/setup-new-buildkite-agent/setup-buildkite.sh

@@ -76,7 +76,7 @@ RestartForceExitStatus=SIGPIPE
 TimeoutStartSec=10
 TimeoutStopSec=0
 KillMode=process
-LimitNOFILE=500000
+LimitNOFILE=700000
 
 [Install]
 WantedBy=multi-user.target

ci/setup-new-buildkite-agent/setup-limits.sh

@@ -8,5 +8,5 @@ source "$HERE"/utils.sh
 ensure_env || exit 1
 
 # Allow more files to be opened by a user
-echo "* - nofile 500000" > /etc/security/limits.d/90-solana-nofiles.conf
+echo "* - nofile 700000" > /etc/security/limits.d/90-solana-nofiles.conf
 

ci/setup-new-buildkite-agent/setup-new-machine.sh

@@ -7,6 +7,7 @@ SOLANA_ROOT="$HERE"/../..
 source "$HERE"/utils.sh
 
 ensure_env || exit 1
+check_ssh_authorized_keys || exit 1
 
 set -ex
 

ci/setup-new-buildkite-agent/setup-ssh.sh

@@ -6,6 +6,11 @@ HERE="$(dirname "$0")"
 source "$HERE"/utils.sh
 
 ensure_env || exit 1
+# This is a last ditch effort to prevent the caller from locking themselves
+# out of the machine. Exiting here will likely leave the system in some
+# half-configured state. To prevent this, duplicate the next line at the top
+# of the entrypoint script.
+check_ssh_authorized_keys || exit 1
 
 set -xe
 # Setup sshd

ci/setup-new-buildkite-agent/utils.sh

@@ -14,3 +14,33 @@ ensure_env() {
   $RC
 }
 
+# Some scripts disable SSH password logins. If no one hash setup authorized_keys
+# this will result in the machine being remotely inaccessible. Check that the
+# user running this script has setup their keys
+check_ssh_authorized_keys() {
+  declare rc=false
+  declare user_home=
+  if [[ -n "$SUDO_USER" ]]; then
+    declare user uid gid home
+    declare passwd_entry
+    passwd_entry="$(grep "$SUDO_USER:[^:]*:$SUDO_UID:$SUDO_GID" /etc/passwd)"
+    IFS=: read -r user _ uid gid _ home _ <<<"$passwd_entry"
+    if [[ "$user" == "$SUDO_USER" && "$uid" == "$SUDO_UID" && "$gid" == "$SUDO_GID" ]]; then
+      user_home="$home"
+    fi
+  else
+    user_home="$HOME"
+  fi
+  declare authorized_keys="${user_home}/.ssh/authorized_keys"
+  if [[ -n "$user_home" ]]; then
+    [[ -s "$authorized_keys" ]] && rc=true
+  fi
+  if ! $rc; then
+    echo "ERROR! This script will disable SSH password logins and you don't"
+    echo "appear to have set up any authorized keys.  Please add you SSH"
+    echo "public key to ${authorized_keys} before continuing!"
+  fi
+  $rc
+}
+
+check_ssh_authorized_keys

ci/test-bench.sh

@@ -6,7 +6,8 @@ source ci/_
 source ci/upload-ci-artifact.sh
 
 eval "$(ci/channel-info.sh)"
-source ci/rust-version.sh all
+
+cargo="$(readlink -f "./cargo")"
 
 set -o pipefail
 export RUST_BACKTRACE=1
@@ -22,40 +23,44 @@ fi
 BENCH_FILE=bench_output.log
 BENCH_ARTIFACT=current_bench_results.log
 
+# solana-keygen required when building C programs
+_ "$cargo" build --manifest-path=keygen/Cargo.toml
+export PATH="$PWD/target/debug":$PATH
+
 # Clear the C dependency files, if dependeny moves these files are not regenerated
 test -d target/debug/bpf && find target/debug/bpf -name '*.d' -delete
 test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
 
 # Ensure all dependencies are built
-_ cargo +$rust_nightly build --release
+_ "$cargo" nightly build --release
 
 # Remove "BENCH_FILE", if it exists so that the following commands can append
 rm -f "$BENCH_FILE"
 
 # Run sdk benches
-_ cargo +$rust_nightly bench --manifest-path sdk/Cargo.toml ${V:+--verbose} \
+_ "$cargo" nightly bench --manifest-path sdk/Cargo.toml ${V:+--verbose} \
   -- -Z unstable-options --format=json | tee -a "$BENCH_FILE"
 
 # Run runtime benches
-_ cargo +$rust_nightly bench --manifest-path runtime/Cargo.toml ${V:+--verbose} \
+_ "$cargo" nightly bench --manifest-path runtime/Cargo.toml ${V:+--verbose} \
   -- -Z unstable-options --format=json | tee -a "$BENCH_FILE"
 
 # Run core benches
-_ cargo +$rust_nightly bench --manifest-path core/Cargo.toml ${V:+--verbose} \
+_ "$cargo" nightly bench --manifest-path core/Cargo.toml ${V:+--verbose} \
   -- -Z unstable-options --format=json | tee -a "$BENCH_FILE"
 
 # Run bpf benches
-_ cargo +$rust_nightly bench --manifest-path programs/bpf/Cargo.toml ${V:+--verbose} --features=bpf_c \
+_ "$cargo" nightly bench --manifest-path programs/bpf/Cargo.toml ${V:+--verbose} --features=bpf_c \
   -- -Z unstable-options --format=json --nocapture | tee -a "$BENCH_FILE"
 
 # Run banking/accounts bench. Doesn't require nightly, but use since it is already built.
-_ cargo +$rust_nightly run --release --manifest-path banking-bench/Cargo.toml ${V:+--verbose} | tee -a "$BENCH_FILE"
-_ cargo +$rust_nightly run --release --manifest-path accounts-bench/Cargo.toml ${V:+--verbose} -- --num_accounts 10000 --num_slots 4 | tee -a "$BENCH_FILE"
+_ "$cargo" nightly run --release --manifest-path banking-bench/Cargo.toml ${V:+--verbose} | tee -a "$BENCH_FILE"
+_ "$cargo" nightly run --release --manifest-path accounts-bench/Cargo.toml ${V:+--verbose} -- --num_accounts 10000 --num_slots 4 | tee -a "$BENCH_FILE"
 
 # `solana-upload-perf` disabled as it can take over 30 minutes to complete for some
 # reason
 exit 0
-_ cargo +$rust_nightly run --release --package solana-upload-perf \
+_ "$cargo" nightly run --release --package solana-upload-perf \
   -- "$BENCH_FILE" "$TARGET_BRANCH" "$UPLOAD_METRICS" | tee "$BENCH_ARTIFACT"
 
 upload-ci-artifact "$BENCH_FILE"

ci/test-checks.sh

@@ -8,6 +8,19 @@ source ci/_
 source ci/rust-version.sh stable
 source ci/rust-version.sh nightly
 eval "$(ci/channel-info.sh)"
+cargo="$(readlink -f "./cargo")"
+
+scripts/increment-cargo-version.sh check
+
+# Disallow uncommitted Cargo.lock changes
+(
+  _ scripts/cargo-for-all-lock-files.sh tree >/dev/null
+  set +e
+  if ! _ git diff --exit-code; then
+    echo -e "\nError: Uncommitted Cargo.lock changes" 1>&2
+    exit 1
+  fi
+)
 
 echo --- build environment
 (
@@ -16,14 +29,16 @@ echo --- build environment
   rustup run "$rust_stable" rustc --version --verbose
   rustup run "$rust_nightly" rustc --version --verbose
 
-  cargo +"$rust_stable" --version --verbose
-  cargo +"$rust_nightly" --version --verbose
+  "$cargo" stable --version --verbose
+  "$cargo" nightly --version --verbose
 
-  cargo +"$rust_stable" clippy --version --verbose
-  cargo +"$rust_nightly" clippy --version --verbose
+  "$cargo" stable clippy --version --verbose
+  "$cargo" nightly clippy --version --verbose
 
-  # audit is done only with stable
-  cargo +"$rust_stable" audit --version
+  # audit is done only with "$cargo stable"
+  "$cargo" stable audit --version
+
+  grcov --version
 )
 
 export RUST_BACKTRACE=1
@@ -32,7 +47,7 @@ export RUSTFLAGS="-D warnings -A incomplete_features"
 # Only force up-to-date lock files on edge
 if [[ $CI_BASE_BRANCH = "$EDGE_CHANNEL" ]]; then
   # Exclude --benches as it's not available in rust stable yet
-  if _ scripts/cargo-for-all-lock-files.sh +"$rust_stable" check --locked --tests --bins --examples; then
+  if _ scripts/cargo-for-all-lock-files.sh stable check --locked --tests --bins --examples; then
     true
   else
     check_status=$?
@@ -43,31 +58,30 @@ if [[ $CI_BASE_BRANCH = "$EDGE_CHANNEL" ]]; then
   fi
 
   # Ensure nightly and --benches
-  _ scripts/cargo-for-all-lock-files.sh +"$rust_nightly" check --locked --all-targets
-
+  _ scripts/cargo-for-all-lock-files.sh nightly check --locked --all-targets
 else
   echo "Note: cargo-for-all-lock-files.sh skipped because $CI_BASE_BRANCH != $EDGE_CHANNEL"
 fi
 
 _ ci/order-crates-for-publishing.py
-_ cargo +"$rust_stable" fmt --all -- --check
 
 # -Z... is needed because of clippy bug: https://github.com/rust-lang/rust-clippy/issues/4612
 # run nightly clippy for `sdk/` as there's a moderate amount of nightly-only code there
-_ cargo +"$rust_nightly" clippy -Zunstable-options --workspace --all-targets -- --deny=warnings
+_ "$cargo" nightly clippy -Zunstable-options --workspace --all-targets -- \
+  --deny=warnings --deny=clippy::integer_arithmetic --allow=clippy::inconsistent_struct_constructor
 
-_ scripts/cargo-for-all-lock-files.sh +"$rust_stable" audit --ignore RUSTSEC-2020-0002 --ignore RUSTSEC-2020-0008
+_ "$cargo" stable fmt --all -- --check
+
+_ ci/do-audit.sh
 
 {
   cd programs/bpf
-  _ cargo +"$rust_stable" audit
   for project in rust/*/ ; do
     echo "+++ do_bpf_checks $project"
     (
       cd "$project"
-      _ cargo +"$rust_stable" fmt -- --check
-      _ cargo +"$rust_nightly" test
-      _ cargo +"$rust_nightly" clippy -- --deny=warnings --allow=clippy::missing_safety_doc
+      _ "$cargo" nightly clippy -- --deny=warnings --allow=clippy::missing_safety_doc
+      _ "$cargo" stable fmt -- --check
     )
   done
 }

ci/test-downstream-builds.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+cd "$(dirname "$0")/.."
+
+export CI_LOCAL_RUN=true
+
+set -ex
+
+scripts/build-downstream-projects.sh

ci/test-sanity.sh

@@ -8,10 +8,16 @@ source ci/_
 (
   echo --- git diff --check
   set -x
+
+  if [[ -n $CI_BASE_BRANCH ]]
+  then branch="$CI_BASE_BRANCH"
+  else branch="master"
+  fi
+
   # Look for failed mergify.io backports by searching leftover conflict markers
   # Also check for any trailing whitespaces!
-  git fetch origin "$CI_BASE_BRANCH"
-  git diff "$(git merge-base HEAD "origin/$CI_BASE_BRANCH")..HEAD" --check --oneline
+  git fetch origin "$branch"
+  git diff "$(git merge-base HEAD "origin/$branch")" --check --oneline
 )
 
 echo
@@ -19,4 +25,29 @@ echo
 _ ci/nits.sh
 _ ci/check-ssh-keys.sh
 
+
+# Ensure the current channel version is not equal ("greater") than
+# the version of the latest tag
+if [[ -z $CI_TAG ]]; then
+  echo "--- channel version check"
+  (
+    eval "$(ci/channel-info.sh)"
+
+    if [[ -n $CHANNEL_LATEST_TAG ]]; then
+      source scripts/read-cargo-variable.sh
+
+      version=$(readCargoVariable version "version/Cargo.toml")
+      echo "version: v$version"
+      echo "latest channel tag: $CHANNEL_LATEST_TAG"
+
+      if [[ $CHANNEL_LATEST_TAG = v$version ]]; then
+        echo "Error: please run ./scripts/increment-cargo-version.sh"
+        exit 1
+      fi
+    else
+      echo "Skipped. CHANNEL_LATEST_TAG (CHANNEL=$CHANNEL) unset"
+    fi
+  )
+fi
+
 echo --- ok

ci/test-stable-bpf.sh

@@ -0,0 +1 @@
+test-stable.sh

ci/test-stable.sh

@@ -2,6 +2,8 @@
 set -e
 cd "$(dirname "$0")/.."
 
+cargo="$(readlink -f "./cargo")"
+
 source ci/_
 
 annotate() {
@@ -19,16 +21,6 @@ export RUST_BACKTRACE=1
 export RUSTFLAGS="-D warnings"
 source scripts/ulimit-n.sh
 
-# Clear cached json keypair files
-rm -rf "$HOME/.config/solana"
-
-# Clear the C dependency files, if dependency moves these files are not regenerated
-test -d target/debug/bpf && find target/debug/bpf -name '*.d' -delete
-test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
-
-# Clear the BPF sysroot files, they are not automatically rebuilt
-rm -rf target/xargo # Issue #3105
-
 # Limit compiler jobs to reduce memory usage
 # on machines with 2gb/thread of memory
 NPROC=$(nproc)
@@ -37,20 +29,59 @@ NPROC=$((NPROC>14 ? 14 : NPROC))
 echo "Executing $testName"
 case $testName in
 test-stable)
-  _ cargo +"$rust_stable" test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
+  _ "$cargo" stable test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
   ;;
-test-stable-perf)
-  # BPF program tests
+test-stable-bpf)
+  # Clear the C dependency files, if dependency moves these files are not regenerated
+  test -d target/debug/bpf && find target/debug/bpf -name '*.d' -delete
+  test -d target/release/bpf && find target/release/bpf -name '*.d' -delete
+
+  # rustfilt required for dumping BPF assembly listings
+  "$cargo" install rustfilt
+
+  # solana-keygen required when building C programs
+  _ "$cargo" build --manifest-path=keygen/Cargo.toml
+  export PATH="$PWD/target/debug":$PATH
+  cargo_build_bpf="$(realpath ./cargo-build-bpf)"
+
+  # BPF solana-sdk legacy compile test
+  "$cargo_build_bpf" --manifest-path sdk/Cargo.toml
+
+  # BPF Program unit tests
+  "$cargo" stable test --manifest-path programs/bpf/Cargo.toml
+  "$cargo_build_bpf" --manifest-path programs/bpf/Cargo.toml --bpf-sdk sdk/bpf
+
+  # BPF program system tests
   _ make -C programs/bpf/c tests
-  _ cargo +"$rust_stable" test \
+  _ "$cargo" stable test \
     --manifest-path programs/bpf/Cargo.toml \
     --no-default-features --features=bpf_c,bpf_rust -- --nocapture
 
+  # Dump BPF program assembly listings
+  for bpf_test in programs/bpf/rust/*; do
+    if pushd "$bpf_test"; then
+      "$cargo_build_bpf" --dump
+      popd
+    fi
+  done
+
+  # BPF program instruction count assertion
+  bpf_target_path=programs/bpf/target
+  _ "$cargo" stable test \
+    --manifest-path programs/bpf/Cargo.toml \
+    --no-default-features --features=bpf_c,bpf_rust assert_instruction_count \
+    -- --nocapture &> "${bpf_target_path}"/deploy/instuction_counts.txt
+
+  bpf_dump_archive="bpf-dumps.tar.bz2"
+  rm -f "$bpf_dump_archive"
+  tar cjvf "$bpf_dump_archive" "${bpf_target_path}"/{deploy/*.txt,bpfel-unknown-unknown/release/*.so}
+  ;;
+test-stable-perf)
   if [[ $(uname) = Linux ]]; then
     # Enable persistence mode to keep the CUDA kernel driver loaded, avoiding a
     # lengthy and unexpected delay the first time CUDA is involved when the driver
     # is not yet loaded.
-    sudo --non-interactive ./net/scripts/enable-nvidia-persistence-mode.sh
+    sudo --non-interactive ./net/scripts/enable-nvidia-persistence-mode.sh || true
 
     rm -rf target/perf-libs
     ./fetch-perf-libs.sh
@@ -62,13 +93,13 @@ test-stable-perf)
     export SOLANA_CUDA=1
   fi
 
-  _ cargo +"$rust_stable" build --bins ${V:+--verbose}
-  _ cargo +"$rust_stable" test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
-  _ cargo +"$rust_stable" run --manifest-path poh-bench/Cargo.toml ${V:+--verbose} -- --hashes-per-tick 10
+  _ "$cargo" stable build --bins ${V:+--verbose}
+  _ "$cargo" stable test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
+  _ "$cargo" stable run --manifest-path poh-bench/Cargo.toml ${V:+--verbose} -- --hashes-per-tick 10
   ;;
 test-local-cluster)
-  _ cargo +"$rust_stable" build --release --bins ${V:+--verbose}
-  _ cargo +"$rust_stable" test --release --package solana-local-cluster ${V:+--verbose} -- --nocapture --test-threads=1
+  _ "$cargo" stable build --release --bins ${V:+--verbose}
+  _ "$cargo" stable test --release --package solana-local-cluster ${V:+--verbose} -- --nocapture --test-threads=1
   exit 0
   ;;
 *)

ci/upload-ci-artifact.sh

@@ -16,3 +16,16 @@ upload-ci-artifact() {
   fi
 }
 
+upload-s3-artifact() {
+  echo "--- artifact: $1 to $2"
+  (
+    set -x
+    docker run \
+      --rm \
+      --env AWS_ACCESS_KEY_ID \
+      --env AWS_SECRET_ACCESS_KEY \
+      --volume "$PWD:/solana" \
+      eremite/aws-cli:2018.12.18 \
+      /usr/bin/s3cmd --acl-public put "$1" "$2"
+  )
+}

clap-utils/Cargo.toml

@@ -1,23 +1,28 @@
 [package]
 name = "solana-clap-utils"
-version = "1.3.13"
+version = "1.6.22"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-clap-utils"
 edition = "2018"
 
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.3.13" }
-solana-sdk = { path = "../sdk", version = "1.3.13" }
-thiserror = "1.0.20"
-tiny-bip39 = "0.7.0"
+solana-remote-wallet = { path = "../remote-wallet", version = "=1.6.22" }
+solana-sdk = { path = "../sdk", version = "=1.6.22" }
+thiserror = "1.0.21"
+tiny-bip39 = "0.8.0"
+uriparse = "0.6.3"
 url = "2.1.0"
 chrono = "0.4"
 
+[dev-dependencies]
+tempfile = "3.1.0"
+
 [lib]
 name = "solana_clap_utils"
 

clap-utils/src/commitment.rs

@@ -1,22 +0,0 @@
-use crate::ArgConstant;
-use clap::Arg;
-
-pub const COMMITMENT_ARG: ArgConstant<'static> = ArgConstant {
-    name: "commitment",
-    long: "commitment",
-    help: "Return information at the selected commitment level",
-};
-
-pub fn commitment_arg<'a, 'b>() -> Arg<'a, 'b> {
-    commitment_arg_with_default("recent")
-}
-
-pub fn commitment_arg_with_default<'a, 'b>(default_value: &'static str) -> Arg<'a, 'b> {
-    Arg::with_name(COMMITMENT_ARG.name)
-        .long(COMMITMENT_ARG.long)
-        .takes_value(true)
-        .possible_values(&["recent", "single", "root", "max"])
-        .default_value(default_value)
-        .value_name("COMMITMENT_LEVEL")
-        .help(COMMITMENT_ARG.help)
-}

clap-utils/src/fee_payer.rs

@@ -1,5 +1,7 @@
-use crate::{input_validators, ArgConstant};
-use clap::Arg;
+use {
+    crate::{input_validators, ArgConstant},
+    clap::Arg,
+};
 
 pub const FEE_PAYER_ARG: ArgConstant<'static> = ArgConstant {
     name: "fee_payer",

clap-utils/src/input_parsers.rs

@@ -1,19 +1,21 @@
-use crate::keypair::{
-    keypair_from_seed_phrase, pubkey_from_path, resolve_signer_from_path, signer_from_path,
-    ASK_KEYWORD, SKIP_SEED_PHRASE_VALIDATION_ARG,
+use {
+    crate::keypair::{
+        keypair_from_seed_phrase, pubkey_from_path, resolve_signer_from_path, signer_from_path,
+        ASK_KEYWORD, SKIP_SEED_PHRASE_VALIDATION_ARG,
+    },
+    chrono::DateTime,
+    clap::ArgMatches,
+    solana_remote_wallet::remote_wallet::RemoteWalletManager,
+    solana_sdk::{
+        clock::UnixTimestamp,
+        commitment_config::CommitmentConfig,
+        genesis_config::ClusterType,
+        native_token::sol_to_lamports,
+        pubkey::Pubkey,
+        signature::{read_keypair_file, Keypair, Signature, Signer},
+    },
+    std::{str::FromStr, sync::Arc},
 };
-use chrono::DateTime;
-use clap::ArgMatches;
-use solana_remote_wallet::remote_wallet::RemoteWalletManager;
-use solana_sdk::{
-    clock::UnixTimestamp,
-    commitment_config::CommitmentConfig,
-    genesis_config::ClusterType,
-    native_token::sol_to_lamports,
-    pubkey::Pubkey,
-    signature::{read_keypair_file, Keypair, Signature, Signer},
-};
-use std::{str::FromStr, sync::Arc};
 
 // Return parsed values from matches at `name`
 pub fn values_of<T>(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<T>>
@@ -55,7 +57,7 @@ pub fn keypair_of(matches: &ArgMatches<'_>, name: &str) -> Option<Keypair> {
     if let Some(value) = matches.value_of(name) {
         if value == ASK_KEYWORD {
             let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
-            keypair_from_seed_phrase(name, skip_validation, true).ok()
+            keypair_from_seed_phrase(name, skip_validation, true, None, true).ok()
         } else {
             read_keypair_file(value).ok()
         }
@@ -70,7 +72,7 @@ pub fn keypairs_of(matches: &ArgMatches<'_>, name: &str) -> Option<Vec<Keypair>>
             .filter_map(|value| {
                 if value == ASK_KEYWORD {
                     let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
-                    keypair_from_seed_phrase(name, skip_validation, true).ok()
+                    keypair_from_seed_phrase(name, skip_validation, true, None, true).ok()
                 } else {
                     read_keypair_file(value).ok()
                 }
@@ -167,12 +169,12 @@ pub fn resolve_signer(
     name: &str,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<Option<String>, Box<dyn std::error::Error>> {
-    Ok(resolve_signer_from_path(
+    resolve_signer_from_path(
         matches,
         matches.value_of(name).unwrap(),
         name,
         wallet_manager,
-    )?)
+    )
 }
 
 pub fn lamports_of_sol(matches: &ArgMatches<'_>, name: &str) -> Option<u64> {
@@ -184,13 +186,9 @@ pub fn cluster_type_of(matches: &ArgMatches<'_>, name: &str) -> Option<ClusterTy
 }
 
 pub fn commitment_of(matches: &ArgMatches<'_>, name: &str) -> Option<CommitmentConfig> {
-    matches.value_of(name).map(|value| match value {
-        "max" => CommitmentConfig::max(),
-        "recent" => CommitmentConfig::recent(),
-        "root" => CommitmentConfig::root(),
-        "single" => CommitmentConfig::single(),
-        _ => CommitmentConfig::default(),
-    })
+    matches
+        .value_of(name)
+        .map(|value| CommitmentConfig::from_str(value).unwrap_or_default())
 }
 
 #[cfg(test)]
@@ -228,8 +226,8 @@ mod tests {
         assert_eq!(values_of(&matches, "multiple"), Some(vec![50, 39]));
         assert_eq!(values_of::<u64>(&matches, "single"), None);
 
-        let pubkey0 = Pubkey::new_rand();
-        let pubkey1 = Pubkey::new_rand();
+        let pubkey0 = solana_sdk::pubkey::new_rand();
+        let pubkey1 = solana_sdk::pubkey::new_rand();
         let matches = app().clone().get_matches_from(vec![
             "test",
             "--multiple",
@@ -251,7 +249,7 @@ mod tests {
         assert_eq!(value_of(&matches, "single"), Some(50));
         assert_eq!(value_of::<u64>(&matches, "multiple"), None);
 
-        let pubkey = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
         let matches = app()
             .clone()
             .get_matches_from(vec!["test", "--single", &pubkey.to_string()]);
@@ -331,8 +329,8 @@ mod tests {
 
     #[test]
     fn test_pubkeys_sigs_of() {
-        let key1 = Pubkey::new_rand();
-        let key2 = Pubkey::new_rand();
+        let key1 = solana_sdk::pubkey::new_rand();
+        let key2 = solana_sdk::pubkey::new_rand();
         let sig1 = Keypair::new().sign_message(&[0u8]);
         let sig2 = Keypair::new().sign_message(&[1u8]);
         let signer1 = format!("{}={}", key1, sig1);

clap-utils/src/input_validators.rs

@@ -1,13 +1,15 @@
-use crate::keypair::{parse_keypair_path, KeypairUrl, ASK_KEYWORD};
-use chrono::DateTime;
-use solana_sdk::{
-    clock::Slot,
-    hash::Hash,
-    pubkey::Pubkey,
-    signature::{read_keypair_file, Signature},
+use {
+    crate::keypair::{parse_signer_source, SignerSourceKind, ASK_KEYWORD},
+    chrono::DateTime,
+    solana_sdk::{
+        clock::{Epoch, Slot},
+        hash::Hash,
+        pubkey::{Pubkey, MAX_SEED_LEN},
+        signature::{read_keypair_file, Signature},
+    },
+    std::fmt::Display,
+    std::str::FromStr,
 };
-use std::fmt::Display;
-use std::str::FromStr;
 
 fn is_parsable_generic<U, T>(string: T) -> Result<(), String>
 where
@@ -32,6 +34,29 @@ where
     is_parsable_generic::<T, String>(string)
 }
 
+// Return an error if string cannot be parsed as numeric type T, and value not within specified
+// range
+pub fn is_within_range<T>(string: String, range_min: T, range_max: T) -> Result<(), String>
+where
+    T: FromStr + Copy + std::fmt::Debug + PartialOrd + std::ops::Add<Output = T> + From<usize>,
+    T::Err: Display,
+{
+    match string.parse::<T>() {
+        Ok(input) => {
+            let range = range_min..range_max + 1.into();
+            if !range.contains(&input) {
+                Err(format!(
+                    "input '{:?}' out of range ({:?}..{:?}]",
+                    input, range_min, range_max
+                ))
+            } else {
+                Ok(())
+            }
+        }
+        Err(err) => Err(format!("error parsing '{}': {}", string, err)),
+    }
+}
+
 // Return an error if a pubkey cannot be parsed.
 pub fn is_pubkey<T>(string: T) -> Result<(), String>
 where
@@ -71,6 +96,26 @@ where
         .map_err(|err| format!("{}", err))
 }
 
+// Return an error if a `SignerSourceKind::Prompt` cannot be parsed
+pub fn is_prompt_signer_source<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    if string.as_ref() == ASK_KEYWORD {
+        return Ok(());
+    }
+    match parse_signer_source(string.as_ref())
+        .map_err(|err| format!("{}", err))?
+        .kind
+    {
+        SignerSourceKind::Prompt => Ok(()),
+        _ => Err(format!(
+            "Unable to parse input as `prompt:` URI scheme or `ASK` keyword: {}",
+            string
+        )),
+    }
+}
+
 // Return an error if string cannot be parsed as pubkey string or keypair file location
 pub fn is_pubkey_or_keypair<T>(string: T) -> Result<(), String>
 where
@@ -85,8 +130,11 @@ pub fn is_valid_pubkey<T>(string: T) -> Result<(), String>
 where
     T: AsRef<str> + Display,
 {
-    match parse_keypair_path(string.as_ref()) {
-        KeypairUrl::Filepath(path) => is_keypair(path),
+    match parse_signer_source(string.as_ref())
+        .map_err(|err| format!("{}", err))?
+        .kind
+    {
+        SignerSourceKind::Filepath(path) => is_keypair(path),
         _ => Ok(()),
     }
 }
@@ -148,6 +196,40 @@ where
     }
 }
 
+pub fn is_url_or_moniker<T>(string: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    match url::Url::parse(&normalize_to_url_if_moniker(string.as_ref())) {
+        Ok(url) => {
+            if url.has_host() {
+                Ok(())
+            } else {
+                Err("no host provided".to_string())
+            }
+        }
+        Err(err) => Err(format!("{}", err)),
+    }
+}
+
+pub fn normalize_to_url_if_moniker<T: AsRef<str>>(url_or_moniker: T) -> String {
+    match url_or_moniker.as_ref() {
+        "m" | "mainnet-beta" => "https://api.mainnet-beta.solana.com",
+        "t" | "testnet" => "https://api.testnet.solana.com",
+        "d" | "devnet" => "https://api.devnet.solana.com",
+        "l" | "localhost" => "http://localhost:8899",
+        url => url,
+    }
+    .to_string()
+}
+
+pub fn is_epoch<T>(epoch: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    is_parsable_generic::<Epoch, _>(epoch)
+}
+
 pub fn is_slot<T>(slot: T) -> Result<(), String>
 where
     T: AsRef<str> + Display,
@@ -257,6 +339,21 @@ where
         .map(|_| ())
 }
 
+pub fn is_derived_address_seed<T>(value: T) -> Result<(), String>
+where
+    T: AsRef<str> + Display,
+{
+    let value = value.as_ref();
+    if value.len() > MAX_SEED_LEN {
+        Err(format!(
+            "Address seed must not be longer than {} bytes",
+            MAX_SEED_LEN
+        ))
+    } else {
+        Ok(())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

clap-utils/src/keypair.rs

@@ -1,33 +1,44 @@
-use crate::{
-    input_parsers::pubkeys_sigs_of,
-    offline::{SIGNER_ARG, SIGN_ONLY_ARG},
-    ArgConstant,
-};
-use bip39::{Language, Mnemonic, Seed};
-use clap::ArgMatches;
-use rpassword::prompt_password_stderr;
-use solana_remote_wallet::{
-    remote_keypair::generate_remote_keypair,
-    remote_wallet::{maybe_wallet_manager, RemoteWalletError, RemoteWalletManager},
-};
-use solana_sdk::{
-    hash::Hash,
-    pubkey::Pubkey,
-    signature::{
-        keypair_from_seed, keypair_from_seed_phrase_and_passphrase, read_keypair,
-        read_keypair_file, Keypair, NullSigner, Presigner, Signature, Signer,
+use {
+    crate::{
+        input_parsers::pubkeys_sigs_of,
+        offline::{SIGNER_ARG, SIGN_ONLY_ARG},
+        ArgConstant,
     },
-};
-use std::{
-    error,
-    io::{stdin, stdout, Write},
-    process::exit,
-    str::FromStr,
-    sync::Arc,
+    bip39::{Language, Mnemonic, Seed},
+    clap::ArgMatches,
+    rpassword::prompt_password_stderr,
+    solana_remote_wallet::{
+        locator::{Locator as RemoteWalletLocator, LocatorError as RemoteWalletLocatorError},
+        remote_keypair::generate_remote_keypair,
+        remote_wallet::{maybe_wallet_manager, RemoteWalletError, RemoteWalletManager},
+    },
+    solana_sdk::{
+        derivation_path::{DerivationPath, DerivationPathError},
+        hash::Hash,
+        message::Message,
+        pubkey::Pubkey,
+        signature::{
+            generate_seed_from_seed_phrase_and_passphrase, keypair_from_seed,
+            keypair_from_seed_and_derivation_path, keypair_from_seed_phrase_and_passphrase,
+            read_keypair, read_keypair_file, Keypair, NullSigner, Presigner, Signature, Signer,
+        },
+    },
+    std::{
+        cell::RefCell,
+        convert::TryFrom,
+        error,
+        io::{stdin, stdout, Write},
+        ops::Deref,
+        process::exit,
+        str::FromStr,
+        sync::Arc,
+    },
+    thiserror::Error,
 };
 
 pub struct SignOnly {
     pub blockhash: Hash,
+    pub message: Option<String>,
     pub present_signers: Vec<(Pubkey, Signature)>,
     pub absent_signers: Vec<Pubkey>,
     pub bad_signers: Vec<Pubkey>,
@@ -58,14 +69,71 @@ impl CliSignerInfo {
             Some(0)
         }
     }
+    pub fn index_of_or_none(&self, pubkey: Option<Pubkey>) -> Option<usize> {
+        if let Some(pubkey) = pubkey {
+            self.signers
+                .iter()
+                .position(|signer| signer.pubkey() == pubkey)
+        } else {
+            None
+        }
+    }
+    pub fn signers_for_message(&self, message: &Message) -> Vec<&dyn Signer> {
+        self.signers
+            .iter()
+            .filter_map(|k| {
+                if message.signer_keys().contains(&&k.pubkey()) {
+                    Some(k.as_ref())
+                } else {
+                    None
+                }
+            })
+            .collect()
+    }
 }
 
+#[derive(Debug, Default)]
 pub struct DefaultSigner {
     pub arg_name: String,
     pub path: String,
+    is_path_checked: RefCell<bool>,
 }
 
 impl DefaultSigner {
+    pub fn new<AN: AsRef<str>, P: AsRef<str>>(arg_name: AN, path: P) -> Self {
+        let arg_name = arg_name.as_ref().to_string();
+        let path = path.as_ref().to_string();
+        Self {
+            arg_name,
+            path,
+            ..Self::default()
+        }
+    }
+
+    fn path(&self) -> Result<&str, Box<dyn std::error::Error>> {
+        if !self.is_path_checked.borrow().deref() {
+            parse_signer_source(&self.path)
+                .and_then(|s| {
+                    if let SignerSourceKind::Filepath(path) = &s.kind {
+                        std::fs::metadata(path).map(|_| ()).map_err(|e| e.into())
+                    } else {
+                        Ok(())
+                    }
+                })
+                .map_err(|_| {
+                    std::io::Error::new(
+                        std::io::ErrorKind::Other,
+                        format!(
+                        "No default signer found, run \"solana-keygen new -o {}\" to create a new one",
+                        self.path
+                    ),
+                    )
+                })?;
+            *self.is_path_checked.borrow_mut() = true;
+        }
+        Ok(&self.path)
+    }
+
     pub fn generate_unique_signers(
         &self,
         bulk_signers: Vec<Option<Box<dyn Signer>>>,
@@ -80,11 +148,9 @@ impl DefaultSigner {
             unique_signers.push(default_signer);
         }
 
-        for signer in bulk_signers.into_iter() {
-            if let Some(signer) = signer {
-                if !unique_signers.iter().any(|s| s == &signer) {
-                    unique_signers.push(signer);
-                }
+        for signer in bulk_signers.into_iter().flatten() {
+            if !unique_signers.iter().any(|s| s == &signer) {
+                unique_signers.push(signer);
             }
         }
         Ok(CliSignerInfo {
@@ -97,29 +163,151 @@ impl DefaultSigner {
         matches: &ArgMatches,
         wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
     ) -> Result<Box<dyn Signer>, Box<dyn std::error::Error>> {
-        signer_from_path(matches, &self.path, &self.arg_name, wallet_manager)
+        signer_from_path(matches, self.path()?, &self.arg_name, wallet_manager)
+    }
+
+    pub fn signer_from_path_with_config(
+        &self,
+        matches: &ArgMatches,
+        wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+        config: &SignerFromPathConfig,
+    ) -> Result<Box<dyn Signer>, Box<dyn std::error::Error>> {
+        signer_from_path_with_config(
+            matches,
+            self.path()?,
+            &self.arg_name,
+            wallet_manager,
+            config,
+        )
     }
 }
 
-pub enum KeypairUrl {
-    Ask,
+pub(crate) struct SignerSource {
+    pub kind: SignerSourceKind,
+    pub derivation_path: Option<DerivationPath>,
+    pub legacy: bool,
+}
+
+impl SignerSource {
+    fn new(kind: SignerSourceKind) -> Self {
+        Self {
+            kind,
+            derivation_path: None,
+            legacy: false,
+        }
+    }
+
+    fn new_legacy(kind: SignerSourceKind) -> Self {
+        Self {
+            kind,
+            derivation_path: None,
+            legacy: true,
+        }
+    }
+}
+
+const SIGNER_SOURCE_PROMPT: &str = "prompt";
+const SIGNER_SOURCE_FILEPATH: &str = "file";
+const SIGNER_SOURCE_USB: &str = "usb";
+const SIGNER_SOURCE_STDIN: &str = "stdin";
+const SIGNER_SOURCE_PUBKEY: &str = "pubkey";
+
+pub(crate) enum SignerSourceKind {
+    Prompt,
     Filepath(String),
-    Usb(String),
+    Usb(RemoteWalletLocator),
     Stdin,
     Pubkey(Pubkey),
 }
 
-pub fn parse_keypair_path(path: &str) -> KeypairUrl {
-    if path == "-" {
-        KeypairUrl::Stdin
-    } else if path == ASK_KEYWORD {
-        KeypairUrl::Ask
-    } else if path.starts_with("usb://") {
-        KeypairUrl::Usb(path.to_string())
-    } else if let Ok(pubkey) = Pubkey::from_str(path) {
-        KeypairUrl::Pubkey(pubkey)
-    } else {
-        KeypairUrl::Filepath(path.to_string())
+impl AsRef<str> for SignerSourceKind {
+    fn as_ref(&self) -> &str {
+        match self {
+            Self::Prompt => SIGNER_SOURCE_PROMPT,
+            Self::Filepath(_) => SIGNER_SOURCE_FILEPATH,
+            Self::Usb(_) => SIGNER_SOURCE_USB,
+            Self::Stdin => SIGNER_SOURCE_STDIN,
+            Self::Pubkey(_) => SIGNER_SOURCE_PUBKEY,
+        }
+    }
+}
+
+impl std::fmt::Debug for SignerSourceKind {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        let s: &str = self.as_ref();
+        write!(f, "{}", s)
+    }
+}
+
+#[derive(Debug, Error)]
+pub(crate) enum SignerSourceError {
+    #[error("unrecognized signer source")]
+    UnrecognizedSource,
+    #[error(transparent)]
+    RemoteWalletLocatorError(#[from] RemoteWalletLocatorError),
+    #[error(transparent)]
+    DerivationPathError(#[from] DerivationPathError),
+    #[error(transparent)]
+    IoError(#[from] std::io::Error),
+}
+
+pub(crate) fn parse_signer_source<S: AsRef<str>>(
+    source: S,
+) -> Result<SignerSource, SignerSourceError> {
+    let source = source.as_ref();
+    let source = {
+        #[cfg(target_family = "windows")]
+        {
+            source.replace("\\", "/")
+        }
+        #[cfg(not(target_family = "windows"))]
+        {
+            source.to_string()
+        }
+    };
+    match uriparse::URIReference::try_from(source.as_str()) {
+        Err(_) => Err(SignerSourceError::UnrecognizedSource),
+        Ok(uri) => {
+            if let Some(scheme) = uri.scheme() {
+                let scheme = scheme.as_str().to_ascii_lowercase();
+                match scheme.as_str() {
+                    SIGNER_SOURCE_PROMPT => Ok(SignerSource {
+                        kind: SignerSourceKind::Prompt,
+                        derivation_path: DerivationPath::from_uri_any_query(&uri)?,
+                        legacy: false,
+                    }),
+                    SIGNER_SOURCE_FILEPATH => Ok(SignerSource::new(SignerSourceKind::Filepath(
+                        uri.path().to_string(),
+                    ))),
+                    SIGNER_SOURCE_USB => Ok(SignerSource {
+                        kind: SignerSourceKind::Usb(RemoteWalletLocator::new_from_uri(&uri)?),
+                        derivation_path: DerivationPath::from_uri_key_query(&uri)?,
+                        legacy: false,
+                    }),
+                    SIGNER_SOURCE_STDIN => Ok(SignerSource::new(SignerSourceKind::Stdin)),
+                    _ => {
+                        #[cfg(target_family = "windows")]
+                        // On Windows, an absolute path's drive letter will be parsed as the URI
+                        // scheme. Assume a filepath source in case of a single character shceme.
+                        if scheme.len() == 1 {
+                            return Ok(SignerSource::new(SignerSourceKind::Filepath(source)));
+                        }
+                        Err(SignerSourceError::UnrecognizedSource)
+                    }
+                }
+            } else {
+                match source.as_str() {
+                    "-" => Ok(SignerSource::new(SignerSourceKind::Stdin)),
+                    ASK_KEYWORD => Ok(SignerSource::new_legacy(SignerSourceKind::Prompt)),
+                    _ => match Pubkey::from_str(source.as_str()) {
+                        Ok(pubkey) => Ok(SignerSource::new(SignerSourceKind::Pubkey(pubkey))),
+                        Err(_) => std::fs::metadata(source.as_str())
+                            .map(|_| SignerSource::new(SignerSourceKind::Filepath(source)))
+                            .map_err(|err| err.into()),
+                    },
+                }
+            }
+        }
     }
 }
 
@@ -136,22 +324,53 @@ pub fn presigner_from_pubkey_sigs(
     })
 }
 
+#[derive(Debug)]
+pub struct SignerFromPathConfig {
+    pub allow_null_signer: bool,
+}
+
+impl Default for SignerFromPathConfig {
+    fn default() -> Self {
+        Self {
+            allow_null_signer: false,
+        }
+    }
+}
+
 pub fn signer_from_path(
     matches: &ArgMatches,
     path: &str,
     keypair_name: &str,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<Box<dyn Signer>, Box<dyn error::Error>> {
-    match parse_keypair_path(path) {
-        KeypairUrl::Ask => {
+    let config = SignerFromPathConfig::default();
+    signer_from_path_with_config(matches, path, keypair_name, wallet_manager, &config)
+}
+
+pub fn signer_from_path_with_config(
+    matches: &ArgMatches,
+    path: &str,
+    keypair_name: &str,
+    wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
+    config: &SignerFromPathConfig,
+) -> Result<Box<dyn Signer>, Box<dyn error::Error>> {
+    let SignerSource {
+        kind,
+        derivation_path,
+        legacy,
+    } = parse_signer_source(path)?;
+    match kind {
+        SignerSourceKind::Prompt => {
             let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
             Ok(Box::new(keypair_from_seed_phrase(
                 keypair_name,
                 skip_validation,
                 false,
+                derivation_path,
+                legacy,
             )?))
         }
-        KeypairUrl::Filepath(path) => match read_keypair_file(&path) {
+        SignerSourceKind::Filepath(path) => match read_keypair_file(&path) {
             Err(e) => Err(std::io::Error::new(
                 std::io::ErrorKind::Other,
                 format!("could not read keypair file \"{}\". Run \"solana-keygen new\" to create a keypair file: {}", path, e),
@@ -159,17 +378,18 @@ pub fn signer_from_path(
             .into()),
             Ok(file) => Ok(Box::new(file)),
         },
-        KeypairUrl::Stdin => {
+        SignerSourceKind::Stdin => {
             let mut stdin = std::io::stdin();
             Ok(Box::new(read_keypair(&mut stdin)?))
         }
-        KeypairUrl::Usb(path) => {
+        SignerSourceKind::Usb(locator) => {
             if wallet_manager.is_none() {
                 *wallet_manager = maybe_wallet_manager()?;
             }
             if let Some(wallet_manager) = wallet_manager {
                 Ok(Box::new(generate_remote_keypair(
-                    path,
+                    locator,
+                    derivation_path.unwrap_or_default(),
                     wallet_manager,
                     matches.is_present("confirm_key"),
                     keypair_name,
@@ -178,13 +398,13 @@ pub fn signer_from_path(
                 Err(RemoteWalletError::NoDeviceFound.into())
             }
         }
-        KeypairUrl::Pubkey(pubkey) => {
+        SignerSourceKind::Pubkey(pubkey) => {
             let presigner = pubkeys_sigs_of(matches, SIGNER_ARG.name)
                 .as_ref()
                 .and_then(|presigners| presigner_from_pubkey_sigs(&pubkey, presigners));
             if let Some(presigner) = presigner {
                 Ok(Box::new(presigner))
-            } else if matches.is_present(SIGN_ONLY_ARG.name) {
+            } else if config.allow_null_signer || matches.is_present(SIGN_ONLY_ARG.name) {
                 Ok(Box::new(NullSigner::new(&pubkey)))
             } else {
                 Err(std::io::Error::new(
@@ -203,8 +423,9 @@ pub fn pubkey_from_path(
     keypair_name: &str,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<Pubkey, Box<dyn error::Error>> {
-    match parse_keypair_path(path) {
-        KeypairUrl::Pubkey(pubkey) => Ok(pubkey),
+    let SignerSource { kind, .. } = parse_signer_source(path)?;
+    match kind {
+        SignerSourceKind::Pubkey(pubkey) => Ok(pubkey),
         _ => Ok(signer_from_path(matches, path, keypair_name, wallet_manager)?.pubkey()),
     }
 }
@@ -215,34 +436,51 @@ pub fn resolve_signer_from_path(
     keypair_name: &str,
     wallet_manager: &mut Option<Arc<RemoteWalletManager>>,
 ) -> Result<Option<String>, Box<dyn error::Error>> {
-    match parse_keypair_path(path) {
-        KeypairUrl::Ask => {
+    let SignerSource {
+        kind,
+        derivation_path,
+        legacy,
+    } = parse_signer_source(path)?;
+    match kind {
+        SignerSourceKind::Prompt => {
             let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
             // This method validates the seed phrase, but returns `None` because there is no path
             // on disk or to a device
-            keypair_from_seed_phrase(keypair_name, skip_validation, false).map(|_| None)
+            keypair_from_seed_phrase(
+                keypair_name,
+                skip_validation,
+                false,
+                derivation_path,
+                legacy,
+            )
+            .map(|_| None)
         }
-        KeypairUrl::Filepath(path) => match read_keypair_file(&path) {
+        SignerSourceKind::Filepath(path) => match read_keypair_file(&path) {
             Err(e) => Err(std::io::Error::new(
                 std::io::ErrorKind::Other,
-                format!("could not read keypair file \"{}\". Run \"solana-keygen new\" to create a keypair file: {}", path, e),
+                format!(
+                    "could not read keypair file \"{}\". \
+                    Run \"solana-keygen new\" to create a keypair file: {}",
+                    path, e
+                ),
             )
             .into()),
             Ok(_) => Ok(Some(path.to_string())),
         },
-        KeypairUrl::Stdin => {
+        SignerSourceKind::Stdin => {
             let mut stdin = std::io::stdin();
             // This method validates the keypair from stdin, but returns `None` because there is no
             // path on disk or to a device
             read_keypair(&mut stdin).map(|_| None)
         }
-        KeypairUrl::Usb(path) => {
+        SignerSourceKind::Usb(locator) => {
             if wallet_manager.is_none() {
                 *wallet_manager = maybe_wallet_manager()?;
             }
             if let Some(wallet_manager) = wallet_manager {
                 let path = generate_remote_keypair(
-                    path,
+                    locator,
+                    derivation_path.unwrap_or_default(),
                     wallet_manager,
                     matches.is_present("confirm_key"),
                     keypair_name,
@@ -257,7 +495,7 @@ pub fn resolve_signer_from_path(
     }
 }
 
-// Keyword used to indicate that the user should be asked for a keypair seed phrase
+// Keyword used to indicate that the user should be prompted for a keypair seed phrase
 pub const ASK_KEYWORD: &str = "ASK";
 
 pub const SKIP_SEED_PHRASE_VALIDATION_ARG: ArgConstant<'static> = ArgConstant {
@@ -278,6 +516,56 @@ pub fn prompt_passphrase(prompt: &str) -> Result<String, Box<dyn error::Error>>
     Ok(passphrase)
 }
 
+/// Parses a path into a SignerSource and returns a Keypair for supporting SignerSourceKinds
+pub fn keypair_from_path(
+    matches: &ArgMatches,
+    path: &str,
+    keypair_name: &str,
+    confirm_pubkey: bool,
+) -> Result<Keypair, Box<dyn error::Error>> {
+    let SignerSource {
+        kind,
+        derivation_path,
+        legacy,
+    } = parse_signer_source(path)?;
+    match kind {
+        SignerSourceKind::Prompt => {
+            let skip_validation = matches.is_present(SKIP_SEED_PHRASE_VALIDATION_ARG.name);
+            Ok(keypair_from_seed_phrase(
+                keypair_name,
+                skip_validation,
+                confirm_pubkey,
+                derivation_path,
+                legacy,
+            )?)
+        }
+        SignerSourceKind::Filepath(path) => match read_keypair_file(&path) {
+            Err(e) => Err(std::io::Error::new(
+                std::io::ErrorKind::Other,
+                format!(
+                    "could not read keypair file \"{}\". \
+                    Run \"solana-keygen new\" to create a keypair file: {}",
+                    path, e
+                ),
+            )
+            .into()),
+            Ok(file) => Ok(file),
+        },
+        SignerSourceKind::Stdin => {
+            let mut stdin = std::io::stdin();
+            Ok(read_keypair(&mut stdin)?)
+        }
+        _ => Err(std::io::Error::new(
+            std::io::ErrorKind::Other,
+            format!(
+                "signer of type `{:?}` does not support Keypair output",
+                kind
+            ),
+        )
+        .into()),
+    }
+}
+
 /// Reads user input from stdin to retrieve a seed phrase and passphrase for keypair derivation
 /// Optionally skips validation of seed phrase
 /// Optionally confirms recovered public key
@@ -285,6 +573,8 @@ pub fn keypair_from_seed_phrase(
     keypair_name: &str,
     skip_validation: bool,
     confirm_pubkey: bool,
+    derivation_path: Option<DerivationPath>,
+    legacy: bool,
 ) -> Result<Keypair, Box<dyn error::Error>> {
     let seed_phrase = prompt_password_stderr(&format!("[{}] seed phrase: ", keypair_name))?;
     let seed_phrase = seed_phrase.trim();
@@ -295,13 +585,39 @@ pub fn keypair_from_seed_phrase(
 
     let keypair = if skip_validation {
         let passphrase = prompt_passphrase(&passphrase_prompt)?;
-        keypair_from_seed_phrase_and_passphrase(&seed_phrase, &passphrase)?
+        if legacy {
+            keypair_from_seed_phrase_and_passphrase(&seed_phrase, &passphrase)?
+        } else {
+            let seed = generate_seed_from_seed_phrase_and_passphrase(&seed_phrase, &passphrase);
+            keypair_from_seed_and_derivation_path(&seed, derivation_path)?
+        }
     } else {
         let sanitized = sanitize_seed_phrase(seed_phrase);
-        let mnemonic = Mnemonic::from_phrase(&sanitized, Language::English)?;
+        let parse_language_fn = || {
+            for language in &[
+                Language::English,
+                Language::ChineseSimplified,
+                Language::ChineseTraditional,
+                Language::Japanese,
+                Language::Spanish,
+                Language::Korean,
+                Language::French,
+                Language::Italian,
+            ] {
+                if let Ok(mnemonic) = Mnemonic::from_phrase(&sanitized, *language) {
+                    return Ok(mnemonic);
+                }
+            }
+            Err("Can't get mnemonic from seed phrases")
+        };
+        let mnemonic = parse_language_fn()?;
         let passphrase = prompt_passphrase(&passphrase_prompt)?;
         let seed = Seed::new(&mnemonic, &passphrase);
-        keypair_from_seed(seed.as_bytes())?
+        if legacy {
+            keypair_from_seed(seed.as_bytes())?
+        } else {
+            keypair_from_seed_and_derivation_path(&seed.as_bytes(), derivation_path)?
+        }
     };
 
     if confirm_pubkey {
@@ -329,6 +645,9 @@ fn sanitize_seed_phrase(seed_phrase: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use solana_remote_wallet::locator::Manufacturer;
+    use solana_sdk::system_instruction;
+    use tempfile::NamedTempFile;
 
     #[test]
     fn test_sanitize_seed_phrase() {
@@ -338,4 +657,153 @@ mod tests {
             sanitize_seed_phrase(seed_phrase)
         );
     }
+
+    #[test]
+    fn test_signer_info_signers_for_message() {
+        let source = Keypair::new();
+        let fee_payer = Keypair::new();
+        let nonsigner1 = Keypair::new();
+        let nonsigner2 = Keypair::new();
+        let recipient = Pubkey::new_unique();
+        let message = Message::new(
+            &[system_instruction::transfer(
+                &source.pubkey(),
+                &recipient,
+                42,
+            )],
+            Some(&fee_payer.pubkey()),
+        );
+        let signers = vec![
+            Box::new(fee_payer) as Box<dyn Signer>,
+            Box::new(source) as Box<dyn Signer>,
+            Box::new(nonsigner1) as Box<dyn Signer>,
+            Box::new(nonsigner2) as Box<dyn Signer>,
+        ];
+        let signer_info = CliSignerInfo { signers };
+        let msg_signers = signer_info.signers_for_message(&message);
+        let signer_pubkeys = msg_signers.iter().map(|s| s.pubkey()).collect::<Vec<_>>();
+        let expect = vec![
+            signer_info.signers[0].pubkey(),
+            signer_info.signers[1].pubkey(),
+        ];
+        assert_eq!(signer_pubkeys, expect);
+    }
+
+    #[test]
+    fn test_parse_signer_source() {
+        assert!(matches!(
+            parse_signer_source("-").unwrap(),
+            SignerSource {
+                kind: SignerSourceKind::Stdin,
+                derivation_path: None,
+                legacy: false,
+            }
+        ));
+        let stdin = "stdin:".to_string();
+        assert!(matches!(
+            parse_signer_source(&stdin).unwrap(),
+            SignerSource {
+                kind: SignerSourceKind::Stdin,
+                derivation_path: None,
+                legacy: false,
+            }
+        ));
+        assert!(matches!(
+            parse_signer_source(ASK_KEYWORD).unwrap(),
+            SignerSource {
+                kind: SignerSourceKind::Prompt,
+                derivation_path: None,
+                legacy: true,
+            }
+        ));
+        let pubkey = Pubkey::new_unique();
+        assert!(
+            matches!(parse_signer_source(&pubkey.to_string()).unwrap(), SignerSource {
+                kind: SignerSourceKind::Pubkey(p),
+                derivation_path: None,
+                legacy: false,
+            }
+            if p == pubkey)
+        );
+
+        // Set up absolute and relative path strs
+        let file0 = NamedTempFile::new().unwrap();
+        let path = file0.path();
+        assert!(path.is_absolute());
+        let absolute_path_str = path.to_str().unwrap();
+
+        let file1 = NamedTempFile::new_in(std::env::current_dir().unwrap()).unwrap();
+        let path = file1.path().file_name().unwrap().to_str().unwrap();
+        let path = std::path::Path::new(path);
+        assert!(path.is_relative());
+        let relative_path_str = path.to_str().unwrap();
+
+        assert!(
+            matches!(parse_signer_source(absolute_path_str).unwrap(), SignerSource {
+                kind: SignerSourceKind::Filepath(p),
+                derivation_path: None,
+                legacy: false,
+            } if p == absolute_path_str)
+        );
+        assert!(
+            matches!(parse_signer_source(&relative_path_str).unwrap(), SignerSource {
+                kind: SignerSourceKind::Filepath(p),
+                derivation_path: None,
+                legacy: false,
+            } if p == relative_path_str)
+        );
+
+        let usb = "usb://ledger".to_string();
+        let expected_locator = RemoteWalletLocator {
+            manufacturer: Manufacturer::Ledger,
+            pubkey: None,
+        };
+        assert!(matches!(parse_signer_source(&usb).unwrap(), SignerSource {
+                kind: SignerSourceKind::Usb(u),
+                derivation_path: None,
+                legacy: false,
+            } if u == expected_locator));
+        let usb = "usb://ledger?key=0/0".to_string();
+        let expected_locator = RemoteWalletLocator {
+            manufacturer: Manufacturer::Ledger,
+            pubkey: None,
+        };
+        let expected_derivation_path = Some(DerivationPath::new_bip44(Some(0), Some(0)));
+        assert!(matches!(parse_signer_source(&usb).unwrap(), SignerSource {
+                kind: SignerSourceKind::Usb(u),
+                derivation_path: d,
+                legacy: false,
+            } if u == expected_locator && d == expected_derivation_path));
+        // Catchall into SignerSource::Filepath fails
+        let junk = "sometextthatisnotapubkeyorfile".to_string();
+        assert!(Pubkey::from_str(&junk).is_err());
+        assert!(matches!(
+            parse_signer_source(&junk),
+            Err(SignerSourceError::IoError(_))
+        ));
+
+        let prompt = "prompt:".to_string();
+        assert!(matches!(
+            parse_signer_source(&prompt).unwrap(),
+            SignerSource {
+                kind: SignerSourceKind::Prompt,
+                derivation_path: None,
+                legacy: false,
+            }
+        ));
+        assert!(
+            matches!(parse_signer_source(&format!("file:{}", absolute_path_str)).unwrap(), SignerSource {
+                kind: SignerSourceKind::Filepath(p),
+                derivation_path: None,
+                legacy: false,
+            } if p == absolute_path_str)
+        );
+        assert!(
+            matches!(parse_signer_source(&format!("file:{}", relative_path_str)).unwrap(), SignerSource {
+                kind: SignerSourceKind::Filepath(p),
+                derivation_path: None,
+                legacy: false,
+            } if p == relative_path_str)
+        );
+    }
 }

clap-utils/src/lib.rs

@@ -23,10 +23,10 @@ impl std::fmt::Debug for DisplayError {
     }
 }
 
-pub mod commitment;
 pub mod fee_payer;
 pub mod input_parsers;
 pub mod input_validators;
 pub mod keypair;
+pub mod memo;
 pub mod nonce;
 pub mod offline;

clap-utils/src/memo.rs

@@ -0,0 +1,15 @@
+use {crate::ArgConstant, clap::Arg};
+
+pub const MEMO_ARG: ArgConstant<'static> = ArgConstant {
+    name: "memo",
+    long: "--with-memo",
+    help: "Specify a memo string to include in the transaction.",
+};
+
+pub fn memo_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(MEMO_ARG.name)
+        .long(MEMO_ARG.long)
+        .takes_value(true)
+        .value_name("MEMO")
+        .help(MEMO_ARG.help)
+}

clap-utils/src/nonce.rs

@@ -1,5 +1,7 @@
-use crate::{input_validators::*, offline::BLOCKHASH_ARG, ArgConstant};
-use clap::{App, Arg};
+use {
+    crate::{input_validators::*, offline::BLOCKHASH_ARG, ArgConstant},
+    clap::{App, Arg},
+};
 
 pub const NONCE_ARG: ArgConstant<'static> = ArgConstant {
     name: "nonce",
@@ -36,12 +38,15 @@ pub fn nonce_authority_arg<'a, 'b>() -> Arg<'a, 'b> {
 }
 
 pub trait NonceArgs {
-    fn nonce_args(self) -> Self;
+    fn nonce_args(self, global: bool) -> Self;
 }
 
 impl NonceArgs for App<'_, '_> {
-    fn nonce_args(self) -> Self {
-        self.arg(nonce_arg())
-            .arg(nonce_authority_arg().requires(NONCE_ARG.name))
+    fn nonce_args(self, global: bool) -> Self {
+        self.arg(nonce_arg().global(global)).arg(
+            nonce_authority_arg()
+                .requires(NONCE_ARG.name)
+                .global(global),
+        )
     }
 }

clap-utils/src/offline.rs

@@ -1,5 +1,7 @@
-use crate::{input_validators::*, ArgConstant};
-use clap::{App, Arg};
+use {
+    crate::{input_validators::*, ArgConstant},
+    clap::{App, Arg},
+};
 
 pub const BLOCKHASH_ARG: ArgConstant<'static> = ArgConstant {
     name: "blockhash",
@@ -19,6 +21,12 @@ pub const SIGNER_ARG: ArgConstant<'static> = ArgConstant {
     help: "Provide a public-key/signature pair for the transaction",
 };
 
+pub const DUMP_TRANSACTION_MESSAGE: ArgConstant<'static> = ArgConstant {
+    name: "dump_transaction_message",
+    long: "dump-transaction-message",
+    help: "Display the base64 encoded binary transaction message in sign-only mode",
+};
+
 pub fn blockhash_arg<'a, 'b>() -> Arg<'a, 'b> {
     Arg::with_name(BLOCKHASH_ARG.name)
         .long(BLOCKHASH_ARG.long)
@@ -47,14 +55,44 @@ fn signer_arg<'a, 'b>() -> Arg<'a, 'b> {
         .help(SIGNER_ARG.help)
 }
 
+pub fn dump_transaction_message<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(DUMP_TRANSACTION_MESSAGE.name)
+        .long(DUMP_TRANSACTION_MESSAGE.long)
+        .takes_value(false)
+        .requires(SIGN_ONLY_ARG.name)
+        .help(DUMP_TRANSACTION_MESSAGE.help)
+}
+
+pub trait ArgsConfig {
+    fn blockhash_arg<'a, 'b>(&self, arg: Arg<'a, 'b>) -> Arg<'a, 'b> {
+        arg
+    }
+    fn sign_only_arg<'a, 'b>(&self, arg: Arg<'a, 'b>) -> Arg<'a, 'b> {
+        arg
+    }
+    fn signer_arg<'a, 'b>(&self, arg: Arg<'a, 'b>) -> Arg<'a, 'b> {
+        arg
+    }
+    fn dump_transaction_message_arg<'a, 'b>(&self, arg: Arg<'a, 'b>) -> Arg<'a, 'b> {
+        arg
+    }
+}
+
 pub trait OfflineArgs {
     fn offline_args(self) -> Self;
+    fn offline_args_config(self, config: &dyn ArgsConfig) -> Self;
 }
 
 impl OfflineArgs for App<'_, '_> {
+    fn offline_args_config(self, config: &dyn ArgsConfig) -> Self {
+        self.arg(config.blockhash_arg(blockhash_arg()))
+            .arg(config.sign_only_arg(sign_only_arg()))
+            .arg(config.signer_arg(signer_arg()))
+            .arg(config.dump_transaction_message_arg(dump_transaction_message()))
+    }
     fn offline_args(self) -> Self {
-        self.arg(blockhash_arg())
-            .arg(sign_only_arg())
-            .arg(signer_arg())
+        struct NullArgsConfig {}
+        impl ArgsConfig for NullArgsConfig {}
+        self.offline_args_config(&NullArgsConfig {})
     }
 }

cli-config/Cargo.toml

@@ -3,15 +3,16 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.3.13"
+version = "1.6.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-cli-config"
 
 [dependencies]
-dirs = "2.0.2"
+dirs-next = "2.0.0"
 lazy_static = "1.4.0"
-serde = "1.0.112"
+serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_yaml = "0.8.13"
 url = "2.1.1"

cli-config/src/config.rs

@@ -5,7 +5,7 @@ use url::Url;
 
 lazy_static! {
     pub static ref CONFIG_FILE: Option<String> = {
-        dirs::home_dir().map(|mut path| {
+        dirs_next::home_dir().map(|mut path| {
             path.extend(&[".config", "solana", "cli", "config.yml"]);
             path.to_str().unwrap().to_string()
         })
@@ -17,15 +17,16 @@ pub struct Config {
     pub json_rpc_url: String,
     pub websocket_url: String,
     pub keypair_path: String,
-
     #[serde(default)]
     pub address_labels: HashMap<String, String>,
+    #[serde(default)]
+    pub commitment: String,
 }
 
 impl Default for Config {
     fn default() -> Self {
         let keypair_path = {
-            let mut keypair_path = dirs::home_dir().expect("home directory");
+            let mut keypair_path = dirs_next::home_dir().expect("home directory");
             keypair_path.extend(&[".config", "solana", "id.json"]);
             keypair_path.to_str().unwrap().to_string()
         };
@@ -41,11 +42,14 @@ impl Default for Config {
             "System Program".to_string(),
         );
 
+        let commitment = "confirmed".to_string();
+
         Self {
             json_rpc_url,
             websocket_url,
             keypair_path,
             address_labels,
+            commitment,
         }
     }
 }
@@ -71,22 +75,12 @@ impl Config {
             .set_scheme(if is_secure { "wss" } else { "ws" })
             .expect("unable to set scheme");
         if let Some(port) = json_rpc_url.port() {
-            ws_url.set_port(Some(port + 1)).expect("unable to set port");
+            let port = port.checked_add(1).expect("port out of range");
+            ws_url.set_port(Some(port)).expect("unable to set port");
         }
         ws_url.to_string()
     }
 
-    pub fn compute_rpc_banks_url(json_rpc_url: &str) -> String {
-        let json_rpc_url: Option<Url> = json_rpc_url.parse().ok();
-        if json_rpc_url.is_none() {
-            return "".to_string();
-        }
-        let mut url = json_rpc_url.unwrap();
-        let port = url.port_or_known_default().unwrap_or(80);
-        url.set_port(Some(port + 3)).expect("unable to set port");
-        url.to_string()
-    }
-
     pub fn import_address_labels<P>(&mut self, filename: P) -> Result<(), io::Error>
     where
         P: AsRef<Path>,
@@ -113,13 +107,13 @@ mod test {
     #[test]
     fn compute_websocket_url() {
         assert_eq!(
-            Config::compute_websocket_url(&"http://devnet.solana.com"),
-            "ws://devnet.solana.com/".to_string()
+            Config::compute_websocket_url(&"http://api.devnet.solana.com"),
+            "ws://api.devnet.solana.com/".to_string()
         );
 
         assert_eq!(
-            Config::compute_websocket_url(&"https://devnet.solana.com"),
-            "wss://devnet.solana.com/".to_string()
+            Config::compute_websocket_url(&"https://api.devnet.solana.com"),
+            "wss://api.devnet.solana.com/".to_string()
         );
 
         assert_eq!(
@@ -133,28 +127,4 @@ mod test {
 
         assert_eq!(Config::compute_websocket_url(&"garbage"), String::new());
     }
-
-    #[test]
-    fn compute_rpc_banks_url() {
-        assert_eq!(
-            Config::compute_rpc_banks_url(&"http://devnet.solana.com"),
-            "http://devnet.solana.com:83/".to_string()
-        );
-
-        assert_eq!(
-            Config::compute_rpc_banks_url(&"https://devnet.solana.com"),
-            "https://devnet.solana.com:446/".to_string()
-        );
-
-        assert_eq!(
-            Config::compute_rpc_banks_url(&"http://example.com:8899"),
-            "http://example.com:8902/".to_string()
-        );
-        assert_eq!(
-            Config::compute_rpc_banks_url(&"https://example.com:1234"),
-            "https://example.com:1237/".to_string()
-        );
-
-        assert_eq!(Config::compute_rpc_banks_url(&"garbage"), String::new());
-    }
 }

cli-output/Cargo.toml

@@ -3,26 +3,30 @@ authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-cli-output"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.3.13"
+version = "1.6.22"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-cli-output"
 
 [dependencies]
+base64 = "0.13.0"
 chrono = { version = "0.4.11", features = ["serde"] }
 console = "0.11.3"
 humantime = "2.0.1"
 Inflector = "0.11.4"
 indicatif = "0.15.0"
-serde = "1.0.112"
+serde = "1.0.122"
 serde_derive = "1.0.103"
 serde_json = "1.0.56"
-solana-clap-utils = { path = "../clap-utils", version = "1.3.13" }
-solana-client = { path = "../client", version = "1.3.13" }
-solana-sdk = { path = "../sdk", version = "1.3.13" }
-solana-stake-program = { path = "../programs/stake", version = "1.3.13" }
-solana-transaction-status = { path = "../transaction-status", version = "1.3.13" }
-solana-vote-program = { path = "../programs/vote", version = "1.3.13" }
+solana-account-decoder = { path = "../account-decoder", version = "=1.6.22" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.22" }
+solana-client = { path = "../client", version = "=1.6.22" }
+solana-sdk = { path = "../sdk", version = "=1.6.22" }
+solana-stake-program = { path = "../programs/stake", version = "=1.6.22" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.6.22" }
+solana-vote-program = { path = "../programs/vote", version = "=1.6.22" }
+spl-memo = { version = "=3.0.1", features = ["no-entrypoint"] }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]