Add w3m's inflation pubkeys (#15142) (#15143)

(cherry picked from commit 2a60dd8492)

Co-authored-by: Michael Vines <mvines@gmail.com>

.appveyor.yml

@@ -1,42 +0,0 @@
-version: '{build}'
-
-branches:
-  only:
-  - master
-  - /^v[0-9.]+\.[0-9.]+/
-
-cache:
-  - '%USERPROFILE%\.cargo'
-  - '%APPVEYOR_BUILD_FOLDER%\target'
-
-clone_folder: d:\projects\solana
-
-build_script:
-  - bash ci/publish-tarball.sh
-
-notifications:
-  - provider: Slack
-    incoming_webhook:
-      secure: GJsBey+F5apAtUm86MHVJ68Uqa6WN1SImcuIc4TsTZrDhA8K1QWUNw9FFQPybUWDyOcS5dly3kubnUqlGt9ux6Ad2efsfRIQYWv0tOVXKeY=
-    channel: ci-status
-    on_build_success: false
-    on_build_failure: true
-    on_build_status_changed: true
-
-deploy:
-  - provider: S3
-    access_key_id:
-      secure: fTbJl6JpFebR40J7cOWZ2mXBa3kIvEiXgzxAj6L3N7A=
-    secret_access_key:
-      secure: vItsBXb2rEFLvkWtVn/Rcxu5a5+2EwC+b7GsA0waJy9hXh6XuBAD0lnHd9re3g/4
-    bucket: release.solana.com
-    region: us-west-1
-    set_public: true
-
-  - provider: GitHub
-    auth_token:
-      secure: 81fEmPZ0cV1wLtNuUrcmtgxKF6ROQF1+/ft5m+fHX21z6PoeCbaNo8cTyLioWBj7
-    draft: false
-    prerelease: false
-    on:
-      appveyor_repo_tag: true

.buildkite/env/secrets.ejson

@@ -1,12 +1,7 @@
 {
     "_public_key": "ae29f4f7ad2fc92de70d470e411c8426d5d48db8817c9e3dae574b122192335f",
+    "_comment": "These credentials are encrypted and pose no risk",
     "environment": {
-      "CODECOV_TOKEN": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:JnxhrIxh09AvqdJgrVSYmb7PxSrh19aE:07WzVExCHEd1lJ1m8QizRRthGri+WBNeZRKjjEvsy5eo4gv3HD7zVEm42tVTGkqITKkBNQ==]",
-      "CRATES_IO_TOKEN": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:d0jJqC32/axwzq/N7kMRmpxKhnRrhtpt:zvcPHwkOzGnjhNkAQSejwdy1Jkr9wR1qXFFCnfIjyt/XQYubzB1tLkoly/qdmeb5]",
-      "GEOLOCATION_API_KEY": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:R4gfB6Ey4i50HyfLt4UZDLBqg3qHEUye:UfZCOgt8XI6Y2g+ivCRVoS1fjFycFs7/GSevvCqh1B50mG0+hzpEyzXQLuKG5OeI]",
-      "GITHUB_TOKEN": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:Vq2dkGTOzfEpRht0BAGHFp/hDogMvXJe:tFXHg1epVt2mq9hkuc5sRHe+KAnVREi/p8S+IZu67XRyzdiA/nGak1k860FXYuuzuaE0QWekaEc=]",
-      "INFLUX_DATABASE": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:5KI9WBkXx3R/W4m256mU5MJOE7N8aAT9:Cb8QFELZ9I60t5zhJ9h55Kcs]",
-      "INFLUX_PASSWORD": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:hQRMpLCrav+OYkNphkeM4hagdVoZv5Iw:AUO76rr6+gF1OLJA8ZLSG8wHKXgYCPNk6gRCV8rBhZBJ4KwDaxpvOhMl7bxxXG6jol7v4aRa/Lk=]",
-      "INFLUX_USERNAME": "EJ[1:yGpTmjdbyjW2kjgIHkFoJv7Ue7EbUvUbqHyw6anGgWg=:R7BNmQjfeqoGDAFTJu9bYTGHol2NgnYN:Q2tOT/EBcFvhFk+DKLKmVU7tLCpVC3Ui]"
+      "CODECOV_TOKEN": "EJ[1:Z7OneT3RdJJ0DipCHQ7rC84snQ+FPbgHwZADQiz54wk=:3K68mE38LJ2RB98VWmjuNLFBNn1XTGR4:cR4r05/TOZQKmEZp1v4CSgUJtC6QJiOaL85QjXW0qZ061fMnsBA8AtAPMDoDq4WCGOZM1A==]"
     }
 }

.buildkite/hooks/post-command

@@ -3,16 +3,19 @@
 #
 # Save target/ for the next CI build on this machine
 #
-(
-  set -x
-  d=$HOME/cargo-target-cache/"$BUILDKITE_LABEL"
-  mkdir -p "$d"
-  set -x
-  rsync -a --delete --link-dest="$PWD" target "$d"
-  du -hs "$d"
-  read -r cacheSizeInGB _ < <(du -s --block-size=1800000000 "$d")
-  echo "--- ${cacheSizeInGB}GB: $d"
-)
+if [[ -z $CARGO_TARGET_CACHE ]]; then
+  echo "+++ CARGO_TARGET_CACHE not defined" # pre-command should have defined it
+else
+  (
+    set -x
+    mkdir -p "$CARGO_TARGET_CACHE"
+    set -x
+    rsync -a --delete --link-dest="$PWD" target "$CARGO_TARGET_CACHE"
+    du -hs "$CARGO_TARGET_CACHE"
+    read -r cacheSizeInGB _ < <(du -s --block-size=1800000000 "$CARGO_TARGET_CACHE")
+    echo "--- ${cacheSizeInGB}GB: $CARGO_TARGET_CACHE"
+  )
+fi
 
 #
 # Add job_stats data point

.buildkite/hooks/pre-command

@@ -11,23 +11,29 @@ export PS4="++"
 #
 # Restore target/ from the previous CI build on this machine
 #
+eval "$(ci/channel-info.sh)"
+export CARGO_TARGET_CACHE=$HOME/cargo-target-cache/"$CHANNEL"-"$BUILDKITE_LABEL"
 (
   set -x
-  d=$HOME/cargo-target-cache/"$BUILDKITE_LABEL"
   MAX_CACHE_SIZE=18 # gigabytes
 
-  if [[ -d $d ]]; then
-    du -hs "$d"
-    read -r cacheSizeInGB _ < <(du -s --block-size=1800000000 "$d")
-    echo "--- ${cacheSizeInGB}GB: $d"
+  if [[ -d $CARGO_TARGET_CACHE ]]; then
+    du -hs "$CARGO_TARGET_CACHE"
+    read -r cacheSizeInGB _ < <(du -s --block-size=1800000000 "$CARGO_TARGET_CACHE")
+    echo "--- ${cacheSizeInGB}GB: $CARGO_TARGET_CACHE"
     if [[ $cacheSizeInGB -gt $MAX_CACHE_SIZE ]]; then
-      echo "--- $d is too large, removing it"
-      rm -rf "$d"
+      echo "--- $CARGO_TARGET_CACHE is too large, removing it"
+      rm -rf "$CARGO_TARGET_CACHE"
     fi
   else
-    echo "--- $d not present"
+    echo "--- $CARGO_TARGET_CACHE not present"
   fi
 
-  mkdir -p "$d"/target
-  rsync -a --delete --link-dest="$d" "$d"/target .
+  mkdir -p "$CARGO_TARGET_CACHE"/target
+  rsync -a --delete --link-dest="$CARGO_TARGET_CACHE" "$CARGO_TARGET_CACHE"/target .
+
+  # Don't reuse BPF target build artifacts due to incremental build issues with
+  # `std:
+  #    "found possibly newer version of crate `std` which `xyz` depends on
+  rm -rf target/bpfel-unknown-unknown
 )

.buildkite/pipeline-upload.sh

@@ -9,23 +9,10 @@
 
 set -e
 cd "$(dirname "$0")"/..
+source ci/_
 
-if [[ -n $BUILDKITE_TAG ]]; then
-  buildkite-agent annotate --style info --context release-tag \
-    "https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG"
-  buildkite-agent pipeline upload ci/buildkite-release.yml
-else
-  if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
-    # Add helpful link back to the corresponding Github Pull Request
-    buildkite-agent annotate --style info --context pr-backlink \
-      "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
-  fi
+_ ci/buildkite-pipeline.sh pipeline.yml
+echo +++ pipeline
+cat pipeline.yml
 
-  if [[ $BUILDKITE_MESSAGE =~ GitBook: ]]; then
-    buildkite-agent annotate --style info --context gitbook-ci-skip \
-      "GitBook commit detected, CI skipped"
-    exit
-  fi
-
-  buildkite-agent pipeline upload ci/buildkite.yml
-fi
+_ buildkite-agent pipeline upload pipeline.yml

.cache/fontconfig/CACHEDIR.TAG

@@ -0,0 +1,4 @@
+Signature: 8a477f597d28d172789f06886806bc55
+# This file is a cache directory tag created by fontconfig.
+# For information about cache directory tags, see:
+#       http://www.brynosaurus.com/cachedir/

.gitbook.yaml

@@ -1,5 +0,0 @@
-root: ./docs/src
-
-structure:
-    readme: introduction.md
-    summary: SUMMARY.md

.github/dependabot.yml

@@ -0,0 +1,41 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+- package-ecosystem: cargo
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "01:00"
+    timezone: America/Los_Angeles
+  #labels:
+  #  - "automerge"
+  open-pull-requests-limit: 3
+  
+- package-ecosystem: npm
+  directory: "/web3.js"
+  schedule:
+    interval: daily
+    time: "01:00"
+    timezone: America/Los_Angeles
+  labels:
+    - "automerge"
+  commit-message:
+    prefix: "chore:"
+  open-pull-requests-limit: 3
+  
+- package-ecosystem: npm
+  directory: "/explorer"
+  schedule:
+    interval: daily
+    time: "01:00"
+    timezone: America/Los_Angeles
+  labels:
+    - "automerge"
+  commit-message:
+    prefix: "chore:"
+    include: "scope"
+  open-pull-requests-limit: 3

.github/stale.yml

@@ -1,24 +1,39 @@
-only: pulls
-
-# Number of days of inactivity before a pull request becomes stale
-daysUntilStale: 7
-
-# Number of days of inactivity before a stale pull request is closed
-daysUntilClose: 7
-
 # Issues with these labels will never be considered stale
 exemptLabels:
   - security
+  - blocked
 
 # Label to use when marking a pull request as stale
 staleLabel: stale
 
-# Comment to post when marking a pull request as stale. Set to `false` to disable
-markComment: >
-  This pull request has been automatically marked as stale because it has not had
-  recent activity. It will be closed if no further activity occurs.
+pulls:
+  # Number of days of inactivity before a pull request becomes stale
+  daysUntilStale: 7
 
-# Comment to post when closing a stale pull request. Set to `false` to disable
-closeComment: >
-  This stale pull request has been automatically closed.
-  Thank you for your contributions.
+  # Number of days of inactivity before a stale pull request is closed
+  daysUntilClose: 7
+  # Comment to post when marking a pull request as stale. Set to `false` to disable
+  markComment: >
+    This pull request has been automatically marked as stale because it has not had
+    recent activity. It will be closed if no further activity occurs.
+
+  # Comment to post when closing a stale pull request. Set to `false` to disable
+  closeComment: >
+    This stale pull request has been automatically closed.
+    Thank you for your contributions.
+
+issues:
+  # Number of days of inactivity before a issue becomes stale
+  daysUntilStale: 365
+
+  # Number of days of inactivity before a stale issue is closed
+  daysUntilClose: 7
+  # Comment to post when marking a issue as stale. Set to `false` to disable
+  markComment: >
+    This issue has been automatically marked as stale because it has not had
+    recent activity. It will be closed if no further activity occurs.
+
+  # Comment to post when closing a stale issue. Set to `false` to disable
+  closeComment: >
+    This stale issue has been automatically closed.
+    Thank you for your contributions.

.gitignore

@@ -1,5 +1,6 @@
 /docs/html/
 /docs/src/tests.ok
+/docs/src/cli/usage.md
 /docs/src/.gitbook/assets/*.svg
 /farf/
 /solana-release/
@@ -22,3 +23,9 @@ log-*/
 /.idea/
 /solana.iml
 /.vscode/
+
+# fetch-spl.sh artifacts
+/spl-genesis-args.sh
+/spl_*.so
+
+.DS_Store

.mergify.yml

@@ -1,9 +1,40 @@
 # Validate your changes with:
 #
-#   $ curl -F 'data=@.mergify.yml' https://gh.mergify.io/validate
+#   $ curl -F 'data=@.mergify.yml' https://gh.mergify.io/validate/
 #
 # https://doc.mergify.io/
 pull_request_rules:
+  - name: automatic merge (squash) on CI success
+    conditions:
+      - status-success=buildkite/solana
+      - status-success=Travis CI - Pull Request
+      - status-success=ci-gate
+      - label=automerge
+      - author≠@dont-squash-my-commits
+    actions:
+      merge:
+        method: squash
+  # Join the dont-squash-my-commits group if you won't like your commits squashed
+  - name: automatic merge (rebase) on CI success
+    conditions:
+      - status-success=buildkite/solana
+      - status-success=Travis CI - Pull Request
+      - status-success=ci-gate
+      - label=automerge
+      - author=@dont-squash-my-commits
+    actions:
+      merge:
+        method: rebase
+  - name: remove automerge label on CI failure
+    conditions:
+      - label=automerge
+      - "#status-failure!=0"
+    actions:
+      label:
+        remove:
+          - automerge
+      comment:
+        message: automerge label removed due to a CI failure
   - name: remove outdated reviews
     conditions:
       - base=master
@@ -19,35 +50,27 @@ pull_request_rules:
       label:
         add:
           - automerge
-  - name: v0.23 backport
-    conditions:
-      - base=master
-      - label=v0.23
-    actions:
-      backport:
-        branches:
-          - v0.23
-  - name: v1.0 backport
-    conditions:
-      - base=master
-      - label=v1.0
-    actions:
-      backport:
-        branches:
-          - v1.0
-  - name: v1.1 backport
-    conditions:
-      - base=master
-      - label=v1.1
-    actions:
-      backport:
-        branches:
-          - v1.1
   - name: v1.2 backport
     conditions:
-      - base=master
       - label=v1.2
     actions:
       backport:
+        ignore_conflicts: true
         branches:
           - v1.2
+  - name: v1.3 backport
+    conditions:
+      - label=v1.3
+    actions:
+      backport:
+        ignore_conflicts: true
+        branches:
+          - v1.3
+  - name: v1.4 backport
+    conditions:
+      - label=v1.4
+    actions:
+      backport:
+        ignore_conflicts: true
+        branches:
+          - v1.4

.travis.yml

@@ -1,43 +1,136 @@
-os:
-  - osx
-  - windows
-
-language: rust
-rust:
-  - stable
-
-install:
-  - source ci/rust-version.sh
-
-script:
-  - source ci/env.sh
-  - ci/publish-tarball.sh
-
 branches:
   only:
     - master
     - /^v\d+\.\d+/
 
 notifications:
+  email: false
   slack:
     on_success: change
+    if: NOT type = pull_request
     secure: F4IjOE05MyaMOdPRL+r8qhs7jBvv4yDM3RmFKE1zNXnfUOqV4X38oQM1EI+YVsgpMQLj/pxnEB7wcTE4Bf86N6moLssEULCpvAuMVoXj4QbWdomLX+01WbFa6fLVeNQIg45NHrz2XzVBhoKOrMNnl+QI5mbR2AlS5oqsudHsXDnyLzZtd4Y5SDMdYG1zVWM01+oNNjgNfjcCGmOE/K0CnOMl6GPi3X9C34tJ19P2XT7MTDsz1/IfEF7fro2Q8DHEYL9dchJMoisXSkem5z7IDQkGzXsWdWT4NnndUvmd1MlTCE9qgoXDqRf95Qh8sB1Dz08HtvgfaosP2XjtNTfDI9BBYS15Ibw9y7PchAJE1luteNjF35EOy6OgmCLw/YpnweqfuNViBZz+yOPWXVC0kxnPIXKZ1wyH9ibeH6E4hr7a8o9SV/6SiWIlbYF+IR9jPXyTCLP/cc3sYljPWxDnhWFwFdRVIi3PbVAhVu7uWtVUO17Oc9gtGPgs/GrhOMkJfwQPXaudRJDpVZowxTX4x9kefNotlMAMRgq+Drbmgt4eEBiCNp0ITWgh17BiE1U09WS3myuduhoct85+FoVeaUkp1sxzHVtGsNQH0hcz7WcpZyOM+AwistJA/qzeEDQao5zi1eKWPbO2xAhi2rV1bDH6bPf/4lDBwLRqSiwvlWU=
 
-deploy:
-  - provider: s3
-    access_key_id: $AWS_ACCESS_KEY_ID
-    secret_access_key: $AWS_SECRET_ACCESS_KEY
-    bucket: release.solana.com
-    region: us-west-1
-    skip_cleanup: true
-    acl: public_read
-    local_dir: travis-s3-upload
-    on:
-      all_branches: true
-  - provider: releases
-    api_key: $GITHUB_TOKEN
-    skip_cleanup: true
-    file_glob: true
-    file: travis-release-upload/*
-    on:
-      tags: true
+os: linux
+dist: bionic
+language: minimal
+
+jobs:
+  include:
+    - name: "Export Github Repositories"
+      if: type IN (push, cron) AND branch = master
+      language: python
+      git:
+        depth: false
+      script:
+        - .travis/export-github-repo.sh web3.js/ solana-web3.js
+        - .travis/export-github-repo.sh explorer/ explorer
+
+    - &release-artifacts
+      if: type IN (api, cron) OR tag IS present
+      name: "macOS release artifacts"
+      os: osx
+      language: rust
+      rust:
+        - stable
+      install:
+        - source ci/rust-version.sh
+        - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
+        - readlink -f .
+      script:
+        - source ci/env.sh
+        - ci/publish-tarball.sh
+      deploy:
+        - provider: s3
+          access_key_id: $AWS_ACCESS_KEY_ID
+          secret_access_key: $AWS_SECRET_ACCESS_KEY
+          bucket: release.solana.com
+          region: us-west-1
+          skip_cleanup: true
+          acl: public_read
+          local_dir: travis-s3-upload
+          on:
+            all_branches: true
+        - provider: releases
+          token: $GITHUB_TOKEN
+          skip_cleanup: true
+          file_glob: true
+          file: travis-release-upload/*
+          on:
+            tags: true
+    - <<: *release-artifacts
+      name: "Windows release artifacts"
+      os: windows
+    #  Linux release artifacts are still built by ci/buildkite-secondary.yml
+    #- <<: *release-artifacts
+    #  name: "Linux release artifacts"
+    #  os: linux
+    #  before_install:
+    #    - sudo apt-get install libssl-dev libudev-dev
+
+    # explorer pull request
+    - name: "explorer"
+      if: type = pull_request AND branch = master
+
+      language: node_js
+      node_js:
+        - "node"
+
+      cache:
+        directories:
+          - ~/.npm
+
+      before_install:
+        - .travis/affects.sh explorer/ .travis || travis_terminate 0
+        - cd explorer
+
+      script:
+        - npm run build
+        - npm run format
+
+    # web3.js pull request
+    - name: "web3.js"
+      if: type = pull_request AND branch = master
+
+      language: node_js
+      node_js:
+        - "lts/*"
+
+      services:
+        - docker
+
+      cache:
+        directories:
+          - ~/.npm
+
+      before_install:
+        - .travis/affects.sh web3.js/ .travis || travis_terminate 0
+        - cd web3.js/
+        - source .travis/before_install.sh
+
+      script:
+        - ../.travis/commitlint.sh
+        - source .travis/script.sh
+
+    # docs pull request
+    - name: "docs"
+      if: type IN (push, pull_request) OR tag IS present
+      language: node_js
+      node_js:
+        - "node"
+
+      services:
+        - docker
+
+      cache:
+        directories:
+          - ~/.npm
+
+      before_install:
+        - source ci/env.sh
+        - .travis/channel_restriction.sh edge beta || travis_terminate 0
+        - .travis/affects.sh docs/ .travis || travis_terminate 0
+        - cd docs/
+        - source .travis/before_install.sh
+
+      script:
+        - source .travis/script.sh

.travis/affects.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# Check if files in the commit range match one or more prefixes
+#
+
+# Always run the job if we are on a tagged release
+if [[ -n "$TRAVIS_TAG" ]]; then
+  exit 0
+fi
+
+(
+  set -x
+  git diff --name-only "$TRAVIS_COMMIT_RANGE"
+)
+
+for file in $(git diff --name-only "$TRAVIS_COMMIT_RANGE"); do
+  for prefix in "$@"; do
+    if [[ $file =~ ^"$prefix" ]]; then
+      exit 0
+    fi
+    done
+done
+
+echo "No modifications to $*"
+exit 1

.travis/channel_restriction.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+#
+# Only proceed if we are on one of the channels passed in, or a tag build
+#
+
+set -ex
+
+[[ -n $CI_TAG ]] && exit 0
+
+eval "$(ci/channel-info.sh)"
+
+for acceptable_channel in "$@"; do
+  if [[ "$CHANNEL" == "$acceptable_channel" ]]; then
+    exit 0
+  fi
+done
+
+echo "Not running from one of the following channels: $*"
+exit 1

.travis/commitlint.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+#
+# Runs commitlint in the provided subdirectory
+#
+
+set -e
+
+basedir=$1
+if [[ -z "$basedir" ]]; then
+  basedir=.
+fi
+
+if [[ ! -d "$basedir" ]]; then
+  echo "Error: not a directory: $basedir"
+  exit 1
+fi
+
+if [[ ! -f "$basedir"/commitlint.config.js ]]; then
+  echo "Error: No commitlint configuration found"
+  exit 1
+fi
+
+if [[ -z $TRAVIS_COMMIT_RANGE ]]; then
+  echo "Error: TRAVIS_COMMIT_RANGE not defined"
+  exit 1
+fi
+
+cd "$basedir"
+echo "Checking commits in TRAVIS_COMMIT_RANGE: $TRAVIS_COMMIT_RANGE"
+while IFS= read -r line; do
+  echo "$line" | npx commitlint
+done < <(git log "$TRAVIS_COMMIT_RANGE" --format=%s -- .)

.travis/export-github-repo.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+#
+# Exports a subdirectory into another github repository
+#
+
+set -e
+if [[ -z $GITHUB_TOKEN ]]; then
+  echo GITHUB_TOKEN not defined
+  exit 1
+fi
+
+cd "$(dirname "$0")/.."
+
+pip3 install git-filter-repo
+
+declare subdir=$1
+declare repo_name=$2
+
+[[ -n "$subdir" ]] || {
+  echo "Error: subdir not specified"
+  exit 1
+}
+[[ -n "$repo_name" ]] || {
+  echo "Error: repo_name not specified"
+  exit 1
+}
+
+echo "Exporting $subdir"
+
+set -x
+rm -rf .github_export/"$repo_name"
+git clone https://"$GITHUB_TOKEN"@github.com/solana-labs/"$repo_name" .github_export/"$repo_name"
+git filter-repo --subdirectory-filter "$subdir" --target .github_export/"$repo_name"
+git -C .github_export/"$repo_name" push https://"$GITHUB_TOKEN"@github.com/solana-labs/"$repo_name"

CONTRIBUTING.md

@@ -45,7 +45,7 @@ $ git pull --rebase upstream master
 
 If there are no functional changes, PRs can be very large and that's no
 problem. If, however, your changes are making meaningful changes or additions,
-then about 1000 lines of changes is about the most you should ask a Solana
+then about 1,000 lines of changes is about the most you should ask a Solana
 maintainer to review.
 
 ### Should I send small PRs as I develop large, new components?
@@ -232,7 +232,7 @@ confused with 3-letter acronyms.
 Solana's architecture is described by docs generated from markdown files in
 the `docs/src/` directory, maintained by an *editor* (currently @garious). To
 add a design proposal, you'll need to include it in the
-[Accepted Design Proposals](https://docs.solana.com/proposals)
+[Accepted Design Proposals](https://docs.solana.com/proposals/accepted-design-proposals)
 section of the Solana docs.  Here's the full process:
 
 1. Propose a design by creating a PR that adds a markdown document to the

Cargo.toml

@@ -3,18 +3,23 @@ members = [
     "bench-exchange",
     "bench-streamer",
     "bench-tps",
+    "accounts-bench",
     "banking-bench",
-    "chacha",
-    "chacha-cuda",
-    "chacha-sys",
+    "banks-client",
+    "banks-interface",
+    "banks-server",
+    "clap-utils",
     "cli-config",
+    "cli-output",
     "client",
     "core",
+    "dos",
+    "download-utils",
     "faucet",
+    "frozen-abi",
     "perf",
     "validator",
     "genesis",
-    "genesis-programs",
     "gossip",
     "install",
     "keygen",
@@ -24,34 +29,43 @@ members = [
     "logger",
     "log-analyzer",
     "merkle-tree",
+    "stake-o-matic",
+    "storage-bigtable",
+    "storage-proto",
+    "streamer",
     "measure",
     "metrics",
     "net-shaper",
+    "notifier",
+    "poh-bench",
+    "program-test",
+    "programs/secp256k1",
     "programs/bpf_loader",
     "programs/budget",
-    "programs/btc_spv",
-    "programs/btc_spv_bin",
     "programs/config",
     "programs/exchange",
     "programs/failure",
     "programs/noop",
     "programs/ownable",
     "programs/stake",
-    "programs/storage",
     "programs/vest",
     "programs/vote",
-    "archiver",
-    "archiver-lib",
-    "archiver-utils",
     "remote-wallet",
+    "ramp-tps",
     "runtime",
     "sdk",
-    "sdk-c",
+    "sdk/cargo-build-bpf",
+    "sdk/cargo-test-bpf",
     "scripts",
+    "stake-accounts",
+    "stake-monitor",
     "sys-tuner",
+    "tokens",
     "transaction-status",
+    "account-decoder",
     "upload-perf",
     "net-utils",
+    "version",
     "vote-signer",
     "cli",
     "rayon-threadlimit",
@@ -60,6 +74,4 @@ members = [
 
 exclude = [
     "programs/bpf",
-    "programs/move_loader",
-    "programs/librapay",
 ]

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2018 Solana Labs, Inc.
+Copyright 2020 Solana Foundation.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

README.md

@@ -1,62 +1,17 @@
+<p align="center">
+  <a href="https://solana.com">
+    <img alt="Solana" src="https://i.imgur.com/OMnvVEz.png" width="250" />
+  </a>
+</p>
+
 [![Solana crate](https://img.shields.io/crates/v/solana-core.svg)](https://crates.io/crates/solana-core)
 [![Solana documentation](https://docs.rs/solana-core/badge.svg)](https://docs.rs/solana-core)
 [![Build status](https://badge.buildkite.com/8cc350de251d61483db98bdfc895b9ea0ac8ffa4a32ee850ed.svg?branch=master)](https://buildkite.com/solana-labs/solana/builds?branch=master)
 [![codecov](https://codecov.io/gh/solana-labs/solana/branch/master/graph/badge.svg)](https://codecov.io/gh/solana-labs/solana)
 
-Blockchain Rebuilt for Scale
-===
+# Building
 
-Solana&trade; is a new blockchain architecture built from the ground up for scale. The architecture supports
-up to 710 thousand transactions per second on a gigabit network.
-
-Documentation
-===
-
-Before you jump into the code, review the documentation [Solana: Blockchain Rebuilt for Scale](https://docs.solana.com).
-
-(The _latest_ development version of the docs is [available here](https://docs.solana.com/v/master).)
-
-Release Binaries
-===
-Official release binaries are available at [Github Releases](https://github.com/solana-labs/solana/releases).
-
-Additionally we provide pre-release binaries for the latest code on the edge and
-beta channels.  Note that these pre-release binaries may be less stable than an
-official release.
-
-### Edge channel
-#### Linux (x86_64-unknown-linux-gnu)
-* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-unknown-linux-gnu.tar.bz2)
-* [solana-install-init](http://release.solana.com/edge/solana-install-init-x86_64-unknown-linux-gnu) as a stand-alone executable
-#### mac OS (x86_64-apple-darwin)
-* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-apple-darwin.tar.bz2)
-* [solana-install-init](http://release.solana.com/edge/solana-install-init-x86_64-apple-darwin) as a stand-alone executable
-#### Windows (x86_64-pc-windows-msvc)
-* [solana.tar.bz2](http://release.solana.com/edge/solana-release-x86_64-pc-windows-msvc.tar.bz2)
-* [solana-install-init.exe](http://release.solana.com/edge/solana-install-init-x86_64-pc-windows-msvc.exe) as a stand-alone executable
-#### All platforms
-* [solana-metrics.tar.bz2](http://release.solana.com.s3.amazonaws.com/edge/solana-metrics.tar.bz2)
-
-### Beta channel
-#### Linux (x86_64-unknown-linux-gnu)
-* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-unknown-linux-gnu.tar.bz2)
-* [solana-install-init](http://release.solana.com/beta/solana-install-init-x86_64-unknown-linux-gnu) as a stand-alone executable
-#### mac OS (x86_64-apple-darwin)
-* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-apple-darwin.tar.bz2)
-* [solana-install-init](http://release.solana.com/beta/solana-install-init-x86_64-apple-darwin) as a stand-alone executable
-#### Windows (x86_64-pc-windows-msvc)
-* [solana.tar.bz2](http://release.solana.com/beta/solana-release-x86_64-pc-windows-msvc.tar.bz2)
-* [solana-install-init.exe](http://release.solana.com/beta/solana-install-init-x86_64-pc-windows-msvc.exe) as a stand-alone executable
-#### All platforms
-* [solana-metrics.tar.bz2](http://release.solana.com.s3.amazonaws.com/beta/solana-metrics.tar.bz2)
-
-Developing
-===
-
-Building
----
-
-Install rustc, cargo and rustfmt:
+## **1. Install rustc, cargo and rustfmt.**
 
 ```bash
 $ curl https://sh.rustup.rs -sSf | sh
@@ -64,7 +19,7 @@ $ source $HOME/.cargo/env
 $ rustup component add rustfmt
 ```
 
-If your rustc version is lower than 1.39.0, please update it:
+Please sure you are always using the latest stable rust version by running:
 
 ```bash
 $ rustup update
@@ -77,112 +32,40 @@ $ sudo apt-get update
 $ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang
 ```
 
-Download the source code:
+## **2. Download the source code.**
 
 ```bash
 $ git clone https://github.com/solana-labs/solana.git
 $ cd solana
 ```
 
-Build
+## **3. Build.**
 
 ```bash
 $ cargo build
 ```
 
-Then to run a minimal local cluster
+## **4. Run a minimal local cluster.**
 ```bash
 $ ./run.sh
 ```
 
-Testing
----
+# Testing
 
-Run the test suite:
+**Run the test suite:**
 
 ```bash
 $ cargo test
 ```
 
-Local Testnet
----
+### Starting a local testnet
+Start your own testnet locally, instructions are in the [online docs](https://docs.solana.com/cluster/bench-tps).
 
-Start your own testnet locally, instructions are in the online docs [Solana: Blockchain Rebuild for Scale: Getting Started](https://docs.solana.com/building-from-source).
+### Accessing the remote development cluster
+* `devnet` - stable public cluster for development accessible via
+devnet.solana.com. Runs 24/7. Learn more about the [public clusters](https://docs.solana.com/clusters)
 
-Remote Testnets
----
-
-* `testnet` - public stable testnet accessible via devnet.solana.com. Runs 24/7
-
-
-## Deploy process
-
-They are deployed with the `ci/testnet-manager.sh` script through a list of [scheduled
-buildkite jobs](https://buildkite.com/solana-labs/testnet-management/settings/schedules).
-Each testnet can be manually manipulated from buildkite as well.
-
-## How do I reset the testnet?
-Manually trigger the [testnet-management](https://buildkite.com/solana-labs/testnet-management) pipeline
-and when prompted select the desired testnet
-
-## How can I scale the tx generation rate?
-
-Increase the TX rate by increasing the number of cores on the client machine which is running
-`bench-tps` or run multiple clients. Decrease by lowering cores or using the rayon env
-variable `RAYON_NUM_THREADS=<xx>`
-
-## How can I test a change on the testnet?
-
-Currently, a merged PR is the only way to test a change on the testnet.  But you
-can run your own testnet using the scripts in the `net/` directory.
-
-## Adjusting the number of clients or validators on the testnet
-Edit `ci/testnet-manager.sh`
-
-
-## Metrics Server Maintenance
-Sometimes the dashboard becomes unresponsive. This happens due to glitch in the metrics server.
-The current solution is to reset the metrics server. Use the following steps.
-
-1. The server is hosted in a GCP VM instance. Check if the VM instance is down by trying to SSH
- into it from the GCP console. The name of the VM is ```metrics-solana-com```.
-2. If the VM is inaccessible, reset it from the GCP console.
-3. Once VM is up (or, was already up), the metrics services can be restarted from build automation.
-    1. Navigate to https://buildkite.com/solana-labs/metrics-dot-solana-dot-com in your web browser
-    2. Click on ```New Build```
-    3. This will show a pop up dialog. Click on ```options``` drop down.
-    4. Type in ```FORCE_START=true``` in ```Environment Variables``` text box.
-    5. Click ```Create Build```
-    6. This will restart the metrics services, and the dashboards should be accessible afterwards.
-
-## Debugging Testnet
-Testnet may exhibit different symptoms of failures. Primary statistics to check are
-1. Rise in Confirmation Time
-2. Nodes are not voting
-3. Panics, and OOM notifications
-
-Check the following if there are any signs of failure.
-1. Did testnet deployment fail?
-    1. View buildkite logs for the last deployment: https://buildkite.com/solana-labs/testnet-management
-    2. Use the relevant branch
-    3. If the deployment failed, look at the build logs. The build artifacts for each remote node is uploaded.
-       It's a good first step to triage from these logs.
-2. You may have to log into remote node if the deployment succeeded, but something failed during runtime.
-    1. Get the private key for the testnet deployment from ```metrics-solana-com``` GCP instance.
-    2. SSH into ```metrics-solana-com``` using GCP console and do the following.
-    ```bash
-    sudo bash
-    cd ~buildkite-agent/.ssh
-    ls
-    ```
-    3. Copy the relevant private key to your local machine
-    4. Find the public IP address of the AWS instance for the remote node using AWS console
-    5. ```ssh -i <private key file> ubuntu@<ip address of remote node>```
-    6. The logs are in ```~solana\solana``` folder
-
-
-Benchmarking
----
+# Benchmarking
 
 First install the nightly build of rustc. `cargo bench` requires use of the
 unstable features only available in the nightly build.
@@ -197,13 +80,11 @@ Run the benchmarks:
 $ cargo +nightly bench
 ```
 
-Release Process
----
+# Release Process
+
 The release process for this project is described [here](RELEASE.md).
 
-
-Code coverage
----
+# Code coverage
 
 To generate code coverage statistics:
 
@@ -212,7 +93,6 @@ $ scripts/coverage.sh
 $ open target/cov/lcov-local/index.html
 ```
 
-
 Why coverage? While most see coverage as a code quality metric, we see it primarily as a developer
 productivity metric. When a developer makes a change to the codebase, presumably it's a *solution* to
 some problem.  Our unit-test suite is how we encode the set of *problems* the codebase solves. Running
@@ -225,7 +105,6 @@ better way to solve the same problem, a Pull Request with your solution would mo
 welcome! Likewise, if rewriting a test can better communicate what code it's protecting, please
 send us that patch!
 
-Disclaimer
-===
+# Disclaimer
 
 All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort.  It is up to the reader to check and validate their accuracy and truthfulness.  Furthermore nothing in this project constitutes a solicitation for investment.

RELEASE.md

@@ -76,21 +76,20 @@ There are three release channels that map to branches as follows:
     git push -u origin <branchname>
     ```
 
-### Update master branch with the next version
+Alternatively use the Github UI.
+
+### Update master branch to the next release minor version
 
 1. After the new branch has been created and pushed, update the Cargo.toml files on **master** to the next semantic version (e.g. 0.9.0 -> 0.10.0) with:
      ```
-     scripts/increment-cargo-version.sh minor
+     $ scripts/increment-cargo-version.sh minor
+     $ ./scripts/cargo-for-all-lock-files.sh update
      ```
-1. Rebuild to get an updated version of `Cargo.lock`:
-    ```
-    cargo build
-    ```
 1. Push all the changed Cargo.toml and Cargo.lock files to the `master` branch with something like:
     ```
     git co -b version_update
     git ls-files -m | xargs git add
-    git commit -m 'Update Cargo.toml versions from X.Y to X.Y+1'
+    git commit -m 'Bump version to X.Y+1.0'
     git push -u origin version_update
     ```
 1. Confirm that your freshly cut release branch is shown as `BETA_CHANNEL` and the previous release branch as `STABLE_CHANNEL`:
@@ -102,84 +101,56 @@ There are three release channels that map to branches as follows:
 
 ### Create the Release Tag on GitHub
 
-1. Go to [GitHub's Releases UI](https://github.com/solana-labs/solana/releases) for tagging a release.
+1. Go to [GitHub Releases](https://github.com/solana-labs/solana/releases) for tagging a release.
 1. Click "Draft new release".  The release tag must exactly match the `version`
    field in `/Cargo.toml` prefixed by `v`.
-   1.  If the Cargo.toml verion field is **0.12.3**, then the release tag must be **v0.12.3**
+   1.  If the Cargo.toml version field is **0.12.3**, then the release tag must be **v0.12.3**
 1. Make sure the Target Branch field matches the branch you want to make a release on.
    1.  If you want to release v0.12.0, the target branch must be v0.12
-1. If this is the first release on the branch (e.g. v0.13.**0**), paste in [this
+1. Fill the release notes.
+   1.  If this is the first release on the branch (e.g. v0.13.**0**), paste in [this
    template](https://raw.githubusercontent.com/solana-labs/solana/master/.github/RELEASE_TEMPLATE.md).  Engineering Lead can provide summary contents for release notes if needed.
-1. Click "Save Draft", then confirm the release notes look good and the tag name and branch are correct.  Go back into edit the release and click "Publish release" when ready.
+   1. If this is a patch release, review all the commits since the previous release on this branch and add details as needed.
+1. Click "Save Draft", then confirm the release notes look good and the tag name and branch are correct.
+1. Ensure all desired commits (usually backports) are landed on the branch by now.
+1. Ensure the release is marked **"This is a pre-release"**.  This flag will need to be be removed manually after confirming the the Linux binary artifacts appear at a later step.
+1. Go back into edit the release and click "Publish release" while being marked as a pre-release.
+1. Confirm there is new git tag with intended version number at the intended revision after running `git fetch` locally.
+
 
 ### Update release branch with the next patch version
 
 1. After the new release has been tagged, update the Cargo.toml files on **release branch** to the next semantic version (e.g. 0.9.0 -> 0.9.1) with:
      ```
-     scripts/increment-cargo-version.sh patch
+     $ scripts/increment-cargo-version.sh patch
+     $ ./scripts/cargo-for-all-lock-files.sh tree
      ```
-1. Rebuild to get an updated version of `Cargo.lock`:
-    ```
-    cargo build
-    ```
 1. Push all the changed Cargo.toml and Cargo.lock files to the **release branch** with something like:
     ```
-    git co -b version_update
-    git ls-files -m | xargs git add
-    git commit -m 'Update Cargo.toml versions from X.Y.Z to X.Y.Z+1'
-    git push -u origin version_update
+    git co -b version_update origin/vX.Y
+    git add -u
+    git commit -m 'Bump version to X.Y.Z+1'
+    git push -u <user-remote> version_update
     ```
+1. Open a PR against origin/vX.Y and then merge the PR after passing CI.
+
+### Prepare for the next release
+1.  Go to [GitHub Releases](https://github.com/solana-labs/solana/releases) and create a new draft release for `X.Y.Z+1` with empty release notes.  This allows people to incrementally add new release notes until it's time for the next release
+    1. Also, point the branch field to the same branch and mark the relese as **"This is a pre-release"**.
+1.  Go to the [Github Milestones](https://github.com/solana-labs/solana/milestones).  Create a new milestone for the `X.Y.Z+1`, move over
+unresolved issues still in the `X.Y.Z` milestone, then close the `X.Y.Z` milestone.
 
 ### Verify release automation success
-1. Go to [Solana Releases](https://github.com/solana-labs/solana/releases) and click on the latest release that you just published.  Verify that all of the build artifacts are present.  This can take up to 90 minutes after creating the tag.
-1. The `solana-secondary` Buildkite pipeline handles creating the binary tarballs and updated crates.  Look for a job under the tag name of the release: https://buildkite.com/solana-labs/solana-secondary
-1. [Crates.io](https://crates.io/crates/solana) should have an updated Solana version.
+Go to [Solana Releases](https://github.com/solana-labs/solana/releases) and click on the latest release that you just published.
+Verify that all of the build artifacts are present, then the uncheck **"This is a pre-release"** for the release.
 
-### Update documentation
-TODO: Documentation update procedure is WIP as we move to gitbook
+Build artifacts can take up to 60 minutes after creating the tag before
+appearing.  To check for progress:
+* The `solana-secondary` Buildkite pipeline handles creating the Linux release artifacts and updated crates.  Look for a job under the tag name of the release: https://buildkite.com/solana-labs/solana-secondary.
+* The macOS and Windows release artifacts are produced by Travis CI: https://travis-ci.com/github/solana-labs/solana/branches
 
-Document the new recommended version by updating `docs/src/running-archiver.md` and `docs/src/validator-testnet.md` on the release (beta) branch to point at the `solana-install` for the upcoming release version.
+[Crates.io](https://crates.io/crates/solana) should have an updated Solana version.  This can take 2-3 hours, and sometimes fails in the `solana-secondary` job.
+If this happens and the error is non-fatal, click "Retry" on the "publish crate" job
 
-### Update software on devnet.solana.com
-
-The testnet running on devnet.solana.com is set to use a fixed release tag
-which is set in the Buildkite testnet-management pipeline.
-This tag needs to be updated and the testnet restarted after a new release
-tag is created.
-
-#### Update testnet schedules
-
-Go to https://buildkite.com/solana-labs and click through: Pipelines ->
-testnet-management -> Pipeline Settings -> Schedules
-Or just click here:
-https://buildkite.com/solana-labs/testnet-management/settings/schedules
-
-There are two scheduled jobs for testnet: a daily restart and an hourly sanity-or-restart. \
-https://buildkite.com/solana-labs/testnet-management/settings/schedules/0efd7856-7143-4713-8817-47e6bdb05387
-https://buildkite.com/solana-labs/testnet-management/settings/schedules/2a926646-d972-42b5-aeb9-bb6759592a53
-
-On each schedule:
-1.  Set TESTNET_TAG environment variable to the desired release tag.
-    1. Example, TESTNET_TAG=v0.13.2
-1.  Set the Build Branch to the branch that TESTNET_TAG is from.
-    1. Example: v0.13
-
-#### Restart the testnet
-
-Trigger a TESTNET_OP=create-and-start to refresh the cluster with the new version
-
-1.  Go to https://buildkite.com/solana-labs/testnet-management
-2.  Click "New Build" and use the following settings, then click "Create Build"
-    1.  Commit: HEAD
-    1.  Branch: [channel branch as set in the schedules]
-    1.  Environment Variables:
-```
-TESTNET=testnet
-TESTNET_TAG=[same value as used in TESTNET_TAG in the schedules]
-TESTNET_OP=create-and-start
-```
-
-### Alert the community
-
-Notify Discord users on #validator-support that a new release for
-devnet.solana.com is available
+### Update software on devnet.solana.com/testnet.solana.com/mainnet-beta.solana.com
+See the documentation at https://github.com/solana-labs/cluster-ops/

account-decoder/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "solana-account-decoder"
+version = "1.4.26"
+description = "Solana account decoder"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+repository = "https://github.com/solana-labs/solana"
+homepage = "https://solana.com/"
+license = "Apache-2.0"
+edition = "2018"
+
+[dependencies]
+base64 = "0.12.3"
+bincode = "1.3.1"
+bs58 = "0.3.1"
+bv = "0.11.1"
+Inflector = "0.11.4"
+lazy_static = "1.4.0"
+serde = "1.0.112"
+serde_derive = "1.0.103"
+serde_json = "1.0.56"
+solana-config-program = { path = "../programs/config", version = "1.4.26" }
+solana-sdk = { path = "../sdk", version = "1.4.26" }
+solana-stake-program = { path = "../programs/stake", version = "1.4.26" }
+solana-vote-program = { path = "../programs/vote", version = "1.4.26" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.0.1", features = ["no-entrypoint"] }
+thiserror = "1.0"
+zstd = "0.5.1"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

account-decoder/src/lib.rs

@@ -0,0 +1,233 @@
+#[macro_use]
+extern crate lazy_static;
+#[macro_use]
+extern crate serde_derive;
+
+pub mod parse_account_data;
+pub mod parse_config;
+pub mod parse_nonce;
+pub mod parse_stake;
+pub mod parse_sysvar;
+pub mod parse_token;
+pub mod parse_vote;
+pub mod validator_info;
+
+use {
+    crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount},
+    solana_sdk::{account::Account, clock::Epoch, fee_calculator::FeeCalculator, pubkey::Pubkey},
+    std::{
+        io::{Read, Write},
+        str::FromStr,
+    },
+};
+
+pub type StringAmount = String;
+
+/// A duplicate representation of an Account for pretty JSON serialization
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct UiAccount {
+    pub lamports: u64,
+    pub data: UiAccountData,
+    pub owner: String,
+    pub executable: bool,
+    pub rent_epoch: Epoch,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase", untagged)]
+pub enum UiAccountData {
+    LegacyBinary(String), // Legacy. Retained for RPC backwards compatibility
+    Json(ParsedAccount),
+    Binary(String, UiAccountEncoding),
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub enum UiAccountEncoding {
+    Binary, // Legacy. Retained for RPC backwards compatibility
+    Base58,
+    Base64,
+    JsonParsed,
+    #[serde(rename = "base64+zstd")]
+    Base64Zstd,
+}
+
+impl UiAccount {
+    pub fn encode(
+        pubkey: &Pubkey,
+        account: Account,
+        encoding: UiAccountEncoding,
+        additional_data: Option<AccountAdditionalData>,
+        data_slice_config: Option<UiDataSliceConfig>,
+    ) -> Self {
+        let data = match encoding {
+            UiAccountEncoding::Binary => UiAccountData::LegacyBinary(
+                bs58::encode(slice_data(&account.data, data_slice_config)).into_string(),
+            ),
+            UiAccountEncoding::Base58 => UiAccountData::Binary(
+                bs58::encode(slice_data(&account.data, data_slice_config)).into_string(),
+                encoding,
+            ),
+            UiAccountEncoding::Base64 => UiAccountData::Binary(
+                base64::encode(slice_data(&account.data, data_slice_config)),
+                encoding,
+            ),
+            UiAccountEncoding::Base64Zstd => {
+                let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 0).unwrap();
+                match encoder
+                    .write_all(slice_data(&account.data, data_slice_config))
+                    .and_then(|()| encoder.finish())
+                {
+                    Ok(zstd_data) => UiAccountData::Binary(base64::encode(zstd_data), encoding),
+                    Err(_) => UiAccountData::Binary(
+                        base64::encode(slice_data(&account.data, data_slice_config)),
+                        UiAccountEncoding::Base64,
+                    ),
+                }
+            }
+            UiAccountEncoding::JsonParsed => {
+                if let Ok(parsed_data) =
+                    parse_account_data(pubkey, &account.owner, &account.data, additional_data)
+                {
+                    UiAccountData::Json(parsed_data)
+                } else {
+                    UiAccountData::Binary(base64::encode(&account.data), UiAccountEncoding::Base64)
+                }
+            }
+        };
+        UiAccount {
+            lamports: account.lamports,
+            data,
+            owner: account.owner.to_string(),
+            executable: account.executable,
+            rent_epoch: account.rent_epoch,
+        }
+    }
+
+    pub fn decode(&self) -> Option<Account> {
+        let data = match &self.data {
+            UiAccountData::Json(_) => None,
+            UiAccountData::LegacyBinary(blob) => bs58::decode(blob).into_vec().ok(),
+            UiAccountData::Binary(blob, encoding) => match encoding {
+                UiAccountEncoding::Base58 => bs58::decode(blob).into_vec().ok(),
+                UiAccountEncoding::Base64 => base64::decode(blob).ok(),
+                UiAccountEncoding::Base64Zstd => base64::decode(blob)
+                    .ok()
+                    .map(|zstd_data| {
+                        let mut data = vec![];
+                        zstd::stream::read::Decoder::new(zstd_data.as_slice())
+                            .and_then(|mut reader| reader.read_to_end(&mut data))
+                            .map(|_| data)
+                            .ok()
+                    })
+                    .flatten(),
+                UiAccountEncoding::Binary | UiAccountEncoding::JsonParsed => None,
+            },
+        }?;
+        Some(Account {
+            lamports: self.lamports,
+            data,
+            owner: Pubkey::from_str(&self.owner).ok()?,
+            executable: self.executable,
+            rent_epoch: self.rent_epoch,
+        })
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiFeeCalculator {
+    pub lamports_per_signature: StringAmount,
+}
+
+impl From<FeeCalculator> for UiFeeCalculator {
+    fn from(fee_calculator: FeeCalculator) -> Self {
+        Self {
+            lamports_per_signature: fee_calculator.lamports_per_signature.to_string(),
+        }
+    }
+}
+
+impl Default for UiFeeCalculator {
+    fn default() -> Self {
+        Self {
+            lamports_per_signature: "0".to_string(),
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct UiDataSliceConfig {
+    pub offset: usize,
+    pub length: usize,
+}
+
+fn slice_data(data: &[u8], data_slice_config: Option<UiDataSliceConfig>) -> &[u8] {
+    if let Some(UiDataSliceConfig { offset, length }) = data_slice_config {
+        if offset >= data.len() {
+            &[]
+        } else if length > data.len() - offset {
+            &data[offset..]
+        } else {
+            &data[offset..offset + length]
+        }
+    } else {
+        data
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_slice_data() {
+        let data = vec![1, 2, 3, 4, 5];
+        let slice_config = Some(UiDataSliceConfig {
+            offset: 0,
+            length: 5,
+        });
+        assert_eq!(slice_data(&data, slice_config), &data[..]);
+
+        let slice_config = Some(UiDataSliceConfig {
+            offset: 0,
+            length: 10,
+        });
+        assert_eq!(slice_data(&data, slice_config), &data[..]);
+
+        let slice_config = Some(UiDataSliceConfig {
+            offset: 1,
+            length: 2,
+        });
+        assert_eq!(slice_data(&data, slice_config), &data[1..3]);
+
+        let slice_config = Some(UiDataSliceConfig {
+            offset: 10,
+            length: 2,
+        });
+        assert_eq!(slice_data(&data, slice_config), &[] as &[u8]);
+    }
+
+    #[test]
+    fn test_base64_zstd() {
+        let encoded_account = UiAccount::encode(
+            &Pubkey::default(),
+            Account {
+                data: vec![0; 1024],
+                ..Account::default()
+            },
+            UiAccountEncoding::Base64Zstd,
+            None,
+            None,
+        );
+        assert!(matches!(
+            encoded_account.data,
+            UiAccountData::Binary(_, UiAccountEncoding::Base64Zstd)
+        ));
+
+        let decoded_account = encoded_account.decode().unwrap();
+        assert_eq!(decoded_account.data, vec![0; 1024]);
+    }
+}

account-decoder/src/parse_account_data.rs

@@ -0,0 +1,145 @@
+use crate::{
+    parse_config::parse_config,
+    parse_nonce::parse_nonce,
+    parse_stake::parse_stake,
+    parse_sysvar::parse_sysvar,
+    parse_token::{parse_token, spl_token_id_v2_0},
+    parse_vote::parse_vote,
+};
+use inflector::Inflector;
+use serde_json::Value;
+use solana_sdk::{instruction::InstructionError, pubkey::Pubkey, system_program, sysvar};
+use std::collections::HashMap;
+use thiserror::Error;
+
+lazy_static! {
+    static ref CONFIG_PROGRAM_ID: Pubkey = solana_config_program::id();
+    static ref STAKE_PROGRAM_ID: Pubkey = solana_stake_program::id();
+    static ref SYSTEM_PROGRAM_ID: Pubkey = system_program::id();
+    static ref SYSVAR_PROGRAM_ID: Pubkey = sysvar::id();
+    static ref TOKEN_PROGRAM_ID: Pubkey = spl_token_id_v2_0();
+    static ref VOTE_PROGRAM_ID: Pubkey = solana_vote_program::id();
+    pub static ref PARSABLE_PROGRAM_IDS: HashMap<Pubkey, ParsableAccount> = {
+        let mut m = HashMap::new();
+        m.insert(*CONFIG_PROGRAM_ID, ParsableAccount::Config);
+        m.insert(*SYSTEM_PROGRAM_ID, ParsableAccount::Nonce);
+        m.insert(*TOKEN_PROGRAM_ID, ParsableAccount::SplToken);
+        m.insert(*STAKE_PROGRAM_ID, ParsableAccount::Stake);
+        m.insert(*SYSVAR_PROGRAM_ID, ParsableAccount::Sysvar);
+        m.insert(*VOTE_PROGRAM_ID, ParsableAccount::Vote);
+        m
+    };
+}
+
+#[derive(Error, Debug)]
+pub enum ParseAccountError {
+    #[error("{0:?} account not parsable")]
+    AccountNotParsable(ParsableAccount),
+
+    #[error("Program not parsable")]
+    ProgramNotParsable,
+
+    #[error("Additional data required to parse: {0}")]
+    AdditionalDataMissing(String),
+
+    #[error("Instruction error")]
+    InstructionError(#[from] InstructionError),
+
+    #[error("Serde json error")]
+    SerdeJsonError(#[from] serde_json::error::Error),
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct ParsedAccount {
+    pub program: String,
+    pub parsed: Value,
+    pub space: u64,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum ParsableAccount {
+    Config,
+    Nonce,
+    SplToken,
+    Stake,
+    Sysvar,
+    Vote,
+}
+
+#[derive(Default)]
+pub struct AccountAdditionalData {
+    pub spl_token_decimals: Option<u8>,
+}
+
+pub fn parse_account_data(
+    pubkey: &Pubkey,
+    program_id: &Pubkey,
+    data: &[u8],
+    additional_data: Option<AccountAdditionalData>,
+) -> Result<ParsedAccount, ParseAccountError> {
+    let program_name = PARSABLE_PROGRAM_IDS
+        .get(program_id)
+        .ok_or_else(|| ParseAccountError::ProgramNotParsable)?;
+    let additional_data = additional_data.unwrap_or_default();
+    let parsed_json = match program_name {
+        ParsableAccount::Config => serde_json::to_value(parse_config(data, pubkey)?)?,
+        ParsableAccount::Nonce => serde_json::to_value(parse_nonce(data)?)?,
+        ParsableAccount::SplToken => {
+            serde_json::to_value(parse_token(data, additional_data.spl_token_decimals)?)?
+        }
+        ParsableAccount::Stake => serde_json::to_value(parse_stake(data)?)?,
+        ParsableAccount::Sysvar => serde_json::to_value(parse_sysvar(data, pubkey)?)?,
+        ParsableAccount::Vote => serde_json::to_value(parse_vote(data)?)?,
+    };
+    Ok(ParsedAccount {
+        program: format!("{:?}", program_name).to_kebab_case(),
+        parsed: parsed_json,
+        space: data.len() as u64,
+    })
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use solana_sdk::nonce::{
+        state::{Data, Versions},
+        State,
+    };
+    use solana_vote_program::vote_state::{VoteState, VoteStateVersions};
+
+    #[test]
+    fn test_parse_account_data() {
+        let account_pubkey = solana_sdk::pubkey::new_rand();
+        let other_program = solana_sdk::pubkey::new_rand();
+        let data = vec![0; 4];
+        assert!(parse_account_data(&account_pubkey, &other_program, &data, None).is_err());
+
+        let vote_state = VoteState::default();
+        let mut vote_account_data: Vec<u8> = vec![0; VoteState::size_of()];
+        let versioned = VoteStateVersions::new_current(vote_state);
+        VoteState::serialize(&versioned, &mut vote_account_data).unwrap();
+        let parsed = parse_account_data(
+            &account_pubkey,
+            &solana_vote_program::id(),
+            &vote_account_data,
+            None,
+        )
+        .unwrap();
+        assert_eq!(parsed.program, "vote".to_string());
+        assert_eq!(parsed.space, VoteState::size_of() as u64);
+
+        let nonce_data = Versions::new_current(State::Initialized(Data::default()));
+        let nonce_account_data = bincode::serialize(&nonce_data).unwrap();
+        let parsed = parse_account_data(
+            &account_pubkey,
+            &system_program::id(),
+            &nonce_account_data,
+            None,
+        )
+        .unwrap();
+        assert_eq!(parsed.program, "nonce".to_string());
+        assert_eq!(parsed.space, State::size() as u64);
+    }
+}

account-decoder/src/parse_config.rs

@@ -0,0 +1,146 @@
+use crate::{
+    parse_account_data::{ParsableAccount, ParseAccountError},
+    validator_info,
+};
+use bincode::deserialize;
+use serde_json::Value;
+use solana_config_program::{get_config_data, ConfigKeys};
+use solana_sdk::pubkey::Pubkey;
+use solana_stake_program::config::Config as StakeConfig;
+
+pub fn parse_config(data: &[u8], pubkey: &Pubkey) -> Result<ConfigAccountType, ParseAccountError> {
+    let parsed_account = if pubkey == &solana_stake_program::config::id() {
+        get_config_data(data)
+            .ok()
+            .and_then(|data| deserialize::<StakeConfig>(data).ok())
+            .map(|config| ConfigAccountType::StakeConfig(config.into()))
+    } else {
+        deserialize::<ConfigKeys>(data).ok().and_then(|key_list| {
+            if !key_list.keys.is_empty() && key_list.keys[0].0 == validator_info::id() {
+                parse_config_data::<String>(data, key_list.keys).and_then(|validator_info| {
+                    Some(ConfigAccountType::ValidatorInfo(UiConfig {
+                        keys: validator_info.keys,
+                        config_data: serde_json::from_str(&validator_info.config_data).ok()?,
+                    }))
+                })
+            } else {
+                None
+            }
+        })
+    };
+    parsed_account.ok_or(ParseAccountError::AccountNotParsable(
+        ParsableAccount::Config,
+    ))
+}
+
+fn parse_config_data<T>(data: &[u8], keys: Vec<(Pubkey, bool)>) -> Option<UiConfig<T>>
+where
+    T: serde::de::DeserializeOwned,
+{
+    let config_data: T = deserialize(&get_config_data(data).ok()?).ok()?;
+    let keys = keys
+        .iter()
+        .map(|key| UiConfigKey {
+            pubkey: key.0.to_string(),
+            signer: key.1,
+        })
+        .collect();
+    Some(UiConfig { keys, config_data })
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase", tag = "type", content = "info")]
+pub enum ConfigAccountType {
+    StakeConfig(UiStakeConfig),
+    ValidatorInfo(UiConfig<Value>),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiConfigKey {
+    pub pubkey: String,
+    pub signer: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiStakeConfig {
+    pub warmup_cooldown_rate: f64,
+    pub slash_penalty: u8,
+}
+
+impl From<StakeConfig> for UiStakeConfig {
+    fn from(config: StakeConfig) -> Self {
+        Self {
+            warmup_cooldown_rate: config.warmup_cooldown_rate,
+            slash_penalty: config.slash_penalty,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiConfig<T> {
+    pub keys: Vec<UiConfigKey>,
+    pub config_data: T,
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::validator_info::ValidatorInfo;
+    use serde_json::json;
+    use solana_config_program::create_config_account;
+
+    #[test]
+    fn test_parse_config() {
+        let stake_config = StakeConfig {
+            warmup_cooldown_rate: 0.25,
+            slash_penalty: 50,
+        };
+        let stake_config_account = create_config_account(vec![], &stake_config, 10);
+        assert_eq!(
+            parse_config(
+                &stake_config_account.data,
+                &solana_stake_program::config::id()
+            )
+            .unwrap(),
+            ConfigAccountType::StakeConfig(UiStakeConfig {
+                warmup_cooldown_rate: 0.25,
+                slash_penalty: 50,
+            }),
+        );
+
+        let validator_info = ValidatorInfo {
+            info: serde_json::to_string(&json!({
+                "name": "Solana",
+            }))
+            .unwrap(),
+        };
+        let info_pubkey = solana_sdk::pubkey::new_rand();
+        let validator_info_config_account = create_config_account(
+            vec![(validator_info::id(), false), (info_pubkey, true)],
+            &validator_info,
+            10,
+        );
+        assert_eq!(
+            parse_config(&validator_info_config_account.data, &info_pubkey).unwrap(),
+            ConfigAccountType::ValidatorInfo(UiConfig {
+                keys: vec![
+                    UiConfigKey {
+                        pubkey: validator_info::id().to_string(),
+                        signer: false,
+                    },
+                    UiConfigKey {
+                        pubkey: info_pubkey.to_string(),
+                        signer: true,
+                    }
+                ],
+                config_data: serde_json::from_str(r#"{"name":"Solana"}"#).unwrap(),
+            }),
+        );
+
+        let bad_data = vec![0; 4];
+        assert!(parse_config(&bad_data, &info_pubkey).is_err());
+    }
+}

account-decoder/src/parse_nonce.rs

@@ -0,0 +1,67 @@
+use crate::{parse_account_data::ParseAccountError, UiFeeCalculator};
+use solana_sdk::{
+    instruction::InstructionError,
+    nonce::{state::Versions, State},
+};
+
+pub fn parse_nonce(data: &[u8]) -> Result<UiNonceState, ParseAccountError> {
+    let nonce_state: Versions = bincode::deserialize(data)
+        .map_err(|_| ParseAccountError::from(InstructionError::InvalidAccountData))?;
+    let nonce_state = nonce_state.convert_to_current();
+    match nonce_state {
+        State::Uninitialized => Ok(UiNonceState::Uninitialized),
+        State::Initialized(data) => Ok(UiNonceState::Initialized(UiNonceData {
+            authority: data.authority.to_string(),
+            blockhash: data.blockhash.to_string(),
+            fee_calculator: data.fee_calculator.into(),
+        })),
+    }
+}
+
+/// A duplicate representation of NonceState for pretty JSON serialization
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase", tag = "type", content = "info")]
+pub enum UiNonceState {
+    Uninitialized,
+    Initialized(UiNonceData),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiNonceData {
+    pub authority: String,
+    pub blockhash: String,
+    pub fee_calculator: UiFeeCalculator,
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use solana_sdk::{
+        hash::Hash,
+        nonce::{
+            state::{Data, Versions},
+            State,
+        },
+        pubkey::Pubkey,
+    };
+
+    #[test]
+    fn test_parse_nonce() {
+        let nonce_data = Versions::new_current(State::Initialized(Data::default()));
+        let nonce_account_data = bincode::serialize(&nonce_data).unwrap();
+        assert_eq!(
+            parse_nonce(&nonce_account_data).unwrap(),
+            UiNonceState::Initialized(UiNonceData {
+                authority: Pubkey::default().to_string(),
+                blockhash: Hash::default().to_string(),
+                fee_calculator: UiFeeCalculator {
+                    lamports_per_signature: 0.to_string(),
+                },
+            }),
+        );
+
+        let bad_data = vec![0; 4];
+        assert!(parse_nonce(&bad_data).is_err());
+    }
+}

account-decoder/src/parse_stake.rs

@@ -0,0 +1,234 @@
+use crate::{
+    parse_account_data::{ParsableAccount, ParseAccountError},
+    StringAmount,
+};
+use bincode::deserialize;
+use solana_sdk::clock::{Epoch, UnixTimestamp};
+use solana_stake_program::stake_state::{Authorized, Delegation, Lockup, Meta, Stake, StakeState};
+
+pub fn parse_stake(data: &[u8]) -> Result<StakeAccountType, ParseAccountError> {
+    let stake_state: StakeState = deserialize(data)
+        .map_err(|_| ParseAccountError::AccountNotParsable(ParsableAccount::Stake))?;
+    let parsed_account = match stake_state {
+        StakeState::Uninitialized => StakeAccountType::Uninitialized,
+        StakeState::Initialized(meta) => StakeAccountType::Initialized(UiStakeAccount {
+            meta: meta.into(),
+            stake: None,
+        }),
+        StakeState::Stake(meta, stake) => StakeAccountType::Delegated(UiStakeAccount {
+            meta: meta.into(),
+            stake: Some(stake.into()),
+        }),
+        StakeState::RewardsPool => StakeAccountType::RewardsPool,
+    };
+    Ok(parsed_account)
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase", tag = "type", content = "info")]
+pub enum StakeAccountType {
+    Uninitialized,
+    Initialized(UiStakeAccount),
+    Delegated(UiStakeAccount),
+    RewardsPool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiStakeAccount {
+    pub meta: UiMeta,
+    pub stake: Option<UiStake>,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiMeta {
+    pub rent_exempt_reserve: StringAmount,
+    pub authorized: UiAuthorized,
+    pub lockup: UiLockup,
+}
+
+impl From<Meta> for UiMeta {
+    fn from(meta: Meta) -> Self {
+        Self {
+            rent_exempt_reserve: meta.rent_exempt_reserve.to_string(),
+            authorized: meta.authorized.into(),
+            lockup: meta.lockup.into(),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiLockup {
+    pub unix_timestamp: UnixTimestamp,
+    pub epoch: Epoch,
+    pub custodian: String,
+}
+
+impl From<Lockup> for UiLockup {
+    fn from(lockup: Lockup) -> Self {
+        Self {
+            unix_timestamp: lockup.unix_timestamp,
+            epoch: lockup.epoch,
+            custodian: lockup.custodian.to_string(),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiAuthorized {
+    pub staker: String,
+    pub withdrawer: String,
+}
+
+impl From<Authorized> for UiAuthorized {
+    fn from(authorized: Authorized) -> Self {
+        Self {
+            staker: authorized.staker.to_string(),
+            withdrawer: authorized.withdrawer.to_string(),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiStake {
+    pub delegation: UiDelegation,
+    pub credits_observed: u64,
+}
+
+impl From<Stake> for UiStake {
+    fn from(stake: Stake) -> Self {
+        Self {
+            delegation: stake.delegation.into(),
+            credits_observed: stake.credits_observed,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiDelegation {
+    pub voter: String,
+    pub stake: StringAmount,
+    pub activation_epoch: StringAmount,
+    pub deactivation_epoch: StringAmount,
+    pub warmup_cooldown_rate: f64,
+}
+
+impl From<Delegation> for UiDelegation {
+    fn from(delegation: Delegation) -> Self {
+        Self {
+            voter: delegation.voter_pubkey.to_string(),
+            stake: delegation.stake.to_string(),
+            activation_epoch: delegation.activation_epoch.to_string(),
+            deactivation_epoch: delegation.deactivation_epoch.to_string(),
+            warmup_cooldown_rate: delegation.warmup_cooldown_rate,
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use bincode::serialize;
+
+    #[test]
+    fn test_parse_stake() {
+        let stake_state = StakeState::Uninitialized;
+        let stake_data = serialize(&stake_state).unwrap();
+        assert_eq!(
+            parse_stake(&stake_data).unwrap(),
+            StakeAccountType::Uninitialized
+        );
+
+        let pubkey = solana_sdk::pubkey::new_rand();
+        let custodian = solana_sdk::pubkey::new_rand();
+        let authorized = Authorized::auto(&pubkey);
+        let lockup = Lockup {
+            unix_timestamp: 0,
+            epoch: 1,
+            custodian,
+        };
+        let meta = Meta {
+            rent_exempt_reserve: 42,
+            authorized,
+            lockup,
+        };
+
+        let stake_state = StakeState::Initialized(meta);
+        let stake_data = serialize(&stake_state).unwrap();
+        assert_eq!(
+            parse_stake(&stake_data).unwrap(),
+            StakeAccountType::Initialized(UiStakeAccount {
+                meta: UiMeta {
+                    rent_exempt_reserve: 42.to_string(),
+                    authorized: UiAuthorized {
+                        staker: pubkey.to_string(),
+                        withdrawer: pubkey.to_string(),
+                    },
+                    lockup: UiLockup {
+                        unix_timestamp: 0,
+                        epoch: 1,
+                        custodian: custodian.to_string(),
+                    }
+                },
+                stake: None,
+            })
+        );
+
+        let voter_pubkey = solana_sdk::pubkey::new_rand();
+        let stake = Stake {
+            delegation: Delegation {
+                voter_pubkey,
+                stake: 20,
+                activation_epoch: 2,
+                deactivation_epoch: std::u64::MAX,
+                warmup_cooldown_rate: 0.25,
+            },
+            credits_observed: 10,
+        };
+
+        let stake_state = StakeState::Stake(meta, stake);
+        let stake_data = serialize(&stake_state).unwrap();
+        assert_eq!(
+            parse_stake(&stake_data).unwrap(),
+            StakeAccountType::Delegated(UiStakeAccount {
+                meta: UiMeta {
+                    rent_exempt_reserve: 42.to_string(),
+                    authorized: UiAuthorized {
+                        staker: pubkey.to_string(),
+                        withdrawer: pubkey.to_string(),
+                    },
+                    lockup: UiLockup {
+                        unix_timestamp: 0,
+                        epoch: 1,
+                        custodian: custodian.to_string(),
+                    }
+                },
+                stake: Some(UiStake {
+                    delegation: UiDelegation {
+                        voter: voter_pubkey.to_string(),
+                        stake: 20.to_string(),
+                        activation_epoch: 2.to_string(),
+                        deactivation_epoch: std::u64::MAX.to_string(),
+                        warmup_cooldown_rate: 0.25,
+                    },
+                    credits_observed: 10,
+                })
+            })
+        );
+
+        let stake_state = StakeState::RewardsPool;
+        let stake_data = serialize(&stake_state).unwrap();
+        assert_eq!(
+            parse_stake(&stake_data).unwrap(),
+            StakeAccountType::RewardsPool
+        );
+
+        let bad_data = vec![1, 2, 3, 4];
+        assert!(parse_stake(&bad_data).is_err());
+    }
+}

account-decoder/src/parse_sysvar.rs

@@ -0,0 +1,329 @@
+use crate::{
+    parse_account_data::{ParsableAccount, ParseAccountError},
+    StringAmount, UiFeeCalculator,
+};
+use bincode::deserialize;
+use bv::BitVec;
+use solana_sdk::{
+    clock::{Clock, Epoch, Slot, UnixTimestamp},
+    epoch_schedule::EpochSchedule,
+    pubkey::Pubkey,
+    rent::Rent,
+    slot_hashes::SlotHashes,
+    slot_history::{self, SlotHistory},
+    stake_history::{StakeHistory, StakeHistoryEntry},
+    sysvar::{self, fees::Fees, recent_blockhashes::RecentBlockhashes, rewards::Rewards},
+};
+
+pub fn parse_sysvar(data: &[u8], pubkey: &Pubkey) -> Result<SysvarAccountType, ParseAccountError> {
+    let parsed_account = {
+        if pubkey == &sysvar::clock::id() {
+            deserialize::<Clock>(data)
+                .ok()
+                .map(|clock| SysvarAccountType::Clock(clock.into()))
+        } else if pubkey == &sysvar::epoch_schedule::id() {
+            deserialize(data).ok().map(SysvarAccountType::EpochSchedule)
+        } else if pubkey == &sysvar::fees::id() {
+            deserialize::<Fees>(data)
+                .ok()
+                .map(|fees| SysvarAccountType::Fees(fees.into()))
+        } else if pubkey == &sysvar::recent_blockhashes::id() {
+            deserialize::<RecentBlockhashes>(data)
+                .ok()
+                .map(|recent_blockhashes| {
+                    let recent_blockhashes = recent_blockhashes
+                        .iter()
+                        .map(|entry| UiRecentBlockhashesEntry {
+                            blockhash: entry.blockhash.to_string(),
+                            fee_calculator: entry.fee_calculator.clone().into(),
+                        })
+                        .collect();
+                    SysvarAccountType::RecentBlockhashes(recent_blockhashes)
+                })
+        } else if pubkey == &sysvar::rent::id() {
+            deserialize::<Rent>(data)
+                .ok()
+                .map(|rent| SysvarAccountType::Rent(rent.into()))
+        } else if pubkey == &sysvar::rewards::id() {
+            deserialize::<Rewards>(data)
+                .ok()
+                .map(|rewards| SysvarAccountType::Rewards(rewards.into()))
+        } else if pubkey == &sysvar::slot_hashes::id() {
+            deserialize::<SlotHashes>(data).ok().map(|slot_hashes| {
+                let slot_hashes = slot_hashes
+                    .iter()
+                    .map(|slot_hash| UiSlotHashEntry {
+                        slot: slot_hash.0,
+                        hash: slot_hash.1.to_string(),
+                    })
+                    .collect();
+                SysvarAccountType::SlotHashes(slot_hashes)
+            })
+        } else if pubkey == &sysvar::slot_history::id() {
+            deserialize::<SlotHistory>(data).ok().map(|slot_history| {
+                SysvarAccountType::SlotHistory(UiSlotHistory {
+                    next_slot: slot_history.next_slot,
+                    bits: format!("{:?}", SlotHistoryBits(slot_history.bits)),
+                })
+            })
+        } else if pubkey == &sysvar::stake_history::id() {
+            deserialize::<StakeHistory>(data).ok().map(|stake_history| {
+                let stake_history = stake_history
+                    .iter()
+                    .map(|entry| UiStakeHistoryEntry {
+                        epoch: entry.0,
+                        stake_history: entry.1.clone(),
+                    })
+                    .collect();
+                SysvarAccountType::StakeHistory(stake_history)
+            })
+        } else {
+            None
+        }
+    };
+    parsed_account.ok_or(ParseAccountError::AccountNotParsable(
+        ParsableAccount::Sysvar,
+    ))
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase", tag = "type", content = "info")]
+pub enum SysvarAccountType {
+    Clock(UiClock),
+    EpochSchedule(EpochSchedule),
+    Fees(UiFees),
+    RecentBlockhashes(Vec<UiRecentBlockhashesEntry>),
+    Rent(UiRent),
+    Rewards(UiRewards),
+    SlotHashes(Vec<UiSlotHashEntry>),
+    SlotHistory(UiSlotHistory),
+    StakeHistory(Vec<UiStakeHistoryEntry>),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct UiClock {
+    pub slot: Slot,
+    pub epoch: Epoch,
+    pub epoch_start_timestamp: UnixTimestamp,
+    pub leader_schedule_epoch: Epoch,
+    pub unix_timestamp: UnixTimestamp,
+}
+
+impl From<Clock> for UiClock {
+    fn from(clock: Clock) -> Self {
+        Self {
+            slot: clock.slot,
+            epoch: clock.epoch,
+            epoch_start_timestamp: clock.epoch_start_timestamp,
+            leader_schedule_epoch: clock.leader_schedule_epoch,
+            unix_timestamp: clock.unix_timestamp,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct UiFees {
+    pub fee_calculator: UiFeeCalculator,
+}
+impl From<Fees> for UiFees {
+    fn from(fees: Fees) -> Self {
+        Self {
+            fee_calculator: fees.fee_calculator.into(),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct UiRent {
+    pub lamports_per_byte_year: StringAmount,
+    pub exemption_threshold: f64,
+    pub burn_percent: u8,
+}
+
+impl From<Rent> for UiRent {
+    fn from(rent: Rent) -> Self {
+        Self {
+            lamports_per_byte_year: rent.lamports_per_byte_year.to_string(),
+            exemption_threshold: rent.exemption_threshold,
+            burn_percent: rent.burn_percent,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct UiRewards {
+    pub validator_point_value: f64,
+}
+
+impl From<Rewards> for UiRewards {
+    fn from(rewards: Rewards) -> Self {
+        Self {
+            validator_point_value: rewards.validator_point_value,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiRecentBlockhashesEntry {
+    pub blockhash: String,
+    pub fee_calculator: UiFeeCalculator,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiSlotHashEntry {
+    pub slot: Slot,
+    pub hash: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiSlotHistory {
+    pub next_slot: Slot,
+    pub bits: String,
+}
+
+struct SlotHistoryBits(BitVec<u64>);
+
+impl std::fmt::Debug for SlotHistoryBits {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        for i in 0..slot_history::MAX_ENTRIES {
+            if self.0.get(i) {
+                write!(f, "1")?;
+            } else {
+                write!(f, "0")?;
+            }
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiStakeHistoryEntry {
+    pub epoch: Epoch,
+    pub stake_history: StakeHistoryEntry,
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use solana_sdk::{
+        account::create_account, fee_calculator::FeeCalculator, hash::Hash,
+        sysvar::recent_blockhashes::IterItem,
+    };
+    use std::iter::FromIterator;
+
+    #[test]
+    fn test_parse_sysvars() {
+        let clock_sysvar = create_account(&Clock::default(), 1);
+        assert_eq!(
+            parse_sysvar(&clock_sysvar.data, &sysvar::clock::id()).unwrap(),
+            SysvarAccountType::Clock(UiClock::default()),
+        );
+
+        let epoch_schedule = EpochSchedule {
+            slots_per_epoch: 12,
+            leader_schedule_slot_offset: 0,
+            warmup: false,
+            first_normal_epoch: 1,
+            first_normal_slot: 12,
+        };
+        let epoch_schedule_sysvar = create_account(&epoch_schedule, 1);
+        assert_eq!(
+            parse_sysvar(&epoch_schedule_sysvar.data, &sysvar::epoch_schedule::id()).unwrap(),
+            SysvarAccountType::EpochSchedule(epoch_schedule),
+        );
+
+        let fees_sysvar = create_account(&Fees::default(), 1);
+        assert_eq!(
+            parse_sysvar(&fees_sysvar.data, &sysvar::fees::id()).unwrap(),
+            SysvarAccountType::Fees(UiFees::default()),
+        );
+
+        let hash = Hash::new(&[1; 32]);
+        let fee_calculator = FeeCalculator {
+            lamports_per_signature: 10,
+        };
+        let recent_blockhashes =
+            RecentBlockhashes::from_iter(vec![IterItem(0, &hash, &fee_calculator)].into_iter());
+        let recent_blockhashes_sysvar = create_account(&recent_blockhashes, 1);
+        assert_eq!(
+            parse_sysvar(
+                &recent_blockhashes_sysvar.data,
+                &sysvar::recent_blockhashes::id()
+            )
+            .unwrap(),
+            SysvarAccountType::RecentBlockhashes(vec![UiRecentBlockhashesEntry {
+                blockhash: hash.to_string(),
+                fee_calculator: fee_calculator.into(),
+            }]),
+        );
+
+        let rent = Rent {
+            lamports_per_byte_year: 10,
+            exemption_threshold: 2.0,
+            burn_percent: 5,
+        };
+        let rent_sysvar = create_account(&rent, 1);
+        assert_eq!(
+            parse_sysvar(&rent_sysvar.data, &sysvar::rent::id()).unwrap(),
+            SysvarAccountType::Rent(rent.into()),
+        );
+
+        let rewards_sysvar = create_account(&Rewards::default(), 1);
+        assert_eq!(
+            parse_sysvar(&rewards_sysvar.data, &sysvar::rewards::id()).unwrap(),
+            SysvarAccountType::Rewards(UiRewards::default()),
+        );
+
+        let mut slot_hashes = SlotHashes::default();
+        slot_hashes.add(1, hash);
+        let slot_hashes_sysvar = create_account(&slot_hashes, 1);
+        assert_eq!(
+            parse_sysvar(&slot_hashes_sysvar.data, &sysvar::slot_hashes::id()).unwrap(),
+            SysvarAccountType::SlotHashes(vec![UiSlotHashEntry {
+                slot: 1,
+                hash: hash.to_string(),
+            }]),
+        );
+
+        let mut slot_history = SlotHistory::default();
+        slot_history.add(42);
+        let slot_history_sysvar = create_account(&slot_history, 1);
+        assert_eq!(
+            parse_sysvar(&slot_history_sysvar.data, &sysvar::slot_history::id()).unwrap(),
+            SysvarAccountType::SlotHistory(UiSlotHistory {
+                next_slot: slot_history.next_slot,
+                bits: format!("{:?}", SlotHistoryBits(slot_history.bits)),
+            }),
+        );
+
+        let mut stake_history = StakeHistory::default();
+        let stake_history_entry = StakeHistoryEntry {
+            effective: 10,
+            activating: 2,
+            deactivating: 3,
+        };
+        stake_history.add(1, stake_history_entry.clone());
+        let stake_history_sysvar = create_account(&stake_history, 1);
+        assert_eq!(
+            parse_sysvar(&stake_history_sysvar.data, &sysvar::stake_history::id()).unwrap(),
+            SysvarAccountType::StakeHistory(vec![UiStakeHistoryEntry {
+                epoch: 1,
+                stake_history: stake_history_entry,
+            }]),
+        );
+
+        let bad_pubkey = solana_sdk::pubkey::new_rand();
+        assert!(parse_sysvar(&stake_history_sysvar.data, &bad_pubkey).is_err());
+
+        let bad_data = vec![0; 4];
+        assert!(parse_sysvar(&bad_data, &sysvar::stake_history::id()).is_err());
+    }
+}

account-decoder/src/parse_token.rs

@@ -0,0 +1,352 @@
+use crate::{
+    parse_account_data::{ParsableAccount, ParseAccountError},
+    StringAmount,
+};
+use solana_sdk::pubkey::Pubkey;
+use spl_token_v2_0::{
+    solana_program::{
+        program_option::COption, program_pack::Pack, pubkey::Pubkey as SplTokenPubkey,
+    },
+    state::{Account, AccountState, Mint, Multisig},
+};
+use std::str::FromStr;
+
+// A helper function to convert spl_token_v2_0::id() as spl_sdk::pubkey::Pubkey to
+// solana_sdk::pubkey::Pubkey
+pub fn spl_token_id_v2_0() -> Pubkey {
+    Pubkey::from_str(&spl_token_v2_0::id().to_string()).unwrap()
+}
+
+// A helper function to convert spl_token_v2_0::native_mint::id() as spl_sdk::pubkey::Pubkey to
+// solana_sdk::pubkey::Pubkey
+pub fn spl_token_v2_0_native_mint() -> Pubkey {
+    Pubkey::from_str(&spl_token_v2_0::native_mint::id().to_string()).unwrap()
+}
+
+// A helper function to convert a solana_sdk::pubkey::Pubkey to spl_sdk::pubkey::Pubkey
+pub fn spl_token_v2_0_pubkey(pubkey: &Pubkey) -> SplTokenPubkey {
+    SplTokenPubkey::from_str(&pubkey.to_string()).unwrap()
+}
+
+// A helper function to convert a spl_sdk::pubkey::Pubkey to solana_sdk::pubkey::Pubkey
+pub fn pubkey_from_spl_token_v2_0(pubkey: &SplTokenPubkey) -> Pubkey {
+    Pubkey::from_str(&pubkey.to_string()).unwrap()
+}
+
+pub fn parse_token(
+    data: &[u8],
+    mint_decimals: Option<u8>,
+) -> Result<TokenAccountType, ParseAccountError> {
+    if data.len() == Account::get_packed_len() {
+        let account = Account::unpack(data)
+            .map_err(|_| ParseAccountError::AccountNotParsable(ParsableAccount::SplToken))?;
+        let decimals = mint_decimals.ok_or_else(|| {
+            ParseAccountError::AdditionalDataMissing(
+                "no mint_decimals provided to parse spl-token account".to_string(),
+            )
+        })?;
+        Ok(TokenAccountType::Account(UiTokenAccount {
+            mint: account.mint.to_string(),
+            owner: account.owner.to_string(),
+            token_amount: token_amount_to_ui_amount(account.amount, decimals),
+            delegate: match account.delegate {
+                COption::Some(pubkey) => Some(pubkey.to_string()),
+                COption::None => None,
+            },
+            state: account.state.into(),
+            is_native: account.is_native(),
+            rent_exempt_reserve: match account.is_native {
+                COption::Some(reserve) => Some(token_amount_to_ui_amount(reserve, decimals)),
+                COption::None => None,
+            },
+            delegated_amount: if account.delegate.is_none() {
+                None
+            } else {
+                Some(token_amount_to_ui_amount(
+                    account.delegated_amount,
+                    decimals,
+                ))
+            },
+            close_authority: match account.close_authority {
+                COption::Some(pubkey) => Some(pubkey.to_string()),
+                COption::None => None,
+            },
+        }))
+    } else if data.len() == Mint::get_packed_len() {
+        let mint = Mint::unpack(data)
+            .map_err(|_| ParseAccountError::AccountNotParsable(ParsableAccount::SplToken))?;
+        Ok(TokenAccountType::Mint(UiMint {
+            mint_authority: match mint.mint_authority {
+                COption::Some(pubkey) => Some(pubkey.to_string()),
+                COption::None => None,
+            },
+            supply: mint.supply.to_string(),
+            decimals: mint.decimals,
+            is_initialized: mint.is_initialized,
+            freeze_authority: match mint.freeze_authority {
+                COption::Some(pubkey) => Some(pubkey.to_string()),
+                COption::None => None,
+            },
+        }))
+    } else if data.len() == Multisig::get_packed_len() {
+        let multisig = Multisig::unpack(data)
+            .map_err(|_| ParseAccountError::AccountNotParsable(ParsableAccount::SplToken))?;
+        Ok(TokenAccountType::Multisig(UiMultisig {
+            num_required_signers: multisig.m,
+            num_valid_signers: multisig.n,
+            is_initialized: multisig.is_initialized,
+            signers: multisig
+                .signers
+                .iter()
+                .filter_map(|pubkey| {
+                    if pubkey != &SplTokenPubkey::default() {
+                        Some(pubkey.to_string())
+                    } else {
+                        None
+                    }
+                })
+                .collect(),
+        }))
+    } else {
+        Err(ParseAccountError::AccountNotParsable(
+            ParsableAccount::SplToken,
+        ))
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase", tag = "type", content = "info")]
+pub enum TokenAccountType {
+    Account(UiTokenAccount),
+    Mint(UiMint),
+    Multisig(UiMultisig),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiTokenAccount {
+    pub mint: String,
+    pub owner: String,
+    pub token_amount: UiTokenAmount,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub delegate: Option<String>,
+    pub state: UiAccountState,
+    pub is_native: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub rent_exempt_reserve: Option<UiTokenAmount>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub delegated_amount: Option<UiTokenAmount>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub close_authority: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub enum UiAccountState {
+    Uninitialized,
+    Initialized,
+    Frozen,
+}
+
+impl From<AccountState> for UiAccountState {
+    fn from(state: AccountState) -> Self {
+        match state {
+            AccountState::Uninitialized => UiAccountState::Uninitialized,
+            AccountState::Initialized => UiAccountState::Initialized,
+            AccountState::Frozen => UiAccountState::Frozen,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiTokenAmount {
+    pub ui_amount: f64,
+    pub decimals: u8,
+    pub amount: StringAmount,
+}
+
+impl UiTokenAmount {
+    pub fn real_number_string(&self) -> String {
+        let decimals = self.decimals as usize;
+        if decimals > 0 {
+            let amount = u64::from_str(&self.amount).unwrap_or(0);
+
+            // Left-pad zeros to decimals + 1, so we at least have an integer zero
+            let mut s = format!("{:01$}", amount, decimals + 1);
+
+            // Add the decimal point (Sorry, "," locales!)
+            s.insert(s.len() - decimals, '.');
+            s
+        } else {
+            self.amount.clone()
+        }
+    }
+
+    pub fn real_number_string_trimmed(&self) -> String {
+        let s = self.real_number_string();
+        let zeros_trimmed = s.trim_end_matches('0');
+        let decimal_trimmed = zeros_trimmed.trim_end_matches('.');
+        decimal_trimmed.to_string()
+    }
+}
+
+pub fn token_amount_to_ui_amount(amount: u64, decimals: u8) -> UiTokenAmount {
+    // Use `amount_to_ui_amount()` once spl_token is bumped to a version that supports it: https://github.com/solana-labs/solana-program-library/pull/211
+    let amount_decimals = amount as f64 / 10_usize.pow(decimals as u32) as f64;
+    UiTokenAmount {
+        ui_amount: amount_decimals,
+        decimals,
+        amount: amount.to_string(),
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiMint {
+    pub mint_authority: Option<String>,
+    pub supply: StringAmount,
+    pub decimals: u8,
+    pub is_initialized: bool,
+    pub freeze_authority: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiMultisig {
+    pub num_required_signers: u8,
+    pub num_valid_signers: u8,
+    pub is_initialized: bool,
+    pub signers: Vec<String>,
+}
+
+pub fn get_token_account_mint(data: &[u8]) -> Option<Pubkey> {
+    if data.len() == Account::get_packed_len() {
+        Some(Pubkey::new(&data[0..32]))
+    } else {
+        None
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_parse_token() {
+        let mint_pubkey = SplTokenPubkey::new(&[2; 32]);
+        let owner_pubkey = SplTokenPubkey::new(&[3; 32]);
+        let mut account_data = vec![0; Account::get_packed_len()];
+        let mut account = Account::unpack_unchecked(&account_data).unwrap();
+        account.mint = mint_pubkey;
+        account.owner = owner_pubkey;
+        account.amount = 42;
+        account.state = AccountState::Initialized;
+        account.is_native = COption::None;
+        account.close_authority = COption::Some(owner_pubkey);
+        Account::pack(account, &mut account_data).unwrap();
+
+        assert!(parse_token(&account_data, None).is_err());
+        assert_eq!(
+            parse_token(&account_data, Some(2)).unwrap(),
+            TokenAccountType::Account(UiTokenAccount {
+                mint: mint_pubkey.to_string(),
+                owner: owner_pubkey.to_string(),
+                token_amount: UiTokenAmount {
+                    ui_amount: 0.42,
+                    decimals: 2,
+                    amount: "42".to_string()
+                },
+                delegate: None,
+                state: UiAccountState::Initialized,
+                is_native: false,
+                rent_exempt_reserve: None,
+                delegated_amount: None,
+                close_authority: Some(owner_pubkey.to_string()),
+            }),
+        );
+
+        let mut mint_data = vec![0; Mint::get_packed_len()];
+        let mut mint = Mint::unpack_unchecked(&mint_data).unwrap();
+        mint.mint_authority = COption::Some(owner_pubkey);
+        mint.supply = 42;
+        mint.decimals = 3;
+        mint.is_initialized = true;
+        mint.freeze_authority = COption::Some(owner_pubkey);
+        Mint::pack(mint, &mut mint_data).unwrap();
+
+        assert_eq!(
+            parse_token(&mint_data, None).unwrap(),
+            TokenAccountType::Mint(UiMint {
+                mint_authority: Some(owner_pubkey.to_string()),
+                supply: 42.to_string(),
+                decimals: 3,
+                is_initialized: true,
+                freeze_authority: Some(owner_pubkey.to_string()),
+            }),
+        );
+
+        let signer1 = SplTokenPubkey::new(&[1; 32]);
+        let signer2 = SplTokenPubkey::new(&[2; 32]);
+        let signer3 = SplTokenPubkey::new(&[3; 32]);
+        let mut multisig_data = vec![0; Multisig::get_packed_len()];
+        let mut signers = [SplTokenPubkey::default(); 11];
+        signers[0] = signer1;
+        signers[1] = signer2;
+        signers[2] = signer3;
+        let mut multisig = Multisig::unpack_unchecked(&multisig_data).unwrap();
+        multisig.m = 2;
+        multisig.n = 3;
+        multisig.is_initialized = true;
+        multisig.signers = signers;
+        Multisig::pack(multisig, &mut multisig_data).unwrap();
+
+        assert_eq!(
+            parse_token(&multisig_data, None).unwrap(),
+            TokenAccountType::Multisig(UiMultisig {
+                num_required_signers: 2,
+                num_valid_signers: 3,
+                is_initialized: true,
+                signers: vec![
+                    signer1.to_string(),
+                    signer2.to_string(),
+                    signer3.to_string()
+                ],
+            }),
+        );
+
+        let bad_data = vec![0; 4];
+        assert!(parse_token(&bad_data, None).is_err());
+    }
+
+    #[test]
+    fn test_get_token_account_mint() {
+        let mint_pubkey = SplTokenPubkey::new(&[2; 32]);
+        let mut account_data = vec![0; Account::get_packed_len()];
+        let mut account = Account::unpack_unchecked(&account_data).unwrap();
+        account.mint = mint_pubkey;
+        Account::pack(account, &mut account_data).unwrap();
+
+        let expected_mint_pubkey = Pubkey::new(&[2; 32]);
+        assert_eq!(
+            get_token_account_mint(&account_data),
+            Some(expected_mint_pubkey)
+        );
+    }
+
+    #[test]
+    fn test_ui_token_amount_real_string() {
+        let token_amount = token_amount_to_ui_amount(1, 0);
+        assert_eq!(&token_amount.real_number_string(), "1");
+        assert_eq!(&token_amount.real_number_string_trimmed(), "1");
+        let token_amount = token_amount_to_ui_amount(1, 9);
+        assert_eq!(&token_amount.real_number_string(), "0.000000001");
+        assert_eq!(&token_amount.real_number_string_trimmed(), "0.000000001");
+        let token_amount = token_amount_to_ui_amount(1_000_000_000, 9);
+        assert_eq!(&token_amount.real_number_string(), "1.000000000");
+        assert_eq!(&token_amount.real_number_string_trimmed(), "1");
+        let token_amount = token_amount_to_ui_amount(1_234_567_890, 3);
+        assert_eq!(&token_amount.real_number_string(), "1234567.890");
+        assert_eq!(&token_amount.real_number_string_trimmed(), "1234567.89");
+    }
+}

account-decoder/src/parse_vote.rs

@@ -0,0 +1,144 @@
+use crate::{parse_account_data::ParseAccountError, StringAmount};
+use solana_sdk::{
+    clock::{Epoch, Slot},
+    pubkey::Pubkey,
+};
+use solana_vote_program::vote_state::{BlockTimestamp, Lockout, VoteState};
+
+pub fn parse_vote(data: &[u8]) -> Result<VoteAccountType, ParseAccountError> {
+    let mut vote_state = VoteState::deserialize(data).map_err(ParseAccountError::from)?;
+    let epoch_credits = vote_state
+        .epoch_credits()
+        .iter()
+        .map(|(epoch, credits, previous_credits)| UiEpochCredits {
+            epoch: *epoch,
+            credits: credits.to_string(),
+            previous_credits: previous_credits.to_string(),
+        })
+        .collect();
+    let votes = vote_state
+        .votes
+        .iter()
+        .map(|lockout| UiLockout {
+            slot: lockout.slot,
+            confirmation_count: lockout.confirmation_count,
+        })
+        .collect();
+    let authorized_voters = vote_state
+        .authorized_voters()
+        .iter()
+        .map(|(epoch, authorized_voter)| UiAuthorizedVoters {
+            epoch: *epoch,
+            authorized_voter: authorized_voter.to_string(),
+        })
+        .collect();
+    let prior_voters = vote_state
+        .prior_voters()
+        .buf()
+        .iter()
+        .filter(|(pubkey, _, _)| pubkey != &Pubkey::default())
+        .map(
+            |(authorized_pubkey, epoch_of_last_authorized_switch, target_epoch)| UiPriorVoters {
+                authorized_pubkey: authorized_pubkey.to_string(),
+                epoch_of_last_authorized_switch: *epoch_of_last_authorized_switch,
+                target_epoch: *target_epoch,
+            },
+        )
+        .collect();
+    Ok(VoteAccountType::Vote(UiVoteState {
+        node_pubkey: vote_state.node_pubkey.to_string(),
+        authorized_withdrawer: vote_state.authorized_withdrawer.to_string(),
+        commission: vote_state.commission,
+        votes,
+        root_slot: vote_state.root_slot,
+        authorized_voters,
+        prior_voters,
+        epoch_credits,
+        last_timestamp: vote_state.last_timestamp,
+    }))
+}
+
+/// A wrapper enum for consistency across programs
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase", tag = "type", content = "info")]
+pub enum VoteAccountType {
+    Vote(UiVoteState),
+}
+
+/// A duplicate representation of VoteState for pretty JSON serialization
+#[derive(Debug, Serialize, Deserialize, Default, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiVoteState {
+    node_pubkey: String,
+    authorized_withdrawer: String,
+    commission: u8,
+    votes: Vec<UiLockout>,
+    root_slot: Option<Slot>,
+    authorized_voters: Vec<UiAuthorizedVoters>,
+    prior_voters: Vec<UiPriorVoters>,
+    epoch_credits: Vec<UiEpochCredits>,
+    last_timestamp: BlockTimestamp,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+struct UiLockout {
+    slot: Slot,
+    confirmation_count: u32,
+}
+
+impl From<&Lockout> for UiLockout {
+    fn from(lockout: &Lockout) -> Self {
+        Self {
+            slot: lockout.slot,
+            confirmation_count: lockout.confirmation_count,
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+struct UiAuthorizedVoters {
+    epoch: Epoch,
+    authorized_voter: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+struct UiPriorVoters {
+    authorized_pubkey: String,
+    epoch_of_last_authorized_switch: Epoch,
+    target_epoch: Epoch,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+struct UiEpochCredits {
+    epoch: Epoch,
+    credits: StringAmount,
+    previous_credits: StringAmount,
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use solana_vote_program::vote_state::VoteStateVersions;
+
+    #[test]
+    fn test_parse_vote() {
+        let vote_state = VoteState::default();
+        let mut vote_account_data: Vec<u8> = vec![0; VoteState::size_of()];
+        let versioned = VoteStateVersions::new_current(vote_state);
+        VoteState::serialize(&versioned, &mut vote_account_data).unwrap();
+        let mut expected_vote_state = UiVoteState::default();
+        expected_vote_state.node_pubkey = Pubkey::default().to_string();
+        expected_vote_state.authorized_withdrawer = Pubkey::default().to_string();
+        assert_eq!(
+            parse_vote(&vote_account_data).unwrap(),
+            VoteAccountType::Vote(expected_vote_state)
+        );
+
+        let bad_data = vec![0; 4];
+        assert!(parse_vote(&bad_data).is_err());
+    }
+}

account-decoder/src/validator_info.rs

@@ -0,0 +1,18 @@
+use solana_config_program::ConfigState;
+
+pub const MAX_SHORT_FIELD_LENGTH: usize = 70;
+pub const MAX_LONG_FIELD_LENGTH: usize = 300;
+pub const MAX_VALIDATOR_INFO: u64 = 576;
+
+solana_sdk::declare_id!("Va1idator1nfo111111111111111111111111111111");
+
+#[derive(Debug, Deserialize, PartialEq, Serialize, Default)]
+pub struct ValidatorInfo {
+    pub info: String,
+}
+
+impl ConfigState for ValidatorInfo {
+    fn max_space() -> u64 {
+        MAX_VALIDATOR_INFO
+    }
+}

accounts-bench/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2018"
+name = "solana-accounts-bench"
+version = "1.4.26"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+publish = false
+
+[dependencies]
+log = "0.4.6"
+rayon = "1.4.0"
+solana-logger = { path = "../logger", version = "1.4.26" }
+solana-runtime = { path = "../runtime", version = "1.4.26" }
+solana-measure = { path = "../measure", version = "1.4.26" }
+solana-sdk = { path = "../sdk", version = "1.4.26" }
+solana-version = { path = "../version", version = "1.4.26" }
+rand = "0.7.0"
+clap = "2.33.1"
+crossbeam-channel = "0.4"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accounts-bench/src/main.rs

@@ -0,0 +1,110 @@
+use clap::{crate_description, crate_name, value_t, App, Arg};
+use rayon::prelude::*;
+use solana_measure::measure::Measure;
+use solana_runtime::{
+    accounts::{create_test_accounts, update_accounts, Accounts},
+    accounts_index::Ancestors,
+};
+use solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey};
+use std::env;
+use std::fs;
+use std::path::PathBuf;
+
+fn main() {
+    solana_logger::setup();
+
+    let matches = App::new(crate_name!())
+        .about(crate_description!())
+        .version(solana_version::version!())
+        .arg(
+            Arg::with_name("num_slots")
+                .long("num_slots")
+                .takes_value(true)
+                .value_name("SLOTS")
+                .help("Number of slots to store to."),
+        )
+        .arg(
+            Arg::with_name("num_accounts")
+                .long("num_accounts")
+                .takes_value(true)
+                .value_name("NUM_ACCOUNTS")
+                .help("Total number of accounts"),
+        )
+        .arg(
+            Arg::with_name("iterations")
+                .long("iterations")
+                .takes_value(true)
+                .value_name("ITERATIONS")
+                .help("Number of bench iterations"),
+        )
+        .arg(
+            Arg::with_name("clean")
+                .long("clean")
+                .takes_value(false)
+                .help("Run clean"),
+        )
+        .get_matches();
+
+    let num_slots = value_t!(matches, "num_slots", usize).unwrap_or(4);
+    let num_accounts = value_t!(matches, "num_accounts", usize).unwrap_or(10_000);
+    let iterations = value_t!(matches, "iterations", usize).unwrap_or(20);
+    let clean = matches.is_present("clean");
+    println!("clean: {:?}", clean);
+
+    let path = PathBuf::from(env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_owned()))
+        .join("accounts-bench");
+    if fs::remove_dir_all(path.clone()).is_err() {
+        println!("Warning: Couldn't remove {:?}", path);
+    }
+    let accounts = Accounts::new(vec![path], &ClusterType::Testnet);
+    println!("Creating {} accounts", num_accounts);
+    let mut create_time = Measure::start("create accounts");
+    let pubkeys: Vec<_> = (0..num_slots)
+        .into_par_iter()
+        .map(|slot| {
+            let mut pubkeys: Vec<Pubkey> = vec![];
+            create_test_accounts(
+                &accounts,
+                &mut pubkeys,
+                num_accounts / num_slots,
+                slot as u64,
+            );
+            pubkeys
+        })
+        .collect();
+    let pubkeys: Vec<_> = pubkeys.into_iter().flatten().collect();
+    create_time.stop();
+    println!(
+        "created {} accounts in {} slots {}",
+        (num_accounts / num_slots) * num_slots,
+        num_slots,
+        create_time
+    );
+    let mut ancestors: Ancestors = vec![(0, 0)].into_iter().collect();
+    for i in 1..num_slots {
+        ancestors.insert(i as u64, i - 1);
+        accounts.add_root(i as u64);
+    }
+    for x in 0..iterations {
+        if clean {
+            let mut time = Measure::start("clean");
+            accounts.accounts_db.clean_accounts(None);
+            time.stop();
+            println!("{}", time);
+            for slot in 0..num_slots {
+                update_accounts(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64);
+                accounts.add_root((x * num_slots + slot) as u64);
+            }
+        } else {
+            let mut pubkeys: Vec<Pubkey> = vec![];
+            let mut time = Measure::start("hash");
+            let hash = accounts
+                .accounts_db
+                .update_accounts_hash(0, &ancestors, true)
+                .0;
+            time.stop();
+            println!("hash: {} {}", hash, time);
+            create_test_accounts(&accounts, &mut pubkeys, 1, 0);
+        }
+    }
+}

archiver-lib/Cargo.toml

@@ -1,39 +0,0 @@
-[package]
-name = "solana-archiver-lib"
-version = "1.0.13"
-description = "Solana Archiver Library"
-authors = ["Solana Maintainers <maintainers@solana.com>"]
-repository = "https://github.com/solana-labs/solana"
-license = "Apache-2.0"
-homepage = "https://solana.com/"
-edition = "2018"
-
-[dependencies]
-bincode = "1.2.1"
-crossbeam-channel = "0.3"
-ed25519-dalek = "=1.0.0-pre.1"
-log = "0.4.8"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
-solana-client = { path = "../client", version = "1.0.13" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.13" }
-thiserror = "1.0"
-serde = "1.0.104"
-serde_json = "1.0.46"
-serde_derive = "1.0.103"
-solana-net-utils = { path = "../net-utils", version = "1.0.13" }
-solana-chacha = { path = "../chacha", version = "1.0.13" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.13" }
-solana-ledger = { path = "../ledger", version = "1.0.13" }
-solana-logger = { path = "../logger", version = "1.0.13" }
-solana-perf = { path = "../perf", version = "1.0.13" }
-solana-sdk = { path = "../sdk", version = "1.0.13" }
-solana-core = { path = "../core", version = "1.0.13" }
-solana-archiver-utils = { path = "../archiver-utils", version = "1.0.13" }
-solana-metrics = { path = "../metrics", version = "1.0.13" }
-
-[dev-dependencies]
-hex = "0.4.0"
-
-[lib]
-name = "solana_archiver_lib"

archiver-lib/src/archiver.rs

@@ -1,937 +0,0 @@
-use crate::result::ArchiverError;
-use crossbeam_channel::unbounded;
-use rand::{thread_rng, Rng, SeedableRng};
-use rand_chacha::ChaChaRng;
-use solana_archiver_utils::sample_file;
-use solana_chacha::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE};
-use solana_client::{
-    rpc_client::RpcClient, rpc_request::RpcRequest, rpc_response::RpcStorageTurn,
-    thin_client::ThinClient,
-};
-use solana_core::{
-    cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE},
-    contact_info::ContactInfo,
-    gossip_service::GossipService,
-    packet::{limited_deserialize, PACKET_DATA_SIZE},
-    repair_service,
-    repair_service::{RepairService, RepairSlotRange, RepairStrategy},
-    serve_repair::ServeRepair,
-    shred_fetch_stage::ShredFetchStage,
-    sigverify_stage::{DisabledSigVerifier, SigVerifyStage},
-    storage_stage::NUM_STORAGE_SAMPLES,
-    streamer::{receiver, responder, PacketReceiver},
-    window_service::WindowService,
-};
-use solana_ledger::{
-    blockstore::Blockstore, leader_schedule_cache::LeaderScheduleCache, shred::Shred,
-};
-use solana_net_utils::bind_in_range;
-use solana_perf::packet::Packets;
-use solana_perf::recycler::Recycler;
-use solana_sdk::packet::Packet;
-use solana_sdk::{
-    account_utils::StateMut,
-    client::{AsyncClient, SyncClient},
-    clock::{get_complete_segment_from_slot, get_segment_from_slot, Slot},
-    commitment_config::CommitmentConfig,
-    hash::Hash,
-    message::Message,
-    signature::{Keypair, Signature, Signer},
-    timing::timestamp,
-    transaction::Transaction,
-    transport::TransportError,
-};
-use solana_storage_program::{
-    storage_contract::StorageContract,
-    storage_instruction::{self, StorageAccountType},
-};
-use std::{
-    io::{self, ErrorKind},
-    net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
-    path::{Path, PathBuf},
-    result,
-    sync::atomic::{AtomicBool, Ordering},
-    sync::mpsc::{channel, Receiver, Sender},
-    sync::{Arc, RwLock},
-    thread::{sleep, spawn, JoinHandle},
-    time::Duration,
-};
-
-type Result<T> = std::result::Result<T, ArchiverError>;
-
-static ENCRYPTED_FILENAME: &str = "ledger.enc";
-
-#[derive(Serialize, Deserialize)]
-pub enum ArchiverRequest {
-    GetSlotHeight(SocketAddr),
-}
-
-pub struct Archiver {
-    thread_handles: Vec<JoinHandle<()>>,
-    exit: Arc<AtomicBool>,
-}
-
-// Shared Archiver Meta struct used internally
-#[derive(Default)]
-struct ArchiverMeta {
-    slot: Slot,
-    slots_per_segment: u64,
-    ledger_path: PathBuf,
-    signature: Signature,
-    ledger_data_file_encrypted: PathBuf,
-    sampling_offsets: Vec<u64>,
-    blockhash: Hash,
-    sha_state: Hash,
-    num_chacha_blocks: usize,
-    client_commitment: CommitmentConfig,
-}
-
-fn get_slot_from_signature(
-    signature: &Signature,
-    storage_turn: u64,
-    slots_per_segment: u64,
-) -> u64 {
-    let signature_vec = signature.as_ref();
-    let mut segment_index = u64::from(signature_vec[0])
-        | (u64::from(signature_vec[1]) << 8)
-        | (u64::from(signature_vec[1]) << 16)
-        | (u64::from(signature_vec[2]) << 24);
-    let max_segment_index =
-        get_complete_segment_from_slot(storage_turn, slots_per_segment).unwrap();
-    segment_index %= max_segment_index as u64;
-    segment_index * slots_per_segment
-}
-
-fn create_request_processor(
-    socket: UdpSocket,
-    exit: &Arc<AtomicBool>,
-    slot_receiver: Receiver<u64>,
-) -> Vec<JoinHandle<()>> {
-    let mut thread_handles = vec![];
-    let (s_reader, r_reader) = channel();
-    let (s_responder, r_responder) = channel();
-    let storage_socket = Arc::new(socket);
-    let recycler = Recycler::default();
-    let t_receiver = receiver(storage_socket.clone(), exit, s_reader, recycler, "archiver");
-    thread_handles.push(t_receiver);
-
-    let t_responder = responder("archiver-responder", storage_socket, r_responder);
-    thread_handles.push(t_responder);
-
-    let exit = exit.clone();
-    let t_processor = spawn(move || {
-        let slot = poll_for_slot(slot_receiver, &exit);
-
-        loop {
-            if exit.load(Ordering::Relaxed) {
-                break;
-            }
-
-            let packets = r_reader.recv_timeout(Duration::from_secs(1));
-
-            if let Ok(packets) = packets {
-                for packet in &packets.packets {
-                    let req: result::Result<ArchiverRequest, Box<bincode::ErrorKind>> =
-                        limited_deserialize(&packet.data[..packet.meta.size]);
-                    match req {
-                        Ok(ArchiverRequest::GetSlotHeight(from)) => {
-                            let packet = Packet::from_data(&from, slot);
-                            let _ = s_responder.send(Packets::new(vec![packet]));
-                        }
-                        Err(e) => {
-                            info!("invalid request: {:?}", e);
-                        }
-                    }
-                }
-            }
-        }
-    });
-    thread_handles.push(t_processor);
-    thread_handles
-}
-
-fn poll_for_slot(receiver: Receiver<u64>, exit: &Arc<AtomicBool>) -> u64 {
-    loop {
-        let slot = receiver.recv_timeout(Duration::from_secs(1));
-        if let Ok(slot) = slot {
-            return slot;
-        }
-        if exit.load(Ordering::Relaxed) {
-            return 0;
-        }
-    }
-}
-
-impl Archiver {
-    /// Returns a Result that contains an archiver on success
-    ///
-    /// # Arguments
-    /// * `ledger_path` - path to where the ledger will be stored.
-    /// Causes panic if none
-    /// * `node` - The archiver node
-    /// * `cluster_entrypoint` - ContactInfo representing an entry into the network
-    /// * `keypair` - Keypair for this archiver
-    #[allow(clippy::new_ret_no_self)]
-    pub fn new(
-        ledger_path: &Path,
-        node: Node,
-        cluster_entrypoint: ContactInfo,
-        keypair: Arc<Keypair>,
-        storage_keypair: Arc<Keypair>,
-        client_commitment: CommitmentConfig,
-    ) -> Result<Self> {
-        let exit = Arc::new(AtomicBool::new(false));
-
-        info!("Archiver: id: {}", keypair.pubkey());
-        info!("Creating cluster info....");
-        let mut cluster_info = ClusterInfo::new(node.info.clone(), keypair.clone());
-        cluster_info.set_entrypoint(cluster_entrypoint.clone());
-        let cluster_info = Arc::new(RwLock::new(cluster_info));
-
-        // Note for now, this ledger will not contain any of the existing entries
-        // in the ledger located at ledger_path, and will only append on newly received
-        // entries after being passed to window_service
-        let blockstore = Arc::new(
-            Blockstore::open(ledger_path).expect("Expected to be able to open database ledger"),
-        );
-
-        let gossip_service = GossipService::new(&cluster_info, None, node.sockets.gossip, &exit);
-
-        info!("Connecting to the cluster via {:?}", cluster_entrypoint);
-        let (nodes, _) =
-            match solana_core::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 1) {
-                Ok(nodes_and_archivers) => nodes_and_archivers,
-                Err(e) => {
-                    //shutdown services before exiting
-                    exit.store(true, Ordering::Relaxed);
-                    gossip_service.join()?;
-                    return Err(e.into());
-                }
-            };
-        let client = solana_core::gossip_service::get_client(&nodes);
-
-        info!("Setting up mining account...");
-        if let Err(e) =
-            Self::setup_mining_account(&client, &keypair, &storage_keypair, client_commitment)
-        {
-            //shutdown services before exiting
-            exit.store(true, Ordering::Relaxed);
-            gossip_service.join()?;
-            return Err(e);
-        };
-
-        let repair_socket = Arc::new(node.sockets.repair);
-        let shred_sockets: Vec<Arc<UdpSocket>> =
-            node.sockets.tvu.into_iter().map(Arc::new).collect();
-        let shred_forward_sockets: Vec<Arc<UdpSocket>> = node
-            .sockets
-            .tvu_forwards
-            .into_iter()
-            .map(Arc::new)
-            .collect();
-        let (shred_fetch_sender, shred_fetch_receiver) = channel();
-        let fetch_stage = ShredFetchStage::new(
-            shred_sockets,
-            shred_forward_sockets,
-            repair_socket.clone(),
-            &shred_fetch_sender,
-            None,
-            &exit,
-        );
-        let (slot_sender, slot_receiver) = channel();
-        let request_processor =
-            create_request_processor(node.sockets.storage.unwrap(), &exit, slot_receiver);
-
-        let t_archiver = {
-            let exit = exit.clone();
-            let node_info = node.info.clone();
-            let mut meta = ArchiverMeta {
-                ledger_path: ledger_path.to_path_buf(),
-                client_commitment,
-                ..ArchiverMeta::default()
-            };
-            spawn(move || {
-                // setup archiver
-                let window_service = match Self::setup(
-                    &mut meta,
-                    cluster_info.clone(),
-                    &blockstore,
-                    &exit,
-                    &node_info,
-                    &storage_keypair,
-                    repair_socket,
-                    shred_fetch_receiver,
-                    slot_sender,
-                ) {
-                    Ok(window_service) => window_service,
-                    Err(e) => {
-                        //shutdown services before exiting
-                        error!("setup failed {:?}; archiver thread exiting...", e);
-                        exit.store(true, Ordering::Relaxed);
-                        request_processor
-                            .into_iter()
-                            .for_each(|t| t.join().unwrap());
-                        fetch_stage.join().unwrap();
-                        gossip_service.join().unwrap();
-                        return;
-                    }
-                };
-
-                info!("setup complete");
-                // run archiver
-                Self::run(
-                    &mut meta,
-                    &blockstore,
-                    cluster_info,
-                    &keypair,
-                    &storage_keypair,
-                    &exit,
-                );
-                // wait until exit
-                request_processor
-                    .into_iter()
-                    .for_each(|t| t.join().unwrap());
-                fetch_stage.join().unwrap();
-                gossip_service.join().unwrap();
-                window_service.join().unwrap()
-            })
-        };
-
-        Ok(Self {
-            thread_handles: vec![t_archiver],
-            exit,
-        })
-    }
-
-    fn run(
-        meta: &mut ArchiverMeta,
-        blockstore: &Arc<Blockstore>,
-        cluster_info: Arc<RwLock<ClusterInfo>>,
-        archiver_keypair: &Arc<Keypair>,
-        storage_keypair: &Arc<Keypair>,
-        exit: &Arc<AtomicBool>,
-    ) {
-        // encrypt segment
-        Self::encrypt_ledger(meta, blockstore).expect("ledger encrypt not successful");
-        let enc_file_path = meta.ledger_data_file_encrypted.clone();
-        // do replicate
-        loop {
-            if exit.load(Ordering::Relaxed) {
-                break;
-            }
-
-            // TODO check if more segments are available - based on space constraints
-            Self::create_sampling_offsets(meta);
-            let sampling_offsets = &meta.sampling_offsets;
-            meta.sha_state =
-                match Self::sample_file_to_create_mining_hash(&enc_file_path, sampling_offsets) {
-                    Ok(hash) => hash,
-                    Err(err) => {
-                        warn!("Error sampling file, exiting: {:?}", err);
-                        break;
-                    }
-                };
-
-            Self::submit_mining_proof(meta, &cluster_info, archiver_keypair, storage_keypair);
-
-            // TODO make this a lot more frequent by picking a "new" blockhash instead of picking a storage blockhash
-            // prep the next proof
-            let (storage_blockhash, _) = match Self::poll_for_blockhash_and_slot(
-                &cluster_info,
-                meta.slots_per_segment,
-                &meta.blockhash,
-                exit,
-            ) {
-                Ok(blockhash_and_slot) => blockhash_and_slot,
-                Err(e) => {
-                    warn!(
-                        "Error couldn't get a newer blockhash than {:?}. {:?}",
-                        meta.blockhash, e
-                    );
-                    break;
-                }
-            };
-            meta.blockhash = storage_blockhash;
-            Self::redeem_rewards(
-                &cluster_info,
-                archiver_keypair,
-                storage_keypair,
-                meta.client_commitment,
-            );
-        }
-        exit.store(true, Ordering::Relaxed);
-    }
-
-    fn redeem_rewards(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
-        archiver_keypair: &Arc<Keypair>,
-        storage_keypair: &Arc<Keypair>,
-        client_commitment: CommitmentConfig,
-    ) {
-        let nodes = cluster_info.read().unwrap().tvu_peers();
-        let client = solana_core::gossip_service::get_client(&nodes);
-
-        if let Ok(Some(account)) =
-            client.get_account_with_commitment(&storage_keypair.pubkey(), client_commitment)
-        {
-            if let Ok(StorageContract::ArchiverStorage { validations, .. }) = account.state() {
-                if !validations.is_empty() {
-                    let ix = storage_instruction::claim_reward(
-                        &archiver_keypair.pubkey(),
-                        &storage_keypair.pubkey(),
-                    );
-                    let message = Message::new_with_payer(&[ix], Some(&archiver_keypair.pubkey()));
-                    if let Err(e) = client.send_message(&[archiver_keypair.as_ref()], message) {
-                        error!("unable to redeem reward, tx failed: {:?}", e);
-                    } else {
-                        info!(
-                            "collected mining rewards: Account balance {:?}",
-                            client.get_balance_with_commitment(
-                                &archiver_keypair.pubkey(),
-                                client_commitment
-                            )
-                        );
-                    }
-                }
-            }
-        } else {
-            info!("Redeem mining reward: No account data found");
-        }
-    }
-
-    // Find a segment to replicate and download it.
-    fn setup(
-        meta: &mut ArchiverMeta,
-        cluster_info: Arc<RwLock<ClusterInfo>>,
-        blockstore: &Arc<Blockstore>,
-        exit: &Arc<AtomicBool>,
-        node_info: &ContactInfo,
-        storage_keypair: &Arc<Keypair>,
-        repair_socket: Arc<UdpSocket>,
-        shred_fetch_receiver: PacketReceiver,
-        slot_sender: Sender<u64>,
-    ) -> Result<WindowService> {
-        let slots_per_segment =
-            match Self::get_segment_config(&cluster_info, meta.client_commitment) {
-                Ok(slots_per_segment) => slots_per_segment,
-                Err(e) => {
-                    error!("unable to get segment size configuration, exiting...");
-                    //shutdown services before exiting
-                    exit.store(true, Ordering::Relaxed);
-                    return Err(e);
-                }
-            };
-        let (segment_blockhash, segment_slot) = match Self::poll_for_segment(
-            &cluster_info,
-            slots_per_segment,
-            &Hash::default(),
-            exit,
-        ) {
-            Ok(blockhash_and_slot) => blockhash_and_slot,
-            Err(e) => {
-                //shutdown services before exiting
-                exit.store(true, Ordering::Relaxed);
-                return Err(e);
-            }
-        };
-        let signature = storage_keypair.sign_message(segment_blockhash.as_ref());
-        let slot = get_slot_from_signature(&signature, segment_slot, slots_per_segment);
-        info!("replicating slot: {}", slot);
-        slot_sender.send(slot)?;
-        meta.slot = slot;
-        meta.slots_per_segment = slots_per_segment;
-        meta.signature = signature;
-        meta.blockhash = segment_blockhash;
-
-        let mut repair_slot_range = RepairSlotRange::default();
-        repair_slot_range.end = slot + slots_per_segment;
-        repair_slot_range.start = slot;
-
-        let (retransmit_sender, _) = channel();
-
-        let (verified_sender, verified_receiver) = unbounded();
-
-        let _sigverify_stage = SigVerifyStage::new(
-            shred_fetch_receiver,
-            verified_sender,
-            DisabledSigVerifier::default(),
-        );
-
-        let window_service = WindowService::new(
-            blockstore.clone(),
-            cluster_info.clone(),
-            verified_receiver,
-            retransmit_sender,
-            repair_socket,
-            &exit,
-            RepairStrategy::RepairRange(repair_slot_range),
-            &Arc::new(LeaderScheduleCache::default()),
-            |_, _, _, _| true,
-        );
-        info!("waiting for ledger download");
-        Self::wait_for_segment_download(
-            slot,
-            slots_per_segment,
-            &blockstore,
-            &exit,
-            &node_info,
-            cluster_info,
-        );
-        Ok(window_service)
-    }
-
-    fn wait_for_segment_download(
-        start_slot: Slot,
-        slots_per_segment: u64,
-        blockstore: &Arc<Blockstore>,
-        exit: &Arc<AtomicBool>,
-        node_info: &ContactInfo,
-        cluster_info: Arc<RwLock<ClusterInfo>>,
-    ) {
-        info!(
-            "window created, waiting for ledger download starting at slot {:?}",
-            start_slot
-        );
-        let mut current_slot = start_slot;
-        'outer: loop {
-            while blockstore.is_full(current_slot) {
-                current_slot += 1;
-                info!("current slot: {}", current_slot);
-                if current_slot >= start_slot + slots_per_segment {
-                    break 'outer;
-                }
-            }
-            if exit.load(Ordering::Relaxed) {
-                break;
-            }
-            sleep(Duration::from_secs(1));
-        }
-
-        info!("Done receiving entries from window_service");
-
-        // Remove archiver from the data plane
-        let mut contact_info = node_info.clone();
-        contact_info.tvu = "0.0.0.0:0".parse().unwrap();
-        contact_info.wallclock = timestamp();
-        // copy over the adopted shred_version from the entrypoint
-        contact_info.shred_version = cluster_info.read().unwrap().my_data().shred_version;
-        {
-            let mut cluster_info_w = cluster_info.write().unwrap();
-            cluster_info_w.insert_self(contact_info);
-        }
-    }
-
-    fn encrypt_ledger(meta: &mut ArchiverMeta, blockstore: &Arc<Blockstore>) -> Result<()> {
-        meta.ledger_data_file_encrypted = meta.ledger_path.join(ENCRYPTED_FILENAME);
-
-        {
-            let mut ivec = [0u8; 64];
-            ivec.copy_from_slice(&meta.signature.as_ref());
-
-            let num_encrypted_bytes = chacha_cbc_encrypt_ledger(
-                blockstore,
-                meta.slot,
-                meta.slots_per_segment,
-                &meta.ledger_data_file_encrypted,
-                &mut ivec,
-            )?;
-
-            meta.num_chacha_blocks = num_encrypted_bytes / CHACHA_BLOCK_SIZE;
-        }
-
-        info!(
-            "Done encrypting the ledger: {:?}",
-            meta.ledger_data_file_encrypted
-        );
-        Ok(())
-    }
-
-    fn create_sampling_offsets(meta: &mut ArchiverMeta) {
-        meta.sampling_offsets.clear();
-        let mut rng_seed = [0u8; 32];
-        rng_seed.copy_from_slice(&meta.blockhash.as_ref());
-        let mut rng = ChaChaRng::from_seed(rng_seed);
-        for _ in 0..NUM_STORAGE_SAMPLES {
-            meta.sampling_offsets
-                .push(rng.gen_range(0, meta.num_chacha_blocks) as u64);
-        }
-    }
-
-    fn sample_file_to_create_mining_hash(
-        enc_file_path: &Path,
-        sampling_offsets: &[u64],
-    ) -> Result<Hash> {
-        let sha_state = sample_file(enc_file_path, sampling_offsets)?;
-        info!("sampled sha_state: {}", sha_state);
-        Ok(sha_state)
-    }
-
-    fn setup_mining_account(
-        client: &ThinClient,
-        keypair: &Keypair,
-        storage_keypair: &Keypair,
-        client_commitment: CommitmentConfig,
-    ) -> Result<()> {
-        // make sure archiver has some balance
-        info!("checking archiver keypair...");
-        if client.poll_balance_with_timeout_and_commitment(
-            &keypair.pubkey(),
-            &Duration::from_millis(100),
-            &Duration::from_secs(5),
-            client_commitment,
-        )? == 0
-        {
-            return Err(ArchiverError::EmptyStorageAccountBalance);
-        }
-
-        info!("checking storage account keypair...");
-        // check if the storage account exists
-        let balance =
-            client.poll_get_balance_with_commitment(&storage_keypair.pubkey(), client_commitment);
-        if balance.is_err() || balance.unwrap() == 0 {
-            let blockhash = match client.get_recent_blockhash_with_commitment(client_commitment) {
-                Ok((blockhash, _)) => blockhash,
-                Err(e) => {
-                    return Err(ArchiverError::TransportError(e));
-                }
-            };
-
-            let ix = storage_instruction::create_storage_account(
-                &keypair.pubkey(),
-                &keypair.pubkey(),
-                &storage_keypair.pubkey(),
-                1,
-                StorageAccountType::Archiver,
-            );
-            let tx = Transaction::new_signed_instructions(&[keypair], ix, blockhash);
-            let signature = client.async_send_transaction(tx)?;
-            client
-                .poll_for_signature_with_commitment(&signature, client_commitment)
-                .map_err(|err| match err {
-                    TransportError::IoError(e) => e,
-                    TransportError::TransactionError(_) => io::Error::new(
-                        ErrorKind::Other,
-                        "setup_mining_account: signature not found",
-                    ),
-                    TransportError::Custom(e) => io::Error::new(ErrorKind::Other, e),
-                })?;
-        }
-        Ok(())
-    }
-
-    fn submit_mining_proof(
-        meta: &ArchiverMeta,
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
-        archiver_keypair: &Arc<Keypair>,
-        storage_keypair: &Arc<Keypair>,
-    ) {
-        // No point if we've got no storage account...
-        let nodes = cluster_info.read().unwrap().tvu_peers();
-        let client = solana_core::gossip_service::get_client(&nodes);
-        let storage_balance = client
-            .poll_get_balance_with_commitment(&storage_keypair.pubkey(), meta.client_commitment);
-        if storage_balance.is_err() || storage_balance.unwrap() == 0 {
-            error!("Unable to submit mining proof, no storage account");
-            return;
-        }
-        // ...or no lamports for fees
-        let balance = client
-            .poll_get_balance_with_commitment(&archiver_keypair.pubkey(), meta.client_commitment);
-        if balance.is_err() || balance.unwrap() == 0 {
-            error!("Unable to submit mining proof, insufficient Archiver Account balance");
-            return;
-        }
-
-        let blockhash = match client.get_recent_blockhash_with_commitment(meta.client_commitment) {
-            Ok((blockhash, _)) => blockhash,
-            Err(_) => {
-                error!("unable to get recent blockhash, can't submit proof");
-                return;
-            }
-        };
-        let instruction = storage_instruction::mining_proof(
-            &storage_keypair.pubkey(),
-            meta.sha_state,
-            get_segment_from_slot(meta.slot, meta.slots_per_segment),
-            Signature::new(&meta.signature.as_ref()),
-            meta.blockhash,
-        );
-        let message = Message::new_with_payer(&[instruction], Some(&archiver_keypair.pubkey()));
-        let mut transaction = Transaction::new(
-            &[archiver_keypair.as_ref(), storage_keypair.as_ref()],
-            message,
-            blockhash,
-        );
-        if let Err(err) = client.send_and_confirm_transaction(
-            &[archiver_keypair.as_ref(), storage_keypair.as_ref()],
-            &mut transaction,
-            10,
-            0,
-        ) {
-            error!("Error: {:?}; while sending mining proof", err);
-        }
-    }
-
-    pub fn close(self) {
-        self.exit.store(true, Ordering::Relaxed);
-        self.join()
-    }
-
-    pub fn join(self) {
-        for handle in self.thread_handles {
-            handle.join().unwrap();
-        }
-    }
-
-    fn get_segment_config(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
-        client_commitment: CommitmentConfig,
-    ) -> Result<u64> {
-        let rpc_peers = {
-            let cluster_info = cluster_info.read().unwrap();
-            cluster_info.all_rpc_peers()
-        };
-        debug!("rpc peers: {:?}", rpc_peers);
-        if !rpc_peers.is_empty() {
-            let rpc_client = {
-                let node_index = thread_rng().gen_range(0, rpc_peers.len());
-                RpcClient::new_socket(rpc_peers[node_index].rpc)
-            };
-            Ok(rpc_client
-                .send(
-                    &RpcRequest::GetSlotsPerSegment,
-                    serde_json::json!([client_commitment]),
-                    0,
-                )
-                .map_err(|err| {
-                    warn!("Error while making rpc request {:?}", err);
-                    ArchiverError::ClientError(err)
-                })?
-                .as_u64()
-                .unwrap())
-        } else {
-            Err(ArchiverError::NoRpcPeers)
-        }
-    }
-
-    /// Waits until the first segment is ready, and returns the current segment
-    fn poll_for_segment(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
-        slots_per_segment: u64,
-        previous_blockhash: &Hash,
-        exit: &Arc<AtomicBool>,
-    ) -> Result<(Hash, u64)> {
-        loop {
-            let (blockhash, turn_slot) = Self::poll_for_blockhash_and_slot(
-                cluster_info,
-                slots_per_segment,
-                previous_blockhash,
-                exit,
-            )?;
-            if get_complete_segment_from_slot(turn_slot, slots_per_segment).is_some() {
-                return Ok((blockhash, turn_slot));
-            }
-        }
-    }
-
-    /// Poll for a different blockhash and associated max_slot than `previous_blockhash`
-    fn poll_for_blockhash_and_slot(
-        cluster_info: &Arc<RwLock<ClusterInfo>>,
-        slots_per_segment: u64,
-        previous_blockhash: &Hash,
-        exit: &Arc<AtomicBool>,
-    ) -> Result<(Hash, u64)> {
-        info!("waiting for the next turn...");
-        loop {
-            let rpc_peers = {
-                let cluster_info = cluster_info.read().unwrap();
-                cluster_info.all_rpc_peers()
-            };
-            debug!("rpc peers: {:?}", rpc_peers);
-            if !rpc_peers.is_empty() {
-                let rpc_client = {
-                    let node_index = thread_rng().gen_range(0, rpc_peers.len());
-                    RpcClient::new_socket(rpc_peers[node_index].rpc)
-                };
-                let response = rpc_client
-                    .send(
-                        &RpcRequest::GetStorageTurn,
-                        serde_json::value::Value::Null,
-                        0,
-                    )
-                    .map_err(|err| {
-                        warn!("Error while making rpc request {:?}", err);
-                        ArchiverError::ClientError(err)
-                    })?;
-                let RpcStorageTurn {
-                    blockhash: storage_blockhash,
-                    slot: turn_slot,
-                } = serde_json::from_value::<RpcStorageTurn>(response)
-                    .map_err(ArchiverError::JsonError)?;
-                let turn_blockhash = storage_blockhash.parse().map_err(|err| {
-                    io::Error::new(
-                        io::ErrorKind::Other,
-                        format!(
-                            "Blockhash parse failure: {:?} on {:?}",
-                            err, storage_blockhash
-                        ),
-                    )
-                })?;
-                if turn_blockhash != *previous_blockhash {
-                    info!("turn slot: {}", turn_slot);
-                    if get_segment_from_slot(turn_slot, slots_per_segment) != 0 {
-                        return Ok((turn_blockhash, turn_slot));
-                    }
-                }
-            }
-            if exit.load(Ordering::Relaxed) {
-                return Err(ArchiverError::IO(io::Error::new(
-                    ErrorKind::Other,
-                    "exit signalled...",
-                )));
-            }
-            sleep(Duration::from_secs(5));
-        }
-    }
-
-    /// Ask an archiver to populate a given blockstore with its segment.
-    /// Return the slot at the start of the archiver's segment
-    ///
-    /// It is recommended to use a temporary blockstore for this since the download will not verify
-    /// shreds received and might impact the chaining of shreds across slots
-    pub fn download_from_archiver(
-        serve_repair: &ServeRepair,
-        archiver_info: &ContactInfo,
-        blockstore: &Arc<Blockstore>,
-        slots_per_segment: u64,
-    ) -> Result<u64> {
-        let ip_addr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0));
-        // Create a client which downloads from the archiver and see that it
-        // can respond with shreds.
-        let start_slot = Self::get_archiver_segment_slot(ip_addr, archiver_info.storage_addr);
-        info!("Archiver download: start at {}", start_slot);
-
-        let exit = Arc::new(AtomicBool::new(false));
-        let (s_reader, r_reader) = channel();
-        let repair_socket = Arc::new(bind_in_range(ip_addr, VALIDATOR_PORT_RANGE).unwrap().1);
-        let t_receiver = receiver(
-            repair_socket.clone(),
-            &exit,
-            s_reader,
-            Recycler::default(),
-            "archiver_reeciver",
-        );
-        let id = serve_repair.keypair().pubkey();
-        info!(
-            "Sending repair requests from: {} to: {}",
-            serve_repair.my_info().id,
-            archiver_info.gossip
-        );
-        let repair_slot_range = RepairSlotRange {
-            start: start_slot,
-            end: start_slot + slots_per_segment,
-        };
-        // try for upto 180 seconds //TODO needs tuning if segments are huge
-        for _ in 0..120 {
-            // Strategy used by archivers
-            let repairs = RepairService::generate_repairs_in_range(
-                blockstore,
-                repair_service::MAX_REPAIR_LENGTH,
-                &repair_slot_range,
-            );
-            //iter over the repairs and send them
-            if let Ok(repairs) = repairs {
-                let reqs: Vec<_> = repairs
-                    .into_iter()
-                    .filter_map(|repair_request| {
-                        serve_repair
-                            .map_repair_request(&repair_request)
-                            .map(|result| ((archiver_info.gossip, result), repair_request))
-                            .ok()
-                    })
-                    .collect();
-
-                for ((to, req), repair_request) in reqs {
-                    if let Ok(local_addr) = repair_socket.local_addr() {
-                        datapoint_info!(
-                            "archiver_download",
-                            ("repair_request", format!("{:?}", repair_request), String),
-                            ("to", to.to_string(), String),
-                            ("from", local_addr.to_string(), String),
-                            ("id", id.to_string(), String)
-                        );
-                    }
-                    repair_socket
-                        .send_to(&req, archiver_info.gossip)
-                        .unwrap_or_else(|e| {
-                            error!("{} repair req send_to({}) error {:?}", id, to, e);
-                            0
-                        });
-                }
-            }
-            let res = r_reader.recv_timeout(Duration::new(1, 0));
-            if let Ok(mut packets) = res {
-                while let Ok(mut more) = r_reader.try_recv() {
-                    packets.packets.append_pinned(&mut more.packets);
-                }
-                let shreds: Vec<Shred> = packets
-                    .packets
-                    .into_iter()
-                    .filter_map(|p| Shred::new_from_serialized_shred(p.data.to_vec()).ok())
-                    .collect();
-                blockstore.insert_shreds(shreds, None, false)?;
-            }
-            // check if all the slots in the segment are complete
-            if Self::segment_complete(start_slot, slots_per_segment, blockstore) {
-                break;
-            }
-            sleep(Duration::from_millis(500));
-        }
-        exit.store(true, Ordering::Relaxed);
-        t_receiver.join().unwrap();
-
-        // check if all the slots in the segment are complete
-        if !Self::segment_complete(start_slot, slots_per_segment, blockstore) {
-            return Err(ArchiverError::SegmentDownloadError);
-        }
-        Ok(start_slot)
-    }
-
-    fn segment_complete(
-        start_slot: Slot,
-        slots_per_segment: u64,
-        blockstore: &Arc<Blockstore>,
-    ) -> bool {
-        for slot in start_slot..(start_slot + slots_per_segment) {
-            if !blockstore.is_full(slot) {
-                return false;
-            }
-        }
-        true
-    }
-
-    fn get_archiver_segment_slot(bind_ip_addr: IpAddr, to: SocketAddr) -> u64 {
-        let (_port, socket) = bind_in_range(bind_ip_addr, VALIDATOR_PORT_RANGE).unwrap();
-        socket
-            .set_read_timeout(Some(Duration::from_secs(5)))
-            .unwrap();
-
-        let req = ArchiverRequest::GetSlotHeight(socket.local_addr().unwrap());
-        let serialized_req = bincode::serialize(&req).unwrap();
-        for _ in 0..10 {
-            socket.send_to(&serialized_req, to).unwrap();
-            let mut buf = [0; 1024];
-            if let Ok((size, _addr)) = socket.recv_from(&mut buf) {
-                // Ignore bad packet and try again
-                if let Ok(slot) = bincode::config()
-                    .limit(PACKET_DATA_SIZE as u64)
-                    .deserialize(&buf[..size])
-                {
-                    return slot;
-                }
-            }
-            sleep(Duration::from_millis(500));
-        }
-        panic!("Couldn't get segment slot from archiver!");
-    }
-}

archiver-lib/src/result.rs

@@ -1,48 +0,0 @@
-use serde_json;
-use solana_client::client_error;
-use solana_ledger::blockstore;
-use solana_sdk::transport;
-use std::any::Any;
-use thiserror::Error;
-
-#[derive(Error, Debug)]
-pub enum ArchiverError {
-    #[error("IO error")]
-    IO(#[from] std::io::Error),
-
-    #[error("blockstore error")]
-    BlockstoreError(#[from] blockstore::BlockstoreError),
-
-    #[error("crossbeam error")]
-    CrossbeamSendError(#[from] crossbeam_channel::SendError<u64>),
-
-    #[error("send error")]
-    SendError(#[from] std::sync::mpsc::SendError<u64>),
-
-    #[error("join error")]
-    JoinError(Box<dyn Any + Send + 'static>),
-
-    #[error("transport error")]
-    TransportError(#[from] transport::TransportError),
-
-    #[error("client error")]
-    ClientError(#[from] client_error::ClientError),
-
-    #[error("Json parsing error")]
-    JsonError(#[from] serde_json::error::Error),
-
-    #[error("Storage account has no balance")]
-    EmptyStorageAccountBalance,
-
-    #[error("No RPC peers..")]
-    NoRpcPeers,
-
-    #[error("Couldn't download full segment")]
-    SegmentDownloadError,
-}
-
-impl std::convert::From<Box<dyn Any + Send + 'static>> for ArchiverError {
-    fn from(e: Box<dyn Any + Send + 'static>) -> ArchiverError {
-        ArchiverError::JoinError(e)
-    }
-}

archiver-utils/Cargo.toml

@@ -1,25 +0,0 @@
-[package]
-name = "solana-archiver-utils"
-version = "1.0.13"
-description = "Solana Archiver Utils"
-authors = ["Solana Maintainers <maintainers@solana.com>"]
-repository = "https://github.com/solana-labs/solana"
-license = "Apache-2.0"
-homepage = "https://solana.com/"
-edition = "2018"
-
-[dependencies]
-log = "0.4.8"
-rand = "0.6.5"
-solana-chacha = { path = "../chacha", version = "1.0.13" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.13" }
-solana-ledger = { path = "../ledger", version = "1.0.13" }
-solana-logger = { path = "../logger", version = "1.0.13" }
-solana-perf = { path = "../perf", version = "1.0.13" }
-solana-sdk = { path = "../sdk", version = "1.0.13" }
-
-[dev-dependencies]
-hex = "0.4.0"
-
-[lib]
-name = "solana_archiver_utils"

archiver-utils/src/lib.rs

@@ -1,120 +0,0 @@
-#[macro_use]
-extern crate log;
-
-use solana_sdk::hash::{Hash, Hasher};
-use std::fs::File;
-use std::io::{self, BufReader, ErrorKind, Read, Seek, SeekFrom};
-use std::mem::size_of;
-use std::path::Path;
-
-pub fn sample_file(in_path: &Path, sample_offsets: &[u64]) -> io::Result<Hash> {
-    let in_file = File::open(in_path)?;
-    let metadata = in_file.metadata()?;
-    let mut buffer_file = BufReader::new(in_file);
-
-    let mut hasher = Hasher::default();
-    let sample_size = size_of::<Hash>();
-    let sample_size64 = sample_size as u64;
-    let mut buf = vec![0; sample_size];
-
-    let file_len = metadata.len();
-    if file_len < sample_size64 {
-        return Err(io::Error::new(ErrorKind::Other, "file too short!"));
-    }
-    for offset in sample_offsets {
-        if *offset > (file_len - sample_size64) / sample_size64 {
-            return Err(io::Error::new(ErrorKind::Other, "offset too large"));
-        }
-        buffer_file.seek(SeekFrom::Start(*offset * sample_size64))?;
-        trace!("sampling @ {} ", *offset);
-        match buffer_file.read(&mut buf) {
-            Ok(size) => {
-                assert_eq!(size, buf.len());
-                hasher.hash(&buf);
-            }
-            Err(e) => {
-                warn!("Error sampling file");
-                return Err(e);
-            }
-        }
-    }
-
-    Ok(hasher.result())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use rand::{thread_rng, Rng};
-    use std::fs::{create_dir_all, remove_file};
-    use std::io::Write;
-    use std::path::PathBuf;
-
-    extern crate hex;
-
-    fn tmp_file_path(name: &str) -> PathBuf {
-        use std::env;
-        let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
-        let mut rand_bits = [0u8; 32];
-        thread_rng().fill(&mut rand_bits[..]);
-
-        let mut path = PathBuf::new();
-        path.push(out_dir);
-        path.push("tmp");
-        create_dir_all(&path).unwrap();
-
-        path.push(format!("{}-{:?}", name, hex::encode(rand_bits)));
-        println!("path: {:?}", path);
-        path
-    }
-
-    #[test]
-    fn test_sample_file() {
-        solana_logger::setup();
-        let in_path = tmp_file_path("test_sample_file_input.txt");
-        let num_strings = 4096;
-        let string = "12foobar";
-        {
-            let mut in_file = File::create(&in_path).unwrap();
-            for _ in 0..num_strings {
-                in_file.write(string.as_bytes()).unwrap();
-            }
-        }
-        let num_samples = (string.len() * num_strings / size_of::<Hash>()) as u64;
-        let samples: Vec<_> = (0..num_samples).collect();
-        let res = sample_file(&in_path, samples.as_slice());
-        let ref_hash: Hash = Hash::new(&[
-            173, 251, 182, 165, 10, 54, 33, 150, 133, 226, 106, 150, 99, 192, 179, 1, 230, 144,
-            151, 126, 18, 191, 54, 67, 249, 140, 230, 160, 56, 30, 170, 52,
-        ]);
-        let res = res.unwrap();
-        assert_eq!(res, ref_hash);
-
-        // Sample just past the end
-        assert!(sample_file(&in_path, &[num_samples]).is_err());
-        remove_file(&in_path).unwrap();
-    }
-
-    #[test]
-    fn test_sample_file_invalid_offset() {
-        let in_path = tmp_file_path("test_sample_file_invalid_offset_input.txt");
-        {
-            let mut in_file = File::create(&in_path).unwrap();
-            for _ in 0..4096 {
-                in_file.write("123456foobar".as_bytes()).unwrap();
-            }
-        }
-        let samples = [0, 200000];
-        let res = sample_file(&in_path, &samples);
-        assert!(res.is_err());
-        remove_file(in_path).unwrap();
-    }
-
-    #[test]
-    fn test_sample_file_missing_file() {
-        let in_path = tmp_file_path("test_sample_file_that_doesnt_exist.txt");
-        let samples = [0, 5];
-        let res = sample_file(&in_path, &samples);
-        assert!(res.is_err());
-    }
-}

archiver/Cargo.toml

@@ -1,20 +0,0 @@
-[package]
-authors = ["Solana Maintainers <maintainers@solana.com>"]
-edition = "2018"
-name = "solana-archiver"
-version = "1.0.13"
-repository = "https://github.com/solana-labs/solana"
-license = "Apache-2.0"
-homepage = "https://solana.com/"
-
-[dependencies]
-clap = "2.33.0"
-console = "0.9.2"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.13" }
-solana-core = { path = "../core", version = "1.0.13" }
-solana-logger = { path = "../logger", version = "1.0.13" }
-solana-metrics = { path = "../metrics", version = "1.0.13" }
-solana-archiver-lib = { path = "../archiver-lib", version = "1.0.13" }
-solana-net-utils = { path = "../net-utils", version = "1.0.13" }
-solana-sdk = { path = "../sdk", version = "1.0.13" }
-

archiver/src/main.rs

@@ -1,131 +0,0 @@
-use clap::{crate_description, crate_name, App, Arg};
-use console::style;
-use solana_archiver_lib::archiver::Archiver;
-use solana_clap_utils::{
-    input_parsers::keypair_of, input_validators::is_keypair_or_ask_keyword,
-    keypair::SKIP_SEED_PHRASE_VALIDATION_ARG,
-};
-use solana_core::{
-    cluster_info::{Node, VALIDATOR_PORT_RANGE},
-    contact_info::ContactInfo,
-};
-use solana_sdk::{
-    commitment_config::CommitmentConfig,
-    signature::{Keypair, Signer},
-};
-use std::{
-    net::{IpAddr, Ipv4Addr, SocketAddr},
-    path::PathBuf,
-    sync::Arc,
-};
-
-fn main() {
-    solana_logger::setup();
-
-    let matches = App::new(crate_name!())
-        .about(crate_description!())
-        .version(solana_clap_utils::version!())
-        .arg(
-            Arg::with_name("identity_keypair")
-                .short("i")
-                .long("identity")
-                .value_name("PATH")
-                .takes_value(true)
-                .validator(is_keypair_or_ask_keyword)
-                .help("File containing an identity (keypair)"),
-        )
-        .arg(
-            Arg::with_name("entrypoint")
-                .short("n")
-                .long("entrypoint")
-                .value_name("HOST:PORT")
-                .takes_value(true)
-                .required(true)
-                .validator(solana_net_utils::is_host_port)
-                .help("Rendezvous with the cluster at this entry point"),
-        )
-        .arg(
-            Arg::with_name("ledger")
-                .short("l")
-                .long("ledger")
-                .value_name("DIR")
-                .takes_value(true)
-                .required(true)
-                .help("use DIR as persistent ledger location"),
-        )
-        .arg(
-            Arg::with_name("storage_keypair")
-                .short("s")
-                .long("storage-keypair")
-                .value_name("PATH")
-                .takes_value(true)
-                .validator(is_keypair_or_ask_keyword)
-                .help("File containing the storage account keypair"),
-        )
-        .arg(
-            Arg::with_name(SKIP_SEED_PHRASE_VALIDATION_ARG.name)
-                .long(SKIP_SEED_PHRASE_VALIDATION_ARG.long)
-                .help(SKIP_SEED_PHRASE_VALIDATION_ARG.help),
-        )
-        .get_matches();
-
-    let ledger_path = PathBuf::from(matches.value_of("ledger").unwrap());
-
-    let identity_keypair = keypair_of(&matches, "identity_keypair").unwrap_or_else(Keypair::new);
-
-    let storage_keypair = keypair_of(&matches, "storage_keypair").unwrap_or_else(|| {
-        clap::Error::with_description(
-            "The `storage-keypair` argument was not found",
-            clap::ErrorKind::ArgumentNotFound,
-        )
-        .exit();
-    });
-
-    let entrypoint_addr = matches
-        .value_of("entrypoint")
-        .map(|entrypoint| {
-            solana_net_utils::parse_host_port(entrypoint)
-                .expect("failed to parse entrypoint address")
-        })
-        .unwrap();
-
-    let gossip_addr = {
-        let ip = solana_net_utils::get_public_ip_addr(&entrypoint_addr).unwrap();
-        let mut addr = SocketAddr::new(ip, 0);
-        addr.set_ip(solana_net_utils::get_public_ip_addr(&entrypoint_addr).unwrap());
-        addr
-    };
-    let node = Node::new_archiver_with_external_ip(
-        &identity_keypair.pubkey(),
-        &gossip_addr,
-        VALIDATOR_PORT_RANGE,
-        IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)),
-    );
-
-    println!(
-        "{} version {} (branch={}, commit={})",
-        style(crate_name!()).bold(),
-        solana_clap_utils::version!(),
-        option_env!("CI_BRANCH").unwrap_or("unknown"),
-        option_env!("CI_COMMIT").unwrap_or("unknown")
-    );
-    solana_metrics::set_host_id(identity_keypair.pubkey().to_string());
-    println!(
-        "replicating the data with identity_keypair={:?} gossip_addr={:?}",
-        identity_keypair.pubkey(),
-        gossip_addr
-    );
-
-    let entrypoint_info = ContactInfo::new_gossip_entry_point(&entrypoint_addr);
-    let archiver = Archiver::new(
-        &ledger_path,
-        node,
-        entrypoint_info,
-        Arc::new(identity_keypair),
-        Arc::new(storage_keypair),
-        CommitmentConfig::recent(),
-    )
-    .unwrap();
-
-    archiver.join();
-}

banking-bench/Cargo.toml

@@ -1,20 +1,29 @@
 [package]
-authors = ["Solana Maintainers <maintainers@solana.com>"]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.0.13"
+version = "1.4.26"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
+publish = false
 
 [dependencies]
+clap = "2.33.1"
+crossbeam-channel = "0.4"
 log = "0.4.6"
-rayon = "1.2.0"
-solana-core = { path = "../core", version = "1.0.13" }
-solana-ledger = { path = "../ledger", version = "1.0.13" }
-solana-logger = { path = "../logger", version = "1.0.13" }
-solana-runtime = { path = "../runtime", version = "1.0.13" }
-solana-measure = { path = "../measure", version = "1.0.13" }
-solana-sdk = { path = "../sdk", version = "1.0.13" }
-rand = "0.6.5"
-crossbeam-channel = "0.3"
+rand = "0.7.0"
+rayon = "1.4.0"
+solana-core = { path = "../core", version = "1.4.26" }
+solana-clap-utils = { path = "../clap-utils", version = "1.4.26" }
+solana-streamer = { path = "../streamer", version = "1.4.26" }
+solana-perf = { path = "../perf", version = "1.4.26" }
+solana-ledger = { path = "../ledger", version = "1.4.26" }
+solana-logger = { path = "../logger", version = "1.4.26" }
+solana-runtime = { path = "../runtime", version = "1.4.26" }
+solana-measure = { path = "../measure", version = "1.4.26" }
+solana-sdk = { path = "../sdk", version = "1.4.26" }
+solana-version = { path = "../version", version = "1.4.26" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -1,30 +1,36 @@
+use clap::{crate_description, crate_name, value_t, App, Arg};
 use crossbeam_channel::unbounded;
 use log::*;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
-use solana_core::banking_stage::{create_test_recorder, BankingStage};
-use solana_core::cluster_info::ClusterInfo;
-use solana_core::cluster_info::Node;
-use solana_core::genesis_utils::{create_genesis_config, GenesisConfigInfo};
-use solana_core::packet::to_packets_chunked;
-use solana_core::poh_recorder::PohRecorder;
-use solana_core::poh_recorder::WorkingBankEntry;
-use solana_ledger::bank_forks::BankForks;
-use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
+use solana_core::{
+    banking_stage::{create_test_recorder, BankingStage},
+    cluster_info::ClusterInfo,
+    cluster_info::Node,
+    poh_recorder::PohRecorder,
+    poh_recorder::WorkingBankEntry,
+};
+use solana_ledger::{
+    blockstore::Blockstore,
+    genesis_utils::{create_genesis_config, GenesisConfigInfo},
+    get_tmp_ledger_path,
+};
 use solana_measure::measure::Measure;
-use solana_runtime::bank::Bank;
-use solana_sdk::hash::Hash;
-use solana_sdk::pubkey::Pubkey;
-use solana_sdk::signature::Keypair;
-use solana_sdk::signature::Signature;
-use solana_sdk::system_transaction;
-use solana_sdk::timing::{duration_as_us, timestamp};
-use solana_sdk::transaction::Transaction;
-use std::sync::atomic::Ordering;
-use std::sync::mpsc::Receiver;
-use std::sync::{Arc, Mutex, RwLock};
-use std::thread::sleep;
-use std::time::{Duration, Instant};
+use solana_perf::packet::to_packets_chunked;
+use solana_runtime::{bank::Bank, bank_forks::BankForks};
+use solana_sdk::{
+    hash::Hash,
+    signature::Keypair,
+    signature::Signature,
+    system_transaction,
+    timing::{duration_as_us, timestamp},
+    transaction::Transaction,
+};
+use std::{
+    sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex},
+    thread::sleep,
+    time::{Duration, Instant},
+};
 
 fn check_txs(
     receiver: &Arc<Receiver<WorkingBankEntry>>,
@@ -57,16 +63,23 @@ fn check_txs(
     no_bank
 }
 
-fn make_accounts_txs(txes: usize, mint_keypair: &Keypair, hash: Hash) -> Vec<Transaction> {
-    let to_pubkey = Pubkey::new_rand();
-    let dummy = system_transaction::transfer(mint_keypair, &to_pubkey, 1, hash);
-    (0..txes)
+fn make_accounts_txs(
+    total_num_transactions: usize,
+    hash: Hash,
+    same_payer: bool,
+) -> Vec<Transaction> {
+    let to_pubkey = solana_sdk::pubkey::new_rand();
+    let payer_key = Keypair::new();
+    let dummy = system_transaction::transfer(&payer_key, &to_pubkey, 1, hash);
+    (0..total_num_transactions)
         .into_par_iter()
         .map(|_| {
             let mut new = dummy.clone();
             let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
-            new.message.account_keys[0] = Pubkey::new_rand();
-            new.message.account_keys[1] = Pubkey::new_rand();
+            if !same_payer {
+                new.message.account_keys[0] = solana_sdk::pubkey::new_rand();
+            }
+            new.message.account_keys[1] = solana_sdk::pubkey::new_rand();
             new.signatures = vec![Signature::new(&sig[0..64])];
             new
         })
@@ -89,13 +102,61 @@ fn bytes_as_usize(bytes: &[u8]) -> usize {
     bytes[0] as usize | (bytes[1] as usize) << 8
 }
 
+#[allow(clippy::cognitive_complexity)]
 fn main() {
     solana_logger::setup();
-    let num_threads = BankingStage::num_threads() as usize;
+
+    let matches = App::new(crate_name!())
+        .about(crate_description!())
+        .version(solana_version::version!())
+        .arg(
+            Arg::with_name("num_chunks")
+                .long("num-chunks")
+                .takes_value(true)
+                .value_name("SIZE")
+                .help("Number of transaction chunks."),
+        )
+        .arg(
+            Arg::with_name("packets_per_chunk")
+                .long("packets-per-chunk")
+                .takes_value(true)
+                .value_name("SIZE")
+                .help("Packets per chunk"),
+        )
+        .arg(
+            Arg::with_name("skip_sanity")
+                .long("skip-sanity")
+                .takes_value(false)
+                .help("Skip transaction sanity execution"),
+        )
+        .arg(
+            Arg::with_name("same_payer")
+                .long("same-payer")
+                .takes_value(false)
+                .help("Use the same payer for transfers"),
+        )
+        .arg(
+            Arg::with_name("iterations")
+                .long("iterations")
+                .takes_value(true)
+                .help("Number of iterations"),
+        )
+        .arg(
+            Arg::with_name("num_threads")
+                .long("num-threads")
+                .takes_value(true)
+                .help("Number of iterations"),
+        )
+        .get_matches();
+
+    let num_threads =
+        value_t!(matches, "num_threads", usize).unwrap_or(BankingStage::num_threads() as usize);
     //   a multiple of packet chunk duplicates to avoid races
-    const CHUNKS: usize = 8 * 2;
-    const PACKETS_PER_BATCH: usize = 192;
-    let txes = PACKETS_PER_BATCH * num_threads * CHUNKS;
+    let num_chunks = value_t!(matches, "num_chunks", usize).unwrap_or(16);
+    let packets_per_chunk = value_t!(matches, "packets_per_chunk", usize).unwrap_or(192);
+    let iterations = value_t!(matches, "iterations", usize).unwrap_or(1000);
+
+    let total_num_transactions = num_chunks * num_threads * packets_per_chunk;
     let mint_total = 1_000_000_000_000;
     let GenesisConfigInfo {
         genesis_config,
@@ -105,38 +166,49 @@ fn main() {
 
     let (verified_sender, verified_receiver) = unbounded();
     let (vote_sender, vote_receiver) = unbounded();
+    let (replay_vote_sender, _replay_vote_receiver) = unbounded();
     let bank0 = Bank::new(&genesis_config);
-    let mut bank_forks = BankForks::new(0, bank0);
+    let mut bank_forks = BankForks::new(bank0);
     let mut bank = bank_forks.working_bank();
 
-    info!("threads: {} txs: {}", num_threads, txes);
+    info!("threads: {} txs: {}", num_threads, total_num_transactions);
 
-    let mut transactions = make_accounts_txs(txes, &mint_keypair, genesis_config.hash());
+    let same_payer = matches.is_present("same_payer");
+    let mut transactions =
+        make_accounts_txs(total_num_transactions, genesis_config.hash(), same_payer);
 
     // fund all the accounts
     transactions.iter().for_each(|tx| {
-        let fund = system_transaction::transfer(
+        let mut fund = system_transaction::transfer(
             &mint_keypair,
             &tx.message.account_keys[0],
-            mint_total / txes as u64,
+            mint_total / total_num_transactions as u64,
             genesis_config.hash(),
         );
+        // Ignore any pesky duplicate signature errors in the case we are using single-payer
+        let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+        fund.signatures = vec![Signature::new(&sig[0..64])];
         let x = bank.process_transaction(&fund);
         x.unwrap();
     });
-    //sanity check, make sure all the transactions can execute sequentially
-    transactions.iter().for_each(|tx| {
-        let res = bank.process_transaction(&tx);
-        assert!(res.is_ok(), "sanity test transactions");
-    });
-    bank.clear_signatures();
-    //sanity check, make sure all the transactions can execute in parallel
-    let res = bank.process_transactions(&transactions);
-    for r in res {
-        assert!(r.is_ok(), "sanity parallel execution");
+
+    let skip_sanity = matches.is_present("skip_sanity");
+    if !skip_sanity {
+        //sanity check, make sure all the transactions can execute sequentially
+        transactions.iter().for_each(|tx| {
+            let res = bank.process_transaction(&tx);
+            assert!(res.is_ok(), "sanity test transactions error: {:?}", res);
+        });
+        bank.clear_signatures();
+        //sanity check, make sure all the transactions can execute in parallel
+        let res = bank.process_transactions(&transactions);
+        for r in res {
+            assert!(r.is_ok(), "sanity parallel execution error: {:?}", r);
+        }
+        bank.clear_signatures();
     }
-    bank.clear_signatures();
-    let mut verified: Vec<_> = to_packets_chunked(&transactions.clone(), PACKETS_PER_BATCH);
+
+    let mut verified: Vec<_> = to_packets_chunked(&transactions, packets_per_chunk);
     let ledger_path = get_tmp_ledger_path!();
     {
         let blockstore = Arc::new(
@@ -145,17 +217,18 @@ fn main() {
         let (exit, poh_recorder, poh_service, signal_receiver) =
             create_test_recorder(&bank, &blockstore, None);
         let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
-        let cluster_info = Arc::new(RwLock::new(cluster_info));
+        let cluster_info = Arc::new(cluster_info);
         let banking_stage = BankingStage::new(
             &cluster_info,
             &poh_recorder,
             verified_receiver,
             vote_receiver,
             None,
+            replay_vote_sender,
         );
         poh_recorder.lock().unwrap().set_bank(&bank);
 
-        let chunk_len = verified.len() / CHUNKS;
+        let chunk_len = verified.len() / num_chunks;
         let mut start = 0;
 
         // This is so that the signal_receiver does not go out of scope after the closure.
@@ -164,17 +237,17 @@ fn main() {
         let signal_receiver = Arc::new(signal_receiver);
         let mut total_us = 0;
         let mut tx_total_us = 0;
+        let base_tx_count = bank.transaction_count();
         let mut txs_processed = 0;
         let mut root = 1;
-        let collector = Pubkey::new_rand();
-        const ITERS: usize = 1_000;
+        let collector = solana_sdk::pubkey::new_rand();
         let config = Config {
-            packets_per_batch: PACKETS_PER_BATCH,
+            packets_per_batch: packets_per_chunk,
             chunk_len,
             num_threads,
         };
         let mut total_sent = 0;
-        for _ in 0..ITERS {
+        for _ in 0..iterations {
             let now = Instant::now();
             let mut sent = 0;
 
@@ -215,7 +288,11 @@ fn main() {
                     sleep(Duration::from_millis(5));
                 }
             }
-            if check_txs(&signal_receiver, txes / CHUNKS, &poh_recorder) {
+            if check_txs(
+                &signal_receiver,
+                total_num_transactions / num_chunks,
+                &poh_recorder,
+            ) {
                 debug!(
                     "resetting bank {} tx count: {} txs_proc: {}",
                     bank.slot(),
@@ -246,7 +323,7 @@ fn main() {
                 poh_recorder.lock().unwrap().set_bank(&bank);
                 assert!(poh_recorder.lock().unwrap().bank().is_some());
                 if bank.slot() > 32 {
-                    bank_forks.set_root(root, &None);
+                    bank_forks.set_root(root, &None, None);
                     root += 1;
                 }
                 debug!(
@@ -267,7 +344,7 @@ fn main() {
             debug!(
                 "time: {} us checked: {} sent: {}",
                 duration_as_us(&now.elapsed()),
-                txes / CHUNKS,
+                total_num_transactions / num_chunks,
                 sent,
             );
             total_sent += sent;
@@ -278,20 +355,26 @@ fn main() {
                     let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
                     tx.signatures[0] = Signature::new(&sig[0..64]);
                 }
-                verified = to_packets_chunked(&transactions.clone(), PACKETS_PER_BATCH);
+                verified = to_packets_chunked(&transactions.clone(), packets_per_chunk);
             }
 
             start += chunk_len;
             start %= verified.len();
         }
+        let txs_processed = bank_forks.working_bank().transaction_count();
+        debug!("processed: {} base: {}", txs_processed, base_tx_count);
         eprintln!(
-            "{{'name': 'banking_bench_total', 'median': '{}'}}",
+            "{{'name': 'banking_bench_total', 'median': '{:.2}'}}",
             (1000.0 * 1000.0 * total_sent as f64) / (total_us as f64),
         );
         eprintln!(
-            "{{'name': 'banking_bench_tx_total', 'median': '{}'}}",
+            "{{'name': 'banking_bench_tx_total', 'median': '{:.2}'}}",
             (1000.0 * 1000.0 * total_sent as f64) / (tx_total_us as f64),
         );
+        eprintln!(
+            "{{'name': 'banking_bench_success_tx_total', 'median': '{:.2}'}}",
+            (1000.0 * 1000.0 * (txs_processed - base_tx_count) as f64) / (total_us as f64),
+        );
 
         drop(verified_sender);
         drop(vote_sender);

banks-client/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "solana-banks-client"
+version = "1.4.26"
+description = "Solana banks client"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+async-trait = "0.1.36"
+bincode = "1.3.1"
+futures = "0.3"
+mio = "0.7.6"
+solana-banks-interface = { path = "../banks-interface", version = "1.4.26" }
+solana-sdk = { path = "../sdk", version = "1.4.26" }
+tarpc = { version = "0.23.0", features = ["full"] }
+tokio = { version = "0.3", features = ["full"] }
+tokio-serde = { version = "0.6", features = ["bincode"] }
+
+[dev-dependencies]
+solana-runtime = { path = "../runtime", version = "1.4.26" }
+solana-banks-server = { path = "../banks-server", version = "1.4.26" }
+
+[lib]
+crate-type = ["lib"]
+name = "solana_banks_client"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

banks-client/src/lib.rs

@@ -0,0 +1,318 @@
+//! A client for the ledger state, from the perspective of an arbitrary validator.
+//!
+//! Use start_tcp_client() to create a client and then import BanksClientExt to
+//! access its methods. Additional "*_with_context" methods are also available,
+//! but they are undocumented, may change over time, and are generally more
+//! cumbersome to use.
+
+use async_trait::async_trait;
+use futures::future::join_all;
+pub use solana_banks_interface::{BanksClient, TransactionStatus};
+use solana_banks_interface::{BanksRequest, BanksResponse};
+use solana_sdk::{
+    account::{from_account, Account},
+    clock::Slot,
+    commitment_config::CommitmentLevel,
+    fee_calculator::FeeCalculator,
+    hash::Hash,
+    pubkey::Pubkey,
+    rent::Rent,
+    signature::Signature,
+    sysvar,
+    transaction::Transaction,
+    transport,
+};
+use std::io::{self, Error, ErrorKind};
+use tarpc::{
+    client, context,
+    rpc::{transport::channel::UnboundedChannel, ClientMessage, Response},
+    serde_transport::tcp,
+};
+use tokio::{net::ToSocketAddrs, time::Duration};
+use tokio_serde::formats::Bincode;
+
+#[async_trait]
+pub trait BanksClientExt {
+    /// Send a transaction and return immediately. The server will resend the
+    /// transaction until either it is accepted by the cluster or the transaction's
+    /// blockhash expires.
+    async fn send_transaction(&mut self, transaction: Transaction) -> io::Result<()>;
+
+    /// Return a recent, rooted blockhash from the server. The cluster will only accept
+    /// transactions with a blockhash that has not yet expired. Use the `get_fees`
+    /// method to get both a blockhash and the blockhash's last valid slot.
+    async fn get_recent_blockhash(&mut self) -> io::Result<Hash>;
+
+    /// Return the fee parameters associated with a recent, rooted blockhash. The cluster
+    /// will use the transaction's blockhash to look up these same fee parameters and
+    /// use them to calculate the transaction fee.
+    async fn get_fees(&mut self) -> io::Result<(FeeCalculator, Hash, Slot)>;
+
+    /// Return the cluster rent
+    async fn get_rent(&mut self) -> io::Result<Rent>;
+
+    /// Send a transaction and return after the transaction has been rejected or
+    /// reached the given level of commitment.
+    async fn process_transaction_with_commitment(
+        &mut self,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> transport::Result<()>;
+
+    /// Send a transaction and return after the transaction has been finalized or rejected.
+    async fn process_transaction(&mut self, transaction: Transaction) -> transport::Result<()>;
+
+    /// Return the status of a transaction with a signature matching the transaction's first
+    /// signature. Return None if the transaction is not found, which may be because the
+    /// blockhash was expired or the fee-paying account had insufficient funds to pay the
+    /// transaction fee. Note that servers rarely store the full transaction history. This
+    /// method may return None if the transaction status has been discarded.
+    async fn get_transaction_status(
+        &mut self,
+        signature: Signature,
+    ) -> io::Result<Option<TransactionStatus>>;
+
+    /// Same as get_transaction_status, but for multiple transactions.
+    async fn get_transaction_statuses(
+        &mut self,
+        signatures: Vec<Signature>,
+    ) -> io::Result<Vec<Option<TransactionStatus>>>;
+
+    /// Return the most recent rooted slot height. All transactions at or below this height
+    /// are said to be finalized. The cluster will not fork to a higher slot height.
+    async fn get_root_slot(&mut self) -> io::Result<Slot>;
+
+    /// Return the account at the given address at the slot corresponding to the given
+    /// commitment level. If the account is not found, None is returned.
+    async fn get_account_with_commitment(
+        &mut self,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> io::Result<Option<Account>>;
+
+    /// Return the account at the given address at the time of the most recent root slot.
+    /// If the account is not found, None is returned.
+    async fn get_account(&mut self, address: Pubkey) -> io::Result<Option<Account>>;
+
+    /// Return the balance in lamports of an account at the given address at the slot
+    /// corresponding to the given commitment level.
+    async fn get_balance_with_commitment(
+        &mut self,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> io::Result<u64>;
+
+    /// Return the balance in lamports of an account at the given address at the time
+    /// of the most recent root slot.
+    async fn get_balance(&mut self, address: Pubkey) -> io::Result<u64>;
+}
+
+#[async_trait]
+impl BanksClientExt for BanksClient {
+    async fn send_transaction(&mut self, transaction: Transaction) -> io::Result<()> {
+        self.send_transaction_with_context(context::current(), transaction)
+            .await
+    }
+
+    async fn get_fees(&mut self) -> io::Result<(FeeCalculator, Hash, Slot)> {
+        self.get_fees_with_commitment_and_context(context::current(), CommitmentLevel::Root)
+            .await
+    }
+
+    async fn get_rent(&mut self) -> io::Result<Rent> {
+        let rent_sysvar = self
+            .get_account(sysvar::rent::id())
+            .await?
+            .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Rent sysvar not present"))?;
+
+        from_account::<Rent>(&rent_sysvar).ok_or_else(|| {
+            io::Error::new(io::ErrorKind::Other, "Failed to deserialize Rent sysvar")
+        })
+    }
+
+    async fn get_recent_blockhash(&mut self) -> io::Result<Hash> {
+        Ok(self.get_fees().await?.1)
+    }
+
+    async fn process_transaction_with_commitment(
+        &mut self,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> transport::Result<()> {
+        let mut ctx = context::current();
+        ctx.deadline += Duration::from_secs(50);
+        let result = self
+            .process_transaction_with_commitment_and_context(ctx, transaction, commitment)
+            .await?;
+        match result {
+            None => Err(Error::new(ErrorKind::TimedOut, "invalid blockhash or fee-payer").into()),
+            Some(transaction_result) => Ok(transaction_result?),
+        }
+    }
+
+    async fn process_transaction(&mut self, transaction: Transaction) -> transport::Result<()> {
+        self.process_transaction_with_commitment(transaction, CommitmentLevel::default())
+            .await
+    }
+
+    async fn get_root_slot(&mut self) -> io::Result<Slot> {
+        self.get_slot_with_context(context::current(), CommitmentLevel::Root)
+            .await
+    }
+
+    async fn get_account_with_commitment(
+        &mut self,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> io::Result<Option<Account>> {
+        self.get_account_with_commitment_and_context(context::current(), address, commitment)
+            .await
+    }
+
+    async fn get_account(&mut self, address: Pubkey) -> io::Result<Option<Account>> {
+        self.get_account_with_commitment(address, CommitmentLevel::default())
+            .await
+    }
+
+    async fn get_balance_with_commitment(
+        &mut self,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> io::Result<u64> {
+        let account = self
+            .get_account_with_commitment_and_context(context::current(), address, commitment)
+            .await?;
+        Ok(account.map(|x| x.lamports).unwrap_or(0))
+    }
+
+    async fn get_balance(&mut self, address: Pubkey) -> io::Result<u64> {
+        self.get_balance_with_commitment(address, CommitmentLevel::default())
+            .await
+    }
+
+    async fn get_transaction_status(
+        &mut self,
+        signature: Signature,
+    ) -> io::Result<Option<TransactionStatus>> {
+        self.get_transaction_status_with_context(context::current(), signature)
+            .await
+    }
+
+    async fn get_transaction_statuses(
+        &mut self,
+        signatures: Vec<Signature>,
+    ) -> io::Result<Vec<Option<TransactionStatus>>> {
+        // tarpc futures oddly hold a mutable reference back to the client so clone the client upfront
+        let mut clients_and_signatures: Vec<_> = signatures
+            .into_iter()
+            .map(|signature| (self.clone(), signature))
+            .collect();
+
+        let futs = clients_and_signatures
+            .iter_mut()
+            .map(|(client, signature)| client.get_transaction_status(*signature));
+
+        let statuses = join_all(futs).await;
+
+        // Convert Vec<Result<_, _>> to Result<Vec<_>>
+        statuses.into_iter().collect()
+    }
+}
+
+pub async fn start_client(
+    transport: UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>>,
+) -> io::Result<BanksClient> {
+    BanksClient::new(client::Config::default(), transport).spawn()
+}
+
+pub async fn start_tcp_client<T: ToSocketAddrs>(addr: T) -> io::Result<BanksClient> {
+    let transport = tcp::connect(addr, Bincode::default).await?;
+    BanksClient::new(client::Config::default(), transport).spawn()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_banks_server::banks_server::start_local_server;
+    use solana_runtime::{bank::Bank, bank_forks::BankForks, genesis_utils::create_genesis_config};
+    use solana_sdk::{message::Message, signature::Signer, system_instruction};
+    use std::sync::{Arc, RwLock};
+    use tarpc::transport;
+    use tokio::{runtime::Runtime, time::sleep};
+
+    #[test]
+    fn test_banks_client_new() {
+        let (client_transport, _server_transport) = transport::channel::unbounded();
+        BanksClient::new(client::Config::default(), client_transport);
+    }
+
+    #[test]
+    fn test_banks_server_transfer_via_server() -> io::Result<()> {
+        // This test shows the preferred way to interact with BanksServer.
+        // It creates a runtime explicitly (no globals via tokio macros) and calls
+        // `runtime.block_on()` just once, to run all the async code.
+
+        let genesis = create_genesis_config(10);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(
+            &genesis.genesis_config,
+        ))));
+
+        let bob_pubkey = solana_sdk::pubkey::new_rand();
+        let mint_pubkey = genesis.mint_keypair.pubkey();
+        let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
+        let message = Message::new(&[instruction], Some(&mint_pubkey));
+
+        Runtime::new()?.block_on(async {
+            let client_transport = start_local_server(&bank_forks).await;
+            let mut banks_client =
+                BanksClient::new(client::Config::default(), client_transport).spawn()?;
+
+            let recent_blockhash = banks_client.get_recent_blockhash().await?;
+            let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
+            banks_client.process_transaction(transaction).await.unwrap();
+            assert_eq!(banks_client.get_balance(bob_pubkey).await?, 1);
+            Ok(())
+        })
+    }
+
+    #[test]
+    fn test_banks_server_transfer_via_client() -> io::Result<()> {
+        // The caller may not want to hold the connection open until the transaction
+        // is processed (or blockhash expires). In this test, we verify the
+        // server-side functionality is available to the client.
+
+        let genesis = create_genesis_config(10);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(
+            &genesis.genesis_config,
+        ))));
+
+        let mint_pubkey = &genesis.mint_keypair.pubkey();
+        let bob_pubkey = solana_sdk::pubkey::new_rand();
+        let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
+        let message = Message::new(&[instruction], Some(&mint_pubkey));
+
+        Runtime::new()?.block_on(async {
+            let client_transport = start_local_server(&bank_forks).await;
+            let mut banks_client =
+                BanksClient::new(client::Config::default(), client_transport).spawn()?;
+            let (_, recent_blockhash, last_valid_slot) = banks_client.get_fees().await?;
+            let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
+            let signature = transaction.signatures[0];
+            banks_client.send_transaction(transaction).await?;
+
+            let mut status = banks_client.get_transaction_status(signature).await?;
+
+            while status.is_none() {
+                let root_slot = banks_client.get_root_slot().await?;
+                if root_slot > last_valid_slot {
+                    break;
+                }
+                sleep(Duration::from_millis(100)).await;
+                status = banks_client.get_transaction_status(signature).await?;
+            }
+            assert!(status.unwrap().err.is_none());
+            assert_eq!(banks_client.get_balance(bob_pubkey).await?, 1);
+            Ok(())
+        })
+    }
+}

banks-interface/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "solana-banks-interface"
+version = "1.4.26"
+description = "Solana banks RPC interface"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+mio = "0.7.6"
+serde = { version = "1.0.112", features = ["derive"] }
+solana-sdk = { path = "../sdk", version = "1.4.26" }
+tarpc = { version = "0.23.0", features = ["full"] }
+
+[lib]
+crate-type = ["lib"]
+name = "solana_banks_interface"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

banks-interface/src/lib.rs

@@ -0,0 +1,57 @@
+use serde::{Deserialize, Serialize};
+use solana_sdk::{
+    account::Account,
+    clock::Slot,
+    commitment_config::CommitmentLevel,
+    fee_calculator::FeeCalculator,
+    hash::Hash,
+    pubkey::Pubkey,
+    signature::Signature,
+    transaction::{self, Transaction, TransactionError},
+};
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub enum TransactionConfirmationStatus {
+    Processed,
+    Confirmed,
+    Finalized,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct TransactionStatus {
+    pub slot: Slot,
+    pub confirmations: Option<usize>, // None = rooted
+    pub err: Option<TransactionError>,
+    pub confirmation_status: Option<TransactionConfirmationStatus>,
+}
+
+#[tarpc::service]
+pub trait Banks {
+    async fn send_transaction_with_context(transaction: Transaction);
+    async fn get_fees_with_commitment_and_context(
+        commitment: CommitmentLevel,
+    ) -> (FeeCalculator, Hash, Slot);
+    async fn get_transaction_status_with_context(signature: Signature)
+        -> Option<TransactionStatus>;
+    async fn get_slot_with_context(commitment: CommitmentLevel) -> Slot;
+    async fn process_transaction_with_commitment_and_context(
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> Option<transaction::Result<()>>;
+    async fn get_account_with_commitment_and_context(
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> Option<Account>;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tarpc::{client, transport};
+
+    #[test]
+    fn test_banks_client_new() {
+        let (client_transport, _server_transport) = transport::channel::unbounded();
+        BanksClient::new(client::Config::default(), client_transport);
+    }
+}

banks-server/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "solana-banks-server"
+version = "1.4.26"
+description = "Solana banks server"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+bincode = "1.3.1"
+futures = "0.3"
+log = "0.4.8"
+mio = "0.7.6"
+solana-banks-interface = { path = "../banks-interface", version = "1.4.26" }
+solana-runtime = { path = "../runtime", version = "1.4.26" }
+solana-sdk = { path = "../sdk", version = "1.4.26" }
+solana-metrics = { path = "../metrics", version = "1.4.26" }
+tarpc = { version = "0.23.0", features = ["full"] }
+tokio = { version = "0.3", features = ["full"] }
+tokio-serde = { version = "0.6", features = ["bincode"] }
+
+[lib]
+crate-type = ["lib"]
+name = "solana_banks_server"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

banks-server/src/banks_server.rs

@@ -0,0 +1,290 @@
+use crate::send_transaction_service::{SendTransactionService, TransactionInfo};
+use bincode::{deserialize, serialize};
+use futures::{
+    future,
+    prelude::stream::{self, StreamExt},
+};
+use solana_banks_interface::{
+    Banks, BanksRequest, BanksResponse, TransactionConfirmationStatus, TransactionStatus,
+};
+use solana_runtime::{bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache};
+use solana_sdk::{
+    account::Account,
+    clock::Slot,
+    commitment_config::CommitmentLevel,
+    fee_calculator::FeeCalculator,
+    hash::Hash,
+    pubkey::Pubkey,
+    signature::Signature,
+    transaction::{self, Transaction},
+};
+use std::{
+    io,
+    net::{Ipv4Addr, SocketAddr},
+    sync::{
+        mpsc::{channel, Receiver, Sender},
+        Arc, RwLock,
+    },
+    thread::Builder,
+    time::Duration,
+};
+use tarpc::{
+    context::Context,
+    rpc::{transport::channel::UnboundedChannel, ClientMessage, Response},
+    serde_transport::tcp,
+    server::{self, Channel, Handler},
+    transport,
+};
+use tokio::time::sleep;
+use tokio_serde::formats::Bincode;
+
+#[derive(Clone)]
+struct BanksServer {
+    bank_forks: Arc<RwLock<BankForks>>,
+    block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+    transaction_sender: Sender<TransactionInfo>,
+}
+
+impl BanksServer {
+    /// Return a BanksServer that forwards transactions to the
+    /// given sender. If unit-testing, those transactions can go to
+    /// a bank in the given BankForks. Otherwise, the receiver should
+    /// forward them to a validator in the leader schedule.
+    fn new(
+        bank_forks: Arc<RwLock<BankForks>>,
+        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+        transaction_sender: Sender<TransactionInfo>,
+    ) -> Self {
+        Self {
+            bank_forks,
+            block_commitment_cache,
+            transaction_sender,
+        }
+    }
+
+    fn run(bank: &Bank, transaction_receiver: Receiver<TransactionInfo>) {
+        while let Ok(info) = transaction_receiver.recv() {
+            let mut transaction_infos = vec![info];
+            while let Ok(info) = transaction_receiver.try_recv() {
+                transaction_infos.push(info);
+            }
+            let transactions: Vec<_> = transaction_infos
+                .into_iter()
+                .map(|info| deserialize(&info.wire_transaction).unwrap())
+                .collect();
+            let _ = bank.process_transactions(&transactions);
+        }
+    }
+
+    /// Useful for unit-testing
+    fn new_loopback(bank_forks: Arc<RwLock<BankForks>>) -> Self {
+        let (transaction_sender, transaction_receiver) = channel();
+        let bank = bank_forks.read().unwrap().working_bank();
+        let slot = bank.slot();
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
+        ));
+        Builder::new()
+            .name("solana-bank-forks-client".to_string())
+            .spawn(move || Self::run(&bank, transaction_receiver))
+            .unwrap();
+        Self::new(bank_forks, block_commitment_cache, transaction_sender)
+    }
+
+    fn slot(&self, commitment: CommitmentLevel) -> Slot {
+        self.block_commitment_cache
+            .read()
+            .unwrap()
+            .slot_with_commitment(commitment)
+    }
+
+    fn bank(&self, commitment: CommitmentLevel) -> Arc<Bank> {
+        self.bank_forks.read().unwrap()[self.slot(commitment)].clone()
+    }
+
+    async fn poll_signature_status(
+        self,
+        signature: &Signature,
+        blockhash: &Hash,
+        last_valid_slot: Slot,
+        commitment: CommitmentLevel,
+    ) -> Option<transaction::Result<()>> {
+        let mut status = self
+            .bank(commitment)
+            .get_signature_status_with_blockhash(signature, blockhash);
+        while status.is_none() {
+            sleep(Duration::from_millis(200)).await;
+            let bank = self.bank(commitment);
+            if bank.slot() > last_valid_slot {
+                break;
+            }
+            status = bank.get_signature_status_with_blockhash(signature, blockhash);
+        }
+        status
+    }
+}
+
+fn verify_transaction(transaction: &Transaction) -> transaction::Result<()> {
+    if let Err(err) = transaction.verify() {
+        Err(err)
+    } else if let Err(err) = transaction.verify_precompiles() {
+        Err(err)
+    } else {
+        Ok(())
+    }
+}
+
+#[tarpc::server]
+impl Banks for BanksServer {
+    async fn send_transaction_with_context(self, _: Context, transaction: Transaction) {
+        let blockhash = &transaction.message.recent_blockhash;
+        let last_valid_slot = self
+            .bank_forks
+            .read()
+            .unwrap()
+            .root_bank()
+            .get_blockhash_last_valid_slot(&blockhash)
+            .unwrap();
+        let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
+        let info =
+            TransactionInfo::new(signature, serialize(&transaction).unwrap(), last_valid_slot);
+        self.transaction_sender.send(info).unwrap();
+    }
+
+    async fn get_fees_with_commitment_and_context(
+        self,
+        _: Context,
+        commitment: CommitmentLevel,
+    ) -> (FeeCalculator, Hash, Slot) {
+        let bank = self.bank(commitment);
+        let (blockhash, fee_calculator) = bank.last_blockhash_with_fee_calculator();
+        let last_valid_slot = bank.get_blockhash_last_valid_slot(&blockhash).unwrap();
+        (fee_calculator, blockhash, last_valid_slot)
+    }
+
+    async fn get_transaction_status_with_context(
+        self,
+        _: Context,
+        signature: Signature,
+    ) -> Option<TransactionStatus> {
+        let bank = self.bank(CommitmentLevel::Recent);
+        let (slot, status) = bank.get_signature_status_slot(&signature)?;
+        let r_block_commitment_cache = self.block_commitment_cache.read().unwrap();
+
+        let optimistically_confirmed_bank = self.bank(CommitmentLevel::SingleGossip);
+        let optimistically_confirmed =
+            optimistically_confirmed_bank.get_signature_status_slot(&signature);
+
+        let confirmations = if r_block_commitment_cache.root() >= slot
+            && r_block_commitment_cache.highest_confirmed_root() >= slot
+        {
+            None
+        } else {
+            r_block_commitment_cache
+                .get_confirmation_count(slot)
+                .or(Some(0))
+        };
+        Some(TransactionStatus {
+            slot,
+            confirmations,
+            err: status.err(),
+            confirmation_status: if confirmations.is_none() {
+                Some(TransactionConfirmationStatus::Finalized)
+            } else if optimistically_confirmed.is_some() {
+                Some(TransactionConfirmationStatus::Confirmed)
+            } else {
+                Some(TransactionConfirmationStatus::Processed)
+            },
+        })
+    }
+
+    async fn get_slot_with_context(self, _: Context, commitment: CommitmentLevel) -> Slot {
+        self.slot(commitment)
+    }
+
+    async fn process_transaction_with_commitment_and_context(
+        self,
+        _: Context,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> Option<transaction::Result<()>> {
+        if let Err(err) = verify_transaction(&transaction) {
+            return Some(Err(err));
+        }
+
+        let blockhash = &transaction.message.recent_blockhash;
+        let last_valid_slot = self
+            .bank_forks
+            .read()
+            .unwrap()
+            .root_bank()
+            .get_blockhash_last_valid_slot(blockhash)
+            .unwrap();
+        let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
+        let info =
+            TransactionInfo::new(signature, serialize(&transaction).unwrap(), last_valid_slot);
+        self.transaction_sender.send(info).unwrap();
+        self.poll_signature_status(&signature, blockhash, last_valid_slot, commitment)
+            .await
+    }
+
+    async fn get_account_with_commitment_and_context(
+        self,
+        _: Context,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> Option<Account> {
+        let bank = self.bank(commitment);
+        bank.get_account(&address)
+    }
+}
+
+pub async fn start_local_server(
+    bank_forks: &Arc<RwLock<BankForks>>,
+) -> UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>> {
+    let banks_server = BanksServer::new_loopback(bank_forks.clone());
+    let (client_transport, server_transport) = transport::channel::unbounded();
+    let server = server::new(server::Config::default())
+        .incoming(stream::once(future::ready(server_transport)))
+        .respond_with(banks_server.serve());
+    tokio::spawn(server);
+    client_transport
+}
+
+pub async fn start_tcp_server(
+    listen_addr: SocketAddr,
+    tpu_addr: SocketAddr,
+    bank_forks: Arc<RwLock<BankForks>>,
+    block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+) -> io::Result<()> {
+    // Note: These settings are copied straight from the tarpc example.
+    let server = tcp::listen(listen_addr, Bincode::default)
+        .await?
+        // Ignore accept errors.
+        .filter_map(|r| future::ready(r.ok()))
+        .map(server::BaseChannel::with_defaults)
+        // Limit channels to 1 per IP.
+        .max_channels_per_key(1, |t| {
+            t.as_ref()
+                .peer_addr()
+                .map(|x| x.ip())
+                .unwrap_or_else(|_| Ipv4Addr::new(0, 0, 0, 0).into())
+        })
+        // serve is generated by the service attribute. It takes as input any type implementing
+        // the generated Banks trait.
+        .map(move |chan| {
+            let (sender, receiver) = channel();
+
+            SendTransactionService::new(tpu_addr, &bank_forks, receiver);
+
+            let server =
+                BanksServer::new(bank_forks.clone(), block_commitment_cache.clone(), sender);
+            chan.respond_with(server.serve()).execute()
+        })
+        // Max 10 channels.
+        .buffer_unordered(10)
+        .for_each(|_| async {});
+
+    server.await;
+    Ok(())
+}

banks-server/src/lib.rs

@@ -0,0 +1,6 @@
+pub mod banks_server;
+pub mod rpc_banks_service;
+pub mod send_transaction_service;
+
+#[macro_use]
+extern crate solana_metrics;

banks-server/src/rpc_banks_service.rs

@@ -0,0 +1,116 @@
+//! The `rpc_banks_service` module implements the Solana Banks RPC API.
+
+use crate::banks_server::start_tcp_server;
+use futures::{future::FutureExt, pin_mut, prelude::stream::StreamExt, select};
+use solana_runtime::{bank_forks::BankForks, commitment::BlockCommitmentCache};
+use std::{
+    net::SocketAddr,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc, RwLock,
+    },
+    thread::{self, Builder, JoinHandle},
+};
+use tokio::{
+    runtime::Runtime,
+    time::{self, Duration},
+};
+
+pub struct RpcBanksService {
+    thread_hdl: JoinHandle<()>,
+}
+
+/// Run the TCP service until `exit` is set to true
+async fn start_abortable_tcp_server(
+    listen_addr: SocketAddr,
+    tpu_addr: SocketAddr,
+    bank_forks: Arc<RwLock<BankForks>>,
+    block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+    exit: Arc<AtomicBool>,
+) {
+    let server = start_tcp_server(
+        listen_addr,
+        tpu_addr,
+        bank_forks.clone(),
+        block_commitment_cache.clone(),
+    )
+    .fuse();
+    let interval = time::interval(Duration::from_millis(100)).fuse();
+    pin_mut!(server, interval);
+    loop {
+        select! {
+            _ = server => {},
+            _ = interval.select_next_some() => {
+                if exit.load(Ordering::Relaxed) {
+                    break;
+                }
+            }
+        }
+    }
+}
+
+impl RpcBanksService {
+    fn run(
+        listen_addr: SocketAddr,
+        tpu_addr: SocketAddr,
+        bank_forks: Arc<RwLock<BankForks>>,
+        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+        exit: Arc<AtomicBool>,
+    ) {
+        let server = start_abortable_tcp_server(
+            listen_addr,
+            tpu_addr,
+            bank_forks,
+            block_commitment_cache,
+            exit,
+        );
+        Runtime::new().unwrap().block_on(server);
+    }
+
+    pub fn new(
+        listen_addr: SocketAddr,
+        tpu_addr: SocketAddr,
+        bank_forks: &Arc<RwLock<BankForks>>,
+        block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
+        exit: &Arc<AtomicBool>,
+    ) -> Self {
+        let bank_forks = bank_forks.clone();
+        let block_commitment_cache = block_commitment_cache.clone();
+        let exit = exit.clone();
+        let thread_hdl = Builder::new()
+            .name("solana-rpc-banks".to_string())
+            .spawn(move || {
+                Self::run(
+                    listen_addr,
+                    tpu_addr,
+                    bank_forks,
+                    block_commitment_cache,
+                    exit,
+                )
+            })
+            .unwrap();
+
+        Self { thread_hdl }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_runtime::bank::Bank;
+
+    #[test]
+    fn test_rpc_banks_server_exit() {
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::default())));
+        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
+        let exit = Arc::new(AtomicBool::new(false));
+        let addr = "127.0.0.1:0".parse().unwrap();
+        let service = RpcBanksService::new(addr, addr, &bank_forks, &block_commitment_cache, &exit);
+        exit.store(true, Ordering::Relaxed);
+        service.join().unwrap();
+    }
+}

banks-server/src/send_transaction_service.rs

@@ -0,0 +1,343 @@
+// TODO: Merge this implementation with the one at `core/src/send_transaction_service.rs`
+use log::*;
+use solana_metrics::{datapoint_warn, inc_new_counter_info};
+use solana_runtime::{bank::Bank, bank_forks::BankForks};
+use solana_sdk::{clock::Slot, signature::Signature};
+use std::{
+    collections::HashMap,
+    net::{SocketAddr, UdpSocket},
+    sync::{
+        mpsc::{Receiver, RecvTimeoutError},
+        Arc, RwLock,
+    },
+    thread::{self, Builder, JoinHandle},
+    time::{Duration, Instant},
+};
+
+/// Maximum size of the transaction queue
+const MAX_TRANSACTION_QUEUE_SIZE: usize = 10_000; // This seems like a lot but maybe it needs to be bigger one day
+
+pub struct SendTransactionService {
+    thread: JoinHandle<()>,
+}
+
+pub struct TransactionInfo {
+    pub signature: Signature,
+    pub wire_transaction: Vec<u8>,
+    pub last_valid_slot: Slot,
+}
+
+impl TransactionInfo {
+    pub fn new(signature: Signature, wire_transaction: Vec<u8>, last_valid_slot: Slot) -> Self {
+        Self {
+            signature,
+            wire_transaction,
+            last_valid_slot,
+        }
+    }
+}
+
+#[derive(Default, Debug, PartialEq)]
+struct ProcessTransactionsResult {
+    rooted: u64,
+    expired: u64,
+    retried: u64,
+    failed: u64,
+    retained: u64,
+}
+
+impl SendTransactionService {
+    pub fn new(
+        tpu_address: SocketAddr,
+        bank_forks: &Arc<RwLock<BankForks>>,
+        receiver: Receiver<TransactionInfo>,
+    ) -> Self {
+        let thread = Self::retry_thread(receiver, bank_forks.clone(), tpu_address);
+        Self { thread }
+    }
+
+    fn retry_thread(
+        receiver: Receiver<TransactionInfo>,
+        bank_forks: Arc<RwLock<BankForks>>,
+        tpu_address: SocketAddr,
+    ) -> JoinHandle<()> {
+        let mut last_status_check = Instant::now();
+        let mut transactions = HashMap::new();
+        let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+
+        Builder::new()
+            .name("send-tx-svc".to_string())
+            .spawn(move || loop {
+                match receiver.recv_timeout(Duration::from_secs(1)) {
+                    Err(RecvTimeoutError::Disconnected) => break,
+                    Err(RecvTimeoutError::Timeout) => {}
+                    Ok(transaction_info) => {
+                        Self::send_transaction(
+                            &send_socket,
+                            &tpu_address,
+                            &transaction_info.wire_transaction,
+                        );
+                        if transactions.len() < MAX_TRANSACTION_QUEUE_SIZE {
+                            transactions.insert(transaction_info.signature, transaction_info);
+                        } else {
+                            datapoint_warn!("send_transaction_service-queue-overflow");
+                        }
+                    }
+                }
+
+                if Instant::now().duration_since(last_status_check).as_secs() >= 5 {
+                    if !transactions.is_empty() {
+                        datapoint_info!(
+                            "send_transaction_service-queue-size",
+                            ("len", transactions.len(), i64)
+                        );
+                        let bank_forks = bank_forks.read().unwrap();
+                        let root_bank = bank_forks.root_bank();
+                        let working_bank = bank_forks.working_bank();
+
+                        let _result = Self::process_transactions(
+                            &working_bank,
+                            &root_bank,
+                            &send_socket,
+                            &tpu_address,
+                            &mut transactions,
+                        );
+                    }
+                    last_status_check = Instant::now();
+                }
+            })
+            .unwrap()
+    }
+
+    fn process_transactions(
+        working_bank: &Arc<Bank>,
+        root_bank: &Arc<Bank>,
+        send_socket: &UdpSocket,
+        tpu_address: &SocketAddr,
+        transactions: &mut HashMap<Signature, TransactionInfo>,
+    ) -> ProcessTransactionsResult {
+        let mut result = ProcessTransactionsResult::default();
+
+        transactions.retain(|signature, transaction_info| {
+            if root_bank.has_signature(signature) {
+                info!("Transaction is rooted: {}", signature);
+                result.rooted += 1;
+                inc_new_counter_info!("send_transaction_service-rooted", 1);
+                false
+            } else if transaction_info.last_valid_slot < root_bank.slot() {
+                info!("Dropping expired transaction: {}", signature);
+                result.expired += 1;
+                inc_new_counter_info!("send_transaction_service-expired", 1);
+                false
+            } else {
+                match working_bank.get_signature_status_slot(signature) {
+                    None => {
+                        // Transaction is unknown to the working bank, it might have been
+                        // dropped or landed in another fork.  Re-send it
+                        info!("Retrying transaction: {}", signature);
+                        result.retried += 1;
+                        inc_new_counter_info!("send_transaction_service-retry", 1);
+                        Self::send_transaction(
+                            &send_socket,
+                            &tpu_address,
+                            &transaction_info.wire_transaction,
+                        );
+                        true
+                    }
+                    Some((_slot, status)) => {
+                        if status.is_err() {
+                            info!("Dropping failed transaction: {}", signature);
+                            result.failed += 1;
+                            inc_new_counter_info!("send_transaction_service-failed", 1);
+                            false
+                        } else {
+                            result.retained += 1;
+                            true
+                        }
+                    }
+                }
+            }
+        });
+
+        result
+    }
+
+    fn send_transaction(
+        send_socket: &UdpSocket,
+        tpu_address: &SocketAddr,
+        wire_transaction: &[u8],
+    ) {
+        if let Err(err) = send_socket.send_to(wire_transaction, tpu_address) {
+            warn!("Failed to send transaction to {}: {:?}", tpu_address, err);
+        }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread.join()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use solana_sdk::{
+        genesis_config::create_genesis_config, pubkey::Pubkey, signature::Signer,
+        system_transaction,
+    };
+    use std::sync::mpsc::channel;
+
+    #[test]
+    fn service_exit() {
+        let tpu_address = "127.0.0.1:0".parse().unwrap();
+        let bank = Bank::default();
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
+        let (sender, receiver) = channel();
+
+        let send_tranaction_service =
+            SendTransactionService::new(tpu_address, &bank_forks, receiver);
+
+        drop(sender);
+        send_tranaction_service.join().unwrap();
+    }
+
+    #[test]
+    fn process_transactions() {
+        let (genesis_config, mint_keypair) = create_genesis_config(4);
+        let bank = Bank::new(&genesis_config);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
+        let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let tpu_address = "127.0.0.1:0".parse().unwrap();
+
+        let root_bank = Arc::new(Bank::new_from_parent(
+            &bank_forks.read().unwrap().working_bank(),
+            &Pubkey::default(),
+            1,
+        ));
+        let rooted_signature = root_bank
+            .transfer(1, &mint_keypair, &mint_keypair.pubkey())
+            .unwrap();
+
+        let working_bank = Arc::new(Bank::new_from_parent(&root_bank, &Pubkey::default(), 2));
+
+        let non_rooted_signature = working_bank
+            .transfer(2, &mint_keypair, &mint_keypair.pubkey())
+            .unwrap();
+
+        let failed_signature = {
+            let blockhash = working_bank.last_blockhash();
+            let transaction =
+                system_transaction::transfer(&mint_keypair, &Pubkey::default(), 1, blockhash);
+            let signature = transaction.signatures[0];
+            working_bank.process_transaction(&transaction).unwrap_err();
+            signature
+        };
+
+        let mut transactions = HashMap::new();
+
+        info!("Expired transactions are dropped..");
+        transactions.insert(
+            Signature::default(),
+            TransactionInfo::new(Signature::default(), vec![], root_bank.slot() - 1),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                expired: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Rooted transactions are dropped...");
+        transactions.insert(
+            rooted_signature,
+            TransactionInfo::new(rooted_signature, vec![], working_bank.slot()),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                rooted: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Failed transactions are dropped...");
+        transactions.insert(
+            failed_signature,
+            TransactionInfo::new(failed_signature, vec![], working_bank.slot()),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                failed: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Non-rooted transactions are kept...");
+        transactions.insert(
+            non_rooted_signature,
+            TransactionInfo::new(non_rooted_signature, vec![], working_bank.slot()),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert_eq!(transactions.len(), 1);
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                retained: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+        transactions.clear();
+
+        info!("Unknown transactions are retried...");
+        transactions.insert(
+            Signature::default(),
+            TransactionInfo::new(Signature::default(), vec![], working_bank.slot()),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert_eq!(transactions.len(), 1);
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                retried: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+    }
+}

bench-exchange/Cargo.toml

@@ -1,34 +1,38 @@
 [package]
-authors = ["Solana Maintainers <maintainers@solana.com>"]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.0.13"
+version = "1.4.26"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 publish = false
 
 [dependencies]
-clap = "2.32.0"
-itertools = "0.8.2"
+clap = "2.33.1"
+itertools = "0.9.0"
 log = "0.4.8"
 num-derive = "0.3"
 num-traits = "0.2"
-rand = "0.6.5"
-rayon = "1.2.0"
-serde_json = "1.0.46"
-serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.13" }
-solana-core = { path = "../core", version = "1.0.13" }
-solana-genesis = { path = "../genesis", version = "1.0.13" }
-solana-client = { path = "../client", version = "1.0.13" }
-solana-faucet = { path = "../faucet", version = "1.0.13" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.0.13" }
-solana-logger = { path = "../logger", version = "1.0.13" }
-solana-metrics = { path = "../metrics", version = "1.0.13" }
-solana-net-utils = { path = "../net-utils", version = "1.0.13" }
-solana-runtime = { path = "../runtime", version = "1.0.13" }
-solana-sdk = { path = "../sdk", version = "1.0.13" }
+rand = "0.7.0"
+rayon = "1.4.0"
+serde_json = "1.0.56"
+serde_yaml = "0.8.13"
+solana-clap-utils = { path = "../clap-utils", version = "1.4.26" }
+solana-core = { path = "../core", version = "1.4.26" }
+solana-genesis = { path = "../genesis", version = "1.4.26" }
+solana-client = { path = "../client", version = "1.4.26" }
+solana-faucet = { path = "../faucet", version = "1.4.26" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.4.26" }
+solana-logger = { path = "../logger", version = "1.4.26" }
+solana-metrics = { path = "../metrics", version = "1.4.26" }
+solana-net-utils = { path = "../net-utils", version = "1.4.26" }
+solana-runtime = { path = "../runtime", version = "1.4.26" }
+solana-sdk = { path = "../sdk", version = "1.4.26" }
+solana-version = { path = "../version", version = "1.4.26" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.0.13" }
+solana-local-cluster = { path = "../local-cluster", version = "1.4.26" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/src/bench.rs

@@ -14,6 +14,7 @@ use solana_metrics::datapoint_info;
 use solana_sdk::{
     client::{Client, SyncClient},
     commitment_config::CommitmentConfig,
+    message::Message,
     pubkey::Pubkey,
     signature::{Keypair, Signer},
     timing::{duration_as_ms, duration_as_s},
@@ -178,19 +179,13 @@ where
 
     info!("Generating {:?} account keys", total_keys);
     let mut account_keypairs = generate_keypairs(total_keys);
-    let src_keypairs: Vec<_> = account_keypairs
-        .drain(0..accounts_in_groups)
-        .map(|keypair| keypair)
-        .collect();
+    let src_keypairs: Vec<_> = account_keypairs.drain(0..accounts_in_groups).collect();
     let src_pubkeys: Vec<Pubkey> = src_keypairs
         .iter()
         .map(|keypair| keypair.pubkey())
         .collect();
 
-    let profit_keypairs: Vec<_> = account_keypairs
-        .drain(0..accounts_in_groups)
-        .map(|keypair| keypair)
-        .collect();
+    let profit_keypairs: Vec<_> = account_keypairs.drain(0..accounts_in_groups).collect();
     let profit_pubkeys: Vec<Pubkey> = profit_keypairs
         .iter()
         .map(|keypair| keypair.pubkey())
@@ -449,7 +444,7 @@ fn swapper<T>(
             }
             account_group = (account_group + 1) % account_groups as usize;
 
-            let (blockhash, _fee_calculator) = client
+            let (blockhash, _fee_calculator, _last_valid_slot) = client
                 .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
                 .expect("Failed to get blockhash");
             let to_swap_txs: Vec<_> = to_swap
@@ -457,16 +452,14 @@ fn swapper<T>(
                 .map(|(signer, swap, profit)| {
                     let s: &Keypair = &signer;
                     let owner = &signer.pubkey();
-                    Transaction::new_signed_instructions(
-                        &[s],
-                        vec![exchange_instruction::swap_request(
-                            owner,
-                            &swap.0.pubkey,
-                            &swap.1.pubkey,
-                            &profit,
-                        )],
-                        blockhash,
-                    )
+                    let instruction = exchange_instruction::swap_request(
+                        owner,
+                        &swap.0.pubkey,
+                        &swap.1.pubkey,
+                        &profit,
+                    );
+                    let message = Message::new(&[instruction], Some(&s.pubkey()));
+                    Transaction::new(&[s], message, blockhash)
                 })
                 .collect();
 
@@ -577,7 +570,7 @@ fn trader<T>(
         }
         account_group = (account_group + 1) % account_groups as usize;
 
-        let (blockhash, _fee_calculator) = client
+        let (blockhash, _fee_calculator, _last_valid_slot) = client
             .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
             .expect("Failed to get blockhash");
 
@@ -588,28 +581,26 @@ fn trader<T>(
                     let owner_pubkey = &owner.pubkey();
                     let trade_pubkey = &trade.pubkey();
                     let space = mem::size_of::<ExchangeState>() as u64;
-                    Transaction::new_signed_instructions(
-                        &[owner.as_ref(), trade],
-                        vec![
-                            system_instruction::create_account(
-                                owner_pubkey,
-                                trade_pubkey,
-                                1,
-                                space,
-                                &id(),
-                            ),
-                            exchange_instruction::trade_request(
-                                owner_pubkey,
-                                trade_pubkey,
-                                *side,
-                                pair,
-                                tokens,
-                                price,
-                                src,
-                            ),
-                        ],
-                        blockhash,
-                    )
+                    let instructions = [
+                        system_instruction::create_account(
+                            owner_pubkey,
+                            trade_pubkey,
+                            1,
+                            space,
+                            &id(),
+                        ),
+                        exchange_instruction::trade_request(
+                            owner_pubkey,
+                            trade_pubkey,
+                            *side,
+                            pair,
+                            tokens,
+                            price,
+                            src,
+                        ),
+                    ];
+                    let message = Message::new(&instructions, Some(&owner_pubkey));
+                    Transaction::new(&[owner.as_ref(), trade], message, blockhash)
                 })
                 .collect();
 
@@ -747,22 +738,19 @@ pub fn fund_keys<T: Client>(client: &T, source: &Keypair, dests: &[Arc<Keypair>]
             let mut to_fund_txs: Vec<_> = chunk
                 .par_iter()
                 .map(|(k, m)| {
-                    (
-                        k.clone(),
-                        Transaction::new_unsigned_instructions(system_instruction::transfer_many(
-                            &k.pubkey(),
-                            &m,
-                        )),
-                    )
+                    let instructions = system_instruction::transfer_many(&k.pubkey(), &m);
+                    let message = Message::new(&instructions, Some(&k.pubkey()));
+                    (k.clone(), Transaction::new_unsigned(message))
                 })
                 .collect();
 
             let mut retries = 0;
             let amount = chunk[0].1[0].1;
             while !to_fund_txs.is_empty() {
-                let receivers = to_fund_txs
+                let receivers: usize = to_fund_txs
                     .iter()
-                    .fold(0, |len, (_, tx)| len + tx.message().instructions.len());
+                    .map(|(_, tx)| tx.message().instructions.len())
+                    .sum();
 
                 debug!(
                     "  {} to {} in {} txs",
@@ -775,7 +763,7 @@ pub fn fund_keys<T: Client>(client: &T, source: &Keypair, dests: &[Arc<Keypair>]
                     to_fund_txs.len(),
                 );
 
-                let (blockhash, _fee_calculator) = client
+                let (blockhash, _fee_calculator, _last_valid_slot) = client
                     .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
                     .expect("blockhash");
                 to_fund_txs.par_iter_mut().for_each(|(k, tx)| {
@@ -847,16 +835,18 @@ pub fn create_token_accounts<T: Client>(
                     );
                     let request_ix =
                         exchange_instruction::account_request(owner_pubkey, &new_keypair.pubkey());
+                    let message = Message::new(&[create_ix, request_ix], Some(&owner_pubkey));
                     (
                         (from_keypair, new_keypair),
-                        Transaction::new_unsigned_instructions(vec![create_ix, request_ix]),
+                        Transaction::new_unsigned(message),
                     )
                 })
                 .collect();
 
-            let accounts = to_create_txs
+            let accounts: usize = to_create_txs
                 .iter()
-                .fold(0, |len, (_, tx)| len + tx.message().instructions.len() / 2);
+                .map(|(_, tx)| tx.message().instructions.len() / 2)
+                .sum();
 
             debug!(
                 "  Creating {} accounts in {} txs",
@@ -866,7 +856,7 @@ pub fn create_token_accounts<T: Client>(
 
             let mut retries = 0;
             while !to_create_txs.is_empty() {
-                let (blockhash, _fee_calculator) = client
+                let (blockhash, _fee_calculator, _last_valid_slot) = client
                     .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
                     .expect("Failed to get blockhash");
                 to_create_txs
@@ -995,7 +985,7 @@ pub fn airdrop_lamports<T: Client>(
 
     let mut tries = 0;
     loop {
-        let (blockhash, _fee_calculator) = client
+        let (blockhash, _fee_calculator, _last_valid_slot) = client
             .get_recent_blockhash_with_commitment(CommitmentConfig::recent())
             .expect("Failed to get blockhash");
         match request_airdrop_transaction(&faucet_addr, &id.pubkey(), amount_to_drop, blockhash) {

bench-exchange/src/main.rs

@@ -11,7 +11,7 @@ fn main() {
     solana_logger::setup();
     solana_metrics::set_panic_hook("bench-exchange");
 
-    let matches = cli::build_args(solana_clap_utils::version!()).get_matches();
+    let matches = cli::build_args(solana_version::version!()).get_matches();
     let cli_config = cli::extract_args(&matches);
 
     let cli::Config {
@@ -54,10 +54,9 @@ fn main() {
         );
     } else {
         info!("Connecting to the cluster");
-        let (nodes, _archivers) =
-            discover_cluster(&entrypoint_addr, num_nodes).unwrap_or_else(|_| {
-                panic!("Failed to discover nodes");
-            });
+        let nodes = discover_cluster(&entrypoint_addr, num_nodes).unwrap_or_else(|_| {
+            panic!("Failed to discover nodes");
+        });
 
         let (client, num_clients) = get_multi_client(&nodes);
 

bench-exchange/tests/bench_exchange.rs

@@ -39,7 +39,7 @@ fn test_exchange_local_cluster() {
     } = config;
     let accounts_in_groups = batch_size * account_groups;
 
-    let cluster = LocalCluster::new(&ClusterConfig {
+    let cluster = LocalCluster::new(&mut ClusterConfig {
         node_stakes: vec![100_000; NUM_NODES],
         cluster_lamports: 100_000_000_000_000,
         validator_configs: vec![ValidatorConfig::default(); NUM_NODES],
@@ -59,7 +59,7 @@ fn test_exchange_local_cluster() {
     let faucet_addr = addr_receiver.recv_timeout(Duration::from_secs(2)).unwrap();
 
     info!("Connecting to the cluster");
-    let (nodes, _) =
+    let nodes =
         discover_cluster(&cluster.entry_point_info.gossip, NUM_NODES).unwrap_or_else(|err| {
             error!("Failed to discover {} nodes: {:?}", NUM_NODES, err);
             exit(1);
@@ -86,7 +86,7 @@ fn test_exchange_bank_client() {
     solana_logger::setup();
     let (genesis_config, identity) = create_genesis_config(100_000_000_000_000);
     let mut bank = Bank::new(&genesis_config);
-    bank.add_instruction_processor(id(), process_instruction);
+    bank.add_builtin("exchange_program", id(), process_instruction);
     let clients = vec![BankClient::new(bank)];
 
     let mut config = Config::default();

bench-streamer/Cargo.toml

@@ -1,15 +1,20 @@
 [package]
-authors = ["Solana Maintainers <maintainers@solana.com>"]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.0.13"
+version = "1.4.26"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
+publish = false
 
 [dependencies]
-clap = "2.33.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.13" }
-solana-core = { path = "../core", version = "1.0.13" }
-solana-logger = { path = "../logger", version = "1.0.13" }
-solana-net-utils = { path = "../net-utils", version = "1.0.13" }
+clap = "2.33.1"
+solana-clap-utils = { path = "../clap-utils", version = "1.4.26" }
+solana-streamer = { path = "../streamer", version = "1.4.26" }
+solana-logger = { path = "../logger", version = "1.4.26" }
+solana-net-utils = { path = "../net-utils", version = "1.4.26" }
+solana-version = { path = "../version", version = "1.4.26" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/src/main.rs

@@ -1,6 +1,6 @@
 use clap::{crate_description, crate_name, App, Arg};
-use solana_core::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
-use solana_core::streamer::{receiver, PacketReceiver};
+use solana_streamer::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
+use solana_streamer::streamer::{receiver, PacketReceiver};
 use std::cmp::max;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
@@ -27,7 +27,7 @@ fn producer(addr: &SocketAddr, exit: Arc<AtomicBool>) -> JoinHandle<()> {
         let mut num = 0;
         for p in &msgs.packets {
             let a = p.meta.addr();
-            assert!(p.meta.size < PACKET_DATA_SIZE);
+            assert!(p.meta.size <= PACKET_DATA_SIZE);
             send.send_to(&p.data[..p.meta.size], &a).unwrap();
             num += 1;
         }
@@ -52,7 +52,7 @@ fn main() -> Result<()> {
 
     let matches = App::new(crate_name!())
         .about(crate_description!())
-        .version(solana_clap_utils::version!())
+        .version(solana_version::version!())
         .arg(
             Arg::with_name("num-recv-sockets")
                 .long("num-recv-sockets")

bench-tps/Cargo.toml

@@ -1,37 +1,37 @@
 [package]
-authors = ["Solana Maintainers <maintainers@solana.com>"]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.0.13"
+version = "1.4.26"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
+publish = false
 
 [dependencies]
-bincode = "1.2.1"
-clap = "2.33.0"
+bincode = "1.3.1"
+clap = "2.33.1"
 log = "0.4.8"
-rayon = "1.2.0"
-serde_json = "1.0.46"
-serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.13" }
-solana-core = { path = "../core", version = "1.0.13" }
-solana-genesis = { path = "../genesis", version = "1.0.13" }
-solana-client = { path = "../client", version = "1.0.13" }
-solana-faucet = { path = "../faucet", version = "1.0.13" }
-solana-librapay = { path = "../programs/librapay", version = "1.0.13", optional = true }
-solana-logger = { path = "../logger", version = "1.0.13" }
-solana-metrics = { path = "../metrics", version = "1.0.13" }
-solana-measure = { path = "../measure", version = "1.0.13" }
-solana-net-utils = { path = "../net-utils", version = "1.0.13" }
-solana-runtime = { path = "../runtime", version = "1.0.13" }
-solana-sdk = { path = "../sdk", version = "1.0.13" }
-solana-move-loader-program = { path = "../programs/move_loader", version = "1.0.13", optional = true }
+rayon = "1.4.0"
+serde_json = "1.0.56"
+serde_yaml = "0.8.13"
+solana-clap-utils = { path = "../clap-utils", version = "1.4.26" }
+solana-core = { path = "../core", version = "1.4.26" }
+solana-genesis = { path = "../genesis", version = "1.4.26" }
+solana-client = { path = "../client", version = "1.4.26" }
+solana-faucet = { path = "../faucet", version = "1.4.26" }
+solana-logger = { path = "../logger", version = "1.4.26" }
+solana-metrics = { path = "../metrics", version = "1.4.26" }
+solana-measure = { path = "../measure", version = "1.4.26" }
+solana-net-utils = { path = "../net-utils", version = "1.4.26" }
+solana-runtime = { path = "../runtime", version = "1.4.26" }
+solana-sdk = { path = "../sdk", version = "1.4.26" }
+solana-version = { path = "../version", version = "1.4.26" }
 
 [dev-dependencies]
-serial_test = "0.3.2"
+serial_test = "0.4.0"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.0.13" }
+solana-local-cluster = { path = "../local-cluster", version = "1.4.26" }
 
-[features]
-move = ["solana-librapay", "solana-move-loader-program"]
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

bench-tps/src/bench.rs

@@ -4,8 +4,6 @@ use rayon::prelude::*;
 use solana_client::perf_utils::{sample_txs, SampleStats};
 use solana_core::gen_keys::GenKeys;
 use solana_faucet::faucet::request_airdrop_transaction;
-#[cfg(feature = "move")]
-use solana_librapay::{create_genesis, upload_mint_script, upload_payment_script};
 use solana_measure::measure::Measure;
 use solana_metrics::{self, datapoint_info};
 use solana_sdk::{
@@ -14,6 +12,7 @@ use solana_sdk::{
     commitment_config::CommitmentConfig,
     fee_calculator::FeeCalculator,
     hash::Hash,
+    message::Message,
     pubkey::Pubkey,
     signature::{Keypair, Signer},
     system_instruction, system_transaction,
@@ -26,9 +25,9 @@ use std::{
     process::exit,
     sync::{
         atomic::{AtomicBool, AtomicIsize, AtomicUsize, Ordering},
-        Arc, RwLock,
+        Arc, Mutex, RwLock,
     },
-    thread::{sleep, Builder},
+    thread::{sleep, Builder, JoinHandle},
     time::{Duration, Instant},
 };
 
@@ -36,9 +35,6 @@ use std::{
 const MAX_TX_QUEUE_AGE: u64 =
     MAX_PROCESSING_AGE as u64 * DEFAULT_TICKS_PER_SLOT / DEFAULT_TICKS_PER_SECOND;
 
-#[cfg(feature = "move")]
-use solana_librapay::librapay_transaction;
-
 pub const MAX_SPENDS_PER_TX: u64 = 4;
 
 #[derive(Debug)]
@@ -50,12 +46,12 @@ pub type Result<T> = std::result::Result<T, BenchTpsError>;
 
 pub type SharedTransactions = Arc<RwLock<VecDeque<Vec<(Transaction, u64)>>>>;
 
-type LibraKeys = (Keypair, Pubkey, Pubkey, Vec<Keypair>);
-
 fn get_recent_blockhash<T: Client>(client: &T) -> (Hash, FeeCalculator) {
     loop {
         match client.get_recent_blockhash_with_commitment(CommitmentConfig::recent()) {
-            Ok((blockhash, fee_calculator)) => return (blockhash, fee_calculator),
+            Ok((blockhash, fee_calculator, _last_valid_slot)) => {
+                return (blockhash, fee_calculator)
+            }
             Err(err) => {
                 info!("Couldn't get recent blockhash: {:?}", err);
                 sleep(Duration::from_secs(1));
@@ -64,105 +60,62 @@ fn get_recent_blockhash<T: Client>(client: &T) -> (Hash, FeeCalculator) {
     }
 }
 
-pub fn do_bench_tps<T>(
-    client: Arc<T>,
-    config: Config,
-    gen_keypairs: Vec<Keypair>,
-    libra_args: Option<LibraKeys>,
-) -> u64
+fn wait_for_target_slots_per_epoch<T>(target_slots_per_epoch: u64, client: &Arc<T>)
 where
     T: 'static + Client + Send + Sync,
 {
-    let Config {
-        id,
-        threads,
-        thread_batch_sleep_ms,
-        duration,
-        tx_count,
-        sustained,
-        ..
-    } = config;
-
-    let mut source_keypair_chunks: Vec<Vec<&Keypair>> = Vec::new();
-    let mut dest_keypair_chunks: Vec<VecDeque<&Keypair>> = Vec::new();
-    assert!(gen_keypairs.len() >= 2 * tx_count);
-    for chunk in gen_keypairs.chunks_exact(2 * tx_count) {
-        source_keypair_chunks.push(chunk[..tx_count].iter().collect());
-        dest_keypair_chunks.push(chunk[tx_count..].iter().collect());
-    }
-
-    let first_tx_count = loop {
-        match client.get_transaction_count() {
-            Ok(count) => break count,
-            Err(err) => {
-                info!("Couldn't get transaction count: {:?}", err);
-                sleep(Duration::from_secs(1));
+    if target_slots_per_epoch != 0 {
+        info!(
+            "Waiting until epochs are {} slots long..",
+            target_slots_per_epoch
+        );
+        loop {
+            if let Ok(epoch_info) = client.get_epoch_info() {
+                if epoch_info.slots_in_epoch >= target_slots_per_epoch {
+                    info!("Done epoch_info: {:?}", epoch_info);
+                    break;
+                }
+                info!(
+                    "Waiting for epoch: {} now: {}",
+                    target_slots_per_epoch, epoch_info.slots_in_epoch
+                );
             }
+            sleep(Duration::from_secs(3));
         }
-    };
-    info!("Initial transaction count {}", first_tx_count);
+    }
+}
 
-    let exit_signal = Arc::new(AtomicBool::new(false));
-
-    // Setup a thread per validator to sample every period
-    // collect the max transaction rate and total tx count seen
-    let maxes = Arc::new(RwLock::new(Vec::new()));
-    let sample_period = 1; // in seconds
+fn create_sampler_thread<T>(
+    client: &Arc<T>,
+    exit_signal: &Arc<AtomicBool>,
+    sample_period: u64,
+    maxes: &Arc<RwLock<Vec<(String, SampleStats)>>>,
+) -> JoinHandle<()>
+where
+    T: 'static + Client + Send + Sync,
+{
     info!("Sampling TPS every {} second...", sample_period);
-    let sample_thread = {
-        let exit_signal = exit_signal.clone();
-        let maxes = maxes.clone();
-        let client = client.clone();
-        Builder::new()
-            .name("solana-client-sample".to_string())
-            .spawn(move || {
-                sample_txs(&exit_signal, &maxes, sample_period, &client);
-            })
-            .unwrap()
-    };
-
-    let shared_txs: SharedTransactions = Arc::new(RwLock::new(VecDeque::new()));
-
-    let recent_blockhash = Arc::new(RwLock::new(get_recent_blockhash(client.as_ref()).0));
-    let shared_tx_active_thread_count = Arc::new(AtomicIsize::new(0));
-    let total_tx_sent_count = Arc::new(AtomicUsize::new(0));
-
-    let blockhash_thread = {
-        let exit_signal = exit_signal.clone();
-        let recent_blockhash = recent_blockhash.clone();
-        let client = client.clone();
-        let id = id.pubkey();
-        Builder::new()
-            .name("solana-blockhash-poller".to_string())
-            .spawn(move || {
-                poll_blockhash(&exit_signal, &recent_blockhash, &client, &id);
-            })
-            .unwrap()
-    };
-
-    let s_threads: Vec<_> = (0..threads)
-        .map(|_| {
-            let exit_signal = exit_signal.clone();
-            let shared_txs = shared_txs.clone();
-            let shared_tx_active_thread_count = shared_tx_active_thread_count.clone();
-            let total_tx_sent_count = total_tx_sent_count.clone();
-            let client = client.clone();
-            Builder::new()
-                .name("solana-client-sender".to_string())
-                .spawn(move || {
-                    do_tx_transfers(
-                        &exit_signal,
-                        &shared_txs,
-                        &shared_tx_active_thread_count,
-                        &total_tx_sent_count,
-                        thread_batch_sleep_ms,
-                        &client,
-                    );
-                })
-                .unwrap()
+    let exit_signal = exit_signal.clone();
+    let maxes = maxes.clone();
+    let client = client.clone();
+    Builder::new()
+        .name("solana-client-sample".to_string())
+        .spawn(move || {
+            sample_txs(&exit_signal, &maxes, sample_period, &client);
         })
-        .collect();
+        .unwrap()
+}
 
+fn generate_chunked_transfers(
+    recent_blockhash: Arc<RwLock<Hash>>,
+    shared_txs: &SharedTransactions,
+    shared_tx_active_thread_count: Arc<AtomicIsize>,
+    source_keypair_chunks: Vec<Vec<&Keypair>>,
+    dest_keypair_chunks: &mut Vec<VecDeque<&Keypair>>,
+    threads: usize,
+    duration: Duration,
+    sustained: bool,
+) {
     // generate and send transactions for the specified duration
     let start = Instant::now();
     let keypair_chunks = source_keypair_chunks.len();
@@ -170,13 +123,12 @@ where
     let mut chunk_index = 0;
     while start.elapsed() < duration {
         generate_txs(
-            &shared_txs,
+            shared_txs,
             &recent_blockhash,
             &source_keypair_chunks[chunk_index],
             &dest_keypair_chunks[chunk_index],
             threads,
             reclaim_lamports_back_to_source_account,
-            &libra_args,
         );
 
         // In sustained mode, overlap the transfers with generation. This has higher average
@@ -206,6 +158,129 @@ where
             reclaim_lamports_back_to_source_account = !reclaim_lamports_back_to_source_account;
         }
     }
+}
+
+fn create_sender_threads<T>(
+    client: &Arc<T>,
+    shared_txs: &SharedTransactions,
+    thread_batch_sleep_ms: usize,
+    total_tx_sent_count: &Arc<AtomicUsize>,
+    threads: usize,
+    exit_signal: &Arc<AtomicBool>,
+    shared_tx_active_thread_count: &Arc<AtomicIsize>,
+) -> Vec<JoinHandle<()>>
+where
+    T: 'static + Client + Send + Sync,
+{
+    (0..threads)
+        .map(|_| {
+            let exit_signal = exit_signal.clone();
+            let shared_txs = shared_txs.clone();
+            let shared_tx_active_thread_count = shared_tx_active_thread_count.clone();
+            let total_tx_sent_count = total_tx_sent_count.clone();
+            let client = client.clone();
+            Builder::new()
+                .name("solana-client-sender".to_string())
+                .spawn(move || {
+                    do_tx_transfers(
+                        &exit_signal,
+                        &shared_txs,
+                        &shared_tx_active_thread_count,
+                        &total_tx_sent_count,
+                        thread_batch_sleep_ms,
+                        &client,
+                    );
+                })
+                .unwrap()
+        })
+        .collect()
+}
+
+pub fn do_bench_tps<T>(client: Arc<T>, config: Config, gen_keypairs: Vec<Keypair>) -> u64
+where
+    T: 'static + Client + Send + Sync,
+{
+    let Config {
+        id,
+        threads,
+        thread_batch_sleep_ms,
+        duration,
+        tx_count,
+        sustained,
+        target_slots_per_epoch,
+        ..
+    } = config;
+
+    let mut source_keypair_chunks: Vec<Vec<&Keypair>> = Vec::new();
+    let mut dest_keypair_chunks: Vec<VecDeque<&Keypair>> = Vec::new();
+    assert!(gen_keypairs.len() >= 2 * tx_count);
+    for chunk in gen_keypairs.chunks_exact(2 * tx_count) {
+        source_keypair_chunks.push(chunk[..tx_count].iter().collect());
+        dest_keypair_chunks.push(chunk[tx_count..].iter().collect());
+    }
+
+    let first_tx_count = loop {
+        match client.get_transaction_count() {
+            Ok(count) => break count,
+            Err(err) => {
+                info!("Couldn't get transaction count: {:?}", err);
+                sleep(Duration::from_secs(1));
+            }
+        }
+    };
+    info!("Initial transaction count {}", first_tx_count);
+
+    let exit_signal = Arc::new(AtomicBool::new(false));
+
+    // Setup a thread per validator to sample every period
+    // collect the max transaction rate and total tx count seen
+    let maxes = Arc::new(RwLock::new(Vec::new()));
+    let sample_period = 1; // in seconds
+    let sample_thread = create_sampler_thread(&client, &exit_signal, sample_period, &maxes);
+
+    let shared_txs: SharedTransactions = Arc::new(RwLock::new(VecDeque::new()));
+
+    let recent_blockhash = Arc::new(RwLock::new(get_recent_blockhash(client.as_ref()).0));
+    let shared_tx_active_thread_count = Arc::new(AtomicIsize::new(0));
+    let total_tx_sent_count = Arc::new(AtomicUsize::new(0));
+
+    let blockhash_thread = {
+        let exit_signal = exit_signal.clone();
+        let recent_blockhash = recent_blockhash.clone();
+        let client = client.clone();
+        let id = id.pubkey();
+        Builder::new()
+            .name("solana-blockhash-poller".to_string())
+            .spawn(move || {
+                poll_blockhash(&exit_signal, &recent_blockhash, &client, &id);
+            })
+            .unwrap()
+    };
+
+    let s_threads = create_sender_threads(
+        &client,
+        &shared_txs,
+        thread_batch_sleep_ms,
+        &total_tx_sent_count,
+        threads,
+        &exit_signal,
+        &shared_tx_active_thread_count,
+    );
+
+    wait_for_target_slots_per_epoch(target_slots_per_epoch, &client);
+
+    let start = Instant::now();
+
+    generate_chunked_transfers(
+        recent_blockhash,
+        &shared_txs,
+        shared_tx_active_thread_count,
+        source_keypair_chunks,
+        &mut dest_keypair_chunks,
+        threads,
+        duration,
+        sustained,
+    );
 
     // Stop the sampling threads so it will collect the stats
     exit_signal.store(true, Ordering::Relaxed);
@@ -250,52 +325,6 @@ fn metrics_submit_lamport_balance(lamport_balance: u64) {
     );
 }
 
-#[cfg(feature = "move")]
-fn generate_move_txs(
-    source: &[&Keypair],
-    dest: &VecDeque<&Keypair>,
-    reclaim: bool,
-    move_keypairs: &[Keypair],
-    libra_pay_program_id: &Pubkey,
-    libra_mint_id: &Pubkey,
-    blockhash: &Hash,
-) -> Vec<(Transaction, u64)> {
-    let count = move_keypairs.len() / 2;
-    let source_move = &move_keypairs[..count];
-    let dest_move = &move_keypairs[count..];
-    let pairs: Vec<_> = if !reclaim {
-        source_move
-            .iter()
-            .zip(dest_move.iter())
-            .zip(source.iter())
-            .collect()
-    } else {
-        dest_move
-            .iter()
-            .zip(source_move.iter())
-            .zip(dest.iter())
-            .collect()
-    };
-
-    pairs
-        .par_iter()
-        .map(|((from, to), payer)| {
-            (
-                librapay_transaction::transfer(
-                    libra_pay_program_id,
-                    libra_mint_id,
-                    &payer,
-                    &from,
-                    &to.pubkey(),
-                    1,
-                    *blockhash,
-                ),
-                timestamp(),
-            )
-        })
-        .collect()
-}
-
 fn generate_system_txs(
     source: &[&Keypair],
     dest: &VecDeque<&Keypair>,
@@ -326,7 +355,6 @@ fn generate_txs(
     dest: &VecDeque<&Keypair>,
     threads: usize,
     reclaim: bool,
-    libra_args: &Option<LibraKeys>,
 ) {
     let blockhash = *blockhash.read().unwrap();
     let tx_count = source.len();
@@ -336,33 +364,7 @@ fn generate_txs(
     );
     let signing_start = Instant::now();
 
-    let transactions = if let Some((
-        _libra_genesis_keypair,
-        _libra_pay_program_id,
-        _libra_mint_program_id,
-        _libra_keys,
-    )) = libra_args
-    {
-        #[cfg(not(feature = "move"))]
-        {
-            return;
-        }
-
-        #[cfg(feature = "move")]
-        {
-            generate_move_txs(
-                source,
-                dest,
-                reclaim,
-                &_libra_keys,
-                _libra_pay_program_id,
-                &_libra_genesis_keypair.pubkey(),
-                &blockhash,
-            )
-        }
-    } else {
-        generate_system_txs(source, dest, reclaim, &blockhash)
-    };
+    let transactions = generate_system_txs(source, dest, reclaim, &blockhash);
 
     let duration = signing_start.elapsed();
     let ns = duration.as_secs() * 1_000_000_000 + u64::from(duration.subsec_nanos());
@@ -563,11 +565,9 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
         let to_fund_txs: Vec<(&Keypair, Transaction)> = to_fund
             .par_iter()
             .map(|(k, t)| {
-                let tx = Transaction::new_unsigned_instructions(system_instruction::transfer_many(
-                    &k.pubkey(),
-                    &t,
-                ));
-                (*k, tx)
+                let instructions = system_instruction::transfer_many(&k.pubkey(), &t);
+                let message = Message::new(&instructions, Some(&k.pubkey()));
+                (*k, Transaction::new_unsigned(message))
             })
             .collect();
         make_txs.stop();
@@ -603,7 +603,9 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
         let too_many_failures = Arc::new(AtomicBool::new(false));
         let loops = if starting_txs < 1000 { 3 } else { 1 };
         // Only loop multiple times for small (quick) transaction batches
+        let time = Arc::new(Mutex::new(Instant::now()));
         for _ in 0..loops {
+            let time = time.clone();
             let failed_verify = Arc::new(AtomicUsize::new(0));
             let client = client.clone();
             let verified_txs = &verified_txs;
@@ -634,11 +636,15 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
                             remaining_count, verified_txs, failed_verify
                         );
                     }
-                    if remaining_count % 100 == 0 {
-                        info!(
-                            "Verifying transfers... {} remaining, {} verified, {} failures",
-                            remaining_count, verified_txs, failed_verify
-                        );
+                    if remaining_count > 0 {
+                        let mut time_l = time.lock().unwrap();
+                        if time_l.elapsed().as_secs() > 2 {
+                            info!(
+                                "Verifying transfers... {} remaining, {} verified, {} failures",
+                                remaining_count, verified_txs, failed_verify
+                            );
+                            *time_l = Instant::now();
+                        }
                     }
 
                     verified
@@ -860,181 +866,13 @@ pub fn generate_keypairs(seed_keypair: &Keypair, count: u64) -> (Vec<Keypair>, u
     (rnd.gen_n_keypairs(total_keys), extra)
 }
 
-#[cfg(feature = "move")]
-fn fund_move_keys<T: Client>(
-    client: &T,
-    funding_key: &Keypair,
-    keypairs: &[Keypair],
-    total: u64,
-    libra_pay_program_id: &Pubkey,
-    libra_mint_program_id: &Pubkey,
-    libra_genesis_key: &Keypair,
-) {
-    let (mut blockhash, _fee_calculator) = get_recent_blockhash(client);
-
-    info!("creating the libra funding account..");
-    let libra_funding_key = Keypair::new();
-    let tx = librapay_transaction::create_account(funding_key, &libra_funding_key, 1, blockhash);
-    client
-        .send_message(&[funding_key, &libra_funding_key], tx.message)
-        .unwrap();
-
-    info!("minting to funding keypair");
-    let tx = librapay_transaction::mint_tokens(
-        &libra_mint_program_id,
-        funding_key,
-        libra_genesis_key,
-        &libra_funding_key.pubkey(),
-        total,
-        blockhash,
-    );
-    client
-        .send_message(&[funding_key, libra_genesis_key], tx.message)
-        .unwrap();
-
-    info!("creating {} move accounts...", keypairs.len());
-    let total_len = keypairs.len();
-    let create_len = 5;
-    let mut funding_time = Measure::start("funding_time");
-    for (i, keys) in keypairs.chunks(create_len).enumerate() {
-        if client
-            .get_balance_with_commitment(&keys[0].pubkey(), CommitmentConfig::recent())
-            .unwrap_or(0)
-            > 0
-        {
-            // already created these accounts.
-            break;
-        }
-
-        let keypairs: Vec<_> = keys.iter().map(|k| k).collect();
-        let tx = librapay_transaction::create_accounts(funding_key, &keypairs, 1, blockhash);
-        let ser_size = bincode::serialized_size(&tx).unwrap();
-        let mut keys = vec![funding_key];
-        keys.extend(&keypairs);
-        client.send_message(&keys, tx.message).unwrap();
-
-        if i % 10 == 0 {
-            info!(
-                "created {} accounts of {} (size {})",
-                i,
-                total_len / create_len,
-                ser_size,
-            );
-        }
-    }
-
-    const NUM_FUNDING_KEYS: usize = 10;
-    let funding_keys: Vec<_> = (0..NUM_FUNDING_KEYS).map(|_| Keypair::new()).collect();
-    let pubkey_amounts: Vec<_> = funding_keys
-        .iter()
-        .map(|key| (key.pubkey(), total / NUM_FUNDING_KEYS as u64))
-        .collect();
-    let tx = Transaction::new_signed_instructions(
-        &[funding_key],
-        system_instruction::transfer_many(&funding_key.pubkey(), &pubkey_amounts),
-        blockhash,
-    );
-    client.send_message(&[funding_key], tx.message).unwrap();
-    let mut balance = 0;
-    for _ in 0..20 {
-        if let Ok(balance_) = client
-            .get_balance_with_commitment(&funding_keys[0].pubkey(), CommitmentConfig::recent())
-        {
-            if balance_ > 0 {
-                balance = balance_;
-                break;
-            }
-        }
-        sleep(Duration::from_millis(100));
-    }
-    assert!(balance > 0);
-    info!(
-        "funded multiple funding accounts with {:?} lanports",
-        balance
-    );
-
-    let libra_funding_keys: Vec<_> = (0..NUM_FUNDING_KEYS).map(|_| Keypair::new()).collect();
-    for (i, key) in libra_funding_keys.iter().enumerate() {
-        let tx = librapay_transaction::create_account(&funding_keys[i], &key, 1, blockhash);
-        client
-            .send_message(&[&funding_keys[i], &key], tx.message)
-            .unwrap();
-
-        let tx = librapay_transaction::transfer(
-            libra_pay_program_id,
-            &libra_genesis_key.pubkey(),
-            &funding_keys[i],
-            &libra_funding_key,
-            &key.pubkey(),
-            total / NUM_FUNDING_KEYS as u64,
-            blockhash,
-        );
-        client
-            .send_message(&[&funding_keys[i], &libra_funding_key], tx.message)
-            .unwrap();
-
-        info!("funded libra funding key {}", i);
-    }
-
-    let keypair_count = keypairs.len();
-    let amount = total / (keypair_count as u64);
-    for (i, keys) in keypairs[..keypair_count]
-        .chunks(NUM_FUNDING_KEYS)
-        .enumerate()
-    {
-        for (j, key) in keys.iter().enumerate() {
-            let tx = librapay_transaction::transfer(
-                libra_pay_program_id,
-                &libra_genesis_key.pubkey(),
-                &funding_keys[j],
-                &libra_funding_keys[j],
-                &key.pubkey(),
-                amount,
-                blockhash,
-            );
-
-            let _sig = client
-                .async_send_transaction(tx.clone())
-                .expect("create_account in generate_and_fund_keypairs");
-        }
-
-        for (j, key) in keys.iter().enumerate() {
-            let mut times = 0;
-            loop {
-                let balance =
-                    librapay_transaction::get_libra_balance(client, &key.pubkey()).unwrap();
-                if balance >= amount {
-                    break;
-                } else if times > 20 {
-                    info!("timed out.. {} key: {} balance: {}", i, j, balance);
-                    break;
-                } else {
-                    times += 1;
-                    sleep(Duration::from_millis(100));
-                }
-            }
-        }
-
-        info!(
-            "funded group {} of {}",
-            i + 1,
-            keypairs.len() / NUM_FUNDING_KEYS
-        );
-        blockhash = get_recent_blockhash(client).0;
-    }
-
-    funding_time.stop();
-    info!("done funding keys, took {} ms", funding_time.as_ms());
-}
-
 pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
     client: Arc<T>,
     faucet_addr: Option<SocketAddr>,
     funding_key: &Keypair,
     keypair_count: usize,
     lamports_per_account: u64,
-    use_move: bool,
-) -> Result<(Vec<Keypair>, Option<LibraKeys>)> {
+) -> Result<Vec<Keypair>> {
     info!("Creating {} keypairs...", keypair_count);
     let (mut keypairs, extra) = generate_keypairs(funding_key, keypair_count as u64);
     info!("Get lamports...");
@@ -1047,12 +885,6 @@ pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
     let last_key = keypairs[keypair_count - 1].pubkey();
     let last_keypair_balance = client.get_balance(&last_key).unwrap_or(0);
 
-    #[cfg(feature = "move")]
-    let mut move_keypairs_ret = None;
-
-    #[cfg(not(feature = "move"))]
-    let move_keypairs_ret = None;
-
     // Repeated runs will eat up keypair balances from transaction fees. In order to quickly
     //   start another bench-tps run without re-funding all of the keypairs, check if the
     //   keypairs still have at least 80% of the expected funds. That should be enough to
@@ -1063,10 +895,7 @@ pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
         let max_fee = fee_rate_governor.max_lamports_per_signature;
         let extra_fees = extra * max_fee;
         let total_keypairs = keypairs.len() as u64 + 1; // Add one for funding keypair
-        let mut total = lamports_per_account * total_keypairs + extra_fees;
-        if use_move {
-            total *= 3;
-        }
+        let total = lamports_per_account * total_keypairs + extra_fees;
 
         let funding_key_balance = client.get_balance(&funding_key.pubkey()).unwrap_or(0);
         info!(
@@ -1078,40 +907,6 @@ pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
             airdrop_lamports(client.as_ref(), &faucet_addr.unwrap(), funding_key, total)?;
         }
 
-        #[cfg(feature = "move")]
-        {
-            if use_move {
-                let libra_genesis_keypair =
-                    create_genesis(&funding_key, client.as_ref(), 10_000_000);
-                let libra_mint_program_id = upload_mint_script(&funding_key, client.as_ref());
-                let libra_pay_program_id = upload_payment_script(&funding_key, client.as_ref());
-
-                // Generate another set of keypairs for move accounts.
-                // Still fund the solana ones which will be used for fees.
-                let seed = [0u8; 32];
-                let mut rnd = GenKeys::new(seed);
-                let move_keypairs = rnd.gen_n_keypairs(keypair_count as u64);
-                fund_move_keys(
-                    client.as_ref(),
-                    funding_key,
-                    &move_keypairs,
-                    total / 3,
-                    &libra_pay_program_id,
-                    &libra_mint_program_id,
-                    &libra_genesis_keypair,
-                );
-                move_keypairs_ret = Some((
-                    libra_genesis_keypair,
-                    libra_pay_program_id,
-                    libra_mint_program_id,
-                    move_keypairs,
-                ));
-
-                // Give solana keys 1/3 and move keys 1/3 the lamports. Keep 1/3 for fees.
-                total /= 3;
-            }
-        }
-
         fund_keys(
             client,
             funding_key,
@@ -1125,7 +920,7 @@ pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
     // 'generate_keypairs' generates extra keys to be able to have size-aligned funding batches for fund_keys.
     keypairs.truncate(keypair_count);
 
-    Ok((keypairs, move_keypairs_ret))
+    Ok(keypairs)
 }
 
 #[cfg(test)]
@@ -1149,11 +944,11 @@ mod tests {
         config.duration = Duration::from_secs(5);
 
         let keypair_count = config.tx_count * config.keypair_multiplier;
-        let (keypairs, _move_keypairs) =
-            generate_and_fund_keypairs(client.clone(), None, &config.id, keypair_count, 20, false)
+        let keypairs =
+            generate_and_fund_keypairs(client.clone(), None, &config.id, keypair_count, 20)
                 .unwrap();
 
-        do_bench_tps(client, config, keypairs, None);
+        do_bench_tps(client, config, keypairs);
     }
 
     #[test]
@@ -1164,9 +959,8 @@ mod tests {
         let keypair_count = 20;
         let lamports = 20;
 
-        let (keypairs, _move_keypairs) =
-            generate_and_fund_keypairs(client.clone(), None, &id, keypair_count, lamports, false)
-                .unwrap();
+        let keypairs =
+            generate_and_fund_keypairs(client.clone(), None, &id, keypair_count, lamports).unwrap();
 
         for kp in &keypairs {
             assert_eq!(
@@ -1188,9 +982,8 @@ mod tests {
         let keypair_count = 20;
         let lamports = 20;
 
-        let (keypairs, _move_keypairs) =
-            generate_and_fund_keypairs(client.clone(), None, &id, keypair_count, lamports, false)
-                .unwrap();
+        let keypairs =
+            generate_and_fund_keypairs(client.clone(), None, &id, keypair_count, lamports).unwrap();
 
         for kp in &keypairs {
             assert_eq!(client.get_balance(&kp.pubkey()).unwrap(), lamports);

bench-tps/src/cli.rs

@@ -1,7 +1,10 @@
 use clap::{crate_description, crate_name, App, Arg, ArgMatches};
 use solana_faucet::faucet::FAUCET_PORT;
 use solana_sdk::fee_calculator::FeeRateGovernor;
-use solana_sdk::signature::{read_keypair_file, Keypair};
+use solana_sdk::{
+    pubkey::Pubkey,
+    signature::{read_keypair_file, Keypair},
+};
 use std::{net::SocketAddr, process::exit, time::Duration};
 
 const NUM_LAMPORTS_PER_ACCOUNT_DEFAULT: u64 = solana_sdk::native_token::LAMPORTS_PER_SOL;
@@ -23,8 +26,9 @@ pub struct Config {
     pub read_from_client_file: bool,
     pub target_lamports_per_signature: u64,
     pub multi_client: bool,
-    pub use_move: bool,
     pub num_lamports_per_account: u64,
+    pub target_slots_per_epoch: u64,
+    pub target_node: Option<Pubkey>,
 }
 
 impl Default for Config {
@@ -45,8 +49,9 @@ impl Default for Config {
             read_from_client_file: false,
             target_lamports_per_signature: FeeRateGovernor::default().target_lamports_per_signature,
             multi_client: true,
-            use_move: false,
             num_lamports_per_account: NUM_LAMPORTS_PER_ACCOUNT_DEFAULT,
+            target_slots_per_epoch: 0,
+            target_node: None,
         }
     }
 }
@@ -107,16 +112,19 @@ pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
                 .long("sustained")
                 .help("Use sustained performance mode vs. peak mode. This overlaps the tx generation with transfers."),
         )
-        .arg(
-            Arg::with_name("use-move")
-                .long("use-move")
-                .help("Use Move language transactions to perform transfers."),
-        )
         .arg(
             Arg::with_name("no-multi-client")
                 .long("no-multi-client")
                 .help("Disable multi-client support, only transact with the entrypoint."),
         )
+        .arg(
+            Arg::with_name("target_node")
+                .long("target-node")
+                .requires("no-multi-client")
+                .takes_value(true)
+                .value_name("PUBKEY")
+                .help("Specify an exact node to send transactions to."),
+        )
         .arg(
             Arg::with_name("tx_count")
                 .long("tx_count")
@@ -172,6 +180,15 @@ pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
                     "Number of lamports per account.",
                 ),
         )
+        .arg(
+            Arg::with_name("target_slots_per_epoch")
+                .long("target-slots-per-epoch")
+                .value_name("SLOTS")
+                .takes_value(true)
+                .help(
+                    "Wait until epochs are this many slots long.",
+                ),
+        )
 }
 
 /// Parses a clap `ArgMatches` structure into a `Config`
@@ -252,12 +269,21 @@ pub fn extract_args<'a>(matches: &ArgMatches<'a>) -> Config {
         args.target_lamports_per_signature = v.to_string().parse().expect("can't parse lamports");
     }
 
-    args.use_move = matches.is_present("use-move");
     args.multi_client = !matches.is_present("no-multi-client");
+    args.target_node = matches
+        .value_of("target_node")
+        .map(|target_str| target_str.parse().unwrap());
 
     if let Some(v) = matches.value_of("num_lamports_per_account") {
         args.num_lamports_per_account = v.to_string().parse().expect("can't parse lamports");
     }
 
+    if let Some(t) = matches.value_of("target_slots_per_epoch") {
+        args.target_slots_per_epoch = t
+            .to_string()
+            .parse()
+            .expect("can't parse target slots per epoch");
+    }
+
     args
 }

bench-tps/src/main.rs

@@ -15,7 +15,7 @@ fn main() {
     solana_logger::setup_with_default("solana=info");
     solana_metrics::set_panic_hook("bench-tps");
 
-    let matches = cli::build_args(solana_clap_utils::version!()).get_matches();
+    let matches = cli::build_args(solana_version::version!()).get_matches();
     let cli_config = cli::extract_args(&matches);
 
     let cli::Config {
@@ -29,9 +29,9 @@ fn main() {
         write_to_client_file,
         read_from_client_file,
         target_lamports_per_signature,
-        use_move,
         multi_client,
         num_lamports_per_account,
+        target_node,
         ..
     } = &cli_config;
 
@@ -67,11 +67,10 @@ fn main() {
     }
 
     info!("Connecting to the cluster");
-    let (nodes, _archivers) =
-        discover_cluster(&entrypoint_addr, *num_nodes).unwrap_or_else(|err| {
-            eprintln!("Failed to discover {} nodes: {:?}", num_nodes, err);
-            exit(1);
-        });
+    let nodes = discover_cluster(&entrypoint_addr, *num_nodes).unwrap_or_else(|err| {
+        eprintln!("Failed to discover {} nodes: {:?}", num_nodes, err);
+        exit(1);
+    });
 
     let client = if *multi_client {
         let (client, num_clients) = get_multi_client(&nodes);
@@ -83,11 +82,24 @@ fn main() {
             exit(1);
         }
         Arc::new(client)
+    } else if let Some(target_node) = target_node {
+        info!("Searching for target_node: {:?}", target_node);
+        let mut target_client = None;
+        for node in nodes {
+            if node.id == *target_node {
+                target_client = Some(Arc::new(get_client(&[node])));
+                break;
+            }
+        }
+        target_client.unwrap_or_else(|| {
+            eprintln!("Target node {} not found", target_node);
+            exit(1);
+        })
     } else {
         Arc::new(get_client(&nodes))
     };
 
-    let (keypairs, move_keypairs) = if *read_from_client_file && !use_move {
+    let keypairs = if *read_from_client_file {
         let path = Path::new(&client_ids_and_stake_file);
         let file = File::open(path).unwrap();
 
@@ -116,8 +128,8 @@ fn main() {
         // Sort keypairs so that do_bench_tps() uses the same subset of accounts for each run.
         // This prevents the amount of storage needed for bench-tps accounts from creeping up
         // across multiple runs.
-        keypairs.sort_by(|x, y| x.pubkey().to_string().cmp(&y.pubkey().to_string()));
-        (keypairs, None)
+        keypairs.sort_by_key(|x| x.pubkey().to_string());
+        keypairs
     } else {
         generate_and_fund_keypairs(
             client.clone(),
@@ -125,7 +137,6 @@ fn main() {
             &id,
             keypair_count,
             *num_lamports_per_account,
-            *use_move,
         )
         .unwrap_or_else(|e| {
             eprintln!("Error could not fund keys: {:?}", e);
@@ -133,5 +144,5 @@ fn main() {
         })
     };
 
-    do_bench_tps(client, cli_config, keypairs, move_keypairs);
+    do_bench_tps(client, cli_config, keypairs);
 }

bench-tps/tests/bench_tps.rs

@@ -6,22 +6,16 @@ use solana_core::cluster_info::VALIDATOR_PORT_RANGE;
 use solana_core::validator::ValidatorConfig;
 use solana_faucet::faucet::run_local_faucet;
 use solana_local_cluster::local_cluster::{ClusterConfig, LocalCluster};
-#[cfg(feature = "move")]
-use solana_sdk::move_loader::solana_move_loader_program;
 use solana_sdk::signature::{Keypair, Signer};
 use std::sync::{mpsc::channel, Arc};
 use std::time::Duration;
 
 fn test_bench_tps_local_cluster(config: Config) {
-    #[cfg(feature = "move")]
-    let native_instruction_processors = vec![solana_move_loader_program()];
-
-    #[cfg(not(feature = "move"))]
     let native_instruction_processors = vec![];
 
     solana_logger::setup();
     const NUM_NODES: usize = 1;
-    let cluster = LocalCluster::new(&ClusterConfig {
+    let cluster = LocalCluster::new(&mut ClusterConfig {
         node_stakes: vec![999_990; NUM_NODES],
         cluster_lamports: 200_000_000,
         validator_configs: vec![ValidatorConfig::default(); NUM_NODES],
@@ -48,17 +42,16 @@ fn test_bench_tps_local_cluster(config: Config) {
     let lamports_per_account = 100;
 
     let keypair_count = config.tx_count * config.keypair_multiplier;
-    let (keypairs, move_keypairs) = generate_and_fund_keypairs(
+    let keypairs = generate_and_fund_keypairs(
         client.clone(),
         Some(faucet_addr),
         &config.id,
         keypair_count,
         lamports_per_account,
-        config.use_move,
     )
     .unwrap();
 
-    let _total = do_bench_tps(client, config, keypairs, move_keypairs);
+    let _total = do_bench_tps(client, config, keypairs);
 
     #[cfg(not(debug_assertions))]
     assert!(_total > 100);
@@ -73,14 +66,3 @@ fn test_bench_tps_local_cluster_solana() {
 
     test_bench_tps_local_cluster(config);
 }
-
-#[test]
-#[serial]
-fn test_bench_tps_local_cluster_move() {
-    let mut config = Config::default();
-    config.tx_count = 100;
-    config.duration = Duration::from_secs(10);
-    config.use_move = true;
-
-    test_bench_tps_local_cluster(config);
-}

cargo

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# shellcheck source=ci/rust-version.sh
+here=$(dirname "$0")
+
+source "${here}"/ci/rust-version.sh all
+
+toolchain=
+case "$1" in
+  stable)
+    # shellcheck disable=SC2054 # rust_stable is sourced from rust-version.sh
+    toolchain="$rust_stable"
+    shift
+    ;;
+  nightly)
+    # shellcheck disable=SC2054 # rust_nightly is sourced from rust-version.sh
+    toolchain="$rust_nightly"
+    shift
+    ;;
+  +*)
+    toolchain="${1#+}"
+    shift
+    ;;
+  *)
+    # shellcheck disable=SC2054 # rust_stable is sourced from rust-version.sh
+    toolchain="$rust_stable"
+    ;;
+esac
+
+set -x
+exec cargo "+${toolchain}" "${@}"

cargo-build-bpf

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+here=$(dirname "$0")
+
+maybe_bpf_sdk="--bpf-sdk $here/sdk/bpf"
+for a in "$@"; do
+  if [[ $a = --bpf-sdk ]]; then
+    maybe_bpf_sdk=
+  fi
+done
+
+set -x
+exec "$here"/cargo run --manifest-path "$here"/sdk/cargo-build-bpf/Cargo.toml -- $maybe_bpf_sdk "$@"

cargo-test-bpf

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+here=$(dirname "$0")
+
+maybe_bpf_sdk="--bpf-sdk $here/sdk/bpf"
+for a in "$@"; do
+  if [[ $a = --bpf-sdk ]]; then
+    maybe_bpf_sdk=
+  fi
+done
+
+export CARGO_BUILD_BPF="$here"/cargo-build-bpf
+set -x
+exec "$here"/cargo run --manifest-path "$here"/sdk/cargo-test-bpf/Cargo.toml -- $maybe_bpf_sdk "$@"

chacha-cuda/Cargo.toml

@@ -1,24 +0,0 @@
-[package]
-name = "solana-chacha-cuda"
-version = "1.0.13"
-description = "Solana Chacha Cuda APIs"
-authors = ["Solana Maintainers <maintainers@solana.com>"]
-repository = "https://github.com/solana-labs/solana"
-license = "Apache-2.0"
-homepage = "https://solana.com/"
-edition = "2018"
-
-[dependencies]
-log = "0.4.8"
-solana-archiver-utils = { path = "../archiver-utils", version = "1.0.13" }
-solana-chacha = { path = "../chacha", version = "1.0.13" }
-solana-ledger = { path = "../ledger", version = "1.0.13" }
-solana-logger = { path = "../logger", version = "1.0.13" }
-solana-perf = { path = "../perf", version = "1.0.13" }
-solana-sdk = { path = "../sdk", version = "1.0.13" }
-
-[dev-dependencies]
-hex-literal = "0.2.1"
-
-[lib]
-name = "solana_chacha_cuda"

chacha-cuda/src/chacha_cuda.rs

@@ -1,280 +0,0 @@
-// Module used by validators to approve storage mining proofs in parallel using the GPU
-
-use solana_chacha::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE};
-use solana_ledger::blockstore::Blockstore;
-use solana_perf::perf_libs;
-use solana_sdk::hash::Hash;
-use std::io;
-use std::mem::size_of;
-use std::sync::Arc;
-
-// Encrypt a file with multiple starting IV states, determined by ivecs.len()
-//
-// Then sample each block at the offsets provided by samples argument with sha256
-// and return the vec of sha states
-pub fn chacha_cbc_encrypt_file_many_keys(
-    blockstore: &Arc<Blockstore>,
-    segment: u64,
-    slots_per_segment: u64,
-    ivecs: &mut [u8],
-    samples: &[u64],
-) -> io::Result<Vec<Hash>> {
-    let api = perf_libs::api().expect("no perf libs");
-    if ivecs.len() % CHACHA_BLOCK_SIZE != 0 {
-        return Err(io::Error::new(
-            io::ErrorKind::Other,
-            format!(
-                "bad IV length({}) not divisible by {} ",
-                ivecs.len(),
-                CHACHA_BLOCK_SIZE,
-            ),
-        ));
-    }
-
-    const BUFFER_SIZE: usize = 8 * 1024;
-    let mut buffer = [0; BUFFER_SIZE];
-    let num_keys = ivecs.len() / CHACHA_BLOCK_SIZE;
-    let mut sha_states = vec![0; num_keys * size_of::<Hash>()];
-    let mut int_sha_states = vec![0; num_keys * 112];
-    let keys: Vec<u8> = vec![0; num_keys * CHACHA_KEY_SIZE]; // keys not used ATM, uniqueness comes from IV
-    let mut current_slot = segment * slots_per_segment;
-    let mut start_index = 0;
-    let start_slot = current_slot;
-    let mut total_size = 0;
-    let mut time: f32 = 0.0;
-    unsafe {
-        (api.chacha_init_sha_state)(int_sha_states.as_mut_ptr(), num_keys as u32);
-    }
-    loop {
-        match blockstore.get_data_shreds(current_slot, start_index, std::u64::MAX, &mut buffer) {
-            Ok((last_index, mut size)) => {
-                debug!(
-                    "chacha_cuda: encrypting segment: {} num_shreds: {} data_len: {}",
-                    segment,
-                    last_index.saturating_sub(start_index),
-                    size
-                );
-
-                if size == 0 {
-                    if current_slot.saturating_sub(start_slot) < slots_per_segment {
-                        current_slot += 1;
-                        start_index = 0;
-                        continue;
-                    } else {
-                        break;
-                    }
-                }
-
-                if size < BUFFER_SIZE {
-                    // round to the nearest key_size boundary
-                    size = (size + CHACHA_KEY_SIZE - 1) & !(CHACHA_KEY_SIZE - 1);
-                }
-
-                unsafe {
-                    (api.chacha_cbc_encrypt_many_sample)(
-                        buffer[..size].as_ptr(),
-                        int_sha_states.as_mut_ptr(),
-                        size,
-                        keys.as_ptr(),
-                        ivecs.as_mut_ptr(),
-                        num_keys as u32,
-                        samples.as_ptr(),
-                        samples.len() as u32,
-                        total_size,
-                        &mut time,
-                    );
-                }
-
-                total_size += size as u64;
-                start_index = last_index + 1;
-            }
-            Err(e) => {
-                info!("Error encrypting file: {:?}", e);
-                break;
-            }
-        }
-    }
-    unsafe {
-        (api.chacha_end_sha_state)(
-            int_sha_states.as_ptr(),
-            sha_states.as_mut_ptr(),
-            num_keys as u32,
-        );
-    }
-    let mut res = Vec::new();
-    for x in 0..num_keys {
-        let start = x * size_of::<Hash>();
-        let end = start + size_of::<Hash>();
-        res.push(Hash::new(&sha_states[start..end]));
-    }
-    Ok(res)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use solana_archiver_utils::sample_file;
-    use solana_chacha::chacha::chacha_cbc_encrypt_ledger;
-    use solana_ledger::entry::create_ticks;
-    use solana_ledger::get_tmp_ledger_path;
-    use solana_sdk::clock::DEFAULT_SLOTS_PER_SEGMENT;
-    use solana_sdk::signature::Keypair;
-    use std::fs::{remove_dir_all, remove_file};
-    use std::path::Path;
-
-    #[test]
-    fn test_encrypt_file_many_keys_single() {
-        solana_logger::setup();
-        if perf_libs::api().is_none() {
-            info!("perf-libs unavailable, skipped");
-            return;
-        }
-
-        let slots_per_segment = 32;
-        let entries = create_ticks(slots_per_segment, 0, Hash::default());
-        let ledger_path = get_tmp_ledger_path!();
-        let ticks_per_slot = 16;
-        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
-
-        blockstore
-            .write_entries(
-                0,
-                0,
-                0,
-                ticks_per_slot,
-                Some(0),
-                true,
-                &Arc::new(Keypair::new()),
-                entries,
-                0,
-            )
-            .unwrap();
-
-        let out_path = Path::new("test_chacha_encrypt_file_many_keys_single_output.txt.enc");
-
-        let samples = [0];
-        let mut ivecs = hex!(
-            "abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234
-                              abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234"
-        );
-
-        let mut cpu_iv = ivecs.clone();
-        chacha_cbc_encrypt_ledger(
-            &blockstore,
-            0,
-            slots_per_segment as u64,
-            out_path,
-            &mut cpu_iv,
-        )
-        .unwrap();
-
-        let ref_hash = sample_file(&out_path, &samples).unwrap();
-
-        let hashes = chacha_cbc_encrypt_file_many_keys(
-            &blockstore,
-            0,
-            slots_per_segment as u64,
-            &mut ivecs,
-            &samples,
-        )
-        .unwrap();
-
-        assert_eq!(hashes[0], ref_hash);
-
-        let _ignored = remove_dir_all(&ledger_path);
-        let _ignored = remove_file(out_path);
-    }
-
-    #[test]
-    fn test_encrypt_file_many_keys_multiple_keys() {
-        solana_logger::setup();
-        if perf_libs::api().is_none() {
-            info!("perf-libs unavailable, skipped");
-            return;
-        }
-
-        let ledger_path = get_tmp_ledger_path!();
-        let ticks_per_slot = 90;
-        let entries = create_ticks(2 * ticks_per_slot, 0, Hash::default());
-        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
-        blockstore
-            .write_entries(
-                0,
-                0,
-                0,
-                ticks_per_slot,
-                Some(0),
-                true,
-                &Arc::new(Keypair::new()),
-                entries,
-                0,
-            )
-            .unwrap();
-
-        let out_path = Path::new("test_chacha_encrypt_file_many_keys_multiple_output.txt.enc");
-
-        let samples = [0, 1, 3, 4, 5, 150];
-        let mut ivecs = Vec::new();
-        let mut ref_hashes: Vec<Hash> = vec![];
-        for i in 0..2 {
-            let mut ivec = hex!(
-                "abc123abc123abc123abc123abc123abc123abababababababababababababab
-                                 abc123abc123abc123abc123abc123abc123abababababababababababababab"
-            );
-            ivec[0] = i;
-            ivecs.extend(ivec.clone().iter());
-            chacha_cbc_encrypt_ledger(
-                &blockstore.clone(),
-                0,
-                DEFAULT_SLOTS_PER_SEGMENT,
-                out_path,
-                &mut ivec,
-            )
-            .unwrap();
-
-            ref_hashes.push(sample_file(&out_path, &samples).unwrap());
-            info!(
-                "ivec: {:?} hash: {:?} ivecs: {:?}",
-                ivec.to_vec(),
-                ref_hashes.last(),
-                ivecs
-            );
-        }
-
-        let hashes = chacha_cbc_encrypt_file_many_keys(
-            &blockstore,
-            0,
-            DEFAULT_SLOTS_PER_SEGMENT,
-            &mut ivecs,
-            &samples,
-        )
-        .unwrap();
-
-        assert_eq!(hashes, ref_hashes);
-
-        let _ignored = remove_dir_all(&ledger_path);
-        let _ignored = remove_file(out_path);
-    }
-
-    #[test]
-    fn test_encrypt_file_many_keys_bad_key_length() {
-        solana_logger::setup();
-        if perf_libs::api().is_none() {
-            info!("perf-libs unavailable, skipped");
-            return;
-        }
-
-        let mut keys = hex!("abc123");
-        let ledger_path = get_tmp_ledger_path!();
-        let samples = [0];
-        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
-        assert!(chacha_cbc_encrypt_file_many_keys(
-            &blockstore,
-            0,
-            DEFAULT_SLOTS_PER_SEGMENT,
-            &mut keys,
-            &samples,
-        )
-        .is_err());
-    }
-}

chacha-cuda/src/lib.rs

@@ -1,8 +0,0 @@
-#[macro_use]
-extern crate log;
-
-#[cfg(test)]
-#[macro_use]
-extern crate hex_literal;
-
-pub mod chacha_cuda;

chacha-sys/build.rs

@@ -1,8 +0,0 @@
-extern crate cc;
-
-fn main() {
-    cc::Build::new()
-        .file("cpu-crypt/chacha20_core.c")
-        .file("cpu-crypt/chacha_cbc.c")
-        .compile("libcpu-crypt");
-}

chacha-sys/cpu-crypt/.gitignore

@@ -1 +0,0 @@
-release/

chacha-sys/cpu-crypt/Makefile

@@ -1,25 +0,0 @@
-V:=debug
-
-LIB:=cpu-crypt
-
-CFLAGS_common:=-Wall -Werror -pedantic -fPIC
-CFLAGS_release:=-march=native -O3 $(CFLAGS_common)
-CFLAGS_debug:=-g $(CFLAGS_common)
-CFLAGS:=$(CFLAGS_$V)
-
-all: $V/lib$(LIB).a
-
-$V/chacha20_core.o: chacha20_core.c chacha.h
-	@mkdir -p $(@D)
-	$(CC) $(CFLAGS) -c $< -o $@
-
-$V/chacha_cbc.o: chacha_cbc.c chacha.h
-	@mkdir -p $(@D)
-	$(CC) $(CFLAGS) -c $< -o $@
-
-$V/lib$(LIB).a: $V/chacha20_core.o $V/chacha_cbc.o
-	$(AR) rcs $@ $^
-
-.PHONY:clean
-clean:
-	rm -rf $V

chacha-sys/cpu-crypt/chacha.h

@@ -1,35 +0,0 @@
-#ifndef HEADER_CHACHA_H
-# define HEADER_CHACHA_H
-
-#include <string.h>
-#include <inttypes.h>
-# include <stddef.h>
-# ifdef  __cplusplus
-extern "C" {
-# endif
-
-typedef unsigned int u32;
-
-#define CHACHA_KEY_SIZE 32
-#define CHACHA_NONCE_SIZE 12
-#define CHACHA_BLOCK_SIZE 64
-#define CHACHA_ROUNDS 500
-
-void chacha20_encrypt(const u32 input[16],
-                      unsigned char output[64],
-                      int num_rounds);
-
-void chacha20_encrypt_ctr(const uint8_t *in, uint8_t *out, size_t in_len,
-                          const uint8_t key[CHACHA_KEY_SIZE], const uint8_t nonce[CHACHA_NONCE_SIZE],
-                          uint32_t counter);
-
-void chacha20_cbc128_encrypt(const unsigned char* in, unsigned char* out,
-                             uint32_t len, const uint8_t* key,
-                             unsigned char* ivec);
-
-
-# ifdef  __cplusplus
-}
-# endif
-
-#endif

chacha-sys/cpu-crypt/chacha20_core.c

@@ -1,102 +0,0 @@
-#include "chacha.h"
-
-#define ROTL32(v, n) (((v) << (n)) | ((v) >> (32 - (n))))
-
-#define ROTATE(v, c) ROTL32((v), (c))
-
-#define XOR(v, w) ((v) ^ (w))
-
-#define PLUS(x, y) ((x) + (y))
-
-#define U32TO8_LITTLE(p, v) \
-{ (p)[0] = ((v)      ) & 0xff; (p)[1] = ((v) >>  8) & 0xff; \
-  (p)[2] = ((v) >> 16) & 0xff; (p)[3] = ((v) >> 24) & 0xff; }
-
-#define U8TO32_LITTLE(p)   \
-     (((u32)((p)[0])      ) | ((u32)((p)[1]) <<  8) | \
-      ((u32)((p)[2]) << 16) | ((u32)((p)[3]) << 24)   )
-
-#define QUARTERROUND(a,b,c,d) \
-  x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]),16); \
-  x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]),12); \
-  x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]), 8); \
-  x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]), 7);
-
-// sigma contains the ChaCha constants, which happen to be an ASCII string.
-static const uint8_t sigma[16] = { 'e', 'x', 'p', 'a', 'n', 'd', ' ', '3',
-                                   '2', '-', 'b', 'y', 't', 'e', ' ', 'k' };
-
-void chacha20_encrypt(const u32 input[16],
-                      unsigned char output[64],
-                      int num_rounds)
-{
-    u32 x[16];
-    int i;
-    memcpy(x, input, sizeof(u32) * 16);
-    for (i = num_rounds; i > 0; i -= 2) {
-        QUARTERROUND( 0, 4, 8,12)
-        QUARTERROUND( 1, 5, 9,13)
-        QUARTERROUND( 2, 6,10,14)
-        QUARTERROUND( 3, 7,11,15)
-        QUARTERROUND( 0, 5,10,15)
-        QUARTERROUND( 1, 6,11,12)
-        QUARTERROUND( 2, 7, 8,13)
-        QUARTERROUND( 3, 4, 9,14)
-    }
-    for (i = 0; i < 16; ++i) {
-        x[i] = PLUS(x[i], input[i]);
-    }
-    for (i = 0; i < 16; ++i) {
-        U32TO8_LITTLE(output + 4 * i, x[i]);
-    }
-}
-
-void chacha20_encrypt_ctr(const uint8_t *in, uint8_t *out, size_t in_len,
-                          const uint8_t key[CHACHA_KEY_SIZE],
-                          const uint8_t nonce[CHACHA_NONCE_SIZE],
-                          uint32_t counter)
-{
-  uint32_t input[16];
-  uint8_t buf[64];
-  size_t todo, i;
-
-  input[0] = U8TO32_LITTLE(sigma + 0);
-  input[1] = U8TO32_LITTLE(sigma + 4);
-  input[2] = U8TO32_LITTLE(sigma + 8);
-  input[3] = U8TO32_LITTLE(sigma + 12);
-
-  input[4] = U8TO32_LITTLE(key + 0);
-  input[5] = U8TO32_LITTLE(key + 4);
-  input[6] = U8TO32_LITTLE(key + 8);
-  input[7] = U8TO32_LITTLE(key + 12);
-
-  input[8] = U8TO32_LITTLE(key + 16);
-  input[9] = U8TO32_LITTLE(key + 20);
-  input[10] = U8TO32_LITTLE(key + 24);
-  input[11] = U8TO32_LITTLE(key + 28);
-
-  input[12] = counter;
-  input[13] = U8TO32_LITTLE(nonce + 0);
-  input[14] = U8TO32_LITTLE(nonce + 4);
-  input[15] = U8TO32_LITTLE(nonce + 8);
-
-  while (in_len > 0) {
-    todo = sizeof(buf);
-    if (in_len < todo) {
-      todo = in_len;
-    }
-
-    chacha20_encrypt(input, buf, 20);
-    for (i = 0; i < todo; i++) {
-      out[i] = in[i] ^ buf[i];
-    }
-
-    out += todo;
-    in += todo;
-    in_len -= todo;
-
-    input[12]++;
-  }
-}
-
-

chacha-sys/cpu-crypt/chacha_cbc.c

@@ -1,72 +0,0 @@
-#include "chacha.h"
-
-#if !defined(STRICT_ALIGNMENT) && !defined(PEDANTIC)
-# define STRICT_ALIGNMENT 0
-#endif
-
-void chacha20_cbc128_encrypt(const unsigned char* in, unsigned char* out,
-                             uint32_t len, const uint8_t* key,
-                             unsigned char* ivec)
-{
-    size_t n;
-    unsigned char *iv = ivec;
-    (void)key;
-
-    if (len == 0) {
-        return;
-    }
-
-#if !defined(OPENSSL_SMALL_FOOTPRINT)
-    if (STRICT_ALIGNMENT &&
-        ((size_t)in | (size_t)out | (size_t)ivec) % sizeof(size_t) != 0) {
-        while (len >= CHACHA_BLOCK_SIZE) {
-            for (n = 0; n < CHACHA_BLOCK_SIZE; ++n) {
-                out[n] = in[n] ^ iv[n];
-                //printf("%x ", out[n]);
-            }
-            chacha20_encrypt((const u32*)out, out, CHACHA_ROUNDS);
-            iv = out;
-            len -= CHACHA_BLOCK_SIZE;
-            in += CHACHA_BLOCK_SIZE;
-            out += CHACHA_BLOCK_SIZE;
-        }
-    } else {
-        while (len >= CHACHA_BLOCK_SIZE) {
-            for (n = 0; n < CHACHA_BLOCK_SIZE; n += sizeof(size_t)) {
-                *(size_t *)(out + n) =
-                    *(size_t *)(in + n) ^ *(size_t *)(iv + n);
-                //printf("%zu ", *(size_t *)(iv + n));
-            }
-            chacha20_encrypt((const u32*)out, out, CHACHA_ROUNDS);
-            iv = out;
-            len -= CHACHA_BLOCK_SIZE;
-            in += CHACHA_BLOCK_SIZE;
-            out += CHACHA_BLOCK_SIZE;
-        }
-    }
-#endif
-    while (len) {
-        for (n = 0; n < CHACHA_BLOCK_SIZE && n < len; ++n) {
-            out[n] = in[n] ^ iv[n];
-        }
-        for (; n < CHACHA_BLOCK_SIZE; ++n) {
-            out[n] = iv[n];
-        }
-        chacha20_encrypt((const u32*)out, out, CHACHA_ROUNDS);
-        iv = out;
-        if (len <= CHACHA_BLOCK_SIZE) {
-            break;
-        }
-        len -= CHACHA_BLOCK_SIZE;
-        in += CHACHA_BLOCK_SIZE;
-        out += CHACHA_BLOCK_SIZE;
-    }
-    memcpy(ivec, iv, CHACHA_BLOCK_SIZE);
-
-}
-
-void chacha20_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t in_len,
-                          const uint8_t key[CHACHA_KEY_SIZE], uint8_t* ivec)
-{
-    chacha20_cbc128_encrypt(in, out, in_len, key, ivec);
-}

chacha-sys/src/lib.rs

@@ -1,21 +0,0 @@
-extern "C" {
-    fn chacha20_cbc_encrypt(
-        input: *const u8,
-        output: *mut u8,
-        in_len: usize,
-        key: *const u8,
-        ivec: *mut u8,
-    );
-}
-
-pub fn chacha_cbc_encrypt(input: &[u8], output: &mut [u8], key: &[u8], ivec: &mut [u8]) {
-    unsafe {
-        chacha20_cbc_encrypt(
-            input.as_ptr(),
-            output.as_mut_ptr(),
-            input.len(),
-            key.as_ptr(),
-            ivec.as_mut_ptr(),
-        );
-    }
-}

chacha/.gitignore

@@ -1 +0,0 @@
-/farf/

chacha/Cargo.toml

@@ -1,25 +0,0 @@
-[package]
-name = "solana-chacha"
-version = "1.0.13"
-description = "Solana Chacha APIs"
-authors = ["Solana Maintainers <maintainers@solana.com>"]
-repository = "https://github.com/solana-labs/solana"
-license = "Apache-2.0"
-homepage = "https://solana.com/"
-edition = "2018"
-
-[dependencies]
-log = "0.4.8"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.13" }
-solana-ledger = { path = "../ledger", version = "1.0.13" }
-solana-logger = { path = "../logger", version = "1.0.13" }
-solana-perf = { path = "../perf", version = "1.0.13" }
-solana-sdk = { path = "../sdk", version = "1.0.13" }
-
-[dev-dependencies]
-hex-literal = "0.2.1"
-
-[lib]
-name = "solana_chacha"

chacha/src/chacha.rs

@@ -1,185 +0,0 @@
-use solana_ledger::blockstore::Blockstore;
-use solana_sdk::clock::Slot;
-use std::fs::File;
-use std::io;
-use std::io::{BufWriter, Write};
-use std::path::Path;
-use std::sync::Arc;
-
-pub use solana_chacha_sys::chacha_cbc_encrypt;
-
-pub const CHACHA_BLOCK_SIZE: usize = 64;
-pub const CHACHA_KEY_SIZE: usize = 32;
-
-pub fn chacha_cbc_encrypt_ledger(
-    blockstore: &Arc<Blockstore>,
-    start_slot: Slot,
-    slots_per_segment: u64,
-    out_path: &Path,
-    ivec: &mut [u8; CHACHA_BLOCK_SIZE],
-) -> io::Result<usize> {
-    let mut out_file =
-        BufWriter::new(File::create(out_path).expect("Can't open ledger encrypted data file"));
-    const BUFFER_SIZE: usize = 8 * 1024;
-    let mut buffer = [0; BUFFER_SIZE];
-    let mut encrypted_buffer = [0; BUFFER_SIZE];
-    let key = [0; CHACHA_KEY_SIZE];
-    let mut total_size = 0;
-    let mut current_slot = start_slot;
-    let mut start_index = 0;
-    loop {
-        match blockstore.get_data_shreds(current_slot, start_index, std::u64::MAX, &mut buffer) {
-            Ok((last_index, mut size)) => {
-                debug!(
-                    "chacha: encrypting slice: {} num_shreds: {} data_len: {}",
-                    current_slot,
-                    last_index.saturating_sub(start_index),
-                    size
-                );
-                debug!("read {} bytes", size);
-
-                if size == 0 {
-                    if current_slot.saturating_sub(start_slot) < slots_per_segment {
-                        current_slot += 1;
-                        start_index = 0;
-                        continue;
-                    } else {
-                        break;
-                    }
-                }
-
-                if size < BUFFER_SIZE {
-                    // round to the nearest key_size boundary
-                    size = (size + CHACHA_KEY_SIZE - 1) & !(CHACHA_KEY_SIZE - 1);
-                }
-                total_size += size;
-
-                chacha_cbc_encrypt(&buffer[..size], &mut encrypted_buffer[..size], &key, ivec);
-                if let Err(res) = out_file.write(&encrypted_buffer[..size]) {
-                    warn!("Error writing file! {:?}", res);
-                    return Err(res);
-                }
-
-                start_index = last_index + 1;
-            }
-            Err(e) => {
-                info!("Error encrypting file: {:?}", e);
-                break;
-            }
-        }
-    }
-    Ok(total_size)
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::chacha::chacha_cbc_encrypt_ledger;
-    use rand::SeedableRng;
-    use rand_chacha::ChaChaRng;
-    use solana_ledger::blockstore::Blockstore;
-    use solana_ledger::entry::Entry;
-    use solana_ledger::get_tmp_ledger_path;
-    use solana_sdk::hash::{hash, Hash, Hasher};
-    use solana_sdk::pubkey::Pubkey;
-    use solana_sdk::signature::{Keypair, Signer};
-    use solana_sdk::system_transaction;
-    use std::fs::remove_file;
-    use std::fs::File;
-    use std::io::Read;
-    use std::sync::Arc;
-
-    fn make_tiny_deterministic_test_entries(num: usize) -> Vec<Entry> {
-        let zero = Hash::default();
-        let one = hash(&zero.as_ref());
-
-        let seed = [2u8; 32];
-
-        let mut generator = ChaChaRng::from_seed(seed);
-        let keypair = Keypair::generate(&mut generator);
-
-        let mut id = one;
-        let mut num_hashes = 0;
-        (0..num)
-            .map(|_| {
-                Entry::new_mut(
-                    &mut id,
-                    &mut num_hashes,
-                    vec![system_transaction::transfer(
-                        &keypair,
-                        &keypair.pubkey(),
-                        1,
-                        one,
-                    )],
-                )
-            })
-            .collect()
-    }
-
-    use std::{env, fs::create_dir_all, path::PathBuf};
-    fn tmp_file_path(name: &str) -> PathBuf {
-        let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
-        let mut path = PathBuf::new();
-        path.push(out_dir);
-        path.push("tmp");
-        create_dir_all(&path).unwrap();
-
-        path.push(format!("{}-{}", name, Pubkey::new_rand()));
-        path
-    }
-
-    #[test]
-    fn test_encrypt_ledger() {
-        solana_logger::setup();
-        let ledger_path = get_tmp_ledger_path!();
-        let ticks_per_slot = 16;
-        let slots_per_segment = 32;
-        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
-        let out_path = tmp_file_path("test_encrypt_ledger");
-
-        let seed = [2u8; 32];
-
-        let mut generator = ChaChaRng::from_seed(seed);
-        let keypair = Keypair::generate(&mut generator);
-
-        let entries = make_tiny_deterministic_test_entries(slots_per_segment);
-        blockstore
-            .write_entries(
-                0,
-                0,
-                0,
-                ticks_per_slot,
-                None,
-                true,
-                &Arc::new(keypair),
-                entries,
-                0,
-            )
-            .unwrap();
-
-        let mut key = hex!(
-            "abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234
-                            abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234"
-        );
-        chacha_cbc_encrypt_ledger(
-            &blockstore,
-            0,
-            slots_per_segment as u64,
-            &out_path,
-            &mut key,
-        )
-        .unwrap();
-        let mut out_file = File::open(&out_path).unwrap();
-        let mut buf = vec![];
-        let size = out_file.read_to_end(&mut buf).unwrap();
-        let mut hasher = Hasher::default();
-        hasher.hash(&buf[..size]);
-
-        //  golden needs to be updated if shred structure changes....
-        let golden: Hash = "2rq8nR6rns2T5zsbQAGBDZb41NVtacneLgkCH17CVxZm"
-            .parse()
-            .unwrap();
-
-        assert_eq!(hasher.result(), golden);
-        remove_file(&out_path).unwrap();
-    }
-}

chacha/src/lib.rs

@@ -1,8 +0,0 @@
-#[macro_use]
-extern crate log;
-
-#[cfg(test)]
-#[macro_use]
-extern crate hex_literal;
-
-pub mod chacha;

ci/README.md

@@ -2,7 +2,7 @@
 Our CI infrastructure is built around [BuildKite](https://buildkite.com) with some
 additional GitHub integration provided by https://github.com/mvines/ci-gate
 
-## Agent Queues
+# Agent Queues
 
 We define two [Agent Queues](https://buildkite.com/docs/agent/v3/queues):
 `queue=default` and `queue=cuda`.  The `default` queue should be favored and
@@ -12,9 +12,52 @@ be run on the `default` queue, and the [buildkite artifact
 system](https://buildkite.com/docs/builds/artifacts) used to transfer build
 products over to a GPU instance for testing.
 
-## Buildkite Agent Management
+# Buildkite Agent Management
 
-### Buildkite Azure Setup
+## Manual Node Setup for Colocated Hardware
+
+This section describes how to set up a new machine that does not have a
+pre-configured image with all the requirements installed.  Used for custom-built
+hardware at a colocation or office facility.  Also works for vanilla Ubuntu cloud
+instances.
+
+### Pre-Requisites
+
+ - Install Ubuntu 18.04 LTS Server
+ - Log in as a local or remote user with `sudo` privileges
+
+### Install Core Requirements
+
+##### Non-GPU enabled machines
+```bash
+sudo ./setup-new-buildkite-agent/setup-new-machine.sh
+```
+
+##### GPU-enabled machines
+ - 1 or more NVIDIA GPUs should be installed in the machine (tested with 2080Ti)
+```bash
+sudo CUDA=1 ./setup-new-buildkite-agent/setup-new-machine.sh
+```
+
+### Configure Node for Buildkite-agent based CI
+
+- Install `buildkite-agent` and set up it user environment with:
+```bash
+sudo ./setup-new-buildkite-agent/setup-buildkite.sh
+```
+- Copy the pubkey contents from `~buildkite-agent/.ssh/id_ecdsa.pub` and
+add the pubkey as an authorized SSH key on github.
+- Edit `/etc/buildkite-agent/buildkite-agent.cfg` and/or `/etc/systemd/system/buildkite-agent@*` to the desired configuration of the agent(s)
+- Copy `ejson` keys from another CI node at `/opt/ejson/keys/`
+to the same location on the new node.
+- Start the new agent(s) with `sudo systemctl enable --now buildkite-agent`
+
+# Reference
+
+This section contains details regarding previous CI setups that have been used,
+and that we may return to one day.
+
+## Buildkite Azure Setup
 
 Create a new Azure-based "queue=default" agent by running the following command:
 ```
@@ -35,7 +78,7 @@ Creating a "queue=cuda" agent follows the same process but additionally:
 2. Edit the tags field in /etc/buildkite-agent/buildkite-agent.cfg to `tags="queue=cuda,queue=default"`
    and decrease the value of the priority field by one
 
-#### Updating the CI Disk Image
+### Updating the CI Disk Image
 
 1. Create a new VM Instance as described above
 1. Modify it as required
@@ -48,12 +91,7 @@ Creating a "queue=cuda" agent follows the same process but additionally:
 1. Goto the `ci` resource group in the Azure portal and remove all resources
    with the XYZ name in them
 
-## Reference
-
-This section contains details regarding previous CI setups that have been used,
-and that we may return to one day.
-
-### Buildkite AWS CloudFormation Setup
+## Buildkite AWS CloudFormation Setup
 
 **AWS CloudFormation is currently inactive, although it may be restored in the
 future**
@@ -62,7 +100,7 @@ AWS CloudFormation can be used to scale machines up and down based on the
 current CI load.  If no machine is currently running it can take up to 60
 seconds to spin up a new instance, please remain calm during this time.
 
-#### AMI
+### AMI
 We use a custom AWS AMI built via https://github.com/solana-labs/elastic-ci-stack-for-aws/tree/solana/cuda.
 
 Use the following process to update this AMI as dependencies change:
@@ -84,13 +122,13 @@ The new AMI should also now be visible in your EC2 Dashboard.  Go to the desired
 AWS CloudFormation stack, update the **ImageId** field to the new AMI id, and
 *apply* the stack changes.
 
-### Buildkite GCP Setup
+## Buildkite GCP Setup
 
 CI runs on Google Cloud Platform via two Compute Engine Instance groups:
 `ci-default` and `ci-cuda`.  Autoscaling is currently disabled and the number of
 VM Instances in each group is manually adjusted.
 
-#### Updating a CI Disk Image
+### Updating a CI Disk Image
 
 Each Instance group has its own disk image, `ci-default-vX` and
 `ci-cuda-vY`, where *X* and *Y* are incremented each time the image is changed.

ci/affects-files.sh

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-#
-# Checks if a CI build affects one or more path patterns.  Each command-line
-# argument is checked in series.
-#
-# Bash regular expressions are permitted in the pattern:
-#     ./affects-files.sh .rs$    -- any file or directory ending in .rs
-#     ./affects-files.sh .rs     -- also matches foo.rs.bar
-#     ./affects-files.sh ^snap/  -- anything under the snap/ subdirectory
-#     ./affects-files.sh snap/   -- also matches foo/snap/
-# Any pattern starting with the ! character will be negated:
-#     ./affects-files.sh !^docs/  -- anything *not* under the docs/ subdirectory
-#
-set -e
-cd "$(dirname "$0")"/..
-
-if [[ -n $CI_PULL_REQUEST ]]; then
-  affectedFiles="$(buildkite-agent meta-data get affected_files)"
-  echo "Affected files in this PR: $affectedFiles"
-
-  IFS=':' read -ra files <<< "$affectedFiles"
-  for pattern in "$@"; do
-    if [[ ${pattern:0:1} = "!" ]]; then
-      for file in "${files[@]}"; do
-        if [[ ! $file =~ ${pattern:1} ]]; then
-          exit 0
-        fi
-      done
-    else
-      for file in "${files[@]}"; do
-        if [[ $file =~ $pattern ]]; then
-          exit 0
-        fi
-      done
-    fi
-  done
-
-  exit 1
-fi
-
-# affected_files metadata is not currently available for non-PR builds, so assume
-# the worse (affected)
-exit 0

ci/buildkite-pipeline.sh

@@ -0,0 +1,278 @@
+#!/usr/bin/env bash
+#
+# Builds a buildkite pipeline based on the environment variables
+#
+
+set -e
+cd "$(dirname "$0")"/..
+
+output_file=${1:-/dev/stderr}
+
+if [[ -n $CI_PULL_REQUEST ]]; then
+  IFS=':' read -ra affected_files <<< "$(buildkite-agent meta-data get affected_files)"
+  if [[ ${#affected_files[*]} -eq 0 ]]; then
+    echo "Unable to determine the files affected by this PR"
+    exit 1
+  fi
+else
+  affected_files=()
+fi
+
+annotate() {
+  if [[ -n $BUILDKITE ]]; then
+    buildkite-agent annotate "$@"
+  fi
+}
+
+# Checks if a CI pull request affects one or more path patterns.  Each
+# pattern argument is checked in series. If one of them found to be affected,
+# return immediately as such.
+#
+# Bash regular expressions are permitted in the pattern:
+#     affects .rs$    -- any file or directory ending in .rs
+#     affects .rs     -- also matches foo.rs.bar
+#     affects ^snap/  -- anything under the snap/ subdirectory
+#     affects snap/   -- also matches foo/snap/
+# Any pattern starting with the ! character will be negated:
+#     affects !^docs/  -- anything *not* under the docs/ subdirectory
+#
+affects() {
+  if [[ -z $CI_PULL_REQUEST ]]; then
+    # affected_files metadata is not currently available for non-PR builds so assume
+    # the worse (affected)
+    return 0
+  fi
+  # Assume everyting needs to be tested when any Dockerfile changes
+  for pattern in ^ci/docker-rust/Dockerfile ^ci/docker-rust-nightly/Dockerfile "$@"; do
+    if [[ ${pattern:0:1} = "!" ]]; then
+      for file in "${affected_files[@]}"; do
+        if [[ ! $file =~ ${pattern:1} ]]; then
+          return 0 # affected
+        fi
+      done
+    else
+      for file in "${affected_files[@]}"; do
+        if [[ $file =~ $pattern ]]; then
+          return 0 # affected
+        fi
+      done
+    fi
+  done
+
+  return 1 # not affected
+}
+
+
+# Checks if a CI pull request affects anything other than the provided path patterns
+#
+# Syntax is the same as `affects()` except that the negation prefix is not
+# supported
+#
+affects_other_than() {
+  if [[ -z $CI_PULL_REQUEST ]]; then
+    # affected_files metadata is not currently available for non-PR builds so assume
+    # the worse (affected)
+    return 0
+  fi
+
+  for file in "${affected_files[@]}"; do
+    declare matched=false
+    for pattern in "$@"; do
+        if [[ $file =~ $pattern ]]; then
+          matched=true
+        fi
+    done
+    if ! $matched; then
+      return 0 # affected
+    fi
+  done
+
+  return 1 # not affected
+}
+
+
+start_pipeline() {
+  echo "# $*" > "$output_file"
+  echo "steps:" >> "$output_file"
+}
+
+command_step() {
+  cat >> "$output_file" <<EOF
+  - name: "$1"
+    command: "$2"
+    timeout_in_minutes: $3
+    artifact_paths: "log-*.txt"
+EOF
+}
+
+
+trigger_secondary_step() {
+  cat  >> "$output_file" <<"EOF"
+  - trigger: "solana-secondary"
+    branches: "!pull/*"
+    async: true
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        TRIGGERED_BUILDKITE_TAG: "${BUILDKITE_TAG}"
+EOF
+}
+
+wait_step() {
+  echo "  - wait" >> "$output_file"
+}
+
+all_test_steps() {
+  command_step checks ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-checks.sh" 20
+  wait_step
+
+  # Coverage...
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-coverage.sh \
+             ^scripts/coverage.sh \
+      ; then
+    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 30
+    wait_step
+  else
+    annotate --style info --context test-coverage \
+      "Coverage skipped as no .rs files were modified"
+  fi
+
+  # Full test suite
+  command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 60
+  wait_step
+
+  # Perf test suite
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-stable-perf.sh \
+             ^ci/test-stable.sh \
+             ^ci/test-local-cluster.sh \
+             ^core/build.rs \
+             ^fetch-perf-libs.sh \
+             ^programs/ \
+             ^sdk/ \
+      ; then
+    cat >> "$output_file" <<"EOF"
+  - command: "ci/test-stable-perf.sh"
+    name: "stable-perf"
+    timeout_in_minutes: 40
+    artifact_paths: "log-*.txt"
+    agents:
+      - "queue=cuda"
+EOF
+  else
+    annotate --style info \
+      "Stable-perf skipped as no relevant files were modified"
+  fi
+
+  # Downstream backwards compatibility
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-stable-perf.sh \
+             ^ci/test-stable.sh \
+             ^ci/test-local-cluster.sh \
+             ^core/build.rs \
+             ^fetch-perf-libs.sh \
+             ^programs/ \
+             ^sdk/ \
+             ^scripts/build-downstream-projects.sh \
+      ; then
+    cat >> "$output_file" <<"EOF"
+  - command: "scripts/build-downstream-projects.sh"
+    name: "downstream-projects"
+    timeout_in_minutes: 30
+EOF
+  else
+    annotate --style info \
+      "downstream-projects skipped as no relevant files were modified"
+  fi
+  # Benches...
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-coverage.sh \
+             ^ci/test-bench.sh \
+      ; then
+    command_step bench "ci/test-bench.sh" 30
+  else
+    annotate --style info --context test-bench \
+      "Bench skipped as no .rs files were modified"
+  fi
+
+  command_step "local-cluster" \
+    ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster.sh" \
+    45
+}
+
+pull_or_push_steps() {
+  command_step sanity "ci/test-sanity.sh" 5
+  wait_step
+
+  # Check for any .sh file changes
+  if affects .sh$; then
+    command_step shellcheck "ci/shellcheck.sh" 5
+    wait_step
+  fi
+
+  # Run the full test suite by default, skipping only if modifications are local
+  # to some particular areas of the tree
+  if affects_other_than ^.buildkite ^.mergify .md$ ^docs/ ^web3.js/ ^explorer/ ^.gitbook; then
+    all_test_steps
+  fi
+
+  # web3.js, explorer and docs changes run on Travis...
+}
+
+
+if [[ -n $BUILDKITE_TAG ]]; then
+  start_pipeline "Tag pipeline for $BUILDKITE_TAG"
+
+  annotate --style info --context release-tag \
+    "https://github.com/solana-labs/solana/releases/$BUILDKITE_TAG"
+
+  # Jump directly to the secondary build to publish release artifacts quickly
+  trigger_secondary_step
+  exit 0
+fi
+
+
+if [[ $BUILDKITE_BRANCH =~ ^pull ]]; then
+  echo "+++ Affected files in this PR"
+  for file in "${affected_files[@]}"; do
+    echo "- $file"
+  done
+
+  start_pipeline "Pull request pipeline for $BUILDKITE_BRANCH"
+
+  # Add helpful link back to the corresponding Github Pull Request
+  annotate --style info --context pr-backlink \
+    "Github Pull Request: https://github.com/solana-labs/solana/$BUILDKITE_BRANCH"
+
+  if [[ $GITHUB_USER = "dependabot-preview[bot]" ]]; then
+    command_step dependabot "ci/dependabot-pr.sh" 5
+    wait_step
+  fi
+  pull_or_push_steps
+  exit 0
+fi
+
+start_pipeline "Push pipeline for ${BUILDKITE_BRANCH:-?unknown branch?}"
+pull_or_push_steps
+wait_step
+trigger_secondary_step
+exit 0

ci/buildkite-release.yml

@@ -1,15 +0,0 @@
-# Build steps that run on a release tag
-#
-# All the steps in `buildkite.yml` are skipped and we jump directly to the
-# secondary build steps since it's assumed the commit that was tagged is known
-# to be good so there's no need to rebuild and retest it.
-steps:
-  - trigger: "solana-secondary"
-    branches: "!pull/*"
-    async: true
-    build:
-      message: "${BUILDKITE_MESSAGE}"
-      commit: "${BUILDKITE_COMMIT}"
-      branch: "${BUILDKITE_BRANCH}"
-      env:
-        TRIGGERED_BUILDKITE_TAG: "${BUILDKITE_TAG}"

ci/buildkite-secondary.yml

@@ -2,6 +2,13 @@
 # Build steps that run after the primary pipeline on pushes and tags.
 # Pull requests to not run these steps.
 steps:
+  - command: "ci/publish-tarball.sh"
+    timeout_in_minutes: 60
+    name: "publish tarball"
+  - command: "ci/publish-bpf-sdk.sh"
+    timeout_in_minutes: 5
+    name: "publish bpf sdk"
+  - wait
   - command: "sdk/docker-solana/build.sh"
     timeout_in_minutes: 60
     name: "publish docker"
@@ -9,12 +16,3 @@ steps:
     timeout_in_minutes: 240
     name: "publish crate"
     branches: "!master"
-  - command: "ci/publish-bpf-sdk.sh"
-    timeout_in_minutes: 5
-    name: "publish bpf sdk"
-  - command: "ci/publish-tarball.sh"
-    timeout_in_minutes: 60
-    name: "publish tarball"
-  - command: "ci/publish-docs.sh"
-    timeout_in_minutes: 15
-    name: "publish docs"

ci/buildkite-tests.yml

@@ -1,37 +0,0 @@
-# These steps are conditionally triggered by ci/buildkite.yml when files
-# other than those in docs/ are modified
-
-steps:
-  - command: "ci/shellcheck.sh"
-    name: "shellcheck"
-    timeout_in_minutes: 5
-
-  - wait
-
-  - command: "ci/test-stable-perf.sh"
-    name: "stable-perf"
-    timeout_in_minutes: 40
-    artifact_paths: "log-*.txt"
-    agents:
-      - "queue=cuda"
-  - command: "ci/test-bench.sh"
-    name: "bench"
-    timeout_in_minutes: 30
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh"
-    name: "stable"
-    timeout_in_minutes: 60
-    artifact_paths: "log-*.txt"
-    agents:
-      - "queue=rpc-test-capable"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
-    name: "move"
-    timeout_in_minutes: 20
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-local-cluster.sh"
-    name: "local-cluster"
-    timeout_in_minutes: 45
-    artifact_paths: "log-*.txt"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh"
-    name: "coverage"
-    timeout_in_minutes: 30
-    agents:
-      - "queue=rpc-test-capable"

ci/buildkite.yml

@@ -1,28 +0,0 @@
-# Build steps that run on pushes and pull requests.
-# If files other than those in docs/ were modified, this will be followed up by
-# ci/buildkite-tests.yml
-#
-# Release tags use buildkite-release.yml instead
-
-steps:
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-checks.sh"
-    name: "checks"
-    timeout_in_minutes: 20
-
-  - wait
-
-  - command: "ci/maybe-trigger-tests.sh"
-    name: "maybe-trigger-tests"
-    timeout_in_minutes: 2
-
-  - wait
-
-  - trigger: "solana-secondary"
-    branches: "!pull/*"
-    async: true
-    build:
-      message: "${BUILDKITE_MESSAGE}"
-      commit: "${BUILDKITE_COMMIT}"
-      branch: "${BUILDKITE_BRANCH}"
-      env:
-        TRIGGERED_BUILDKITE_TAG: "${BUILDKITE_TAG}"

ci/channel-info.sh

@@ -89,12 +89,20 @@ BETA_CHANNEL_LATEST_TAG=${beta_tag:+v$beta_tag}
 STABLE_CHANNEL_LATEST_TAG=${stable_tag:+v$stable_tag}
 
 
-if [[ $CI_BRANCH = "$STABLE_CHANNEL" ]]; then
-  CHANNEL=stable
-elif [[ $CI_BRANCH = "$EDGE_CHANNEL" ]]; then
-  CHANNEL=edge
-elif [[ $CI_BRANCH = "$BETA_CHANNEL" ]]; then
-  CHANNEL=beta
+if [[ -n $CI_BASE_BRANCH ]]; then
+  BRANCH="$CI_BASE_BRANCH"
+elif [[ -n $CI_BRANCH ]]; then
+  BRANCH="$CI_BRANCH"
+fi
+
+if [[ -z "$CHANNEL" ]]; then
+  if [[ $BRANCH = "$STABLE_CHANNEL" ]]; then
+    CHANNEL=stable
+  elif [[ $BRANCH = "$EDGE_CHANNEL" ]]; then
+    CHANNEL=edge
+  elif [[ $BRANCH = "$BETA_CHANNEL" ]]; then
+    CHANNEL=beta
+  fi
 fi
 
 echo EDGE_CHANNEL="$EDGE_CHANNEL"

ci/dependabot-pr.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd "$(dirname "$0")/.."
+
+if ! echo "$BUILDKITE_BRANCH" | grep -E '^pull/[0-9]+/head$'; then
+  echo "not pull request!?" >&2
+  exit 1
+fi
+
+source ci/rust-version.sh stable
+
+ci/docker-run.sh $rust_nightly_docker_image ci/dependabot-updater.sh
+
+if [[ $(git status --short :**/Cargo.lock | wc -l) -eq 0 ]]; then
+  echo --- ok
+  exit 0
+fi
+
+echo --- "(FAILING) Backpropagating dependabot-triggered Cargo.lock updates"
+
+name="dependabot-buildkite"
+api_base="https://api.github.com/repos/solana-labs/solana/pulls"
+pr_num=$(echo "$BUILDKITE_BRANCH" | grep -Eo '[0-9]+')
+branch=$(curl -s "$api_base/$pr_num" | python -c 'import json,sys;print json.load(sys.stdin)["head"]["ref"]')
+
+git add :**/Cargo.lock
+EMAIL="dependabot-buildkite@noreply.solana.com" \
+  GIT_AUTHOR_NAME="$name" \
+  GIT_COMMITTER_NAME="$name" \
+  git commit -m "[auto-commit] Update all Cargo lock files"
+git push origin "HEAD:$branch"
+
+echo "Source branch is updated; failing this build for the next"
+exit 1

ci/dependabot-updater.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -ex
+cd "$(dirname "$0")/.."
+source ci/_
+
+commit_range="$(git merge-base HEAD origin/master)..HEAD"
+parsed_update_args="$(
+  git log "$commit_range" --author "dependabot-preview" --oneline -n1 |
+    grep -o '[Bb]ump.*$' |
+    sed -r 's/[Bb]ump ([^ ]+) from ([^ ]+) to ([^ ]+)/-p \1:\2 --precise \3/'
+)"
+# relaxed_parsed_update_args is temporal measure...
+relaxed_parsed_update_args="$(
+  git log "$commit_range" --author "dependabot-preview" --oneline -n1 |
+    grep -o '[Bb]ump.*$' |
+    sed -r 's/[Bb]ump ([^ ]+) from [^ ]+ to ([^ ]+)/-p \1 --precise \2/'
+)"
+package=$(echo "$parsed_update_args" | awk '{print $2}' | grep -o "^[^:]*")
+if [[ -n $parsed_update_args ]]; then
+  # find other Cargo.lock files and update them, excluding the default Cargo.lock
+  # shellcheck disable=SC2086
+  for lock in $(git grep --files-with-matches '^name = "'$package'"$' :**/Cargo.lock); do
+    # it's possible our current versions are out of sync across lock files,
+    # in that case try to sync them up with $relaxed_parsed_update_args
+    _ scripts/cargo-for-all-lock-files.sh \
+      "$lock" -- \
+      update $parsed_update_args ||
+      _ scripts/cargo-for-all-lock-files.sh \
+        "$lock" -- \
+        update $relaxed_parsed_update_args
+  done
+fi
+
+echo --- ok

ci/docker-run.sh

@@ -49,7 +49,7 @@ else
   # ~/.cargo
   ARGS+=(--volume "$PWD:/home")
 fi
-ARGS+=(--env "CARGO_HOME=/home/.cargo")
+ARGS+=(--env "HOME=/home" --env "CARGO_HOME=/home/.cargo")
 
 # kcov tries to set the personality of the binary which docker
 # doesn't allow by default.
@@ -60,6 +60,12 @@ if [[ -z "$SOLANA_DOCKER_RUN_NOSETUID" ]]; then
   ARGS+=(--user "$(id -u):$(id -g)")
 fi
 
+if [[ -n $SOLANA_ALLOCATE_TTY ]]; then
+  # Colored output, progress bar and Ctrl-C:
+  # https://stackoverflow.com/a/41099052/10242004
+  ARGS+=(--interactive --tty)
+fi
+
 # Environment variables to propagate into the container
 ARGS+=(
   --env BUILDKITE
@@ -67,6 +73,7 @@ ARGS+=(
   --env BUILDKITE_JOB_ID
   --env CI
   --env CI_BRANCH
+  --env CI_BASE_BRANCH
   --env CI_TAG
   --env CI_BUILD_ID
   --env CI_COMMIT

ci/docker-rust-nightly/Dockerfile

@@ -1,9 +1,10 @@
-FROM solanalabs/rust:1.42.0
+FROM solanalabs/rust:1.46.0
 ARG date
 
 RUN set -x \
  && rustup install nightly-$date \
  && rustup component add clippy --toolchain=nightly-$date \
+ && rustup component add rustfmt --toolchain=nightly-$date \
  && rustup show \
  && rustc --version \
  && cargo --version \

ci/docker-rust-nightly/README.md

@@ -2,23 +2,27 @@ Docker image containing rust nightly and some preinstalled crates used in CI.
 
 This image may be manually updated by running `CI=true ./build.sh` if you are a member
 of the [Solana Labs](https://hub.docker.com/u/solanalabs/) Docker Hub
-organization, but it is also automatically updated periodically by
-[this automation](https://buildkite.com/solana-labs/solana-ci-docker-rust-nightly).
+organization.
 
 ## Moving to a newer nightly
 
+NOTE: Follow instructions in docker-rust/README.md before this when updating the stable
+rust version as well.
+
 We pin the version of nightly (see the `ARG nightly=xyz` line in `Dockerfile`)
 to avoid the build breaking at unexpected times, as occasionally nightly will
 introduce breaking changes.
 
 To update the pinned version:
+1. Edit `Dockerfile` to match the desired stable rust version to base on if needed.
 1. Run `ci/docker-rust-nightly/build.sh` to rebuild the nightly image locally,
    or potentially `ci/docker-rust-nightly/build.sh YYYY-MM-DD` if there's a
    specific YYYY-MM-DD that is desired (default is today's build).
    Check https://rust-lang.github.io/rustup-components-history/ for build
    status
-1. Update `ci/rust-version.sh` to reflect the new nightly `YYY-MM-DD`
-1. Run `SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-coverage.sh`
+1. Update `ci/rust-version.sh` to reflect the new nightly `YYYY-MM-DD`
+1. Run `SOLANA_ALLOCATE_TTY=1 SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-checks.sh`
+   and `SOLANA_ALLOCATE_TTY=1 SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-coverage.sh [args]...`
    to confirm the new nightly image builds.  Fix any issues as needed
 1. Run `docker login` to enable pushing images to Docker Hub, if you're authorized.
 1. Run `CI=true ci/docker-rust-nightly/build.sh YYYY-MM-DD` to push the new nightly image to dockerhub.com.

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.42.0
+FROM rust:1.46.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0
@@ -32,6 +32,7 @@ RUN set -x \
  && cargo install cargo-audit \
  && cargo install svgbob_cli \
  && cargo install mdbook \
+ && cargo install mdbook-linkcheck \
  && rustc --version \
  && cargo --version \
  && curl -OL https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP \

ci/docker-rust/README.md

@@ -1,7 +1,11 @@
 Docker image containing rust and some preinstalled packages used in CI.
 
+NOTE: Recreate rust-nightly docker image after this when updating the stable rust
+version! Both of docker images must be updated in tandem.
+
 This image manually maintained:
 1. Edit `Dockerfile` to match the desired rust version
-2. Run `./build.sh` to publish the new image, if you are a member of the [Solana
+1. Run `docker login` to enable pushing images to Docker Hub, if you're authorized.
+1. Run `./build.sh` to publish the new image, if you are a member of the [Solana
    Labs](https://hub.docker.com/u/solanalabs/) Docker Hub organization.
 

ci/env.sh

@@ -8,10 +8,11 @@ if [[ -n $CI ]]; then
   export CI=1
   if [[ -n $TRAVIS ]]; then
     export CI_BRANCH=$TRAVIS_BRANCH
+    export CI_BASE_BRANCH=$TRAVIS_BRANCH
     export CI_BUILD_ID=$TRAVIS_BUILD_ID
     export CI_COMMIT=$TRAVIS_COMMIT
     export CI_JOB_ID=$TRAVIS_JOB_ID
-    if $TRAVIS_PULL_REQUEST; then
+    if [[ $TRAVIS_PULL_REQUEST != false ]]; then
       export CI_PULL_REQUEST=true
     else
       export CI_PULL_REQUEST=
@@ -28,8 +29,10 @@ if [[ -n $CI ]]; then
     # to how solana-ci-gate is used to trigger PR builds rather than using the
     # standard Buildkite PR trigger.
     if [[ $CI_BRANCH =~ pull/* ]]; then
+      export CI_BASE_BRANCH=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
       export CI_PULL_REQUEST=true
     else
+      export CI_BASE_BRANCH=$BUILDKITE_BRANCH
       export CI_PULL_REQUEST=
     fi
     export CI_OS_NAME=linux