Optimize packet dedup (#22571)

* Use bloom filter to dedup packets

* dedup first

* Update bloom/src/bloom.rs

Co-authored-by: Trent Nelson <trent.a.b.nelson@gmail.com>

* Update core/src/sigverify_stage.rs

Co-authored-by: Trent Nelson <trent.a.b.nelson@gmail.com>

* Update core/src/sigverify_stage.rs

Co-authored-by: Trent Nelson <trent.a.b.nelson@gmail.com>

* Update core/src/sigverify_stage.rs

Co-authored-by: Trent Nelson <trent.a.b.nelson@gmail.com>

* fixup

* fixup

* fixup

Co-authored-by: Trent Nelson <trent.a.b.nelson@gmail.com>
(cherry picked from commit d343713f61)

# Conflicts:
#	Cargo.lock
#	core/Cargo.toml
#	core/src/banking_stage.rs
#	core/src/sigverify_stage.rs
#	gossip/Cargo.toml
#	perf/Cargo.toml
#	programs/bpf/Cargo.lock
#	runtime/Cargo.toml

.buildkite/env/secrets.ejson

@@ -2,6 +2,6 @@
     "_public_key": "ae29f4f7ad2fc92de70d470e411c8426d5d48db8817c9e3dae574b122192335f",
     "_comment": "These credentials are encrypted and pose no risk",
     "environment": {
-      "CODECOV_TOKEN": "EJ[1:Z7OneT3RdJJ0DipCHQ7rC84snQ+FPbgHwZADQiz54wk=:3K68mE38LJ2RB98VWmjuNLFBNn1XTGR4:cR4r05/TOZQKmEZp1v4CSgUJtC6QJiOaL85QjXW0qZ061fMnsBA8AtAPMDoDq4WCGOZM1A==]"
+      "CODECOV_TOKEN": "EJ[1:KToenD1Sr3w82lHGxz1n+j3hwNlLk/1pYrjZHlvY6kE=:hN1Q25omtJ+4yYVn+qzIsPLKT3O6J9XN:DMLNLXi/pkWgvwF6gNIcNF222sgsRR9LnwLZYj0P0wGj7q6w8YQnd1Rskj+sRroI/z5pQg==]"
     }
 }

.buildkite/hooks/pre-command

@@ -31,4 +31,12 @@ export CARGO_TARGET_CACHE=$HOME/cargo-target-cache/"$CHANNEL"-"$BUILDKITE_LABEL"
 
   mkdir -p "$CARGO_TARGET_CACHE"/target
   rsync -a --delete --link-dest="$CARGO_TARGET_CACHE" "$CARGO_TARGET_CACHE"/target .
+
+  # Don't reuse BPF target build artifacts due to incremental build issues with
+  # `std:
+  #    "found possibly newer version of crate `std` which `xyz` depends on
+  rm -rf target/bpfel-unknown-unknown
+  if [[ $BUILDKITE_LABEL = "stable-perf" ]]; then
+    rm -rf target/release
+  fi
 )

.github/workflows/explorer.yml

@@ -0,0 +1,28 @@
+name: Explorer_build&test_on_PR
+on:
+  pull_request:
+    branches:
+     - master
+    paths:
+      - 'explorer/**'
+jobs:
+  check-explorer:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: explorer
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          cache: 'npm'
+          cache-dependency-path: explorer/package-lock.json
+      - run: npm i -g npm@7
+      - run: npm ci
+      - run: npm run format
+      - run: npm run build
+      - run: npm run test

.github/workflows/explorer_preview.yml

@@ -0,0 +1,20 @@
+name : explorer_preview
+on:
+  workflow_run:
+    workflows: ["Explorer_build&test_on_PR"]
+    types: 
+      - completed
+jobs:
+   explorer_preview:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: amondnet/vercel-action@v20
+        with:
+          vercel-token: ${{ secrets.VERCEL_TOKEN }} # Required
+          github-token: ${{ secrets.GITHUB_TOKEN }} #Optional 
+          vercel-org-id: ${{ secrets.ORG_ID}}  #Required
+          vercel-project-id: ${{ secrets.PROJECT_ID}} #Required 
+          working-directory: ./explorer
+          scope: ${{ secrets.TEAM_ID }}

.github/workflows/explorer_production.yml

@@ -0,0 +1,46 @@
+name: Explorer_production_build&test
+on:
+  push:
+   branches: [master]
+   paths:
+      - 'explorer/**'
+jobs:
+    Explorer_production_build_test:
+      runs-on: ubuntu-latest
+      defaults:
+        run:
+          working-directory: explorer
+      steps:
+       - uses: actions/checkout@v2
+         with:
+          ref: ${{ github.event.pull_request.head.sha }}
+       - uses: actions/setup-node@v2
+         with:
+           node-version: '14'
+           cache: 'npm'
+           cache-dependency-path: explorer/package-lock.json
+       - run: npm i -g npm@7
+       - run: npm ci
+       - run: npm run format
+       - run: npm run build
+       - run: npm run test
+       
+    Explorer_production_deploy:
+      needs: Explorer_production_build_test
+      runs-on: ubuntu-latest
+      defaults:
+        run:
+          working-directory: explorer
+      steps:
+       - uses: actions/checkout@v2
+         with:
+           ref: ${{ github.event.pull_request.head.sha }}
+       - uses: amondnet/vercel-action@v20
+         with:
+           vercel-token: ${{ secrets.VERCEL_TOKEN }} # Required
+           github-token: ${{ secrets.GITHUB_TOKEN }} #Optional 
+           vercel-args: '--prod' #for production
+           vercel-org-id: ${{ secrets.ORG_ID}}  #Required
+           vercel-project-id: ${{ secrets.PROJECT_ID}} #Required 
+           working-directory: ./explorer
+           scope: ${{ secrets.TEAM_ID }}

.github/workflows/solana-action.yml.txt

@@ -0,0 +1,208 @@
+name : minimal 
+
+on:
+  push:
+    branches:  [master]
+  pull_request:
+    branches:  [master]
+
+jobs:
+  Export_Github_Repositories:
+    runs-on: ubuntu-latest
+    env: 
+       VERCEL_TOKEN: ${{secrets.VERCEL_TOKEN}}
+       GITHUB_TOKEN: ${{secrets.PAT_ANM}} 
+       COMMIT_RANGE: ${{ github.event.before}}...${{ github.event.after}}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - run: echo "COMMIT_DIFF_RANGE=$(echo $COMMIT_RANGE)" >> $GITHUB_ENV
+#       - run: echo "$COMMIT_DIFF_RANGE"
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          GITHUB_TOKEN: ${{secrets.PAT_ANM}}
+        if: ${{ github.event_name == 'push' && 'cron'&& github.ref == 'refs/heads/master'}}
+        
+      - name: cmd
+        run : |
+            .travis/export-github-repo.sh web3.js/ solana-web3.js
+            
+  macos-artifacts:
+    needs: [Export_Github_Repositories]
+    strategy:
+      fail-fast: false
+    runs-on: macos-latest
+    if : ${{ github.event_name == 'api' && 'cron' || 'push' || startsWith(github.ref, 'refs/tags/v')}}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Setup | Rust
+        uses: ATiltedTree/setup-rust@v1
+        with:
+          rust-version: stable
+      - name:  release artifact
+        run: |
+           source ci/rust-version.sh
+           brew install coreutils
+           export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
+           greadlink -f .
+           source ci/env.sh
+           rustup set profile default
+           ci/publish-tarball.sh
+        shell: bash
+        
+      - name: Cache modules
+        uses: actions/cache@master
+        id: yarn-cache
+        with:
+          path: node_modules
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: ${{ runner.os }}-yarn-
+          
+
+# - To stop from uploading on the production
+#      - uses: ochanje210/simple-s3-upload-action@master
+#        with:
+#          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_KEY_ID }}
+#          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+#          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+#          SOURCE_DIR: 'travis-s3-upload1'
+#          DEST_DIR: 'giitsol'
+
+#      - uses: ochanje210/simple-s3-upload-action@master
+#        with:
+#          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_KEY_ID }}
+#          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+#          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+#          SOURCE_DIR: './docs/'
+#          DEST_DIR: 'giitsol'
+
+  
+  windows-artifact:
+    needs: [Export_Github_Repositories]
+    strategy:
+      fail-fast: false 
+    runs-on: windows-latest
+    if : ${{ github.event_name == 'api' && 'cron' || 'push' || startsWith(github.ref, 'refs/tags/v')}}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Setup | Rust
+        uses: ATiltedTree/setup-rust@v1
+        with:
+          rust-version: stable
+  release-artifact:
+    needs: windows-artifact
+    runs-on: windows-latest
+    if : ${{ github.event_name == 'api' && 'cron' || github.ref == 'refs/heads/master'}}
+    steps:
+      - name: release artifact 
+        run: |
+             git clone git://git.openssl.org/openssl.git
+             cd openssl
+             make
+             make test
+             make install
+             openssl version
+#            choco install openssl 
+#            vcpkg integrate install
+#            refreshenv
+
+      - name: Checkout repository
+        uses: actions/checkout@v2 
+      - uses: actions/checkout@v2
+      - run: choco install msys2
+      - uses: actions/checkout@v2
+      - run: |
+           openssl version
+           bash ci/rust-version.sh
+           readlink -f .
+           bash ci/env.sh
+           rustup set profile default
+           bash ci/publish-tarball.sh
+        shell: bash
+        
+      - name: Cache modules
+        uses: actions/cache@v1
+        id: yarn-cache
+        with:
+          path: node_modules
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: ${{ runner.os }}-yarn-
+
+# - To stop from uploading on the production      
+#      - name: Config. aws cred
+#        uses: aws-actions/configure-aws-credentials@v1
+#        with:
+#          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+#          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+#          aws-region: us-east-2
+#      - name: Deploy
+#        uses: shallwefootball/s3-upload-action@master
+#        with:
+#            folder: build
+#            aws_bucket: ${{ secrets.AWS_S3_BUCKET }}
+#            aws_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+#            aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+#            destination_dir: /
+#            bucket-region: us-east-2
+#            delete-removed: true
+#            no-cache: true
+#            private: true
+  
+#  Docs:
+#    needs: [windows-artifact,release-artifact]
+#    runs-on: ubuntu-latest
+#    env: 
+#       GITHUB_TOKEN: ${{secrets.PAT_NEW}} 
+#       GITHUB_EVENT_BEFORE: ${{ github.event.before }}
+#       GITHUB_EVENT_AFTER: ${{ github.event.after }}
+#       COMMIT_RANGE: ${{ github.event.before}}...${{ github.event.after}}
+#    steps:
+#      - name: Checkout repo
+#        uses: actions/checkout@v2
+#        with:
+#          fetch-depth: 2
+#      - name: docs
+#        if: ${{github.event_name == 'pull_request' || startsWith(github.ref, 'refs/tags/v')}}
+#        run: |
+#             touch .env
+#             echo "COMMIT_RANGE=($COMMIT_RANGE)" > .env
+#             source ci/env.sh
+#             .travis/channel_restriction.sh edge beta || exit 0
+#             .travis/affects.sh docs/ .travis || exit 0
+#             cd docs/
+#             source .travis/before_install.sh
+#             source .travis/script.sh
+#      - name: setup-node
+#        uses: actions/checkout@v2
+#      - name: setup-node
+#        uses: actions/setup-node@v2
+#        with:
+#          node-version: 'lts/*'
+#      - name: Cache
+#        uses: actions/cache@v1
+#        with:
+#          path: ~/.npm
+#          key: ${{ runner.OS }}-npm-cache-${{ hashFiles('**/package-lock.json') }}
+#          restore-keys: |
+#            ${{ runner.OS }}-npm-cache-2  
+
+#  auto_bump:
+#     needs: [windows-artifact,release-artifact,Docs]
+#     runs-on: ubuntu-latest
+#     steps:
+#       - name : checkout repo
+#         uses: actions/checkout@v2
+#         with:
+#          fetch-depth: '0'
+#       - name: Bump version and push tag
+#         uses: anothrNick/github-tag-action@1.26.0
+#         env:
+#           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+#           WITH_V: true
+#           DEFAULT_BUMP: patch

.github/workflows/web3.yml

@@ -0,0 +1,84 @@
+name: Web3
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - "web3.js/**"
+  pull_request:
+    branches: [ master ]
+    paths:
+      - "web3.js/**"
+
+jobs:
+  # needed for grouping check-web3 strategies into one check for mergify
+  all-web3-checks:
+    runs-on: ubuntu-latest
+    needs:
+      - check-web3
+    steps:
+      - run: echo "Done"
+
+  web3-commit-lint:
+    runs-on: ubuntu-latest
+    # Set to true in order to avoid cancelling other workflow jobs.
+    # Mergify will still require web3-commit-lint for automerge
+    continue-on-error: true
+
+    defaults:
+      run:
+        working-directory: web3.js
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # maybe needed for base sha below
+          fetch-depth: 0
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+          cache: 'npm'
+          cache-dependency-path: web3.js/package-lock.json
+      - run: npm ci
+      - name: commit-lint
+        if: ${{ github.event_name == 'pull_request' }}
+        run: bash commitlint.sh
+        env:
+          COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
+
+  check-web3:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: web3.js
+
+    strategy:
+      matrix:
+        node: [ '12', '14', '16' ]
+
+    name: Node ${{ matrix.node }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node }}
+          cache: 'npm'
+          cache-dependency-path: web3.js/package-lock.json
+      - run: npm i -g npm@7
+      - run: npm ci
+      - run: npm run lint
+      - run: |
+          npm run build
+          ls -l lib
+          test -r lib/index.iife.js
+          test -r lib/index.cjs.js
+          test -r lib/index.esm.js
+      - run: npm run doc
+      - run: npm run codecov
+      - run: |
+          sh -c "$(curl -sSfL https://release.solana.com/edge/install)"
+          echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH
+          PATH="$HOME/.local/share/solana/install/active_release/bin:$PATH"
+          solana --version
+      - run: npm run test:live-with-test-validator

.gitignore

@@ -1,19 +1,18 @@
-/docs/html/
-/docs/src/tests.ok
-/docs/src/cli/usage.md
-/docs/src/.gitbook/assets/*.svg
 /farf/
 /solana-release/
 /solana-release.tar.bz2
 /solana-metrics/
 /solana-metrics.tar.bz2
 /target/
+/test-ledger/
 
 **/*.rs.bk
 .cargo
 
 /config/
 
+.cache
+
 # log files
 *.log
 log-*.txt
@@ -29,3 +28,5 @@ log-*/
 /spl_*.so
 
 .DS_Store
+# scripts that may be generated by cargo *-bpf commands
+**/cargo-*-bpf-child-script-*.sh

.mergify.yml

@@ -4,24 +4,71 @@
 #
 # https://doc.mergify.io/
 pull_request_rules:
+  - name: label changes from community
+    conditions:
+      - author≠@core-contributors
+      - author≠mergify[bot]
+      - author≠dependabot[bot]
+    actions:
+      label:
+        add:
+          - community
+  - name: request review for community changes
+    conditions:
+      - author≠@core-contributors
+      - author≠mergify[bot]
+      - author≠dependabot[bot]
+      # Only request reviews from the pr subscribers group if no one
+      # has reviewed the community PR yet. These checks only match
+      # reviewers with admin, write or maintain permission on the repository.
+      - "#approved-reviews-by=0"
+      - "#commented-reviews-by=0"
+      - "#changes-requested-reviews-by=0"
+    actions:
+      request_reviews:
+        teams:
+          - "@solana-labs/community-pr-subscribers"
   - name: automatic merge (squash) on CI success
     conditions:
-      - status-success=buildkite/solana
-      - status-success=Travis CI - Pull Request
-      - status-success=ci-gate
-      - label=automerge
-      - author≠@dont-squash-my-commits
+      - and:
+        - status-success=buildkite/solana
+        - status-success=ci-gate
+        - label=automerge
+        - author≠@dont-squash-my-commits
+        - or:
+          # only require travis success if docs files changed
+          - status-success=Travis CI - Pull Request
+          - -files~=^docs/
+        - or:
+          # only require explorer checks if explorer files changed
+          - status-success=check-explorer
+          - -files~=^explorer/
+        - or:
+          - and:
+            - status-success=all-web3-checks
+            - status-success=web3-commit-lint
+          # only require web3 checks if web3.js files changed
+          - -files~=^web3.js/
     actions:
       merge:
         method: squash
   # Join the dont-squash-my-commits group if you won't like your commits squashed
   - name: automatic merge (rebase) on CI success
     conditions:
-      - status-success=buildkite/solana
-      - status-success=Travis CI - Pull Request
-      - status-success=ci-gate
-      - label=automerge
-      - author=@dont-squash-my-commits
+      - and:
+        - status-success=buildkite/solana
+        - status-success=Travis CI - Pull Request
+        - status-success=ci-gate
+        - label=automerge
+        - author=@dont-squash-my-commits
+        - or:
+          # only require explorer checks if explorer files changed
+          - status-success=check-explorer
+          - -files~=^explorer/
+        - or:
+          # only require web3 checks if web3.js files changed
+          - status-success=all-web3-checks
+          - -files~=^web3.js/
     actions:
       merge:
         method: rebase
@@ -50,27 +97,19 @@ pull_request_rules:
       label:
         add:
           - automerge
-  - name: v1.2 backport
+  - name: v1.8 backport
     conditions:
-      - label=v1.2
+      - label=v1.8
     actions:
       backport:
         ignore_conflicts: true
         branches:
-          - v1.2
-  - name: v1.3 backport
+          - v1.8
+  - name: v1.9 backport
     conditions:
-      - label=v1.3
+      - label=v1.9
     actions:
       backport:
         ignore_conflicts: true
         branches:
-          - v1.3
-  - name: v1.4 backport
-    conditions:
-      - label=v1.4
-    actions:
-      backport:
-        ignore_conflicts: true
-        branches:
-          - v1.4
+          - v1.9

.travis.yml

@@ -23,19 +23,25 @@ jobs:
         depth: false
       script:
         - .travis/export-github-repo.sh web3.js/ solana-web3.js
-        - .travis/export-github-repo.sh explorer/ explorer
 
     - &release-artifacts
       if: type IN (api, cron) OR tag IS present
       name: "macOS release artifacts"
       os: osx
+      osx_image: xcode12
       language: rust
       rust:
         - stable
       install:
         - source ci/rust-version.sh
+        - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
+        - readlink -f .
+        - brew install gnu-tar
+        - PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"
+        - tar --version
       script:
         - source ci/env.sh
+        - rustup set profile default
         - ci/publish-tarball.sh
       deploy:
         - provider: s3
@@ -58,6 +64,12 @@ jobs:
     - <<: *release-artifacts
       name: "Windows release artifacts"
       os: windows
+      install:
+        - choco install openssl
+        - export OPENSSL_DIR="C:\Program Files\OpenSSL-Win64"
+        - source ci/rust-version.sh
+        - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
+        - readlink -f .
     #  Linux release artifacts are still built by ci/buildkite-secondary.yml
     #- <<: *release-artifacts
     #  name: "Linux release artifacts"
@@ -65,56 +77,12 @@ jobs:
     #  before_install:
     #    - sudo apt-get install libssl-dev libudev-dev
 
-    # explorer pull request
-    - name: "explorer"
-      if: type = pull_request AND branch = master
-
-      language: node_js
-      node_js:
-        - "node"
-
-      cache:
-        directories:
-          - ~/.npm
-
-      before_install:
-        - .travis/affects.sh explorer/ .travis || travis_terminate 0
-        - cd explorer
-
-      script:
-        - npm run build
-        - npm run format
-
-    # web3.js pull request
-    - name: "web3.js"
-      if: type = pull_request AND branch = master
-
-      language: node_js
-      node_js:
-        - "lts/*"
-
-      services:
-        - docker
-
-      cache:
-        directories:
-          - ~/.npm
-
-      before_install:
-        - .travis/affects.sh web3.js/ .travis || travis_terminate 0
-        - cd web3.js/
-        - source .travis/before_install.sh
-
-      script:
-        - ../.travis/commitlint.sh
-        - source .travis/script.sh
-
     # docs pull request
     - name: "docs"
       if: type IN (push, pull_request) OR tag IS present
       language: node_js
       node_js:
-        - "node"
+        - "lts/*"
 
       services:
         - docker

Cargo.toml

@@ -1,6 +1,9 @@
 [workspace]
 members = [
-    "bench-exchange",
+    "accountsdb-plugin-interface",
+    "accountsdb-plugin-manager",
+    "accountsdb-plugin-postgres",
+    "accounts-cluster-bench",
     "bench-streamer",
     "bench-tps",
     "accounts-bench",
@@ -8,6 +11,8 @@ members = [
     "banks-client",
     "banks-interface",
     "banks-server",
+    "bucket_map",
+    "bloom",
     "clap-utils",
     "cli-config",
     "cli-output",
@@ -15,10 +20,13 @@ members = [
     "core",
     "dos",
     "download-utils",
+    "entry",
     "faucet",
+    "frozen-abi",
     "perf",
     "validator",
     "genesis",
+    "genesis-utils",
     "gossip",
     "install",
     "keygen",
@@ -27,46 +35,59 @@ members = [
     "local-cluster",
     "logger",
     "log-analyzer",
+    "merkle-root-bench",
     "merkle-tree",
-    "stake-o-matic",
     "storage-bigtable",
+    "storage-proto",
     "streamer",
     "measure",
     "metrics",
     "net-shaper",
     "notifier",
+    "poh",
     "poh-bench",
-    "programs/secp256k1",
+    "program-test",
+    "programs/address-lookup-table",
+    "programs/address-lookup-table-tests",
     "programs/bpf_loader",
-    "programs/budget",
+    "programs/bpf_loader/gen-syscall-list",
+    "programs/compute-budget",
     "programs/config",
-    "programs/exchange",
-    "programs/failure",
-    "programs/noop",
-    "programs/ownable",
     "programs/stake",
-    "programs/vest",
     "programs/vote",
+    "rbpf-cli",
     "remote-wallet",
-    "ramp-tps",
+    "rpc",
     "runtime",
+    "runtime/store-tool",
     "sdk",
-    "scripts",
+    "sdk/cargo-build-bpf",
+    "sdk/cargo-test-bpf",
+    "send-transaction-service",
     "stake-accounts",
-    "stake-monitor",
     "sys-tuner",
     "tokens",
+    "transaction-dos",
     "transaction-status",
     "account-decoder",
     "upload-perf",
     "net-utils",
     "version",
-    "vote-signer",
     "cli",
     "rayon-threadlimit",
     "watchtower",
+    "replica-node",
+    "replica-lib",
+    "test-validator",
+    "rpc-test",
+    "client-test",
 ]
 
 exclude = [
     "programs/bpf",
 ]
+
+# TODO: Remove once the "simd-accel" feature from the reed-solomon-erasure
+# dependency is supported on Apple M1. v2 of the feature resolver is needed to
+# specify arch-specific features.
+resolver = "2"

README.md

@@ -1,6 +1,6 @@
 <p align="center">
   <a href="https://solana.com">
-    <img alt="Solana" src="https://i.imgur.com/OMnvVEz.png" width="250" />
+    <img alt="Solana" src="https://i.imgur.com/IKyzQ6T.png" width="250" />
   </a>
 </p>
 
@@ -19,17 +19,29 @@ $ source $HOME/.cargo/env
 $ rustup component add rustfmt
 ```
 
-Please sure you are always using the latest stable rust version by running:
+When building the master branch, please make sure you are using the latest stable rust version by running:
 
 ```bash
 $ rustup update
 ```
 
+When building a specific release branch, you should check the rust version in `ci/rust-version.sh` and if necessary, install that version by running:
+```bash
+$ rustup install VERSION
+```
+Note that if this is not the latest rust version on your machine, cargo commands may require an [override](https://rust-lang.github.io/rustup/overrides.html) in order to use the correct version.
+
 On Linux systems you may need to install libssl-dev, pkg-config, zlib1g-dev, etc.  On Ubuntu:
 
 ```bash
 $ sudo apt-get update
-$ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang
+$ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang make
+```
+
+On Mac M1s, make sure you set up your terminal & homebrew [to use](https://5balloons.info/correct-way-to-install-and-use-homebrew-on-m1-macs/) Rosetta. You can install it with:
+
+```bash
+$ softwareupdate --install-rosetta
 ```
 
 ## **2. Download the source code.**
@@ -45,11 +57,6 @@ $ cd solana
 $ cargo build
 ```
 
-## **4. Run a minimal local cluster.**
-```bash
-$ ./run.sh
-```
-
 # Testing
 
 **Run the test suite:**
@@ -61,8 +68,9 @@ $ cargo test
 ### Starting a local testnet
 Start your own testnet locally, instructions are in the [online docs](https://docs.solana.com/cluster/bench-tps).
 
-### Accessing the remote testnet
-* `testnet` - public stable testnet accessible via devnet.solana.com. Runs 24/7
+### Accessing the remote development cluster
+* `devnet` - stable public cluster for development accessible via
+devnet.solana.com. Runs 24/7. Learn more about the [public clusters](https://docs.solana.com/clusters)
 
 # Benchmarking
 
@@ -106,4 +114,41 @@ send us that patch!
 
 # Disclaimer
 
-All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort.  It is up to the reader to check and validate their accuracy and truthfulness.  Furthermore nothing in this project constitutes a solicitation for investment.
+All claims, content, designs, algorithms, estimates, roadmaps,
+specifications, and performance measurements described in this project
+are done with the Solana Foundation's ("SF") good faith efforts. It is up to
+the reader to check and validate their accuracy and truthfulness.
+Furthermore, nothing in this project constitutes a solicitation for
+investment.
+
+Any content produced by SF or developer resources that SF provides, are
+for educational and inspirational purposes only. SF does not encourage,
+induce or sanction the deployment, integration or use of any such
+applications (including the code comprising the Solana blockchain
+protocol) in violation of applicable laws or regulations and hereby
+prohibits any such deployment, integration or use. This includes use of
+any such applications by the reader (a) in violation of export control
+or sanctions laws of the United States or any other applicable
+jurisdiction, (b) if the reader is located in or ordinarily resident in
+a country or territory subject to comprehensive sanctions administered
+by the U.S. Office of Foreign Assets Control (OFAC), or (c) if the
+reader is or is working on behalf of a Specially Designated National
+(SDN) or a person subject to similar blocking or denied party
+prohibitions.
+
+The reader should be aware that U.S. export control and sanctions laws
+prohibit U.S. persons (and other persons that are subject to such laws)
+from transacting with persons in certain countries and territories or
+that are on the SDN list. As a project based primarily on open-source
+software, it is possible that such sanctioned persons may nevertheless
+bypass prohibitions, obtain the code comprising the Solana blockchain
+protocol (or other project code or applications) and deploy, integrate,
+or otherwise use it. Accordingly, there is a risk to individuals that
+other persons using the Solana blockchain protocol may be sanctioned
+persons and that transactions with such persons would be a violation of
+U.S. export controls and sanctions law. This risk applies to
+individuals, organizations, and other ecosystem participants that
+deploy, integrate, or use the Solana blockchain protocol code directly
+(e.g., as a node operator), and individuals that transact on the Solana
+blockchain through light clients, third party interfaces, and/or wallet
+software.

SECURITY.md

@@ -0,0 +1,169 @@
+# Security Policy
+
+1. [Reporting security problems](#reporting)
+4. [Security Bug Bounties](#bounty)
+2. [Incident Response Process](#process)
+
+<a name="reporting"></a>
+## Reporting security problems to Solana
+
+**DO NOT CREATE AN ISSUE** to report a security problem. Instead, please send an
+email to security@solana.com and provide your github username so we can add you
+to a new draft security advisory for further discussion.
+
+Expect a response as fast as possible, within one business day at the latest.
+
+<a name="bounty"></a>
+## Security Bug Bounties
+We offer bounties for critical security issues. Please see below for more details.
+
+Loss of Funds:
+$2,000,000 USD in locked SOL tokens (locked for 12 months)
+* Theft of funds without users signature from any account
+* Theft of funds without users interaction in system, token, stake, vote programs
+* Theft of funds that requires users signature - creating a vote program that drains the delegated stakes.
+
+Consensus/Safety Violations:
+$1,000,000 USD in locked SOL tokens (locked for 12 months)
+* Consensus safety violation
+* Tricking a validator to accept an optimistic confirmation or rooted slot without a double vote, etc..
+
+Other Attacks:
+ $400,000 USD in locked SOL tokens (locked for 12 months)
+* Protocol liveness attacks,
+* Eclipse attacks,
+* Remote attacks that partition the network,
+
+DoS Attacks:
+$100,000 USD in locked SOL tokens (locked for 12 months)
+* Remote resource exaustion via Non-RPC protocols
+
+RPC DoS/Crashes:
+$5,000 USD in locked SOL tokens (locked for 12 months)
+* RPC attacks
+
+Out of Scope:
+The following components are out of scope for the bounty program
+* Metrics: `/metrics` in the monorepo as well as https://metrics.solana.com
+* Explorer: `/explorer` in the monorepo as well as https://explorer.solana.com
+* Any encrypted credentials, auth tokens, etc. checked into the repo
+* Bugs in dependencies. Please take them upstream!
+* Attacks that require social engineering
+
+Eligibility:
+* The participant submitting the bug report shall follow the process outlined within this document
+* Valid exploits can be eligible even if they are not successfully executed on the cluster
+* Multiple submissions for the same class of exploit are still eligible for compensation, though may be compensated at a lower rate, however these will be assessed on a case-by-case basis
+* Participants must complete KYC and sign the participation agreement here when the registrations are open https://solana.com/validator-registration. Security exploits will still be assessed and open for submission at all times. This needs only be done prior to distribution of tokens.
+
+Payment of Bug Bounties:
+* Payments for eligible bug reports are distributed monthly.
+* Bounties for all bug reports submitted in a given month are paid out in the middle of the
+following month.
+* The SOL/USD conversion rate used for payments is the market price at the end of
+  the last day of the month for the month in which the bug was submitted.
+* The reference for this price is the Closing Price given by Coingecko.com on
+  that date given here:
+  https://www.coingecko.com/en/coins/solana/historical_data/usd#panel
+* For example, for all bugs submitted in March 2021, the SOL/USD price for bug
+  payouts is the Close price on 2021-03-31 of $19.49.  This applies to all bugs
+  submitted in March 2021, to be paid in mid-April 2021.
+* Bug bounties are paid out in
+[stake accounts](https://solana.com/staking) with a
+[lockup](https://docs.solana.com/staking/stake-accounts#lockups)
+expiring 12 months from the last day of the month in which the bug was submitted.
+
+<a name="process"></a>
+## Incident Response Process
+
+In case an incident is discovered or reported, the following process will be
+followed to contain, respond and remediate:
+
+### 1. Establish a new draft security advisory
+In response to an email to security@solana.com, a member of the `solana-labs/admins` group will
+1. Create a new draft security advisory for the incident at https://github.com/solana-labs/solana/security/advisories
+1. Add the reporter's github user and the `solana-labs/security-incident-response` group to the draft security advisory
+1. Create a private fork of the repository (grey button towards the bottom of the page)
+1. Respond to the reporter by email, sharing a link to the draft security advisory
+
+### 2. Triage
+Within the draft security advisory, discuss and determine the severity of the
+issue. If necessary, members of the `solana-labs/security-incident-response`
+group may add other github users to the advisory to assist.
+
+If it is determined that this not a critical network issue then the advisory
+should be closed and if more follow-up is required a normal Solana public github
+issue should be created.
+
+### 3. Prepare Fixes
+For the affected branches, typically all three (edge, beta and stable), prepare
+a fix for the issue and push them to the corresponding branch in the private
+repository associated with the draft security advisory.
+
+There is no CI available in the private repository so you must build from source
+and manually verify fixes.
+
+Code review from the reporter is ideal, as well as from multiple members of the
+core development team.
+
+### 4. Notify Security Group Validators
+Once an ETA is available for the fix, a member of the
+`solana-labs/security-incident-response` group should notify the validators so
+they can prepare for an update using the "Solana Red Alert" notification system.
+
+The teams are all over the world and it's critical to provide actionable
+information at the right time. Don't be the person that wakes everybody up at
+2am when a fix won't be available for hours.
+
+### 5. Ship the patch
+Once the fix is accepted, a member of the
+`solana-labs/security-incident-response` group should prepare a single patch
+file for each affected branch. The commit title for the patch should only
+contain the advisory id, and not disclose any further details about the
+incident.
+
+Copy the patches to https://release.solana.com/ under a subdirectory named after
+the advisory id (example:
+https://release.solana.com/GHSA-hx59-f5g4-jghh/v1.4.patch). Contact a member of
+the `solana-labs/admins` group if you require access to release.solana.com
+
+Using the "Solana Red Alert" channel:
+1. Notify validators that there's an issue and a patch will be provided in X minutes
+2. If X minutes expires and there's no patch, notify of the delay and provide a
+   new ETA
+3. Provide links to patches of https://release.solana.com/ for each affected branch
+
+Validators can be expected to build the patch from source against the latest
+release for the affected branch.
+
+Since the software version will not change after the patch is applied, request
+that each validator notify in the existing channel once they've updated. Manually
+monitor the roll out until a sufficient amount of stake has updated - typically
+at least 33.3% or 66.6% depending on the issue.
+
+### 6. Public Disclosure and Release
+Once the fix has been deployed to the security group validators, the patches from the security
+advisory may be merged into the main source repository. A new official release
+for each affected branch should be shipped and all validators requested to
+upgrade as quickly as possible.
+
+### 7. Security Advisory Bounty Accounting and Cleanup
+
+If this issue is eligible for a bounty, prefix the title of the security
+advisory with one of the following, depending on the severity:
+* `[Bounty Category: Critical: Loss of Funds]`
+* `[Bounty Category: Critical: Loss of Availability]`
+* `[Bounty Category: Critical: DoS]`
+* `[Bounty Category: Critical: Other]`
+* `[Bounty Category: Non-critical]`
+* `[Bounty Category: RPC]`
+
+Confirm with the reporter that they agree with the severity assessment, and
+discuss as required to reach a conclusion.
+
+We currently do not use the Github workflow to publish security advisories.
+Once the issue and fix have been disclosed, and a bounty category is assessed if
+appropriate, the GitHub security advisory is no longer needed and can be closed.
+
+Bounties are currently awarded once a quarter (TODO: link to this process, or
+inline the workflow)

account-decoder/Cargo.toml

@@ -1,29 +1,30 @@
 [package]
 name = "solana-account-decoder"
-version = "1.4.0"
+version = "1.9.5"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-account-decoder"
 license = "Apache-2.0"
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 base64 = "0.12.3"
-bincode = "1.3.1"
-bs58 = "0.3.1"
+bincode = "1.3.3"
+bs58 = "0.4.0"
 bv = "0.11.1"
 Inflector = "0.11.4"
 lazy_static = "1.4.0"
-serde = "1.0.112"
+serde = "1.0.130"
 serde_derive = "1.0.103"
-serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-stake-program = { path = "../programs/stake", version = "1.4.0" }
-solana-vote-program = { path = "../programs/vote", version = "1.4.0" }
-spl-token-v2-0 = { package = "spl-token", version = "=2.0.6", features = ["skip-no-mangle"] }
+serde_json = "1.0.72"
+solana-config-program = { path = "../programs/config", version = "=1.9.5" }
+solana-sdk = { path = "../sdk", version = "=1.9.5" }
+solana-vote-program = { path = "../programs/vote", version = "=1.9.5" }
+spl-token = { version = "=3.2.0", features = ["no-entrypoint"] }
 thiserror = "1.0"
+zstd = "0.9.0"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

account-decoder/src/lib.rs

@@ -1,9 +1,11 @@
+#![allow(clippy::integer_arithmetic)]
 #[macro_use]
 extern crate lazy_static;
 #[macro_use]
 extern crate serde_derive;
 
 pub mod parse_account_data;
+pub mod parse_bpf_loader;
 pub mod parse_config;
 pub mod parse_nonce;
 pub mod parse_stake;
@@ -12,11 +14,23 @@ pub mod parse_token;
 pub mod parse_vote;
 pub mod validator_info;
 
-use crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount};
-use solana_sdk::{account::Account, clock::Epoch, fee_calculator::FeeCalculator, pubkey::Pubkey};
-use std::str::FromStr;
+use {
+    crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount},
+    solana_sdk::{
+        account::{ReadableAccount, WritableAccount},
+        clock::Epoch,
+        fee_calculator::FeeCalculator,
+        pubkey::Pubkey,
+    },
+    std::{
+        io::{Read, Write},
+        str::FromStr,
+    },
+};
 
 pub type StringAmount = String;
+pub type StringDecimals = String;
+pub const MAX_BASE58_BYTES: usize = 128;
 
 /// A duplicate representation of an Account for pretty JSON serialization
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -37,71 +51,111 @@ pub enum UiAccountData {
     Binary(String, UiAccountEncoding),
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash)]
 #[serde(rename_all = "camelCase")]
 pub enum UiAccountEncoding {
     Binary, // Legacy. Retained for RPC backwards compatibility
     Base58,
     Base64,
     JsonParsed,
+    #[serde(rename = "base64+zstd")]
+    Base64Zstd,
 }
 
 impl UiAccount {
-    pub fn encode(
+    fn encode_bs58<T: ReadableAccount>(
+        account: &T,
+        data_slice_config: Option<UiDataSliceConfig>,
+    ) -> String {
+        if account.data().len() <= MAX_BASE58_BYTES {
+            bs58::encode(slice_data(account.data(), data_slice_config)).into_string()
+        } else {
+            "error: data too large for bs58 encoding".to_string()
+        }
+    }
+
+    pub fn encode<T: ReadableAccount>(
         pubkey: &Pubkey,
-        account: Account,
+        account: &T,
         encoding: UiAccountEncoding,
         additional_data: Option<AccountAdditionalData>,
         data_slice_config: Option<UiDataSliceConfig>,
     ) -> Self {
         let data = match encoding {
-            UiAccountEncoding::Binary => UiAccountData::LegacyBinary(
-                bs58::encode(slice_data(&account.data, data_slice_config)).into_string(),
-            ),
-            UiAccountEncoding::Base58 => UiAccountData::Binary(
-                bs58::encode(slice_data(&account.data, data_slice_config)).into_string(),
-                encoding,
-            ),
+            UiAccountEncoding::Binary => {
+                let data = Self::encode_bs58(account, data_slice_config);
+                UiAccountData::LegacyBinary(data)
+            }
+            UiAccountEncoding::Base58 => {
+                let data = Self::encode_bs58(account, data_slice_config);
+                UiAccountData::Binary(data, encoding)
+            }
             UiAccountEncoding::Base64 => UiAccountData::Binary(
-                base64::encode(slice_data(&account.data, data_slice_config)),
+                base64::encode(slice_data(account.data(), data_slice_config)),
                 encoding,
             ),
+            UiAccountEncoding::Base64Zstd => {
+                let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 0).unwrap();
+                match encoder
+                    .write_all(slice_data(account.data(), data_slice_config))
+                    .and_then(|()| encoder.finish())
+                {
+                    Ok(zstd_data) => UiAccountData::Binary(base64::encode(zstd_data), encoding),
+                    Err(_) => UiAccountData::Binary(
+                        base64::encode(slice_data(account.data(), data_slice_config)),
+                        UiAccountEncoding::Base64,
+                    ),
+                }
+            }
             UiAccountEncoding::JsonParsed => {
                 if let Ok(parsed_data) =
-                    parse_account_data(pubkey, &account.owner, &account.data, additional_data)
+                    parse_account_data(pubkey, account.owner(), account.data(), additional_data)
                 {
                     UiAccountData::Json(parsed_data)
                 } else {
-                    UiAccountData::Binary(base64::encode(&account.data), UiAccountEncoding::Base64)
+                    UiAccountData::Binary(
+                        base64::encode(&account.data()),
+                        UiAccountEncoding::Base64,
+                    )
                 }
             }
         };
         UiAccount {
-            lamports: account.lamports,
+            lamports: account.lamports(),
             data,
-            owner: account.owner.to_string(),
-            executable: account.executable,
-            rent_epoch: account.rent_epoch,
+            owner: account.owner().to_string(),
+            executable: account.executable(),
+            rent_epoch: account.rent_epoch(),
         }
     }
 
-    pub fn decode(&self) -> Option<Account> {
+    pub fn decode<T: WritableAccount>(&self) -> Option<T> {
         let data = match &self.data {
             UiAccountData::Json(_) => None,
             UiAccountData::LegacyBinary(blob) => bs58::decode(blob).into_vec().ok(),
             UiAccountData::Binary(blob, encoding) => match encoding {
                 UiAccountEncoding::Base58 => bs58::decode(blob).into_vec().ok(),
                 UiAccountEncoding::Base64 => base64::decode(blob).ok(),
+                UiAccountEncoding::Base64Zstd => base64::decode(blob)
+                    .ok()
+                    .map(|zstd_data| {
+                        let mut data = vec![];
+                        zstd::stream::read::Decoder::new(zstd_data.as_slice())
+                            .and_then(|mut reader| reader.read_to_end(&mut data))
+                            .map(|_| data)
+                            .ok()
+                    })
+                    .flatten(),
                 UiAccountEncoding::Binary | UiAccountEncoding::JsonParsed => None,
             },
         }?;
-        Some(Account {
-            lamports: self.lamports,
+        Some(T::create(
+            self.lamports,
             data,
-            owner: Pubkey::from_str(&self.owner).ok()?,
-            executable: self.executable,
-            rent_epoch: self.rent_epoch,
-        })
+            Pubkey::from_str(&self.owner).ok()?,
+            self.executable,
+            self.rent_epoch,
+        ))
     }
 }
 
@@ -127,7 +181,7 @@ impl Default for UiFeeCalculator {
     }
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct UiDataSliceConfig {
     pub offset: usize,
@@ -150,7 +204,10 @@ fn slice_data(data: &[u8], data_slice_config: Option<UiDataSliceConfig>) -> &[u8
 
 #[cfg(test)]
 mod test {
-    use super::*;
+    use {
+        super::*,
+        solana_sdk::account::{Account, AccountSharedData},
+    };
 
     #[test]
     fn test_slice_data() {
@@ -179,4 +236,27 @@ mod test {
         });
         assert_eq!(slice_data(&data, slice_config), &[] as &[u8]);
     }
+
+    #[test]
+    fn test_base64_zstd() {
+        let encoded_account = UiAccount::encode(
+            &Pubkey::default(),
+            &AccountSharedData::from(Account {
+                data: vec![0; 1024],
+                ..Account::default()
+            }),
+            UiAccountEncoding::Base64Zstd,
+            None,
+            None,
+        );
+        assert!(matches!(
+            encoded_account.data,
+            UiAccountData::Binary(_, UiAccountEncoding::Base64Zstd)
+        ));
+
+        let decoded_account = encoded_account.decode::<Account>().unwrap();
+        assert_eq!(decoded_account.data(), &vec![0; 1024]);
+        let decoded_account = encoded_account.decode::<AccountSharedData>().unwrap();
+        assert_eq!(decoded_account.data(), &vec![0; 1024]);
+    }
 }

account-decoder/src/parse_account_data.rs

@@ -1,26 +1,34 @@
-use crate::{
-    parse_config::parse_config,
-    parse_nonce::parse_nonce,
-    parse_stake::parse_stake,
-    parse_sysvar::parse_sysvar,
-    parse_token::{parse_token, spl_token_id_v2_0},
-    parse_vote::parse_vote,
+use {
+    crate::{
+        parse_bpf_loader::parse_bpf_upgradeable_loader,
+        parse_config::parse_config,
+        parse_nonce::parse_nonce,
+        parse_stake::parse_stake,
+        parse_sysvar::parse_sysvar,
+        parse_token::{parse_token, spl_token_id},
+        parse_vote::parse_vote,
+    },
+    inflector::Inflector,
+    serde_json::Value,
+    solana_sdk::{instruction::InstructionError, pubkey::Pubkey, stake, system_program, sysvar},
+    std::collections::HashMap,
+    thiserror::Error,
 };
-use inflector::Inflector;
-use serde_json::Value;
-use solana_sdk::{instruction::InstructionError, pubkey::Pubkey, system_program, sysvar};
-use std::collections::HashMap;
-use thiserror::Error;
 
 lazy_static! {
+    static ref BPF_UPGRADEABLE_LOADER_PROGRAM_ID: Pubkey = solana_sdk::bpf_loader_upgradeable::id();
     static ref CONFIG_PROGRAM_ID: Pubkey = solana_config_program::id();
-    static ref STAKE_PROGRAM_ID: Pubkey = solana_stake_program::id();
+    static ref STAKE_PROGRAM_ID: Pubkey = stake::program::id();
     static ref SYSTEM_PROGRAM_ID: Pubkey = system_program::id();
     static ref SYSVAR_PROGRAM_ID: Pubkey = sysvar::id();
-    static ref TOKEN_PROGRAM_ID: Pubkey = spl_token_id_v2_0();
+    static ref TOKEN_PROGRAM_ID: Pubkey = spl_token_id();
     static ref VOTE_PROGRAM_ID: Pubkey = solana_vote_program::id();
     pub static ref PARSABLE_PROGRAM_IDS: HashMap<Pubkey, ParsableAccount> = {
         let mut m = HashMap::new();
+        m.insert(
+            *BPF_UPGRADEABLE_LOADER_PROGRAM_ID,
+            ParsableAccount::BpfUpgradeableLoader,
+        );
         m.insert(*CONFIG_PROGRAM_ID, ParsableAccount::Config);
         m.insert(*SYSTEM_PROGRAM_ID, ParsableAccount::Nonce);
         m.insert(*TOKEN_PROGRAM_ID, ParsableAccount::SplToken);
@@ -60,6 +68,7 @@ pub struct ParsedAccount {
 #[derive(Debug, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum ParsableAccount {
+    BpfUpgradeableLoader,
     Config,
     Nonce,
     SplToken,
@@ -81,9 +90,12 @@ pub fn parse_account_data(
 ) -> Result<ParsedAccount, ParseAccountError> {
     let program_name = PARSABLE_PROGRAM_IDS
         .get(program_id)
-        .ok_or_else(|| ParseAccountError::ProgramNotParsable)?;
+        .ok_or(ParseAccountError::ProgramNotParsable)?;
     let additional_data = additional_data.unwrap_or_default();
     let parsed_json = match program_name {
+        ParsableAccount::BpfUpgradeableLoader => {
+            serde_json::to_value(parse_bpf_upgradeable_loader(data)?)?
+        }
         ParsableAccount::Config => serde_json::to_value(parse_config(data, pubkey)?)?,
         ParsableAccount::Nonce => serde_json::to_value(parse_nonce(data)?)?,
         ParsableAccount::SplToken => {
@@ -102,23 +114,25 @@ pub fn parse_account_data(
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use solana_sdk::nonce::{
-        state::{Data, Versions},
-        State,
+    use {
+        super::*,
+        solana_sdk::nonce::{
+            state::{Data, Versions},
+            State,
+        },
+        solana_vote_program::vote_state::{VoteState, VoteStateVersions},
     };
-    use solana_vote_program::vote_state::{VoteState, VoteStateVersions};
 
     #[test]
     fn test_parse_account_data() {
-        let account_pubkey = Pubkey::new_rand();
-        let other_program = Pubkey::new_rand();
+        let account_pubkey = solana_sdk::pubkey::new_rand();
+        let other_program = solana_sdk::pubkey::new_rand();
         let data = vec![0; 4];
         assert!(parse_account_data(&account_pubkey, &other_program, &data, None).is_err());
 
         let vote_state = VoteState::default();
         let mut vote_account_data: Vec<u8> = vec![0; VoteState::size_of()];
-        let versioned = VoteStateVersions::Current(Box::new(vote_state));
+        let versioned = VoteStateVersions::new_current(vote_state);
         VoteState::serialize(&versioned, &mut vote_account_data).unwrap();
         let parsed = parse_account_data(
             &account_pubkey,

account-decoder/src/parse_bpf_loader.rs

@@ -0,0 +1,181 @@
+use {
+    crate::{
+        parse_account_data::{ParsableAccount, ParseAccountError},
+        UiAccountData, UiAccountEncoding,
+    },
+    bincode::{deserialize, serialized_size},
+    solana_sdk::{bpf_loader_upgradeable::UpgradeableLoaderState, pubkey::Pubkey},
+};
+
+pub fn parse_bpf_upgradeable_loader(
+    data: &[u8],
+) -> Result<BpfUpgradeableLoaderAccountType, ParseAccountError> {
+    let account_state: UpgradeableLoaderState = deserialize(data).map_err(|_| {
+        ParseAccountError::AccountNotParsable(ParsableAccount::BpfUpgradeableLoader)
+    })?;
+    let parsed_account = match account_state {
+        UpgradeableLoaderState::Uninitialized => BpfUpgradeableLoaderAccountType::Uninitialized,
+        UpgradeableLoaderState::Buffer { authority_address } => {
+            let offset = if authority_address.is_some() {
+                UpgradeableLoaderState::buffer_data_offset().unwrap()
+            } else {
+                // This case included for code completeness; in practice, a Buffer account will
+                // always have authority_address.is_some()
+                UpgradeableLoaderState::buffer_data_offset().unwrap()
+                    - serialized_size(&Pubkey::default()).unwrap() as usize
+            };
+            BpfUpgradeableLoaderAccountType::Buffer(UiBuffer {
+                authority: authority_address.map(|pubkey| pubkey.to_string()),
+                data: UiAccountData::Binary(
+                    base64::encode(&data[offset as usize..]),
+                    UiAccountEncoding::Base64,
+                ),
+            })
+        }
+        UpgradeableLoaderState::Program {
+            programdata_address,
+        } => BpfUpgradeableLoaderAccountType::Program(UiProgram {
+            program_data: programdata_address.to_string(),
+        }),
+        UpgradeableLoaderState::ProgramData {
+            slot,
+            upgrade_authority_address,
+        } => {
+            let offset = if upgrade_authority_address.is_some() {
+                UpgradeableLoaderState::programdata_data_offset().unwrap()
+            } else {
+                UpgradeableLoaderState::programdata_data_offset().unwrap()
+                    - serialized_size(&Pubkey::default()).unwrap() as usize
+            };
+            BpfUpgradeableLoaderAccountType::ProgramData(UiProgramData {
+                slot,
+                authority: upgrade_authority_address.map(|pubkey| pubkey.to_string()),
+                data: UiAccountData::Binary(
+                    base64::encode(&data[offset as usize..]),
+                    UiAccountEncoding::Base64,
+                ),
+            })
+        }
+    };
+    Ok(parsed_account)
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase", tag = "type", content = "info")]
+pub enum BpfUpgradeableLoaderAccountType {
+    Uninitialized,
+    Buffer(UiBuffer),
+    Program(UiProgram),
+    ProgramData(UiProgramData),
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiBuffer {
+    pub authority: Option<String>,
+    pub data: UiAccountData,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiProgram {
+    pub program_data: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct UiProgramData {
+    pub slot: u64,
+    pub authority: Option<String>,
+    pub data: UiAccountData,
+}
+
+#[cfg(test)]
+mod test {
+    use {super::*, bincode::serialize, solana_sdk::pubkey::Pubkey};
+
+    #[test]
+    fn test_parse_bpf_upgradeable_loader_accounts() {
+        let bpf_loader_state = UpgradeableLoaderState::Uninitialized;
+        let account_data = serialize(&bpf_loader_state).unwrap();
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Uninitialized
+        );
+
+        let program = vec![7u8; 64]; // Arbitrary program data
+
+        let authority = Pubkey::new_unique();
+        let bpf_loader_state = UpgradeableLoaderState::Buffer {
+            authority_address: Some(authority),
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Buffer(UiBuffer {
+                authority: Some(authority.to_string()),
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+
+        // This case included for code completeness; in practice, a Buffer account will always have
+        // authority_address.is_some()
+        let bpf_loader_state = UpgradeableLoaderState::Buffer {
+            authority_address: None,
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Buffer(UiBuffer {
+                authority: None,
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+
+        let programdata_address = Pubkey::new_unique();
+        let bpf_loader_state = UpgradeableLoaderState::Program {
+            programdata_address,
+        };
+        let account_data = serialize(&bpf_loader_state).unwrap();
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::Program(UiProgram {
+                program_data: programdata_address.to_string(),
+            })
+        );
+
+        let authority = Pubkey::new_unique();
+        let slot = 42;
+        let bpf_loader_state = UpgradeableLoaderState::ProgramData {
+            slot,
+            upgrade_authority_address: Some(authority),
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::ProgramData(UiProgramData {
+                slot,
+                authority: Some(authority.to_string()),
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+
+        let bpf_loader_state = UpgradeableLoaderState::ProgramData {
+            slot,
+            upgrade_authority_address: None,
+        };
+        let mut account_data = serialize(&bpf_loader_state).unwrap();
+        account_data.extend_from_slice(&program);
+        assert_eq!(
+            parse_bpf_upgradeable_loader(&account_data).unwrap(),
+            BpfUpgradeableLoaderAccountType::ProgramData(UiProgramData {
+                slot,
+                authority: None,
+                data: UiAccountData::Binary(base64::encode(&program), UiAccountEncoding::Base64),
+            })
+        );
+    }
+}

account-decoder/src/parse_config.rs

@@ -1,15 +1,19 @@
-use crate::{
-    parse_account_data::{ParsableAccount, ParseAccountError},
-    validator_info,
+use {
+    crate::{
+        parse_account_data::{ParsableAccount, ParseAccountError},
+        validator_info,
+    },
+    bincode::deserialize,
+    serde_json::Value,
+    solana_config_program::{get_config_data, ConfigKeys},
+    solana_sdk::{
+        pubkey::Pubkey,
+        stake::config::{self as stake_config, Config as StakeConfig},
+    },
 };
-use bincode::deserialize;
-use serde_json::Value;
-use solana_config_program::{get_config_data, ConfigKeys};
-use solana_sdk::pubkey::Pubkey;
-use solana_stake_program::config::Config as StakeConfig;
 
 pub fn parse_config(data: &[u8], pubkey: &Pubkey) -> Result<ConfigAccountType, ParseAccountError> {
-    let parsed_account = if pubkey == &solana_stake_program::config::id() {
+    let parsed_account = if pubkey == &stake_config::id() {
         get_config_data(data)
             .ok()
             .and_then(|data| deserialize::<StakeConfig>(data).ok())
@@ -37,7 +41,7 @@ fn parse_config_data<T>(data: &[u8], keys: Vec<(Pubkey, bool)>) -> Option<UiConf
 where
     T: serde::de::DeserializeOwned,
 {
-    let config_data: T = deserialize(&get_config_data(data).ok()?).ok()?;
+    let config_data: T = deserialize(get_config_data(data).ok()?).ok()?;
     let keys = keys
         .iter()
         .map(|key| UiConfigKey {
@@ -87,10 +91,10 @@ pub struct UiConfig<T> {
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use crate::validator_info::ValidatorInfo;
-    use serde_json::json;
-    use solana_config_program::create_config_account;
+    use {
+        super::*, crate::validator_info::ValidatorInfo, serde_json::json,
+        solana_config_program::create_config_account, solana_sdk::account::ReadableAccount,
+    };
 
     #[test]
     fn test_parse_config() {
@@ -100,11 +104,7 @@ mod test {
         };
         let stake_config_account = create_config_account(vec![], &stake_config, 10);
         assert_eq!(
-            parse_config(
-                &stake_config_account.data,
-                &solana_stake_program::config::id()
-            )
-            .unwrap(),
+            parse_config(stake_config_account.data(), &stake_config::id()).unwrap(),
             ConfigAccountType::StakeConfig(UiStakeConfig {
                 warmup_cooldown_rate: 0.25,
                 slash_penalty: 50,
@@ -117,14 +117,14 @@ mod test {
             }))
             .unwrap(),
         };
-        let info_pubkey = Pubkey::new_rand();
+        let info_pubkey = solana_sdk::pubkey::new_rand();
         let validator_info_config_account = create_config_account(
             vec![(validator_info::id(), false), (info_pubkey, true)],
             &validator_info,
             10,
         );
         assert_eq!(
-            parse_config(&validator_info_config_account.data, &info_pubkey).unwrap(),
+            parse_config(validator_info_config_account.data(), &info_pubkey).unwrap(),
             ConfigAccountType::ValidatorInfo(UiConfig {
                 keys: vec![
                     UiConfigKey {

account-decoder/src/parse_nonce.rs

@@ -1,7 +1,9 @@
-use crate::{parse_account_data::ParseAccountError, UiFeeCalculator};
-use solana_sdk::{
-    instruction::InstructionError,
-    nonce::{state::Versions, State},
+use {
+    crate::{parse_account_data::ParseAccountError, UiFeeCalculator},
+    solana_sdk::{
+        instruction::InstructionError,
+        nonce::{state::Versions, State},
+    },
 };
 
 pub fn parse_nonce(data: &[u8]) -> Result<UiNonceState, ParseAccountError> {
@@ -9,7 +11,13 @@ pub fn parse_nonce(data: &[u8]) -> Result<UiNonceState, ParseAccountError> {
         .map_err(|_| ParseAccountError::from(InstructionError::InvalidAccountData))?;
     let nonce_state = nonce_state.convert_to_current();
     match nonce_state {
-        State::Uninitialized => Ok(UiNonceState::Uninitialized),
+        // This prevents parsing an allocated System-owned account with empty data of any non-zero
+        // length as `uninitialized` nonce. An empty account of the wrong length can never be
+        // initialized as a nonce account, and an empty account of the correct length may not be an
+        // uninitialized nonce account, since it can be assigned to another program.
+        State::Uninitialized => Err(ParseAccountError::from(
+            InstructionError::InvalidAccountData,
+        )),
         State::Initialized(data) => Ok(UiNonceState::Initialized(UiNonceData {
             authority: data.authority.to_string(),
             blockhash: data.blockhash.to_string(),
@@ -36,14 +44,16 @@ pub struct UiNonceData {
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use solana_sdk::{
-        hash::Hash,
-        nonce::{
-            state::{Data, Versions},
-            State,
+    use {
+        super::*,
+        solana_sdk::{
+            hash::Hash,
+            nonce::{
+                state::{Data, Versions},
+                State,
+            },
+            pubkey::Pubkey,
         },
-        pubkey::Pubkey,
     };
 
     #[test]

account-decoder/src/parse_stake.rs

@@ -1,10 +1,14 @@
-use crate::{
-    parse_account_data::{ParsableAccount, ParseAccountError},
-    StringAmount,
+use {
+    crate::{
+        parse_account_data::{ParsableAccount, ParseAccountError},
+        StringAmount,
+    },
+    bincode::deserialize,
+    solana_sdk::{
+        clock::{Epoch, UnixTimestamp},
+        stake::state::{Authorized, Delegation, Lockup, Meta, Stake, StakeState},
+    },
 };
-use bincode::deserialize;
-use solana_sdk::clock::{Epoch, UnixTimestamp};
-use solana_stake_program::stake_state::{Authorized, Delegation, Lockup, Meta, Stake, StakeState};
 
 pub fn parse_stake(data: &[u8]) -> Result<StakeAccountType, ParseAccountError> {
     let stake_state: StakeState = deserialize(data)
@@ -132,9 +136,7 @@ impl From<Delegation> for UiDelegation {
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use bincode::serialize;
-    use solana_sdk::pubkey::Pubkey;
+    use {super::*, bincode::serialize};
 
     #[test]
     fn test_parse_stake() {
@@ -145,8 +147,8 @@ mod test {
             StakeAccountType::Uninitialized
         );
 
-        let pubkey = Pubkey::new_rand();
-        let custodian = Pubkey::new_rand();
+        let pubkey = solana_sdk::pubkey::new_rand();
+        let custodian = solana_sdk::pubkey::new_rand();
         let authorized = Authorized::auto(&pubkey);
         let lockup = Lockup {
             unix_timestamp: 0,
@@ -180,7 +182,7 @@ mod test {
             })
         );
 
-        let voter_pubkey = Pubkey::new_rand();
+        let voter_pubkey = solana_sdk::pubkey::new_rand();
         let stake = Stake {
             delegation: Delegation {
                 voter_pubkey,

account-decoder/src/parse_sysvar.rs

@@ -1,21 +1,26 @@
-use crate::{
-    parse_account_data::{ParsableAccount, ParseAccountError},
-    StringAmount, UiFeeCalculator,
-};
-use bincode::deserialize;
-use bv::BitVec;
-use solana_sdk::{
-    clock::{Clock, Epoch, Slot, UnixTimestamp},
-    epoch_schedule::EpochSchedule,
-    pubkey::Pubkey,
-    rent::Rent,
-    slot_hashes::SlotHashes,
-    slot_history::{self, SlotHistory},
-    stake_history::{StakeHistory, StakeHistoryEntry},
-    sysvar::{self, fees::Fees, recent_blockhashes::RecentBlockhashes, rewards::Rewards},
+#[allow(deprecated)]
+use solana_sdk::sysvar::{fees::Fees, recent_blockhashes::RecentBlockhashes};
+use {
+    crate::{
+        parse_account_data::{ParsableAccount, ParseAccountError},
+        StringAmount, UiFeeCalculator,
+    },
+    bincode::deserialize,
+    bv::BitVec,
+    solana_sdk::{
+        clock::{Clock, Epoch, Slot, UnixTimestamp},
+        epoch_schedule::EpochSchedule,
+        pubkey::Pubkey,
+        rent::Rent,
+        slot_hashes::SlotHashes,
+        slot_history::{self, SlotHistory},
+        stake_history::{StakeHistory, StakeHistoryEntry},
+        sysvar::{self, rewards::Rewards},
+    },
 };
 
 pub fn parse_sysvar(data: &[u8], pubkey: &Pubkey) -> Result<SysvarAccountType, ParseAccountError> {
+    #[allow(deprecated)]
     let parsed_account = {
         if pubkey == &sysvar::clock::id() {
             deserialize::<Clock>(data)
@@ -91,7 +96,9 @@ pub fn parse_sysvar(data: &[u8], pubkey: &Pubkey) -> Result<SysvarAccountType, P
 pub enum SysvarAccountType {
     Clock(UiClock),
     EpochSchedule(EpochSchedule),
+    #[allow(deprecated)]
     Fees(UiFees),
+    #[allow(deprecated)]
     RecentBlockhashes(Vec<UiRecentBlockhashesEntry>),
     Rent(UiRent),
     Rewards(UiRewards),
@@ -105,6 +112,7 @@ pub enum SysvarAccountType {
 pub struct UiClock {
     pub slot: Slot,
     pub epoch: Epoch,
+    pub epoch_start_timestamp: UnixTimestamp,
     pub leader_schedule_epoch: Epoch,
     pub unix_timestamp: UnixTimestamp,
 }
@@ -114,6 +122,7 @@ impl From<Clock> for UiClock {
         Self {
             slot: clock.slot,
             epoch: clock.epoch,
+            epoch_start_timestamp: clock.epoch_start_timestamp,
             leader_schedule_epoch: clock.leader_schedule_epoch,
             unix_timestamp: clock.unix_timestamp,
         }
@@ -125,6 +134,7 @@ impl From<Clock> for UiClock {
 pub struct UiFees {
     pub fee_calculator: UiFeeCalculator,
 }
+#[allow(deprecated)]
 impl From<Fees> for UiFees {
     fn from(fees: Fees) -> Self {
         Self {
@@ -210,17 +220,18 @@ pub struct UiStakeHistoryEntry {
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use solana_sdk::{
-        fee_calculator::FeeCalculator,
-        hash::Hash,
-        sysvar::{recent_blockhashes::IterItem, Sysvar},
+    #[allow(deprecated)]
+    use solana_sdk::sysvar::recent_blockhashes::IterItem;
+    use {
+        super::*,
+        solana_sdk::{account::create_account_for_test, fee_calculator::FeeCalculator, hash::Hash},
     };
-    use std::iter::FromIterator;
 
     #[test]
     fn test_parse_sysvars() {
-        let clock_sysvar = Clock::default().create_account(1);
+        let hash = Hash::new(&[1; 32]);
+
+        let clock_sysvar = create_account_for_test(&Clock::default());
         assert_eq!(
             parse_sysvar(&clock_sysvar.data, &sysvar::clock::id()).unwrap(),
             SysvarAccountType::Clock(UiClock::default()),
@@ -233,49 +244,48 @@ mod test {
             first_normal_epoch: 1,
             first_normal_slot: 12,
         };
-        let epoch_schedule_sysvar = epoch_schedule.create_account(1);
+        let epoch_schedule_sysvar = create_account_for_test(&epoch_schedule);
         assert_eq!(
             parse_sysvar(&epoch_schedule_sysvar.data, &sysvar::epoch_schedule::id()).unwrap(),
             SysvarAccountType::EpochSchedule(epoch_schedule),
         );
 
-        let fees_sysvar = Fees::default().create_account(1);
-        assert_eq!(
-            parse_sysvar(&fees_sysvar.data, &sysvar::fees::id()).unwrap(),
-            SysvarAccountType::Fees(UiFees::default()),
-        );
+        #[allow(deprecated)]
+        {
+            let fees_sysvar = create_account_for_test(&Fees::default());
+            assert_eq!(
+                parse_sysvar(&fees_sysvar.data, &sysvar::fees::id()).unwrap(),
+                SysvarAccountType::Fees(UiFees::default()),
+            );
 
-        let hash = Hash::new(&[1; 32]);
-        let fee_calculator = FeeCalculator {
-            lamports_per_signature: 10,
-        };
-        let recent_blockhashes =
-            RecentBlockhashes::from_iter(vec![IterItem(0, &hash, &fee_calculator)].into_iter());
-        let recent_blockhashes_sysvar = recent_blockhashes.create_account(1);
-        assert_eq!(
-            parse_sysvar(
-                &recent_blockhashes_sysvar.data,
-                &sysvar::recent_blockhashes::id()
-            )
-            .unwrap(),
-            SysvarAccountType::RecentBlockhashes(vec![UiRecentBlockhashesEntry {
-                blockhash: hash.to_string(),
-                fee_calculator: fee_calculator.into(),
-            }]),
-        );
+            let recent_blockhashes: RecentBlockhashes =
+                vec![IterItem(0, &hash, 10)].into_iter().collect();
+            let recent_blockhashes_sysvar = create_account_for_test(&recent_blockhashes);
+            assert_eq!(
+                parse_sysvar(
+                    &recent_blockhashes_sysvar.data,
+                    &sysvar::recent_blockhashes::id()
+                )
+                .unwrap(),
+                SysvarAccountType::RecentBlockhashes(vec![UiRecentBlockhashesEntry {
+                    blockhash: hash.to_string(),
+                    fee_calculator: FeeCalculator::new(10).into(),
+                }]),
+            );
+        }
 
         let rent = Rent {
             lamports_per_byte_year: 10,
             exemption_threshold: 2.0,
             burn_percent: 5,
         };
-        let rent_sysvar = rent.create_account(1);
+        let rent_sysvar = create_account_for_test(&rent);
         assert_eq!(
             parse_sysvar(&rent_sysvar.data, &sysvar::rent::id()).unwrap(),
             SysvarAccountType::Rent(rent.into()),
         );
 
-        let rewards_sysvar = Rewards::default().create_account(1);
+        let rewards_sysvar = create_account_for_test(&Rewards::default());
         assert_eq!(
             parse_sysvar(&rewards_sysvar.data, &sysvar::rewards::id()).unwrap(),
             SysvarAccountType::Rewards(UiRewards::default()),
@@ -283,7 +293,7 @@ mod test {
 
         let mut slot_hashes = SlotHashes::default();
         slot_hashes.add(1, hash);
-        let slot_hashes_sysvar = slot_hashes.create_account(1);
+        let slot_hashes_sysvar = create_account_for_test(&slot_hashes);
         assert_eq!(
             parse_sysvar(&slot_hashes_sysvar.data, &sysvar::slot_hashes::id()).unwrap(),
             SysvarAccountType::SlotHashes(vec![UiSlotHashEntry {
@@ -294,7 +304,7 @@ mod test {
 
         let mut slot_history = SlotHistory::default();
         slot_history.add(42);
-        let slot_history_sysvar = slot_history.create_account(1);
+        let slot_history_sysvar = create_account_for_test(&slot_history);
         assert_eq!(
             parse_sysvar(&slot_history_sysvar.data, &sysvar::slot_history::id()).unwrap(),
             SysvarAccountType::SlotHistory(UiSlotHistory {
@@ -310,7 +320,7 @@ mod test {
             deactivating: 3,
         };
         stake_history.add(1, stake_history_entry.clone());
-        let stake_history_sysvar = stake_history.create_account(1);
+        let stake_history_sysvar = create_account_for_test(&stake_history);
         assert_eq!(
             parse_sysvar(&stake_history_sysvar.data, &sysvar::stake_history::id()).unwrap(),
             SysvarAccountType::StakeHistory(vec![UiStakeHistoryEntry {
@@ -319,7 +329,7 @@ mod test {
             }]),
         );
 
-        let bad_pubkey = Pubkey::new_rand();
+        let bad_pubkey = solana_sdk::pubkey::new_rand();
         assert!(parse_sysvar(&stake_history_sysvar.data, &bad_pubkey).is_err());
 
         let bad_data = vec![0; 4];

account-decoder/src/parse_token.rs

@@ -1,24 +1,38 @@
-use crate::{
-    parse_account_data::{ParsableAccount, ParseAccountError},
-    StringAmount,
+use {
+    crate::{
+        parse_account_data::{ParsableAccount, ParseAccountError},
+        StringAmount, StringDecimals,
+    },
+    solana_sdk::pubkey::Pubkey,
+    spl_token::{
+        solana_program::{
+            program_option::COption, program_pack::Pack, pubkey::Pubkey as SplTokenPubkey,
+        },
+        state::{Account, AccountState, Mint, Multisig},
+    },
+    std::str::FromStr,
 };
-use solana_sdk::pubkey::Pubkey;
-use spl_token_v2_0::{
-    solana_sdk::{program_option::COption, program_pack::Pack, pubkey::Pubkey as SplTokenPubkey},
-    state::{Account, AccountState, Mint, Multisig},
-};
-use std::str::FromStr;
 
-// A helper function to convert spl_token_v2_0::id() as spl_sdk::pubkey::Pubkey to
+// A helper function to convert spl_token::id() as spl_sdk::pubkey::Pubkey to
 // solana_sdk::pubkey::Pubkey
-pub fn spl_token_id_v2_0() -> Pubkey {
-    Pubkey::from_str(&spl_token_v2_0::id().to_string()).unwrap()
+pub fn spl_token_id() -> Pubkey {
+    Pubkey::new_from_array(spl_token::id().to_bytes())
 }
 
-// A helper function to convert spl_token_v2_0::native_mint::id() as spl_sdk::pubkey::Pubkey to
+// A helper function to convert spl_token::native_mint::id() as spl_sdk::pubkey::Pubkey to
 // solana_sdk::pubkey::Pubkey
-pub fn spl_token_v2_0_native_mint() -> Pubkey {
-    Pubkey::from_str(&spl_token_v2_0::native_mint::id().to_string()).unwrap()
+pub fn spl_token_native_mint() -> Pubkey {
+    Pubkey::new_from_array(spl_token::native_mint::id().to_bytes())
+}
+
+// A helper function to convert a solana_sdk::pubkey::Pubkey to spl_sdk::pubkey::Pubkey
+pub fn spl_token_pubkey(pubkey: &Pubkey) -> SplTokenPubkey {
+    SplTokenPubkey::new_from_array(pubkey.to_bytes())
+}
+
+// A helper function to convert a spl_sdk::pubkey::Pubkey to solana_sdk::pubkey::Pubkey
+pub fn pubkey_from_spl_token(pubkey: &SplTokenPubkey) -> Pubkey {
+    Pubkey::new_from_array(pubkey.to_bytes())
 }
 
 pub fn parse_token(
@@ -104,6 +118,7 @@ pub fn parse_token(
 
 #[derive(Debug, Serialize, Deserialize, PartialEq)]
 #[serde(rename_all = "camelCase", tag = "type", content = "info")]
+#[allow(clippy::large_enum_variant)]
 pub enum TokenAccountType {
     Account(UiTokenAccount),
     Mint(UiMint),
@@ -146,46 +161,66 @@ impl From<AccountState> for UiAccountState {
     }
 }
 
+pub fn real_number_string(amount: u64, decimals: u8) -> StringDecimals {
+    let decimals = decimals as usize;
+    if decimals > 0 {
+        // Left-pad zeros to decimals + 1, so we at least have an integer zero
+        let mut s = format!("{:01$}", amount, decimals + 1);
+        // Add the decimal point (Sorry, "," locales!)
+        s.insert(s.len() - decimals, '.');
+        s
+    } else {
+        amount.to_string()
+    }
+}
+
+pub fn real_number_string_trimmed(amount: u64, decimals: u8) -> StringDecimals {
+    let mut s = real_number_string(amount, decimals);
+    if decimals > 0 {
+        let zeros_trimmed = s.trim_end_matches('0');
+        s = zeros_trimmed.trim_end_matches('.').to_string();
+    }
+    s
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct UiTokenAmount {
-    pub ui_amount: f64,
+    pub ui_amount: Option<f64>,
     pub decimals: u8,
     pub amount: StringAmount,
+    pub ui_amount_string: StringDecimals,
 }
 
 impl UiTokenAmount {
     pub fn real_number_string(&self) -> String {
-        let decimals = self.decimals as usize;
-        if decimals > 0 {
-            let amount = u64::from_str(&self.amount).unwrap_or(0);
-
-            // Left-pad zeros to decimals + 1, so we at least have an integer zero
-            let mut s = format!("{:01$}", amount, decimals + 1);
-
-            // Add the decimal point (Sorry, "," locales!)
-            s.insert(s.len() - decimals, '.');
-            s
-        } else {
-            self.amount.clone()
-        }
+        real_number_string(
+            u64::from_str(&self.amount).unwrap_or_default(),
+            self.decimals as u8,
+        )
     }
 
     pub fn real_number_string_trimmed(&self) -> String {
-        let s = self.real_number_string();
-        let zeros_trimmed = s.trim_end_matches('0');
-        let decimal_trimmed = zeros_trimmed.trim_end_matches('.');
-        decimal_trimmed.to_string()
+        if !self.ui_amount_string.is_empty() {
+            self.ui_amount_string.clone()
+        } else {
+            real_number_string_trimmed(
+                u64::from_str(&self.amount).unwrap_or_default(),
+                self.decimals as u8,
+            )
+        }
     }
 }
 
 pub fn token_amount_to_ui_amount(amount: u64, decimals: u8) -> UiTokenAmount {
-    // Use `amount_to_ui_amount()` once spl_token is bumped to a version that supports it: https://github.com/solana-labs/solana-program-library/pull/211
-    let amount_decimals = amount as f64 / 10_usize.pow(decimals as u32) as f64;
+    let amount_decimals = 10_usize
+        .checked_pow(decimals as u32)
+        .map(|dividend| amount as f64 / dividend as f64);
     UiTokenAmount {
         ui_amount: amount_decimals,
         decimals,
         amount: amount.to_string(),
+        ui_amount_string: real_number_string_trimmed(amount, decimals),
     }
 }
 
@@ -241,9 +276,10 @@ mod test {
                 mint: mint_pubkey.to_string(),
                 owner: owner_pubkey.to_string(),
                 token_amount: UiTokenAmount {
-                    ui_amount: 0.42,
+                    ui_amount: Some(0.42),
                     decimals: 2,
-                    amount: "42".to_string()
+                    amount: "42".to_string(),
+                    ui_amount_string: "0.42".to_string()
                 },
                 delegate: None,
                 state: UiAccountState::Initialized,
@@ -324,17 +360,87 @@ mod test {
 
     #[test]
     fn test_ui_token_amount_real_string() {
+        assert_eq!(&real_number_string(1, 0), "1");
+        assert_eq!(&real_number_string_trimmed(1, 0), "1");
         let token_amount = token_amount_to_ui_amount(1, 0);
-        assert_eq!(&token_amount.real_number_string(), "1");
-        assert_eq!(&token_amount.real_number_string_trimmed(), "1");
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(1, 0)
+        );
+        assert_eq!(token_amount.ui_amount, Some(1.0));
+        assert_eq!(&real_number_string(10, 0), "10");
+        assert_eq!(&real_number_string_trimmed(10, 0), "10");
+        let token_amount = token_amount_to_ui_amount(10, 0);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(10, 0)
+        );
+        assert_eq!(token_amount.ui_amount, Some(10.0));
+        assert_eq!(&real_number_string(1, 9), "0.000000001");
+        assert_eq!(&real_number_string_trimmed(1, 9), "0.000000001");
         let token_amount = token_amount_to_ui_amount(1, 9);
-        assert_eq!(&token_amount.real_number_string(), "0.000000001");
-        assert_eq!(&token_amount.real_number_string_trimmed(), "0.000000001");
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(1, 9)
+        );
+        assert_eq!(token_amount.ui_amount, Some(0.000000001));
+        assert_eq!(&real_number_string(1_000_000_000, 9), "1.000000000");
+        assert_eq!(&real_number_string_trimmed(1_000_000_000, 9), "1");
         let token_amount = token_amount_to_ui_amount(1_000_000_000, 9);
-        assert_eq!(&token_amount.real_number_string(), "1.000000000");
-        assert_eq!(&token_amount.real_number_string_trimmed(), "1");
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(1_000_000_000, 9)
+        );
+        assert_eq!(token_amount.ui_amount, Some(1.0));
+        assert_eq!(&real_number_string(1_234_567_890, 3), "1234567.890");
+        assert_eq!(&real_number_string_trimmed(1_234_567_890, 3), "1234567.89");
         let token_amount = token_amount_to_ui_amount(1_234_567_890, 3);
-        assert_eq!(&token_amount.real_number_string(), "1234567.890");
-        assert_eq!(&token_amount.real_number_string_trimmed(), "1234567.89");
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(1_234_567_890, 3)
+        );
+        assert_eq!(token_amount.ui_amount, Some(1234567.89));
+        assert_eq!(
+            &real_number_string(1_234_567_890, 25),
+            "0.0000000000000001234567890"
+        );
+        assert_eq!(
+            &real_number_string_trimmed(1_234_567_890, 25),
+            "0.000000000000000123456789"
+        );
+        let token_amount = token_amount_to_ui_amount(1_234_567_890, 20);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(1_234_567_890, 20)
+        );
+        assert_eq!(token_amount.ui_amount, None);
+    }
+
+    #[test]
+    fn test_ui_token_amount_real_string_zero() {
+        assert_eq!(&real_number_string(0, 0), "0");
+        assert_eq!(&real_number_string_trimmed(0, 0), "0");
+        let token_amount = token_amount_to_ui_amount(0, 0);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(0, 0)
+        );
+        assert_eq!(token_amount.ui_amount, Some(0.0));
+        assert_eq!(&real_number_string(0, 9), "0.000000000");
+        assert_eq!(&real_number_string_trimmed(0, 9), "0");
+        let token_amount = token_amount_to_ui_amount(0, 9);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(0, 9)
+        );
+        assert_eq!(token_amount.ui_amount, Some(0.0));
+        assert_eq!(&real_number_string(0, 25), "0.0000000000000000000000000");
+        assert_eq!(&real_number_string_trimmed(0, 25), "0");
+        let token_amount = token_amount_to_ui_amount(0, 20);
+        assert_eq!(
+            token_amount.ui_amount_string,
+            real_number_string_trimmed(0, 20)
+        );
+        assert_eq!(token_amount.ui_amount, None);
     }
 }

account-decoder/src/parse_vote.rs

@@ -1,9 +1,11 @@
-use crate::{parse_account_data::ParseAccountError, StringAmount};
-use solana_sdk::{
-    clock::{Epoch, Slot},
-    pubkey::Pubkey,
+use {
+    crate::{parse_account_data::ParseAccountError, StringAmount},
+    solana_sdk::{
+        clock::{Epoch, Slot},
+        pubkey::Pubkey,
+    },
+    solana_vote_program::vote_state::{BlockTimestamp, Lockout, VoteState},
 };
-use solana_vote_program::vote_state::{BlockTimestamp, Lockout, VoteState};
 
 pub fn parse_vote(data: &[u8]) -> Result<VoteAccountType, ParseAccountError> {
     let mut vote_state = VoteState::deserialize(data).map_err(ParseAccountError::from)?;
@@ -121,18 +123,19 @@ struct UiEpochCredits {
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use solana_vote_program::vote_state::VoteStateVersions;
+    use {super::*, solana_vote_program::vote_state::VoteStateVersions};
 
     #[test]
     fn test_parse_vote() {
         let vote_state = VoteState::default();
         let mut vote_account_data: Vec<u8> = vec![0; VoteState::size_of()];
-        let versioned = VoteStateVersions::Current(Box::new(vote_state));
+        let versioned = VoteStateVersions::new_current(vote_state);
         VoteState::serialize(&versioned, &mut vote_account_data).unwrap();
-        let mut expected_vote_state = UiVoteState::default();
-        expected_vote_state.node_pubkey = Pubkey::default().to_string();
-        expected_vote_state.authorized_withdrawer = Pubkey::default().to_string();
+        let expected_vote_state = UiVoteState {
+            node_pubkey: Pubkey::default().to_string(),
+            authorized_withdrawer: Pubkey::default().to_string(),
+            ..UiVoteState::default()
+        };
         assert_eq!(
             parse_vote(&vote_account_data).unwrap(),
             VoteAccountType::Vote(expected_vote_state)

accounts-bench/Cargo.toml

@@ -1,24 +1,22 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2018"
+edition = "2021"
 name = "solana-accounts-bench"
-version = "1.4.0"
+version = "1.9.5"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 publish = false
 
 [dependencies]
-log = "0.4.6"
-rayon = "1.4.0"
-solana-logger = { path = "../logger", version = "1.4.0" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-measure = { path = "../measure", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-version = { path = "../version", version = "1.4.0" }
-rand = "0.7.0"
+log = "0.4.14"
+rayon = "1.5.1"
+solana-logger = { path = "../logger", version = "=1.9.5" }
+solana-runtime = { path = "../runtime", version = "=1.9.5" }
+solana-measure = { path = "../measure", version = "=1.9.5" }
+solana-sdk = { path = "../sdk", version = "=1.9.5" }
+solana-version = { path = "../version", version = "=1.9.5" }
 clap = "2.33.1"
-crossbeam-channel = "0.4"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accounts-bench/src/main.rs

@@ -1,14 +1,19 @@
-use clap::{crate_description, crate_name, value_t, App, Arg};
-use rayon::prelude::*;
-use solana_measure::measure::Measure;
-use solana_runtime::{
-    accounts::{create_test_accounts, update_accounts, Accounts},
-    accounts_index::Ancestors,
+#![allow(clippy::integer_arithmetic)]
+#[macro_use]
+extern crate log;
+use {
+    clap::{crate_description, crate_name, value_t, App, Arg},
+    rayon::prelude::*,
+    solana_measure::measure::Measure,
+    solana_runtime::{
+        accounts::{create_test_accounts, update_accounts_bench, Accounts},
+        accounts_db::AccountShrinkThreshold,
+        accounts_index::AccountSecondaryIndexes,
+        ancestors::Ancestors,
+    },
+    solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey},
+    std::{env, fs, path::PathBuf},
 };
-use solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey};
-use std::env;
-use std::fs;
-use std::path::PathBuf;
 
 fn main() {
     solana_logger::setup();
@@ -53,10 +58,17 @@ fn main() {
 
     let path = PathBuf::from(env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_owned()))
         .join("accounts-bench");
+    println!("cleaning file system: {:?}", path);
     if fs::remove_dir_all(path.clone()).is_err() {
         println!("Warning: Couldn't remove {:?}", path);
     }
-    let accounts = Accounts::new(vec![path], &ClusterType::Testnet);
+    let accounts = Accounts::new_with_config_for_benches(
+        vec![path],
+        &ClusterType::Testnet,
+        AccountSecondaryIndexes::default(),
+        false,
+        AccountShrinkThreshold::default(),
+    );
     println!("Creating {} accounts", num_accounts);
     let mut create_time = Measure::start("create accounts");
     let pubkeys: Vec<_> = (0..num_slots)
@@ -80,28 +92,62 @@ fn main() {
         num_slots,
         create_time
     );
-    let mut ancestors: Ancestors = vec![(0, 0)].into_iter().collect();
+    let mut ancestors = Vec::with_capacity(num_slots);
+    ancestors.push(0);
     for i in 1..num_slots {
-        ancestors.insert(i as u64, i - 1);
+        ancestors.push(i as u64);
         accounts.add_root(i as u64);
     }
+    let ancestors = Ancestors::from(ancestors);
+    let mut elapsed = vec![0; iterations];
+    let mut elapsed_store = vec![0; iterations];
     for x in 0..iterations {
         if clean {
             let mut time = Measure::start("clean");
-            accounts.accounts_db.clean_accounts(None);
+            accounts.accounts_db.clean_accounts(None, false, None);
             time.stop();
             println!("{}", time);
             for slot in 0..num_slots {
-                update_accounts(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64);
+                update_accounts_bench(&accounts, &pubkeys, ((x + 1) * num_slots + slot) as u64);
                 accounts.add_root((x * num_slots + slot) as u64);
             }
         } else {
             let mut pubkeys: Vec<Pubkey> = vec![];
             let mut time = Measure::start("hash");
-            let hash = accounts.accounts_db.update_accounts_hash(0, &ancestors).0;
+            let results = accounts.accounts_db.update_accounts_hash(0, &ancestors);
             time.stop();
-            println!("hash: {} {}", hash, time);
+            let mut time_store = Measure::start("hash using store");
+            let results_store = accounts.accounts_db.update_accounts_hash_with_index_option(
+                false,
+                false,
+                solana_sdk::clock::Slot::default(),
+                &ancestors,
+                None,
+                false,
+                None,
+                false,
+            );
+            time_store.stop();
+            if results != results_store {
+                error!("results different: \n{:?}\n{:?}", results, results_store);
+            }
+            println!(
+                "hash,{},{},{},{}%",
+                results.0,
+                time,
+                time_store,
+                (time_store.as_us() as f64 / time.as_us() as f64 * 100.0f64) as u32
+            );
             create_test_accounts(&accounts, &mut pubkeys, 1, 0);
+            elapsed[x] = time.as_us();
+            elapsed_store[x] = time_store.as_us();
         }
     }
+
+    for x in elapsed {
+        info!("update_accounts_hash(us),{}", x);
+    }
+    for x in elapsed_store {
+        info!("calculate_accounts_hash_without_index(us),{}", x);
+    }
 }

accounts-cluster-bench/.gitignore

@@ -0,0 +1 @@
+/farf/

accounts-cluster-bench/Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2021"
+name = "solana-accounts-cluster-bench"
+version = "1.9.5"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+publish = false
+
+[dependencies]
+clap = "2.33.1"
+log = "0.4.14"
+rand = "0.7.0"
+rayon = "1.5.1"
+solana-account-decoder = { path = "../account-decoder", version = "=1.9.5" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.9.5" }
+solana-client = { path = "../client", version = "=1.9.5" }
+solana-core = { path = "../core", version = "=1.9.5" }
+solana-faucet = { path = "../faucet", version = "=1.9.5" }
+solana-gossip = { path = "../gossip", version = "=1.9.5" }
+solana-logger = { path = "../logger", version = "=1.9.5" }
+solana-measure = { path = "../measure", version = "=1.9.5" }
+solana-net-utils = { path = "../net-utils", version = "=1.9.5" }
+solana-runtime = { path = "../runtime", version = "=1.9.5" }
+solana-sdk = { path = "../sdk", version = "=1.9.5" }
+solana-streamer = { path = "../streamer", version = "=1.9.5" }
+solana-test-validator = { path = "../test-validator", version = "=1.9.5" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.9.5" }
+solana-version = { path = "../version", version = "=1.9.5" }
+spl-token = { version = "=3.2.0", features = ["no-entrypoint"] }
+
+[dev-dependencies]
+solana-local-cluster = { path = "../local-cluster", version = "=1.9.5" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accounts-cluster-bench/src/main.rs

@@ -0,0 +1,813 @@
+#![allow(clippy::integer_arithmetic)]
+use {
+    clap::{crate_description, crate_name, value_t, values_t_or_exit, App, Arg},
+    log::*,
+    rand::{thread_rng, Rng},
+    rayon::prelude::*,
+    solana_account_decoder::parse_token::spl_token_pubkey,
+    solana_clap_utils::input_parsers::pubkey_of,
+    solana_client::{rpc_client::RpcClient, transaction_executor::TransactionExecutor},
+    solana_faucet::faucet::{request_airdrop_transaction, FAUCET_PORT},
+    solana_gossip::gossip_service::discover,
+    solana_runtime::inline_spl_token,
+    solana_sdk::{
+        commitment_config::CommitmentConfig,
+        instruction::{AccountMeta, Instruction},
+        message::Message,
+        pubkey::Pubkey,
+        rpc_port::DEFAULT_RPC_PORT,
+        signature::{read_keypair_file, Keypair, Signer},
+        system_instruction, system_program,
+        transaction::Transaction,
+    },
+    solana_streamer::socket::SocketAddrSpace,
+    solana_transaction_status::parse_token::spl_token_instruction,
+    std::{
+        cmp::min,
+        net::SocketAddr,
+        process::exit,
+        sync::{
+            atomic::{AtomicU64, Ordering},
+            Arc,
+        },
+        thread::sleep,
+        time::{Duration, Instant},
+    },
+};
+
+pub fn airdrop_lamports(
+    client: &RpcClient,
+    faucet_addr: &SocketAddr,
+    id: &Keypair,
+    desired_balance: u64,
+) -> bool {
+    let starting_balance = client.get_balance(&id.pubkey()).unwrap_or(0);
+    info!("starting balance {}", starting_balance);
+
+    if starting_balance < desired_balance {
+        let airdrop_amount = desired_balance - starting_balance;
+        info!(
+            "Airdropping {:?} lamports from {} for {}",
+            airdrop_amount,
+            faucet_addr,
+            id.pubkey(),
+        );
+
+        let blockhash = client.get_latest_blockhash().unwrap();
+        match request_airdrop_transaction(faucet_addr, &id.pubkey(), airdrop_amount, blockhash) {
+            Ok(transaction) => {
+                let mut tries = 0;
+                loop {
+                    tries += 1;
+                    let result = client.send_and_confirm_transaction(&transaction);
+
+                    if result.is_ok() {
+                        break;
+                    }
+                    if tries >= 5 {
+                        panic!(
+                            "Error requesting airdrop: to addr: {:?} amount: {} {:?}",
+                            faucet_addr, airdrop_amount, result
+                        )
+                    }
+                }
+            }
+            Err(err) => {
+                panic!(
+                    "Error requesting airdrop: {:?} to addr: {:?} amount: {}",
+                    err, faucet_addr, airdrop_amount
+                );
+            }
+        };
+
+        let current_balance = client.get_balance(&id.pubkey()).unwrap_or_else(|e| {
+            panic!("airdrop error {}", e);
+        });
+        info!("current balance {}...", current_balance);
+
+        if current_balance - starting_balance != airdrop_amount {
+            info!(
+                "Airdrop failed? {} {} {} {}",
+                id.pubkey(),
+                current_balance,
+                starting_balance,
+                airdrop_amount,
+            );
+        }
+    }
+    true
+}
+
+struct SeedTracker {
+    max_created: Arc<AtomicU64>,
+    max_closed: Arc<AtomicU64>,
+}
+
+fn make_create_message(
+    keypair: &Keypair,
+    base_keypair: &Keypair,
+    max_created_seed: Arc<AtomicU64>,
+    num_instructions: usize,
+    balance: u64,
+    maybe_space: Option<u64>,
+    mint: Option<Pubkey>,
+) -> Message {
+    let space = maybe_space.unwrap_or_else(|| thread_rng().gen_range(0, 1000));
+
+    let instructions: Vec<_> = (0..num_instructions)
+        .into_iter()
+        .map(|_| {
+            let program_id = if mint.is_some() {
+                inline_spl_token::id()
+            } else {
+                system_program::id()
+            };
+            let seed = max_created_seed.fetch_add(1, Ordering::Relaxed).to_string();
+            let to_pubkey =
+                Pubkey::create_with_seed(&base_keypair.pubkey(), &seed, &program_id).unwrap();
+            let mut instructions = vec![system_instruction::create_account_with_seed(
+                &keypair.pubkey(),
+                &to_pubkey,
+                &base_keypair.pubkey(),
+                &seed,
+                balance,
+                space,
+                &program_id,
+            )];
+            if let Some(mint_address) = mint {
+                instructions.push(spl_token_instruction(
+                    spl_token::instruction::initialize_account(
+                        &spl_token::id(),
+                        &spl_token_pubkey(&to_pubkey),
+                        &spl_token_pubkey(&mint_address),
+                        &spl_token_pubkey(&base_keypair.pubkey()),
+                    )
+                    .unwrap(),
+                ));
+            }
+
+            instructions
+        })
+        .flatten()
+        .collect();
+
+    Message::new(&instructions, Some(&keypair.pubkey()))
+}
+
+fn make_close_message(
+    keypair: &Keypair,
+    base_keypair: &Keypair,
+    max_created: Arc<AtomicU64>,
+    max_closed: Arc<AtomicU64>,
+    num_instructions: usize,
+    balance: u64,
+    spl_token: bool,
+) -> Message {
+    let instructions: Vec<_> = (0..num_instructions)
+        .into_iter()
+        .filter_map(|_| {
+            let program_id = if spl_token {
+                inline_spl_token::id()
+            } else {
+                system_program::id()
+            };
+            let max_created_seed = max_created.load(Ordering::Relaxed);
+            let max_closed_seed = max_closed.load(Ordering::Relaxed);
+            if max_closed_seed >= max_created_seed {
+                return None;
+            }
+            let seed = max_closed.fetch_add(1, Ordering::Relaxed).to_string();
+            let address =
+                Pubkey::create_with_seed(&base_keypair.pubkey(), &seed, &program_id).unwrap();
+            if spl_token {
+                Some(spl_token_instruction(
+                    spl_token::instruction::close_account(
+                        &spl_token::id(),
+                        &spl_token_pubkey(&address),
+                        &spl_token_pubkey(&keypair.pubkey()),
+                        &spl_token_pubkey(&base_keypair.pubkey()),
+                        &[],
+                    )
+                    .unwrap(),
+                ))
+            } else {
+                Some(system_instruction::transfer_with_seed(
+                    &address,
+                    &base_keypair.pubkey(),
+                    seed,
+                    &program_id,
+                    &keypair.pubkey(),
+                    balance,
+                ))
+            }
+        })
+        .collect();
+
+    Message::new(&instructions, Some(&keypair.pubkey()))
+}
+
+#[allow(clippy::too_many_arguments)]
+fn run_accounts_bench(
+    entrypoint_addr: SocketAddr,
+    faucet_addr: SocketAddr,
+    payer_keypairs: &[&Keypair],
+    iterations: usize,
+    maybe_space: Option<u64>,
+    batch_size: usize,
+    close_nth_batch: u64,
+    maybe_lamports: Option<u64>,
+    num_instructions: usize,
+    mint: Option<Pubkey>,
+    reclaim_accounts: bool,
+) {
+    assert!(num_instructions > 0);
+    let client =
+        RpcClient::new_socket_with_commitment(entrypoint_addr, CommitmentConfig::confirmed());
+
+    info!("Targeting {}", entrypoint_addr);
+
+    let mut latest_blockhash = Instant::now();
+    let mut last_log = Instant::now();
+    let mut count = 0;
+    let mut blockhash = client.get_latest_blockhash().expect("blockhash");
+    let mut tx_sent_count = 0;
+    let mut total_accounts_created = 0;
+    let mut total_accounts_closed = 0;
+    let mut balances: Vec<_> = payer_keypairs
+        .iter()
+        .map(|keypair| client.get_balance(&keypair.pubkey()).unwrap_or(0))
+        .collect();
+    let mut last_balance = Instant::now();
+
+    let default_max_lamports = 1000;
+    let min_balance = maybe_lamports.unwrap_or_else(|| {
+        let space = maybe_space.unwrap_or(default_max_lamports);
+        client
+            .get_minimum_balance_for_rent_exemption(space as usize)
+            .expect("min balance")
+    });
+
+    let base_keypair = Keypair::new();
+    let seed_tracker = SeedTracker {
+        max_created: Arc::new(AtomicU64::default()),
+        max_closed: Arc::new(AtomicU64::default()),
+    };
+
+    info!("Starting balance(s): {:?}", balances);
+
+    let executor = TransactionExecutor::new(entrypoint_addr);
+
+    // Create and close messages both require 2 signatures, fake a 2 signature message to calculate fees
+    let mut message = Message::new(
+        &[
+            Instruction::new_with_bytes(
+                Pubkey::new_unique(),
+                &[],
+                vec![AccountMeta::new(Pubkey::new_unique(), true)],
+            ),
+            Instruction::new_with_bytes(
+                Pubkey::new_unique(),
+                &[],
+                vec![AccountMeta::new(Pubkey::new_unique(), true)],
+            ),
+        ],
+        None,
+    );
+
+    loop {
+        if latest_blockhash.elapsed().as_millis() > 10_000 {
+            blockhash = client.get_latest_blockhash().expect("blockhash");
+            latest_blockhash = Instant::now();
+        }
+
+        message.recent_blockhash = blockhash;
+        let fee = client
+            .get_fee_for_message(&message)
+            .expect("get_fee_for_message");
+        let lamports = min_balance + fee;
+
+        for (i, balance) in balances.iter_mut().enumerate() {
+            if *balance < lamports || last_balance.elapsed().as_millis() > 2000 {
+                if let Ok(b) = client.get_balance(&payer_keypairs[i].pubkey()) {
+                    *balance = b;
+                }
+                last_balance = Instant::now();
+                if *balance < lamports * 2 {
+                    info!(
+                        "Balance {} is less than needed: {}, doing aidrop...",
+                        balance, lamports
+                    );
+                    if !airdrop_lamports(
+                        &client,
+                        &faucet_addr,
+                        payer_keypairs[i],
+                        lamports * 100_000,
+                    ) {
+                        warn!("failed airdrop, exiting");
+                        return;
+                    }
+                }
+            }
+        }
+
+        // Create accounts
+        let sigs_len = executor.num_outstanding();
+        if sigs_len < batch_size {
+            let num_to_create = batch_size - sigs_len;
+            if num_to_create >= payer_keypairs.len() {
+                info!("creating {} new", num_to_create);
+                let chunk_size = num_to_create / payer_keypairs.len();
+                if chunk_size > 0 {
+                    for (i, keypair) in payer_keypairs.iter().enumerate() {
+                        let txs: Vec<_> = (0..chunk_size)
+                            .into_par_iter()
+                            .map(|_| {
+                                let message = make_create_message(
+                                    keypair,
+                                    &base_keypair,
+                                    seed_tracker.max_created.clone(),
+                                    num_instructions,
+                                    min_balance,
+                                    maybe_space,
+                                    mint,
+                                );
+                                let signers: Vec<&Keypair> = vec![keypair, &base_keypair];
+                                Transaction::new(&signers, message, blockhash)
+                            })
+                            .collect();
+                        balances[i] = balances[i].saturating_sub(lamports * txs.len() as u64);
+                        info!("txs: {}", txs.len());
+                        let new_ids = executor.push_transactions(txs);
+                        info!("ids: {}", new_ids.len());
+                        tx_sent_count += new_ids.len();
+                        total_accounts_created += num_instructions * new_ids.len();
+                    }
+                }
+            }
+
+            if close_nth_batch > 0 {
+                let num_batches_to_close =
+                    total_accounts_created as u64 / (close_nth_batch * batch_size as u64);
+                let expected_closed = num_batches_to_close * batch_size as u64;
+                let max_closed_seed = seed_tracker.max_closed.load(Ordering::Relaxed);
+                // Close every account we've created with seed between max_closed_seed..expected_closed
+                if max_closed_seed < expected_closed {
+                    let txs: Vec<_> = (0..expected_closed - max_closed_seed)
+                        .into_par_iter()
+                        .map(|_| {
+                            let message = make_close_message(
+                                payer_keypairs[0],
+                                &base_keypair,
+                                seed_tracker.max_created.clone(),
+                                seed_tracker.max_closed.clone(),
+                                1,
+                                min_balance,
+                                mint.is_some(),
+                            );
+                            let signers: Vec<&Keypair> = vec![payer_keypairs[0], &base_keypair];
+                            Transaction::new(&signers, message, blockhash)
+                        })
+                        .collect();
+                    balances[0] = balances[0].saturating_sub(fee * txs.len() as u64);
+                    info!("close txs: {}", txs.len());
+                    let new_ids = executor.push_transactions(txs);
+                    info!("close ids: {}", new_ids.len());
+                    tx_sent_count += new_ids.len();
+                    total_accounts_closed += new_ids.len() as u64;
+                }
+            }
+        } else {
+            let _ = executor.drain_cleared();
+        }
+
+        count += 1;
+        if last_log.elapsed().as_millis() > 3000 || count >= iterations {
+            info!(
+                "total_accounts_created: {} total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance(s): {:?}",
+                total_accounts_created, total_accounts_closed, tx_sent_count, count, balances
+            );
+            last_log = Instant::now();
+        }
+        if iterations != 0 && count >= iterations {
+            break;
+        }
+        if executor.num_outstanding() >= batch_size {
+            sleep(Duration::from_millis(500));
+        }
+    }
+    executor.close();
+
+    if reclaim_accounts {
+        let executor = TransactionExecutor::new(entrypoint_addr);
+        loop {
+            let max_closed_seed = seed_tracker.max_closed.load(Ordering::Relaxed);
+            let max_created_seed = seed_tracker.max_created.load(Ordering::Relaxed);
+
+            if latest_blockhash.elapsed().as_millis() > 10_000 {
+                blockhash = client.get_latest_blockhash().expect("blockhash");
+                latest_blockhash = Instant::now();
+            }
+            message.recent_blockhash = blockhash;
+            let fee = client
+                .get_fee_for_message(&message)
+                .expect("get_fee_for_message");
+
+            let sigs_len = executor.num_outstanding();
+            if sigs_len < batch_size && max_closed_seed < max_created_seed {
+                let num_to_close = min(
+                    batch_size - sigs_len,
+                    (max_created_seed - max_closed_seed) as usize,
+                );
+                if num_to_close >= payer_keypairs.len() {
+                    info!("closing {} accounts", num_to_close);
+                    let chunk_size = num_to_close / payer_keypairs.len();
+                    info!("{:?} chunk_size", chunk_size);
+                    if chunk_size > 0 {
+                        for (i, keypair) in payer_keypairs.iter().enumerate() {
+                            let txs: Vec<_> = (0..chunk_size)
+                                .into_par_iter()
+                                .filter_map(|_| {
+                                    let message = make_close_message(
+                                        keypair,
+                                        &base_keypair,
+                                        seed_tracker.max_created.clone(),
+                                        seed_tracker.max_closed.clone(),
+                                        num_instructions,
+                                        min_balance,
+                                        mint.is_some(),
+                                    );
+                                    if message.instructions.is_empty() {
+                                        return None;
+                                    }
+                                    let signers: Vec<&Keypair> = vec![keypair, &base_keypair];
+                                    Some(Transaction::new(&signers, message, blockhash))
+                                })
+                                .collect();
+                            balances[i] = balances[i].saturating_sub(fee * txs.len() as u64);
+                            info!("close txs: {}", txs.len());
+                            let new_ids = executor.push_transactions(txs);
+                            info!("close ids: {}", new_ids.len());
+                            tx_sent_count += new_ids.len();
+                            total_accounts_closed += (num_instructions * new_ids.len()) as u64;
+                        }
+                    }
+                }
+            } else {
+                let _ = executor.drain_cleared();
+            }
+            count += 1;
+            if last_log.elapsed().as_millis() > 3000 || max_closed_seed >= max_created_seed {
+                info!(
+                    "total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance(s): {:?}",
+                    total_accounts_closed, tx_sent_count, count, balances
+                );
+                last_log = Instant::now();
+            }
+
+            if max_closed_seed >= max_created_seed {
+                break;
+            }
+            if executor.num_outstanding() >= batch_size {
+                sleep(Duration::from_millis(500));
+            }
+        }
+        executor.close();
+    }
+}
+
+fn main() {
+    solana_logger::setup_with_default("solana=info");
+    let matches = App::new(crate_name!())
+        .about(crate_description!())
+        .version(solana_version::version!())
+        .arg(
+            Arg::with_name("entrypoint")
+                .long("entrypoint")
+                .takes_value(true)
+                .value_name("HOST:PORT")
+                .help("RPC entrypoint address. Usually <ip>:8899"),
+        )
+        .arg(
+            Arg::with_name("faucet_addr")
+                .long("faucet")
+                .takes_value(true)
+                .value_name("HOST:PORT")
+                .help("Faucet entrypoint address. Usually <ip>:9900"),
+        )
+        .arg(
+            Arg::with_name("space")
+                .long("space")
+                .takes_value(true)
+                .value_name("BYTES")
+                .help("Size of accounts to create"),
+        )
+        .arg(
+            Arg::with_name("lamports")
+                .long("lamports")
+                .takes_value(true)
+                .value_name("LAMPORTS")
+                .help("How many lamports to fund each account"),
+        )
+        .arg(
+            Arg::with_name("identity")
+                .long("identity")
+                .takes_value(true)
+                .multiple(true)
+                .value_name("FILE")
+                .help("keypair file"),
+        )
+        .arg(
+            Arg::with_name("batch_size")
+                .long("batch-size")
+                .takes_value(true)
+                .value_name("BYTES")
+                .help("Number of transactions to send per batch"),
+        )
+        .arg(
+            Arg::with_name("close_nth_batch")
+                .long("close-frequency")
+                .takes_value(true)
+                .value_name("BYTES")
+                .help(
+                    "Every `n` batches, create a batch of close transactions for
+                    the earliest remaining batch of accounts created.
+                    Note: Should be > 1 to avoid situations where the close \
+                    transactions will be submitted before the corresponding \
+                    create transactions have been confirmed",
+                ),
+        )
+        .arg(
+            Arg::with_name("num_instructions")
+                .long("num-instructions")
+                .takes_value(true)
+                .value_name("NUM")
+                .help("Number of accounts to create on each transaction"),
+        )
+        .arg(
+            Arg::with_name("iterations")
+                .long("iterations")
+                .takes_value(true)
+                .value_name("NUM")
+                .help("Number of iterations to make. 0 = unlimited iterations."),
+        )
+        .arg(
+            Arg::with_name("check_gossip")
+                .long("check-gossip")
+                .help("Just use entrypoint address directly"),
+        )
+        .arg(
+            Arg::with_name("mint")
+                .long("mint")
+                .takes_value(true)
+                .help("Mint address to initialize account"),
+        )
+        .arg(
+            Arg::with_name("reclaim_accounts")
+                .long("reclaim-accounts")
+                .takes_value(false)
+                .help("Reclaim accounts after session ends; incompatible with --iterations 0"),
+        )
+        .get_matches();
+
+    let skip_gossip = !matches.is_present("check_gossip");
+
+    let port = if skip_gossip { DEFAULT_RPC_PORT } else { 8001 };
+    let mut entrypoint_addr = SocketAddr::from(([127, 0, 0, 1], port));
+    if let Some(addr) = matches.value_of("entrypoint") {
+        entrypoint_addr = solana_net_utils::parse_host_port(addr).unwrap_or_else(|e| {
+            eprintln!("failed to parse entrypoint address: {}", e);
+            exit(1)
+        });
+    }
+    let mut faucet_addr = SocketAddr::from(([127, 0, 0, 1], FAUCET_PORT));
+    if let Some(addr) = matches.value_of("faucet_addr") {
+        faucet_addr = solana_net_utils::parse_host_port(addr).unwrap_or_else(|e| {
+            eprintln!("failed to parse entrypoint address: {}", e);
+            exit(1)
+        });
+    }
+
+    let space = value_t!(matches, "space", u64).ok();
+    let lamports = value_t!(matches, "lamports", u64).ok();
+    let batch_size = value_t!(matches, "batch_size", usize).unwrap_or(4);
+    let close_nth_batch = value_t!(matches, "close_nth_batch", u64).unwrap_or(0);
+    let iterations = value_t!(matches, "iterations", usize).unwrap_or(10);
+    let num_instructions = value_t!(matches, "num_instructions", usize).unwrap_or(1);
+    if num_instructions == 0 || num_instructions > 500 {
+        eprintln!("bad num_instructions: {}", num_instructions);
+        exit(1);
+    }
+
+    let mint = pubkey_of(&matches, "mint");
+
+    let payer_keypairs: Vec<_> = values_t_or_exit!(matches, "identity", String)
+        .iter()
+        .map(|keypair_string| {
+            read_keypair_file(keypair_string)
+                .unwrap_or_else(|_| panic!("bad keypair {:?}", keypair_string))
+        })
+        .collect();
+    let mut payer_keypair_refs: Vec<&Keypair> = vec![];
+    for keypair in payer_keypairs.iter() {
+        payer_keypair_refs.push(keypair);
+    }
+
+    let rpc_addr = if !skip_gossip {
+        info!("Finding cluster entry: {:?}", entrypoint_addr);
+        let (gossip_nodes, _validators) = discover(
+            None, // keypair
+            Some(&entrypoint_addr),
+            None,                    // num_nodes
+            Duration::from_secs(60), // timeout
+            None,                    // find_node_by_pubkey
+            Some(&entrypoint_addr),  // find_node_by_gossip_addr
+            None,                    // my_gossip_addr
+            0,                       // my_shred_version
+            SocketAddrSpace::Unspecified,
+        )
+        .unwrap_or_else(|err| {
+            eprintln!("Failed to discover {} node: {:?}", entrypoint_addr, err);
+            exit(1);
+        });
+
+        info!("done found {} nodes", gossip_nodes.len());
+        gossip_nodes[0].rpc
+    } else {
+        info!("Using {:?} as the RPC address", entrypoint_addr);
+        entrypoint_addr
+    };
+
+    run_accounts_bench(
+        rpc_addr,
+        faucet_addr,
+        &payer_keypair_refs,
+        iterations,
+        space,
+        batch_size,
+        close_nth_batch,
+        lamports,
+        num_instructions,
+        mint,
+        matches.is_present("reclaim_accounts"),
+    );
+}
+
+#[cfg(test)]
+pub mod test {
+    use {
+        super::*,
+        solana_core::validator::ValidatorConfig,
+        solana_faucet::faucet::run_local_faucet,
+        solana_local_cluster::{
+            local_cluster::{ClusterConfig, LocalCluster},
+            validator_configs::make_identical_validator_configs,
+        },
+        solana_measure::measure::Measure,
+        solana_sdk::{native_token::sol_to_lamports, poh_config::PohConfig},
+        solana_test_validator::TestValidator,
+        spl_token::{
+            solana_program::program_pack::Pack,
+            state::{Account, Mint},
+        },
+    };
+
+    #[test]
+    fn test_accounts_cluster_bench() {
+        solana_logger::setup();
+        let validator_config = ValidatorConfig::default();
+        let num_nodes = 1;
+        let mut config = ClusterConfig {
+            cluster_lamports: 10_000_000,
+            poh_config: PohConfig::new_sleep(Duration::from_millis(50)),
+            node_stakes: vec![100; num_nodes],
+            validator_configs: make_identical_validator_configs(&validator_config, num_nodes),
+            ..ClusterConfig::default()
+        };
+
+        let faucet_addr = SocketAddr::from(([127, 0, 0, 1], 9900));
+        let cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
+        let iterations = 10;
+        let maybe_space = None;
+        let batch_size = 100;
+        let close_nth_batch = 100;
+        let maybe_lamports = None;
+        let num_instructions = 2;
+        let mut start = Measure::start("total accounts run");
+        run_accounts_bench(
+            cluster.entry_point_info.rpc,
+            faucet_addr,
+            &[&cluster.funding_keypair],
+            iterations,
+            maybe_space,
+            batch_size,
+            close_nth_batch,
+            maybe_lamports,
+            num_instructions,
+            None,
+            false,
+        );
+        start.stop();
+        info!("{}", start);
+    }
+
+    #[test]
+    fn test_create_then_reclaim_spl_token_accounts() {
+        solana_logger::setup();
+        let mint_keypair = Keypair::new();
+        let mint_pubkey = mint_keypair.pubkey();
+        let faucet_addr = run_local_faucet(mint_keypair, None);
+        let test_validator = TestValidator::with_custom_fees(
+            mint_pubkey,
+            1,
+            Some(faucet_addr),
+            SocketAddrSpace::Unspecified,
+        );
+        let rpc_client =
+            RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
+
+        // Created funder
+        let funder = Keypair::new();
+        let latest_blockhash = rpc_client.get_latest_blockhash().unwrap();
+        let signature = rpc_client
+            .request_airdrop_with_blockhash(
+                &funder.pubkey(),
+                sol_to_lamports(1.0),
+                &latest_blockhash,
+            )
+            .unwrap();
+        rpc_client
+            .confirm_transaction_with_spinner(
+                &signature,
+                &latest_blockhash,
+                CommitmentConfig::confirmed(),
+            )
+            .unwrap();
+
+        // Create Mint
+        let spl_mint_keypair = Keypair::new();
+        let spl_mint_len = Mint::get_packed_len();
+        let spl_mint_rent = rpc_client
+            .get_minimum_balance_for_rent_exemption(spl_mint_len)
+            .unwrap();
+        let transaction = Transaction::new_signed_with_payer(
+            &[
+                system_instruction::create_account(
+                    &funder.pubkey(),
+                    &spl_mint_keypair.pubkey(),
+                    spl_mint_rent,
+                    spl_mint_len as u64,
+                    &inline_spl_token::id(),
+                ),
+                spl_token_instruction(
+                    spl_token::instruction::initialize_mint(
+                        &spl_token::id(),
+                        &spl_token_pubkey(&spl_mint_keypair.pubkey()),
+                        &spl_token_pubkey(&spl_mint_keypair.pubkey()),
+                        None,
+                        2,
+                    )
+                    .unwrap(),
+                ),
+            ],
+            Some(&funder.pubkey()),
+            &[&funder, &spl_mint_keypair],
+            latest_blockhash,
+        );
+        let _sig = rpc_client
+            .send_and_confirm_transaction(&transaction)
+            .unwrap();
+
+        let account_len = Account::get_packed_len();
+        let minimum_balance = rpc_client
+            .get_minimum_balance_for_rent_exemption(account_len)
+            .unwrap();
+
+        let iterations = 5;
+        let batch_size = 100;
+        let close_nth_batch = 0;
+        let num_instructions = 4;
+        let mut start = Measure::start("total accounts run");
+        let keypair0 = Keypair::new();
+        let keypair1 = Keypair::new();
+        let keypair2 = Keypair::new();
+        run_accounts_bench(
+            test_validator
+                .rpc_url()
+                .replace("http://", "")
+                .parse()
+                .unwrap(),
+            faucet_addr,
+            &[&keypair0, &keypair1, &keypair2],
+            iterations,
+            Some(account_len as u64),
+            batch_size,
+            close_nth_batch,
+            Some(minimum_balance),
+            num_instructions,
+            Some(spl_mint_keypair.pubkey()),
+            true,
+        );
+        start.stop();
+        info!("{}", start);
+    }
+}

accountsdb-plugin-interface/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2021"
+name = "solana-accountsdb-plugin-interface"
+description = "The Solana AccountsDb plugin interface."
+version = "1.9.5"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-accountsdb-plugin-interface"
+
+[dependencies]
+log = "0.4.11"
+thiserror = "1.0.30"
+solana-sdk = { path = "../sdk", version = "=1.9.5" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.9.5" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-interface/README.md

@@ -0,0 +1,20 @@
+<p align="center">
+  <a href="https://solana.com">
+    <img alt="Solana" src="https://i.imgur.com/IKyzQ6T.png" width="250" />
+  </a>
+</p>
+
+# Solana AccountsDb Plugin Interface
+
+This crate enables an AccountsDb plugin to be plugged into the Solana Validator runtime to take actions
+at the time of each account update; for example, saving the account state to an external database. The plugin must implement the `AccountsDbPlugin` trait. Please see the detail of the `accountsdb_plugin_interface.rs` for the interface definition.
+
+The plugin should produce a `cdylib` dynamic library, which must expose a `C` function `_create_plugin()` that
+instantiates the implementation of the interface.
+
+The `solana-accountsdb-plugin-postgres` crate provides an example of how to create a plugin which saves the accounts data into an
+external PostgreSQL databases.
+
+More information about Solana is available in the [Solana documentation](https://docs.solana.com/).
+
+Still have questions?  Ask us on [Discord](https://discordapp.com/invite/pquxPsq)

accountsdb-plugin-interface/src/accountsdb_plugin_interface.rs

@@ -0,0 +1,220 @@
+/// The interface for AccountsDb plugins. A plugin must implement
+/// the AccountsDbPlugin trait to work with the runtime.
+/// In addition, the dynamic library must export a "C" function _create_plugin which
+/// creates the implementation of the plugin.
+use {
+    solana_sdk::{clock::UnixTimestamp, signature::Signature, transaction::SanitizedTransaction},
+    solana_transaction_status::{Reward, TransactionStatusMeta},
+    std::{any::Any, error, io},
+    thiserror::Error,
+};
+
+impl Eq for ReplicaAccountInfo<'_> {}
+
+#[derive(Clone, PartialEq, Debug)]
+/// Information about an account being updated
+pub struct ReplicaAccountInfo<'a> {
+    /// The Pubkey for the account
+    pub pubkey: &'a [u8],
+
+    /// The lamports for the account
+    pub lamports: u64,
+
+    /// The Pubkey of the owner program account
+    pub owner: &'a [u8],
+
+    /// This account's data contains a loaded program (and is now read-only)
+    pub executable: bool,
+
+    /// The epoch at which this account will next owe rent
+    pub rent_epoch: u64,
+
+    /// The data held in this account.
+    pub data: &'a [u8],
+
+    /// A global monotonically increasing atomic number, which can be used
+    /// to tell the order of the account update. For example, when an
+    /// account is updated in the same slot multiple times, the update
+    /// with higher write_version should supersede the one with lower
+    /// write_version.
+    pub write_version: u64,
+}
+
+/// A wrapper to future-proof ReplicaAccountInfo handling.
+/// If there were a change to the structure of ReplicaAccountInfo,
+/// there would be new enum entry for the newer version, forcing
+/// plugin implementations to handle the change.
+pub enum ReplicaAccountInfoVersions<'a> {
+    V0_0_1(&'a ReplicaAccountInfo<'a>),
+}
+
+/// Information about a transaction
+#[derive(Clone, Debug)]
+pub struct ReplicaTransactionInfo<'a> {
+    /// The first signature of the transaction, used for identifying the transaction.
+    pub signature: &'a Signature,
+
+    /// Indicates if the transaction is a simple vote transaction.
+    pub is_vote: bool,
+
+    /// The sanitized transaction.
+    pub transaction: &'a SanitizedTransaction,
+
+    /// Metadata of the transaction status.
+    pub transaction_status_meta: &'a TransactionStatusMeta,
+}
+
+/// A wrapper to future-proof ReplicaTransactionInfo handling.
+/// If there were a change to the structure of ReplicaTransactionInfo,
+/// there would be new enum entry for the newer version, forcing
+/// plugin implementations to handle the change.
+pub enum ReplicaTransactionInfoVersions<'a> {
+    V0_0_1(&'a ReplicaTransactionInfo<'a>),
+}
+
+#[derive(Clone, Debug)]
+pub struct ReplicaBlockInfo<'a> {
+    pub slot: u64,
+    pub blockhash: &'a str,
+    pub rewards: &'a [Reward],
+    pub block_time: Option<UnixTimestamp>,
+    pub block_height: Option<u64>,
+}
+
+pub enum ReplicaBlockInfoVersions<'a> {
+    V0_0_1(&'a ReplicaBlockInfo<'a>),
+}
+
+/// Errors returned by plugin calls
+#[derive(Error, Debug)]
+pub enum AccountsDbPluginError {
+    /// Error opening the configuration file; for example, when the file
+    /// is not found or when the validator process has no permission to read it.
+    #[error("Error opening config file. Error detail: ({0}).")]
+    ConfigFileOpenError(#[from] io::Error),
+
+    /// Error in reading the content of the config file or the content
+    /// is not in the expected format.
+    #[error("Error reading config file. Error message: ({msg})")]
+    ConfigFileReadError { msg: String },
+
+    /// Error when updating the account.
+    #[error("Error updating account. Error message: ({msg})")]
+    AccountsUpdateError { msg: String },
+
+    /// Error when updating the slot status
+    #[error("Error updating slot status. Error message: ({msg})")]
+    SlotStatusUpdateError { msg: String },
+
+    /// Any custom error defined by the plugin.
+    #[error("Plugin-defined custom error. Error message: ({0})")]
+    Custom(Box<dyn error::Error + Send + Sync>),
+}
+
+/// The current status of a slot
+#[derive(Debug, Clone)]
+pub enum SlotStatus {
+    /// The highest slot of the heaviest fork processed by the node. Ledger state at this slot is
+    /// not derived from a confirmed or finalized block, but if multiple forks are present, is from
+    /// the fork the validator believes is most likely to finalize.
+    Processed,
+
+    /// The highest slot having reached max vote lockout.
+    Rooted,
+
+    /// The highest slot that has been voted on by supermajority of the cluster, ie. is confirmed.
+    Confirmed,
+}
+
+impl SlotStatus {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            SlotStatus::Confirmed => "confirmed",
+            SlotStatus::Processed => "processed",
+            SlotStatus::Rooted => "rooted",
+        }
+    }
+}
+
+pub type Result<T> = std::result::Result<T, AccountsDbPluginError>;
+
+/// Defines an AccountsDb plugin, to stream data from the runtime.
+/// AccountsDb plugins must describe desired behavior for load and unload,
+/// as well as how they will handle streamed data.
+pub trait AccountsDbPlugin: Any + Send + Sync + std::fmt::Debug {
+    fn name(&self) -> &'static str;
+
+    /// The callback called when a plugin is loaded by the system,
+    /// used for doing whatever initialization is required by the plugin.
+    /// The _config_file contains the name of the
+    /// of the config file. The config must be in JSON format and
+    /// include a field "libpath" indicating the full path
+    /// name of the shared library implementing this interface.
+    fn on_load(&mut self, _config_file: &str) -> Result<()> {
+        Ok(())
+    }
+
+    /// The callback called right before a plugin is unloaded by the system
+    /// Used for doing cleanup before unload.
+    fn on_unload(&mut self) {}
+
+    /// Called when an account is updated at a slot.
+    /// When `is_startup` is true, it indicates the account is loaded from
+    /// snapshots when the validator starts up. When `is_startup` is false,
+    /// the account is updated during transaction processing.
+    #[allow(unused_variables)]
+    fn update_account(
+        &mut self,
+        account: ReplicaAccountInfoVersions,
+        slot: u64,
+        is_startup: bool,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    /// Called when all accounts are notified of during startup.
+    fn notify_end_of_startup(&mut self) -> Result<()> {
+        Ok(())
+    }
+
+    /// Called when a slot status is updated
+    #[allow(unused_variables)]
+    fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    /// Called when a transaction is updated at a slot.
+    #[allow(unused_variables)]
+    fn notify_transaction(
+        &mut self,
+        transaction: ReplicaTransactionInfoVersions,
+        slot: u64,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    /// Called when block's metadata is updated.
+    #[allow(unused_variables)]
+    fn notify_block_metadata(&mut self, blockinfo: ReplicaBlockInfoVersions) -> Result<()> {
+        Ok(())
+    }
+
+    /// Check if the plugin is interested in account data
+    /// Default is true -- if the plugin is not interested in
+    /// account data, please return false.
+    fn account_data_notifications_enabled(&self) -> bool {
+        true
+    }
+
+    /// Check if the plugin is interested in transaction data
+    /// Default is false -- if the plugin is not interested in
+    /// transaction data, please return false.
+    fn transaction_notifications_enabled(&self) -> bool {
+        false
+    }
+}

accountsdb-plugin-interface/src/lib.rs

@@ -0,0 +1 @@
+pub mod accountsdb_plugin_interface;

accountsdb-plugin-manager/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2021"
+name = "solana-accountsdb-plugin-manager"
+description = "The Solana AccountsDb plugin manager."
+version = "1.9.5"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-validator"
+
+[dependencies]
+bs58 = "0.4.0"
+crossbeam-channel = "0.5"
+libloading = "0.7.2"
+log = "0.4.11"
+serde = "1.0.130"
+serde_derive = "1.0.103"
+serde_json = "1.0.72"
+solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.9.5" }
+solana-logger = { path = "../logger", version = "=1.9.5" }
+solana-measure = { path = "../measure", version = "=1.9.5" }
+solana-metrics = { path = "../metrics", version = "=1.9.5" }
+solana-rpc = { path = "../rpc", version = "=1.9.5" }
+solana-runtime = { path = "../runtime", version = "=1.9.5" }
+solana-sdk = { path = "../sdk", version = "=1.9.5" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.9.5" }
+thiserror = "1.0.30"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-manager/src/accounts_update_notifier.rs

@@ -0,0 +1,180 @@
+/// Module responsible for notifying plugins of account updates
+use {
+    crate::accountsdb_plugin_manager::AccountsDbPluginManager,
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        ReplicaAccountInfo, ReplicaAccountInfoVersions,
+    },
+    solana_measure::measure::Measure,
+    solana_metrics::*,
+    solana_runtime::{
+        accounts_update_notifier_interface::AccountsUpdateNotifierInterface,
+        append_vec::{StoredAccountMeta, StoredMeta},
+    },
+    solana_sdk::{
+        account::{AccountSharedData, ReadableAccount},
+        clock::Slot,
+    },
+    std::sync::{Arc, RwLock},
+};
+#[derive(Debug)]
+pub(crate) struct AccountsUpdateNotifierImpl {
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+}
+
+impl AccountsUpdateNotifierInterface for AccountsUpdateNotifierImpl {
+    fn notify_account_update(&self, slot: Slot, meta: &StoredMeta, account: &AccountSharedData) {
+        if let Some(account_info) = self.accountinfo_from_shared_account_data(meta, account) {
+            self.notify_plugins_of_account_update(account_info, slot, false);
+        }
+    }
+
+    fn notify_account_restore_from_snapshot(&self, slot: Slot, account: &StoredAccountMeta) {
+        let mut measure_all = Measure::start("accountsdb-plugin-notify-account-restore-all");
+        let mut measure_copy = Measure::start("accountsdb-plugin-copy-stored-account-info");
+
+        let account = self.accountinfo_from_stored_account_meta(account);
+        measure_copy.stop();
+
+        inc_new_counter_debug!(
+            "accountsdb-plugin-copy-stored-account-info-us",
+            measure_copy.as_us() as usize,
+            100000,
+            100000
+        );
+
+        if let Some(account_info) = account {
+            self.notify_plugins_of_account_update(account_info, slot, true);
+        }
+        measure_all.stop();
+
+        inc_new_counter_debug!(
+            "accountsdb-plugin-notify-account-restore-all-us",
+            measure_all.as_us() as usize,
+            100000,
+            100000
+        );
+    }
+
+    fn notify_end_of_restore_from_snapshot(&self) {
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-end-of-restore-from-snapshot");
+            match plugin.notify_end_of_startup() {
+                Err(err) => {
+                    error!(
+                        "Failed to notify the end of restore from snapshot, error: {} to plugin {}",
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully notified the end of restore from snapshot to plugin {}",
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-end-of-restore-from-snapshot",
+                measure.as_us() as usize
+            );
+        }
+    }
+}
+
+impl AccountsUpdateNotifierImpl {
+    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
+        AccountsUpdateNotifierImpl { plugin_manager }
+    }
+
+    fn accountinfo_from_shared_account_data<'a>(
+        &self,
+        meta: &'a StoredMeta,
+        account: &'a AccountSharedData,
+    ) -> Option<ReplicaAccountInfo<'a>> {
+        Some(ReplicaAccountInfo {
+            pubkey: meta.pubkey.as_ref(),
+            lamports: account.lamports(),
+            owner: account.owner().as_ref(),
+            executable: account.executable(),
+            rent_epoch: account.rent_epoch(),
+            data: account.data(),
+            write_version: meta.write_version,
+        })
+    }
+
+    fn accountinfo_from_stored_account_meta<'a>(
+        &self,
+        stored_account_meta: &'a StoredAccountMeta,
+    ) -> Option<ReplicaAccountInfo<'a>> {
+        Some(ReplicaAccountInfo {
+            pubkey: stored_account_meta.meta.pubkey.as_ref(),
+            lamports: stored_account_meta.account_meta.lamports,
+            owner: stored_account_meta.account_meta.owner.as_ref(),
+            executable: stored_account_meta.account_meta.executable,
+            rent_epoch: stored_account_meta.account_meta.rent_epoch,
+            data: stored_account_meta.data,
+            write_version: stored_account_meta.meta.write_version,
+        })
+    }
+
+    fn notify_plugins_of_account_update(
+        &self,
+        account: ReplicaAccountInfo,
+        slot: Slot,
+        is_startup: bool,
+    ) {
+        let mut measure2 = Measure::start("accountsdb-plugin-notify_plugins_of_account_update");
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-update-account");
+            match plugin.update_account(
+                ReplicaAccountInfoVersions::V0_0_1(&account),
+                slot,
+                is_startup,
+            ) {
+                Err(err) => {
+                    error!(
+                        "Failed to update account {} at slot {}, error: {} to plugin {}",
+                        bs58::encode(account.pubkey).into_string(),
+                        slot,
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully updated account {} at slot {} to plugin {}",
+                        bs58::encode(account.pubkey).into_string(),
+                        slot,
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-update-account-us",
+                measure.as_us() as usize,
+                100000,
+                100000
+            );
+        }
+        measure2.stop();
+        inc_new_counter_debug!(
+            "accountsdb-plugin-notify_plugins_of_account_update-us",
+            measure2.as_us() as usize,
+            100000,
+            100000
+        );
+    }
+}

accountsdb-plugin-manager/src/accountsdb_plugin_manager.rs

@@ -0,0 +1,75 @@
+/// Managing the AccountsDb plugins
+use {
+    libloading::{Library, Symbol},
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::AccountsDbPlugin,
+    std::error::Error,
+};
+
+#[derive(Default, Debug)]
+pub struct AccountsDbPluginManager {
+    pub plugins: Vec<Box<dyn AccountsDbPlugin>>,
+    libs: Vec<Library>,
+}
+
+impl AccountsDbPluginManager {
+    pub fn new() -> Self {
+        AccountsDbPluginManager {
+            plugins: Vec::default(),
+            libs: Vec::default(),
+        }
+    }
+
+    /// # Safety
+    ///
+    /// This function loads the dynamically linked library specified in the path. The library
+    /// must do necessary initializations.
+    pub unsafe fn load_plugin(
+        &mut self,
+        libpath: &str,
+        config_file: &str,
+    ) -> Result<(), Box<dyn Error>> {
+        type PluginConstructor = unsafe fn() -> *mut dyn AccountsDbPlugin;
+        let lib = Library::new(libpath)?;
+        let constructor: Symbol<PluginConstructor> = lib.get(b"_create_plugin")?;
+        let plugin_raw = constructor();
+        let mut plugin = Box::from_raw(plugin_raw);
+        plugin.on_load(config_file)?;
+        self.plugins.push(plugin);
+        self.libs.push(lib);
+        Ok(())
+    }
+
+    /// Unload all plugins and loaded plugin libraries, making sure to fire
+    /// their `on_plugin_unload()` methods so they can do any necessary cleanup.
+    pub fn unload(&mut self) {
+        for mut plugin in self.plugins.drain(..) {
+            info!("Unloading plugin for {:?}", plugin.name());
+            plugin.on_unload();
+        }
+
+        for lib in self.libs.drain(..) {
+            drop(lib);
+        }
+    }
+
+    /// Check if there is any plugin interested in account data
+    pub fn account_data_notifications_enabled(&self) -> bool {
+        for plugin in &self.plugins {
+            if plugin.account_data_notifications_enabled() {
+                return true;
+            }
+        }
+        false
+    }
+
+    /// Check if there is any plugin interested in transaction data
+    pub fn transaction_notifications_enabled(&self) -> bool {
+        for plugin in &self.plugins {
+            if plugin.transaction_notifications_enabled() {
+                return true;
+            }
+        }
+        false
+    }
+}

accountsdb-plugin-manager/src/accountsdb_plugin_service.rs

@@ -0,0 +1,211 @@
+use {
+    crate::{
+        accounts_update_notifier::AccountsUpdateNotifierImpl,
+        accountsdb_plugin_manager::AccountsDbPluginManager,
+        block_metadata_notifier::BlockMetadataNotifierImpl,
+        block_metadata_notifier_interface::BlockMetadataNotifierLock,
+        slot_status_notifier::SlotStatusNotifierImpl, slot_status_observer::SlotStatusObserver,
+        transaction_notifier::TransactionNotifierImpl,
+    },
+    crossbeam_channel::Receiver,
+    log::*,
+    serde_json,
+    solana_rpc::{
+        optimistically_confirmed_bank_tracker::BankNotification,
+        transaction_notifier_interface::TransactionNotifierLock,
+    },
+    solana_runtime::accounts_update_notifier_interface::AccountsUpdateNotifier,
+    std::{
+        fs::File,
+        io::Read,
+        path::{Path, PathBuf},
+        sync::{Arc, RwLock},
+        thread,
+    },
+    thiserror::Error,
+};
+
+#[derive(Error, Debug)]
+pub enum AccountsdbPluginServiceError {
+    #[error("Cannot open the the plugin config file")]
+    CannotOpenConfigFile(String),
+
+    #[error("Cannot read the the plugin config file")]
+    CannotReadConfigFile(String),
+
+    #[error("The config file is not in a valid Json format")]
+    InvalidConfigFileFormat(String),
+
+    #[error("Plugin library path is not specified in the config file")]
+    LibPathNotSet,
+
+    #[error("Invalid plugin path")]
+    InvalidPluginPath,
+
+    #[error("Cannot load plugin shared library")]
+    PluginLoadError(String),
+}
+
+/// The service managing the AccountsDb plugin workflow.
+pub struct AccountsDbPluginService {
+    slot_status_observer: Option<SlotStatusObserver>,
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+    accounts_update_notifier: Option<AccountsUpdateNotifier>,
+    transaction_notifier: Option<TransactionNotifierLock>,
+    block_metadata_notifier: Option<BlockMetadataNotifierLock>,
+}
+
+impl AccountsDbPluginService {
+    /// Creates and returns the AccountsDbPluginService.
+    /// # Arguments
+    /// * `confirmed_bank_receiver` - The receiver for confirmed bank notification
+    /// * `accountsdb_plugin_config_file` - The config file path for the plugin. The
+    ///    config file controls the plugin responsible
+    ///    for transporting the data to external data stores. It is defined in JSON format.
+    ///    The `libpath` field should be pointed to the full path of the dynamic shared library
+    ///    (.so file) to be loaded. The shared library must implement the `AccountsDbPlugin`
+    ///    trait. And the shared library shall export a `C` function `_create_plugin` which
+    ///    shall create the implementation of `AccountsDbPlugin` and returns to the caller.
+    ///    The rest of the JSON fields' definition is up to to the concrete plugin implementation
+    ///    It is usually used to configure the connection information for the external data store.
+
+    pub fn new(
+        confirmed_bank_receiver: Receiver<BankNotification>,
+        accountsdb_plugin_config_files: &[PathBuf],
+    ) -> Result<Self, AccountsdbPluginServiceError> {
+        info!(
+            "Starting AccountsDbPluginService from config files: {:?}",
+            accountsdb_plugin_config_files
+        );
+        let mut plugin_manager = AccountsDbPluginManager::new();
+
+        for accountsdb_plugin_config_file in accountsdb_plugin_config_files {
+            Self::load_plugin(&mut plugin_manager, accountsdb_plugin_config_file)?;
+        }
+        let account_data_notifications_enabled =
+            plugin_manager.account_data_notifications_enabled();
+        let transaction_notifications_enabled = plugin_manager.transaction_notifications_enabled();
+
+        let plugin_manager = Arc::new(RwLock::new(plugin_manager));
+
+        let accounts_update_notifier: Option<AccountsUpdateNotifier> =
+            if account_data_notifications_enabled {
+                let accounts_update_notifier =
+                    AccountsUpdateNotifierImpl::new(plugin_manager.clone());
+                Some(Arc::new(RwLock::new(accounts_update_notifier)))
+            } else {
+                None
+            };
+
+        let transaction_notifier: Option<TransactionNotifierLock> =
+            if transaction_notifications_enabled {
+                let transaction_notifier = TransactionNotifierImpl::new(plugin_manager.clone());
+                Some(Arc::new(RwLock::new(transaction_notifier)))
+            } else {
+                None
+            };
+
+        let (slot_status_observer, block_metadata_notifier): (
+            Option<SlotStatusObserver>,
+            Option<BlockMetadataNotifierLock>,
+        ) = if account_data_notifications_enabled || transaction_notifications_enabled {
+            let slot_status_notifier = SlotStatusNotifierImpl::new(plugin_manager.clone());
+            let slot_status_notifier = Arc::new(RwLock::new(slot_status_notifier));
+            (
+                Some(SlotStatusObserver::new(
+                    confirmed_bank_receiver,
+                    slot_status_notifier,
+                )),
+                Some(Arc::new(RwLock::new(BlockMetadataNotifierImpl::new(
+                    plugin_manager.clone(),
+                )))),
+            )
+        } else {
+            (None, None)
+        };
+
+        info!("Started AccountsDbPluginService");
+        Ok(AccountsDbPluginService {
+            slot_status_observer,
+            plugin_manager,
+            accounts_update_notifier,
+            transaction_notifier,
+            block_metadata_notifier,
+        })
+    }
+
+    fn load_plugin(
+        plugin_manager: &mut AccountsDbPluginManager,
+        accountsdb_plugin_config_file: &Path,
+    ) -> Result<(), AccountsdbPluginServiceError> {
+        let mut file = match File::open(accountsdb_plugin_config_file) {
+            Ok(file) => file,
+            Err(err) => {
+                return Err(AccountsdbPluginServiceError::CannotOpenConfigFile(format!(
+                    "Failed to open the plugin config file {:?}, error: {:?}",
+                    accountsdb_plugin_config_file, err
+                )));
+            }
+        };
+
+        let mut contents = String::new();
+        if let Err(err) = file.read_to_string(&mut contents) {
+            return Err(AccountsdbPluginServiceError::CannotReadConfigFile(format!(
+                "Failed to read the plugin config file {:?}, error: {:?}",
+                accountsdb_plugin_config_file, err
+            )));
+        }
+
+        let result: serde_json::Value = match serde_json::from_str(&contents) {
+            Ok(value) => value,
+            Err(err) => {
+                return Err(AccountsdbPluginServiceError::InvalidConfigFileFormat(
+                    format!(
+                        "The config file {:?} is not in a valid Json format, error: {:?}",
+                        accountsdb_plugin_config_file, err
+                    ),
+                ));
+            }
+        };
+
+        let libpath = result["libpath"]
+            .as_str()
+            .ok_or(AccountsdbPluginServiceError::LibPathNotSet)?;
+        let config_file = accountsdb_plugin_config_file
+            .as_os_str()
+            .to_str()
+            .ok_or(AccountsdbPluginServiceError::InvalidPluginPath)?;
+
+        unsafe {
+            let result = plugin_manager.load_plugin(libpath, config_file);
+            if let Err(err) = result {
+                let msg = format!(
+                    "Failed to load the plugin library: {:?}, error: {:?}",
+                    libpath, err
+                );
+                return Err(AccountsdbPluginServiceError::PluginLoadError(msg));
+            }
+        }
+        Ok(())
+    }
+
+    pub fn get_accounts_update_notifier(&self) -> Option<AccountsUpdateNotifier> {
+        self.accounts_update_notifier.clone()
+    }
+
+    pub fn get_transaction_notifier(&self) -> Option<TransactionNotifierLock> {
+        self.transaction_notifier.clone()
+    }
+
+    pub fn get_block_metadata_notifier(&self) -> Option<BlockMetadataNotifierLock> {
+        self.block_metadata_notifier.clone()
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        if let Some(mut slot_status_observer) = self.slot_status_observer {
+            slot_status_observer.join()?;
+        }
+        self.plugin_manager.write().unwrap().unload();
+        Ok(())
+    }
+}

accountsdb-plugin-manager/src/block_metadata_notifier.rs

@@ -0,0 +1,105 @@
+use {
+    crate::{
+        accountsdb_plugin_manager::AccountsDbPluginManager,
+        block_metadata_notifier_interface::BlockMetadataNotifier,
+    },
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        ReplicaBlockInfo, ReplicaBlockInfoVersions,
+    },
+    solana_measure::measure::Measure,
+    solana_metrics::*,
+    solana_runtime::bank::RewardInfo,
+    solana_sdk::{clock::UnixTimestamp, pubkey::Pubkey},
+    solana_transaction_status::{Reward, Rewards},
+    std::sync::{Arc, RwLock},
+};
+
+pub(crate) struct BlockMetadataNotifierImpl {
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+}
+
+impl BlockMetadataNotifier for BlockMetadataNotifierImpl {
+    /// Notify the block metadata
+    fn notify_block_metadata(
+        &self,
+        slot: u64,
+        blockhash: &str,
+        rewards: &RwLock<Vec<(Pubkey, RewardInfo)>>,
+        block_time: Option<UnixTimestamp>,
+        block_height: Option<u64>,
+    ) {
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+        let rewards = Self::build_rewards(rewards);
+
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-update-slot");
+            let block_info =
+                Self::build_replica_block_info(slot, blockhash, &rewards, block_time, block_height);
+            let block_info = ReplicaBlockInfoVersions::V0_0_1(&block_info);
+            match plugin.notify_block_metadata(block_info) {
+                Err(err) => {
+                    error!(
+                        "Failed to update block metadata at slot {}, error: {} to plugin {}",
+                        slot,
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully updated block metadata at slot {} to plugin {}",
+                        slot,
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-update-block-metadata-us",
+                measure.as_us() as usize,
+                1000,
+                1000
+            );
+        }
+    }
+}
+
+impl BlockMetadataNotifierImpl {
+    fn build_rewards(rewards: &RwLock<Vec<(Pubkey, RewardInfo)>>) -> Rewards {
+        let rewards = rewards.read().unwrap();
+        rewards
+            .iter()
+            .map(|(pubkey, reward)| Reward {
+                pubkey: pubkey.to_string(),
+                lamports: reward.lamports,
+                post_balance: reward.post_balance,
+                reward_type: Some(reward.reward_type),
+                commission: reward.commission,
+            })
+            .collect()
+    }
+
+    fn build_replica_block_info<'a>(
+        slot: u64,
+        blockhash: &'a str,
+        rewards: &'a [Reward],
+        block_time: Option<UnixTimestamp>,
+        block_height: Option<u64>,
+    ) -> ReplicaBlockInfo<'a> {
+        ReplicaBlockInfo {
+            slot,
+            blockhash,
+            rewards,
+            block_time,
+            block_height,
+        }
+    }
+
+    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
+        Self { plugin_manager }
+    }
+}

accountsdb-plugin-manager/src/block_metadata_notifier_interface.rs

@@ -0,0 +1,20 @@
+use {
+    solana_runtime::bank::RewardInfo,
+    solana_sdk::{clock::UnixTimestamp, pubkey::Pubkey},
+    std::sync::{Arc, RwLock},
+};
+
+/// Interface for notifying block metadata changes
+pub trait BlockMetadataNotifier {
+    /// Notify the block metadata
+    fn notify_block_metadata(
+        &self,
+        slot: u64,
+        blockhash: &str,
+        rewards: &RwLock<Vec<(Pubkey, RewardInfo)>>,
+        block_time: Option<UnixTimestamp>,
+        block_height: Option<u64>,
+    );
+}
+
+pub type BlockMetadataNotifierLock = Arc<RwLock<dyn BlockMetadataNotifier + Sync + Send>>;

accountsdb-plugin-manager/src/lib.rs

@@ -0,0 +1,8 @@
+pub mod accounts_update_notifier;
+pub mod accountsdb_plugin_manager;
+pub mod accountsdb_plugin_service;
+pub mod block_metadata_notifier;
+pub mod block_metadata_notifier_interface;
+pub mod slot_status_notifier;
+pub mod slot_status_observer;
+pub mod transaction_notifier;

accountsdb-plugin-manager/src/slot_status_notifier.rs

@@ -0,0 +1,81 @@
+use {
+    crate::accountsdb_plugin_manager::AccountsDbPluginManager,
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::SlotStatus,
+    solana_measure::measure::Measure,
+    solana_metrics::*,
+    solana_sdk::clock::Slot,
+    std::sync::{Arc, RwLock},
+};
+
+pub trait SlotStatusNotifierInterface {
+    /// Notified when a slot is optimistically confirmed
+    fn notify_slot_confirmed(&self, slot: Slot, parent: Option<Slot>);
+
+    /// Notified when a slot is marked frozen.
+    fn notify_slot_processed(&self, slot: Slot, parent: Option<Slot>);
+
+    /// Notified when a slot is rooted.
+    fn notify_slot_rooted(&self, slot: Slot, parent: Option<Slot>);
+}
+
+pub type SlotStatusNotifier = Arc<RwLock<dyn SlotStatusNotifierInterface + Sync + Send>>;
+
+pub struct SlotStatusNotifierImpl {
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+}
+
+impl SlotStatusNotifierInterface for SlotStatusNotifierImpl {
+    fn notify_slot_confirmed(&self, slot: Slot, parent: Option<Slot>) {
+        self.notify_slot_status(slot, parent, SlotStatus::Confirmed);
+    }
+
+    fn notify_slot_processed(&self, slot: Slot, parent: Option<Slot>) {
+        self.notify_slot_status(slot, parent, SlotStatus::Processed);
+    }
+
+    fn notify_slot_rooted(&self, slot: Slot, parent: Option<Slot>) {
+        self.notify_slot_status(slot, parent, SlotStatus::Rooted);
+    }
+}
+
+impl SlotStatusNotifierImpl {
+    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
+        Self { plugin_manager }
+    }
+
+    pub fn notify_slot_status(&self, slot: Slot, parent: Option<Slot>, slot_status: SlotStatus) {
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-update-slot");
+            match plugin.update_slot_status(slot, parent, slot_status.clone()) {
+                Err(err) => {
+                    error!(
+                        "Failed to update slot status at slot {}, error: {} to plugin {}",
+                        slot,
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully updated slot status at slot {} to plugin {}",
+                        slot,
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-update-slot-us",
+                measure.as_us() as usize,
+                1000,
+                1000
+            );
+        }
+    }
+}

accountsdb-plugin-manager/src/slot_status_observer.rs

@@ -0,0 +1,80 @@
+use {
+    crate::slot_status_notifier::SlotStatusNotifier,
+    crossbeam_channel::Receiver,
+    solana_rpc::optimistically_confirmed_bank_tracker::BankNotification,
+    std::{
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc,
+        },
+        thread::{self, Builder, JoinHandle},
+    },
+};
+
+#[derive(Debug)]
+pub(crate) struct SlotStatusObserver {
+    bank_notification_receiver_service: Option<JoinHandle<()>>,
+    exit_updated_slot_server: Arc<AtomicBool>,
+}
+
+impl SlotStatusObserver {
+    pub fn new(
+        bank_notification_receiver: Receiver<BankNotification>,
+        slot_status_notifier: SlotStatusNotifier,
+    ) -> Self {
+        let exit_updated_slot_server = Arc::new(AtomicBool::new(false));
+
+        Self {
+            bank_notification_receiver_service: Some(Self::run_bank_notification_receiver(
+                bank_notification_receiver,
+                exit_updated_slot_server.clone(),
+                slot_status_notifier,
+            )),
+            exit_updated_slot_server,
+        }
+    }
+
+    pub fn join(&mut self) -> thread::Result<()> {
+        self.exit_updated_slot_server.store(true, Ordering::Relaxed);
+        self.bank_notification_receiver_service
+            .take()
+            .map(JoinHandle::join)
+            .unwrap()
+    }
+
+    fn run_bank_notification_receiver(
+        bank_notification_receiver: Receiver<BankNotification>,
+        exit: Arc<AtomicBool>,
+        slot_status_notifier: SlotStatusNotifier,
+    ) -> JoinHandle<()> {
+        Builder::new()
+            .name("bank_notification_receiver".to_string())
+            .spawn(move || {
+                while !exit.load(Ordering::Relaxed) {
+                    if let Ok(slot) = bank_notification_receiver.recv() {
+                        match slot {
+                            BankNotification::OptimisticallyConfirmed(slot) => {
+                                slot_status_notifier
+                                    .read()
+                                    .unwrap()
+                                    .notify_slot_confirmed(slot, None);
+                            }
+                            BankNotification::Frozen(bank) => {
+                                slot_status_notifier
+                                    .read()
+                                    .unwrap()
+                                    .notify_slot_processed(bank.slot(), Some(bank.parent_slot()));
+                            }
+                            BankNotification::Root(bank) => {
+                                slot_status_notifier
+                                    .read()
+                                    .unwrap()
+                                    .notify_slot_rooted(bank.slot(), Some(bank.parent_slot()));
+                            }
+                        }
+                    }
+                }
+            })
+            .unwrap()
+    }
+}

accountsdb-plugin-manager/src/transaction_notifier.rs

@@ -0,0 +1,92 @@
+/// Module responsible for notifying plugins of transactions
+use {
+    crate::accountsdb_plugin_manager::AccountsDbPluginManager,
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        ReplicaTransactionInfo, ReplicaTransactionInfoVersions,
+    },
+    solana_measure::measure::Measure,
+    solana_metrics::*,
+    solana_rpc::transaction_notifier_interface::TransactionNotifier,
+    solana_sdk::{clock::Slot, signature::Signature, transaction::SanitizedTransaction},
+    solana_transaction_status::TransactionStatusMeta,
+    std::sync::{Arc, RwLock},
+};
+
+/// This implementation of TransactionNotifier is passed to the rpc's TransactionStatusService
+/// at the validator startup. TransactionStatusService invokes the notify_transaction method
+/// for new transactions. The implementation in turn invokes the notify_transaction of each
+/// plugin enabled with transaction notification managed by the AccountsDbPluginManager.
+pub(crate) struct TransactionNotifierImpl {
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+}
+
+impl TransactionNotifier for TransactionNotifierImpl {
+    fn notify_transaction(
+        &self,
+        slot: Slot,
+        signature: &Signature,
+        transaction_status_meta: &TransactionStatusMeta,
+        transaction: &SanitizedTransaction,
+    ) {
+        let mut measure = Measure::start("accountsdb-plugin-notify_plugins_of_transaction_info");
+        let transaction_log_info =
+            Self::build_replica_transaction_info(signature, transaction_status_meta, transaction);
+
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+
+        for plugin in plugin_manager.plugins.iter_mut() {
+            if !plugin.transaction_notifications_enabled() {
+                continue;
+            }
+            match plugin.notify_transaction(
+                ReplicaTransactionInfoVersions::V0_0_1(&transaction_log_info),
+                slot,
+            ) {
+                Err(err) => {
+                    error!(
+                        "Failed to notify transaction, error: ({}) to plugin {}",
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully notified transaction to plugin {}",
+                        plugin.name()
+                    );
+                }
+            }
+        }
+        measure.stop();
+        inc_new_counter_debug!(
+            "accountsdb-plugin-notify_plugins_of_transaction_info-us",
+            measure.as_us() as usize,
+            10000,
+            10000
+        );
+    }
+}
+
+impl TransactionNotifierImpl {
+    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
+        Self { plugin_manager }
+    }
+
+    fn build_replica_transaction_info<'a>(
+        signature: &'a Signature,
+        transaction_status_meta: &'a TransactionStatusMeta,
+        transaction: &'a SanitizedTransaction,
+    ) -> ReplicaTransactionInfo<'a> {
+        ReplicaTransactionInfo {
+            signature,
+            is_vote: transaction.is_simple_vote_transaction(),
+            transaction,
+            transaction_status_meta,
+        }
+    }
+}

accountsdb-plugin-postgres/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2021"
+name = "solana-accountsdb-plugin-postgres"
+description = "The Solana AccountsDb plugin for PostgreSQL database."
+version = "1.9.5"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-validator"
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+bs58 = "0.4.0"
+chrono = { version = "0.4.11", features = ["serde"] }
+crossbeam-channel = "0.5"
+log = "0.4.14"
+postgres = { version = "0.19.2", features = ["with-chrono-0_4"] }
+postgres-types = { version = "0.2.2", features = ["derive"] }
+serde = "1.0.130"
+serde_derive = "1.0.103"
+serde_json = "1.0.72"
+solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.9.5" }
+solana-logger = { path = "../logger", version = "=1.9.5" }
+solana-measure = { path = "../measure", version = "=1.9.5" }
+solana-metrics = { path = "../metrics", version = "=1.9.5" }
+solana-runtime = { path = "../runtime", version = "=1.9.5" }
+solana-sdk = { path = "../sdk", version = "=1.9.5" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.9.5" }
+thiserror = "1.0.30"
+tokio-postgres = "0.7.4"
+
+[dev-dependencies]
+solana-account-decoder = { path = "../account-decoder", version = "=1.9.5" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-postgres/README.md

@@ -0,0 +1,5 @@
+This is an example implementing the AccountsDb plugin for PostgreSQL database.
+Please see the `src/accountsdb_plugin_postgres.rs` for the format of the plugin's configuration file.
+
+To create the schema objects for the database, please use `scripts/create_schema.sql`.
+`scripts/drop_schema.sql` can be used to tear down the schema objects.

accountsdb-plugin-postgres/scripts/create_schema.sql

@@ -0,0 +1,201 @@
+/**
+ * This plugin implementation for PostgreSQL requires the following tables
+ */
+-- The table storing accounts
+
+
+CREATE TABLE account (
+    pubkey BYTEA PRIMARY KEY,
+    owner BYTEA,
+    lamports BIGINT NOT NULL,
+    slot BIGINT NOT NULL,
+    executable BOOL NOT NULL,
+    rent_epoch BIGINT NOT NULL,
+    data BYTEA,
+    write_version BIGINT NOT NULL,
+    updated_on TIMESTAMP NOT NULL
+);
+
+-- The table storing slot information
+CREATE TABLE slot (
+    slot BIGINT PRIMARY KEY,
+    parent BIGINT,
+    status VARCHAR(16) NOT NULL,
+    updated_on TIMESTAMP NOT NULL
+);
+
+-- Types for Transactions
+
+Create TYPE "TransactionErrorCode" AS ENUM (
+    'AccountInUse',
+    'AccountLoadedTwice',
+    'AccountNotFound',
+    'ProgramAccountNotFound',
+    'InsufficientFundsForFee',
+    'InvalidAccountForFee',
+    'AlreadyProcessed',
+    'BlockhashNotFound',
+    'InstructionError',
+    'CallChainTooDeep',
+    'MissingSignatureForFee',
+    'InvalidAccountIndex',
+    'SignatureFailure',
+    'InvalidProgramForExecution',
+    'SanitizeFailure',
+    'ClusterMaintenance',
+    'AccountBorrowOutstanding',
+    'WouldExceedMaxAccountCostLimit',
+    'WouldExceedMaxBlockCostLimit',
+    'UnsupportedVersion',
+    'InvalidWritableAccount',
+    'WouldExceedMaxAccountDataCostLimit',
+    'TooManyAccountLocks',
+    'AddressLookupTableNotFound',
+    'InvalidAddressLookupTableOwner',
+    'InvalidAddressLookupTableData',
+    'InvalidAddressLookupTableIndex',
+    'InvalidRentPayingAccount'
+);
+
+CREATE TYPE "TransactionError" AS (
+    error_code "TransactionErrorCode",
+    error_detail VARCHAR(256)
+);
+
+CREATE TYPE "CompiledInstruction" AS (
+    program_id_index SMALLINT,
+    accounts SMALLINT[],
+    data BYTEA
+);
+
+CREATE TYPE "InnerInstructions" AS (
+    index SMALLINT,
+    instructions "CompiledInstruction"[]
+);
+
+CREATE TYPE "TransactionTokenBalance" AS (
+    account_index SMALLINT,
+    mint VARCHAR(44),
+    ui_token_amount DOUBLE PRECISION,
+    owner VARCHAR(44)
+);
+
+Create TYPE "RewardType" AS ENUM (
+    'Fee',
+    'Rent',
+    'Staking',
+    'Voting'
+);
+
+CREATE TYPE "Reward" AS (
+    pubkey VARCHAR(44),
+    lamports BIGINT,
+    post_balance BIGINT,
+    reward_type "RewardType",
+    commission SMALLINT
+);
+
+CREATE TYPE "TransactionStatusMeta" AS (
+    error "TransactionError",
+    fee BIGINT,
+    pre_balances BIGINT[],
+    post_balances BIGINT[],
+    inner_instructions "InnerInstructions"[],
+    log_messages TEXT[],
+    pre_token_balances "TransactionTokenBalance"[],
+    post_token_balances "TransactionTokenBalance"[],
+    rewards "Reward"[]
+);
+
+CREATE TYPE "TransactionMessageHeader" AS (
+    num_required_signatures SMALLINT,
+    num_readonly_signed_accounts SMALLINT,
+    num_readonly_unsigned_accounts SMALLINT
+);
+
+CREATE TYPE "TransactionMessage" AS (
+    header "TransactionMessageHeader",
+    account_keys BYTEA[],
+    recent_blockhash BYTEA,
+    instructions "CompiledInstruction"[]
+);
+
+CREATE TYPE "TransactionMessageAddressTableLookup" AS (
+    account_key BYTEA,
+    writable_indexes SMALLINT[],
+    readonly_indexes SMALLINT[]
+);
+
+CREATE TYPE "TransactionMessageV0" AS (
+    header "TransactionMessageHeader",
+    account_keys BYTEA[],
+    recent_blockhash BYTEA,
+    instructions "CompiledInstruction"[],
+    address_table_lookups "TransactionMessageAddressTableLookup"[]
+);
+
+CREATE TYPE "LoadedAddresses" AS (
+    writable BYTEA[],
+    readonly BYTEA[]
+);
+
+CREATE TYPE "LoadedMessageV0" AS (
+    message "TransactionMessageV0",
+    loaded_addresses "LoadedAddresses"
+);
+
+-- The table storing transactions
+CREATE TABLE transaction (
+    slot BIGINT NOT NULL,
+    signature BYTEA NOT NULL,
+    is_vote BOOL NOT NULL,
+    message_type SMALLINT, -- 0: legacy, 1: v0 message
+    legacy_message "TransactionMessage",
+    v0_loaded_message "LoadedMessageV0",
+    signatures BYTEA[],
+    message_hash BYTEA,
+    meta "TransactionStatusMeta",
+    updated_on TIMESTAMP NOT NULL,
+    CONSTRAINT transaction_pk PRIMARY KEY (slot, signature)
+);
+
+-- The table storing block metadata
+CREATE TABLE block (
+    slot BIGINT PRIMARY KEY,
+    blockhash VARCHAR(44),
+    rewards "Reward"[],
+    block_time BIGINT,
+    block_height BIGINT,
+    updated_on TIMESTAMP NOT NULL
+);
+
+/**
+ * The following is for keeping historical data for accounts and is not required for plugin to work.
+ */
+-- The table storing historical data for accounts
+CREATE TABLE account_audit (
+    pubkey BYTEA,
+    owner BYTEA,
+    lamports BIGINT NOT NULL,
+    slot BIGINT NOT NULL,
+    executable BOOL NOT NULL,
+    rent_epoch BIGINT NOT NULL,
+    data BYTEA,
+    write_version BIGINT NOT NULL,
+    updated_on TIMESTAMP NOT NULL
+);
+
+CREATE INDEX account_audit_account_key ON  account_audit (pubkey, write_version);
+
+CREATE FUNCTION audit_account_update() RETURNS trigger AS $audit_account_update$
+    BEGIN
+		INSERT INTO account_audit (pubkey, owner, lamports, slot, executable, rent_epoch, data, write_version, updated_on)
+            VALUES (OLD.pubkey, OLD.owner, OLD.lamports, OLD.slot,
+                    OLD.executable, OLD.rent_epoch, OLD.data, OLD.write_version, OLD.updated_on);
+        RETURN NEW;
+    END;
+
+$audit_account_update$ LANGUAGE plpgsql;
+
+CREATE TRIGGER account_update_trigger AFTER UPDATE OR DELETE ON account
+    FOR EACH ROW EXECUTE PROCEDURE audit_account_update();

accountsdb-plugin-postgres/scripts/drop_schema.sql

@@ -0,0 +1,26 @@
+/**
+ * Script for cleaning up the schema for PostgreSQL used for the AccountsDb plugin.
+ */
+
+DROP TRIGGER account_update_trigger ON account;
+DROP FUNCTION audit_account_update;
+DROP TABLE account_audit;
+DROP TABLE account;
+DROP TABLE slot;
+DROP TABLE transaction;
+DROP TABLE block;
+
+DROP TYPE "TransactionError" CASCADE;
+DROP TYPE "TransactionErrorCode" CASCADE;
+DROP TYPE "LoadedMessageV0" CASCADE;
+DROP TYPE "LoadedAddresses" CASCADE;
+DROP TYPE "TransactionMessageV0" CASCADE;
+DROP TYPE "TransactionMessage" CASCADE;
+DROP TYPE "TransactionMessageHeader" CASCADE;
+DROP TYPE "TransactionMessageAddressTableLookup" CASCADE;
+DROP TYPE "TransactionStatusMeta" CASCADE;
+DROP TYPE "RewardType" CASCADE;
+DROP TYPE "Reward" CASCADE;
+DROP TYPE "TransactionTokenBalance" CASCADE;
+DROP TYPE "InnerInstructions" CASCADE;
+DROP TYPE "CompiledInstruction" CASCADE;

accountsdb-plugin-postgres/scripts/postgresql.conf

@@ -0,0 +1,802 @@
+# This a reference configuration file for the PostgreSQL database version 14.
+
+# -----------------------------
+# PostgreSQL configuration file
+# -----------------------------
+#
+# This file consists of lines of the form:
+#
+#   name = value
+#
+# (The "=" is optional.)  Whitespace may be used.  Comments are introduced with
+# "#" anywhere on a line.  The complete list of parameter names and allowed
+# values can be found in the PostgreSQL documentation.
+#
+# The commented-out settings shown in this file represent the default values.
+# Re-commenting a setting is NOT sufficient to revert it to the default value;
+# you need to reload the server.
+#
+# This file is read on server startup and when the server receives a SIGHUP
+# signal.  If you edit the file on a running system, you have to SIGHUP the
+# server for the changes to take effect, run "pg_ctl reload", or execute
+# "SELECT pg_reload_conf()".  Some parameters, which are marked below,
+# require a server shutdown and restart to take effect.
+#
+# Any parameter can also be given as a command-line option to the server, e.g.,
+# "postgres -c log_connections=on".  Some parameters can be changed at run time
+# with the "SET" SQL command.
+#
+# Memory units:  B  = bytes            Time units:  us  = microseconds
+#                kB = kilobytes                     ms  = milliseconds
+#                MB = megabytes                     s   = seconds
+#                GB = gigabytes                     min = minutes
+#                TB = terabytes                     h   = hours
+#                                                   d   = days
+
+
+#------------------------------------------------------------------------------
+# FILE LOCATIONS
+#------------------------------------------------------------------------------
+
+# The default values of these variables are driven from the -D command-line
+# option or PGDATA environment variable, represented here as ConfigDir.
+
+data_directory = '/var/lib/postgresql/14/main'		# use data in another directory
+					# (change requires restart)
+
+hba_file = '/etc/postgresql/14/main/pg_hba.conf'	# host-based authentication file
+					# (change requires restart)
+ident_file = '/etc/postgresql/14/main/pg_ident.conf'	# ident configuration file
+					# (change requires restart)
+
+# If external_pid_file is not explicitly set, no extra PID file is written.
+external_pid_file = '/var/run/postgresql/14-main.pid'			# write an extra PID file
+					# (change requires restart)
+
+
+#------------------------------------------------------------------------------
+# CONNECTIONS AND AUTHENTICATION
+#------------------------------------------------------------------------------
+
+# - Connection Settings -
+
+#listen_addresses = 'localhost'		# what IP address(es) to listen on;
+					# comma-separated list of addresses;
+					# defaults to 'localhost'; use '*' for all
+					# (change requires restart)
+listen_addresses = '*'
+port = 5433				# (change requires restart)
+max_connections = 200			# (change requires restart)
+#superuser_reserved_connections = 3	# (change requires restart)
+unix_socket_directories = '/var/run/postgresql'	# comma-separated list of directories
+					# (change requires restart)
+#unix_socket_group = ''			# (change requires restart)
+#unix_socket_permissions = 0777		# begin with 0 to use octal notation
+					# (change requires restart)
+#bonjour = off				# advertise server via Bonjour
+					# (change requires restart)
+#bonjour_name = ''			# defaults to the computer name
+					# (change requires restart)
+
+# - TCP settings -
+# see "man tcp" for details
+
+#tcp_keepalives_idle = 0		# TCP_KEEPIDLE, in seconds;
+					# 0 selects the system default
+#tcp_keepalives_interval = 0		# TCP_KEEPINTVL, in seconds;
+					# 0 selects the system default
+#tcp_keepalives_count = 0		# TCP_KEEPCNT;
+					# 0 selects the system default
+#tcp_user_timeout = 0			# TCP_USER_TIMEOUT, in milliseconds;
+					# 0 selects the system default
+
+#client_connection_check_interval = 0	# time between checks for client
+					# disconnection while running queries;
+					# 0 for never
+
+# - Authentication -
+
+#authentication_timeout = 1min		# 1s-600s
+#password_encryption = scram-sha-256	# scram-sha-256 or md5
+#db_user_namespace = off
+
+# GSSAPI using Kerberos
+#krb_server_keyfile = 'FILE:${sysconfdir}/krb5.keytab'
+#krb_caseins_users = off
+
+# - SSL -
+
+ssl = on
+#ssl_ca_file = ''
+ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem'
+#ssl_crl_file = ''
+#ssl_crl_dir = ''
+ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key'
+#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers
+#ssl_prefer_server_ciphers = on
+#ssl_ecdh_curve = 'prime256v1'
+#ssl_min_protocol_version = 'TLSv1.2'
+#ssl_max_protocol_version = ''
+#ssl_dh_params_file = ''
+#ssl_passphrase_command = ''
+#ssl_passphrase_command_supports_reload = off
+
+
+#------------------------------------------------------------------------------
+# RESOURCE USAGE (except WAL)
+#------------------------------------------------------------------------------
+
+# - Memory -
+
+shared_buffers = 1GB			# min 128kB
+					# (change requires restart)
+#huge_pages = try			# on, off, or try
+					# (change requires restart)
+#huge_page_size = 0			# zero for system default
+					# (change requires restart)
+#temp_buffers = 8MB			# min 800kB
+#max_prepared_transactions = 0		# zero disables the feature
+					# (change requires restart)
+# Caution: it is not advisable to set max_prepared_transactions nonzero unless
+# you actively intend to use prepared transactions.
+#work_mem = 4MB				# min 64kB
+#hash_mem_multiplier = 1.0		# 1-1000.0 multiplier on hash table work_mem
+#maintenance_work_mem = 64MB		# min 1MB
+#autovacuum_work_mem = -1		# min 1MB, or -1 to use maintenance_work_mem
+#logical_decoding_work_mem = 64MB	# min 64kB
+#max_stack_depth = 2MB			# min 100kB
+#shared_memory_type = mmap		# the default is the first option
+					# supported by the operating system:
+					#   mmap
+					#   sysv
+					#   windows
+					# (change requires restart)
+dynamic_shared_memory_type = posix	# the default is the first option
+					# supported by the operating system:
+					#   posix
+					#   sysv
+					#   windows
+					#   mmap
+					# (change requires restart)
+#min_dynamic_shared_memory = 0MB	# (change requires restart)
+
+# - Disk -
+
+#temp_file_limit = -1			# limits per-process temp file space
+					# in kilobytes, or -1 for no limit
+
+# - Kernel Resources -
+
+#max_files_per_process = 1000		# min 64
+					# (change requires restart)
+
+# - Cost-Based Vacuum Delay -
+
+#vacuum_cost_delay = 0			# 0-100 milliseconds (0 disables)
+#vacuum_cost_page_hit = 1		# 0-10000 credits
+#vacuum_cost_page_miss = 2		# 0-10000 credits
+#vacuum_cost_page_dirty = 20		# 0-10000 credits
+#vacuum_cost_limit = 200		# 1-10000 credits
+
+# - Background Writer -
+
+#bgwriter_delay = 200ms			# 10-10000ms between rounds
+#bgwriter_lru_maxpages = 100		# max buffers written/round, 0 disables
+#bgwriter_lru_multiplier = 2.0		# 0-10.0 multiplier on buffers scanned/round
+#bgwriter_flush_after = 512kB		# measured in pages, 0 disables
+
+# - Asynchronous Behavior -
+
+#backend_flush_after = 0		# measured in pages, 0 disables
+effective_io_concurrency = 1000		# 1-1000; 0 disables prefetching
+#maintenance_io_concurrency = 10	# 1-1000; 0 disables prefetching
+#max_worker_processes = 8		# (change requires restart)
+#max_parallel_workers_per_gather = 2	# taken from max_parallel_workers
+#max_parallel_maintenance_workers = 2	# taken from max_parallel_workers
+#max_parallel_workers = 8		# maximum number of max_worker_processes that
+					# can be used in parallel operations
+#parallel_leader_participation = on
+#old_snapshot_threshold = -1		# 1min-60d; -1 disables; 0 is immediate
+					# (change requires restart)
+
+
+#------------------------------------------------------------------------------
+# WRITE-AHEAD LOG
+#------------------------------------------------------------------------------
+
+# - Settings -
+
+wal_level = minimal 			# minimal, replica, or logical
+					# (change requires restart)
+fsync = off				# flush data to disk for crash safety
+					# (turning this off can cause
+					# unrecoverable data corruption)
+synchronous_commit = off		# synchronization level;
+					# off, local, remote_write, remote_apply, or on
+#wal_sync_method = fsync		# the default is the first option
+					# supported by the operating system:
+					#   open_datasync
+					#   fdatasync (default on Linux and FreeBSD)
+					#   fsync
+					#   fsync_writethrough
+					#   open_sync
+full_page_writes = off			# recover from partial page writes
+#wal_log_hints = off			# also do full page writes of non-critical updates
+					# (change requires restart)
+#wal_compression = off			# enable compression of full-page writes
+#wal_init_zero = on			# zero-fill new WAL files
+#wal_recycle = on			# recycle WAL files
+#wal_buffers = -1			# min 32kB, -1 sets based on shared_buffers
+					# (change requires restart)
+#wal_writer_delay = 200ms		# 1-10000 milliseconds
+#wal_writer_flush_after = 1MB		# measured in pages, 0 disables
+#wal_skip_threshold = 2MB
+
+#commit_delay = 0			# range 0-100000, in microseconds
+#commit_siblings = 5			# range 1-1000
+
+# - Checkpoints -
+
+#checkpoint_timeout = 5min		# range 30s-1d
+#checkpoint_completion_target = 0.9	# checkpoint target duration, 0.0 - 1.0
+#checkpoint_flush_after = 256kB		# measured in pages, 0 disables
+#checkpoint_warning = 30s		# 0 disables
+max_wal_size = 1GB
+min_wal_size = 80MB
+
+# - Archiving -
+
+#archive_mode = off		# enables archiving; off, on, or always
+				# (change requires restart)
+#archive_command = ''		# command to use to archive a logfile segment
+				# placeholders: %p = path of file to archive
+				#               %f = file name only
+				# e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f'
+#archive_timeout = 0		# force a logfile segment switch after this
+				# number of seconds; 0 disables
+
+# - Archive Recovery -
+
+# These are only used in recovery mode.
+
+#restore_command = ''		# command to use to restore an archived logfile segment
+				# placeholders: %p = path of file to restore
+				#               %f = file name only
+				# e.g. 'cp /mnt/server/archivedir/%f %p'
+#archive_cleanup_command = ''	# command to execute at every restartpoint
+#recovery_end_command = ''	# command to execute at completion of recovery
+
+# - Recovery Target -
+
+# Set these only when performing a targeted recovery.
+
+#recovery_target = ''		# 'immediate' to end recovery as soon as a
+                                # consistent state is reached
+				# (change requires restart)
+#recovery_target_name = ''	# the named restore point to which recovery will proceed
+				# (change requires restart)
+#recovery_target_time = ''	# the time stamp up to which recovery will proceed
+				# (change requires restart)
+#recovery_target_xid = ''	# the transaction ID up to which recovery will proceed
+				# (change requires restart)
+#recovery_target_lsn = ''	# the WAL LSN up to which recovery will proceed
+				# (change requires restart)
+#recovery_target_inclusive = on # Specifies whether to stop:
+				# just after the specified recovery target (on)
+				# just before the recovery target (off)
+				# (change requires restart)
+#recovery_target_timeline = 'latest'	# 'current', 'latest', or timeline ID
+				# (change requires restart)
+#recovery_target_action = 'pause'	# 'pause', 'promote', 'shutdown'
+				# (change requires restart)
+
+
+#------------------------------------------------------------------------------
+# REPLICATION
+#------------------------------------------------------------------------------
+
+# - Sending Servers -
+
+# Set these on the primary and on any standby that will send replication data.
+
+max_wal_senders = 0		# max number of walsender processes
+				# (change requires restart)
+#max_replication_slots = 10	# max number of replication slots
+				# (change requires restart)
+#wal_keep_size = 0		# in megabytes; 0 disables
+#max_slot_wal_keep_size = -1	# in megabytes; -1 disables
+#wal_sender_timeout = 60s	# in milliseconds; 0 disables
+#track_commit_timestamp = off	# collect timestamp of transaction commit
+				# (change requires restart)
+
+# - Primary Server -
+
+# These settings are ignored on a standby server.
+
+#synchronous_standby_names = ''	# standby servers that provide sync rep
+				# method to choose sync standbys, number of sync standbys,
+				# and comma-separated list of application_name
+				# from standby(s); '*' = all
+#vacuum_defer_cleanup_age = 0	# number of xacts by which cleanup is delayed
+
+# - Standby Servers -
+
+# These settings are ignored on a primary server.
+
+#primary_conninfo = ''			# connection string to sending server
+#primary_slot_name = ''			# replication slot on sending server
+#promote_trigger_file = ''		# file name whose presence ends recovery
+#hot_standby = on			# "off" disallows queries during recovery
+					# (change requires restart)
+#max_standby_archive_delay = 30s	# max delay before canceling queries
+					# when reading WAL from archive;
+					# -1 allows indefinite delay
+#max_standby_streaming_delay = 30s	# max delay before canceling queries
+					# when reading streaming WAL;
+					# -1 allows indefinite delay
+#wal_receiver_create_temp_slot = off	# create temp slot if primary_slot_name
+					# is not set
+#wal_receiver_status_interval = 10s	# send replies at least this often
+					# 0 disables
+#hot_standby_feedback = off		# send info from standby to prevent
+					# query conflicts
+#wal_receiver_timeout = 60s		# time that receiver waits for
+					# communication from primary
+					# in milliseconds; 0 disables
+#wal_retrieve_retry_interval = 5s	# time to wait before retrying to
+					# retrieve WAL after a failed attempt
+#recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
+
+# - Subscribers -
+
+# These settings are ignored on a publisher.
+
+#max_logical_replication_workers = 4	# taken from max_worker_processes
+					# (change requires restart)
+#max_sync_workers_per_subscription = 2	# taken from max_logical_replication_workers
+
+
+#------------------------------------------------------------------------------
+# QUERY TUNING
+#------------------------------------------------------------------------------
+
+# - Planner Method Configuration -
+
+#enable_async_append = on
+#enable_bitmapscan = on
+#enable_gathermerge = on
+#enable_hashagg = on
+#enable_hashjoin = on
+#enable_incremental_sort = on
+#enable_indexscan = on
+#enable_indexonlyscan = on
+#enable_material = on
+#enable_memoize = on
+#enable_mergejoin = on
+#enable_nestloop = on
+#enable_parallel_append = on
+#enable_parallel_hash = on
+#enable_partition_pruning = on
+#enable_partitionwise_join = off
+#enable_partitionwise_aggregate = off
+#enable_seqscan = on
+#enable_sort = on
+#enable_tidscan = on
+
+# - Planner Cost Constants -
+
+#seq_page_cost = 1.0			# measured on an arbitrary scale
+#random_page_cost = 4.0			# same scale as above
+#cpu_tuple_cost = 0.01			# same scale as above
+#cpu_index_tuple_cost = 0.005		# same scale as above
+#cpu_operator_cost = 0.0025		# same scale as above
+#parallel_setup_cost = 1000.0	# same scale as above
+#parallel_tuple_cost = 0.1		# same scale as above
+#min_parallel_table_scan_size = 8MB
+#min_parallel_index_scan_size = 512kB
+#effective_cache_size = 4GB
+
+#jit_above_cost = 100000		# perform JIT compilation if available
+					# and query more expensive than this;
+					# -1 disables
+#jit_inline_above_cost = 500000		# inline small functions if query is
+					# more expensive than this; -1 disables
+#jit_optimize_above_cost = 500000	# use expensive JIT optimizations if
+					# query is more expensive than this;
+					# -1 disables
+
+# - Genetic Query Optimizer -
+
+#geqo = on
+#geqo_threshold = 12
+#geqo_effort = 5			# range 1-10
+#geqo_pool_size = 0			# selects default based on effort
+#geqo_generations = 0			# selects default based on effort
+#geqo_selection_bias = 2.0		# range 1.5-2.0
+#geqo_seed = 0.0			# range 0.0-1.0
+
+# - Other Planner Options -
+
+#default_statistics_target = 100	# range 1-10000
+#constraint_exclusion = partition	# on, off, or partition
+#cursor_tuple_fraction = 0.1		# range 0.0-1.0
+#from_collapse_limit = 8
+#jit = on				# allow JIT compilation
+#join_collapse_limit = 8		# 1 disables collapsing of explicit
+					# JOIN clauses
+#plan_cache_mode = auto			# auto, force_generic_plan or
+					# force_custom_plan
+
+
+#------------------------------------------------------------------------------
+# REPORTING AND LOGGING
+#------------------------------------------------------------------------------
+
+# - Where to Log -
+
+#log_destination = 'stderr'		# Valid values are combinations of
+					# stderr, csvlog, syslog, and eventlog,
+					# depending on platform.  csvlog
+					# requires logging_collector to be on.
+
+# This is used when logging to stderr:
+#logging_collector = off		# Enable capturing of stderr and csvlog
+					# into log files. Required to be on for
+					# csvlogs.
+					# (change requires restart)
+
+# These are only used if logging_collector is on:
+#log_directory = 'log'			# directory where log files are written,
+					# can be absolute or relative to PGDATA
+#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'	# log file name pattern,
+					# can include strftime() escapes
+#log_file_mode = 0600			# creation mode for log files,
+					# begin with 0 to use octal notation
+#log_rotation_age = 1d			# Automatic rotation of logfiles will
+					# happen after that time.  0 disables.
+#log_rotation_size = 10MB		# Automatic rotation of logfiles will
+					# happen after that much log output.
+					# 0 disables.
+#log_truncate_on_rotation = off		# If on, an existing log file with the
+					# same name as the new log file will be
+					# truncated rather than appended to.
+					# But such truncation only occurs on
+					# time-driven rotation, not on restarts
+					# or size-driven rotation.  Default is
+					# off, meaning append to existing files
+					# in all cases.
+
+# These are relevant when logging to syslog:
+#syslog_facility = 'LOCAL0'
+#syslog_ident = 'postgres'
+#syslog_sequence_numbers = on
+#syslog_split_messages = on
+
+# This is only relevant when logging to eventlog (Windows):
+# (change requires restart)
+#event_source = 'PostgreSQL'
+
+# - When to Log -
+
+#log_min_messages = warning		# values in order of decreasing detail:
+					#   debug5
+					#   debug4
+					#   debug3
+					#   debug2
+					#   debug1
+					#   info
+					#   notice
+					#   warning
+					#   error
+					#   log
+					#   fatal
+					#   panic
+
+#log_min_error_statement = error	# values in order of decreasing detail:
+					#   debug5
+					#   debug4
+					#   debug3
+					#   debug2
+					#   debug1
+					#   info
+					#   notice
+					#   warning
+					#   error
+					#   log
+					#   fatal
+					#   panic (effectively off)
+
+#log_min_duration_statement = -1	# -1 is disabled, 0 logs all statements
+					# and their durations, > 0 logs only
+					# statements running at least this number
+					# of milliseconds
+
+#log_min_duration_sample = -1		# -1 is disabled, 0 logs a sample of statements
+					# and their durations, > 0 logs only a sample of
+					# statements running at least this number
+					# of milliseconds;
+					# sample fraction is determined by log_statement_sample_rate
+
+#log_statement_sample_rate = 1.0	# fraction of logged statements exceeding
+					# log_min_duration_sample to be logged;
+					# 1.0 logs all such statements, 0.0 never logs
+
+
+#log_transaction_sample_rate = 0.0	# fraction of transactions whose statements
+					# are logged regardless of their duration; 1.0 logs all
+					# statements from all transactions, 0.0 never logs
+
+# - What to Log -
+
+#debug_print_parse = off
+#debug_print_rewritten = off
+#debug_print_plan = off
+#debug_pretty_print = on
+#log_autovacuum_min_duration = -1	# log autovacuum activity;
+					# -1 disables, 0 logs all actions and
+					# their durations, > 0 logs only
+					# actions running at least this number
+					# of milliseconds.
+#log_checkpoints = off
+#log_connections = off
+#log_disconnections = off
+#log_duration = off
+#log_error_verbosity = default		# terse, default, or verbose messages
+#log_hostname = off
+log_line_prefix = '%m [%p] %q%u@%d '		# special values:
+					#   %a = application name
+					#   %u = user name
+					#   %d = database name
+					#   %r = remote host and port
+					#   %h = remote host
+					#   %b = backend type
+					#   %p = process ID
+					#   %P = process ID of parallel group leader
+					#   %t = timestamp without milliseconds
+					#   %m = timestamp with milliseconds
+					#   %n = timestamp with milliseconds (as a Unix epoch)
+					#   %Q = query ID (0 if none or not computed)
+					#   %i = command tag
+					#   %e = SQL state
+					#   %c = session ID
+					#   %l = session line number
+					#   %s = session start timestamp
+					#   %v = virtual transaction ID
+					#   %x = transaction ID (0 if none)
+					#   %q = stop here in non-session
+					#        processes
+					#   %% = '%'
+					# e.g. '<%u%%%d> '
+#log_lock_waits = off			# log lock waits >= deadlock_timeout
+#log_recovery_conflict_waits = off	# log standby recovery conflict waits
+					# >= deadlock_timeout
+#log_parameter_max_length = -1		# when logging statements, limit logged
+					# bind-parameter values to N bytes;
+					# -1 means print in full, 0 disables
+#log_parameter_max_length_on_error = 0	# when logging an error, limit logged
+					# bind-parameter values to N bytes;
+					# -1 means print in full, 0 disables
+#log_statement = 'none'			# none, ddl, mod, all
+#log_replication_commands = off
+#log_temp_files = -1			# log temporary files equal or larger
+					# than the specified size in kilobytes;
+					# -1 disables, 0 logs all temp files
+log_timezone = 'Etc/UTC'
+
+
+#------------------------------------------------------------------------------
+# PROCESS TITLE
+#------------------------------------------------------------------------------
+
+cluster_name = '14/main'			# added to process titles if nonempty
+					# (change requires restart)
+#update_process_title = on
+
+
+#------------------------------------------------------------------------------
+# STATISTICS
+#------------------------------------------------------------------------------
+
+# - Query and Index Statistics Collector -
+
+#track_activities = on
+#track_activity_query_size = 1024	# (change requires restart)
+#track_counts = on
+#track_io_timing = off
+#track_wal_io_timing = off
+#track_functions = none			# none, pl, all
+stats_temp_directory = '/var/run/postgresql/14-main.pg_stat_tmp'
+
+
+# - Monitoring -
+
+#compute_query_id = auto
+#log_statement_stats = off
+#log_parser_stats = off
+#log_planner_stats = off
+#log_executor_stats = off
+
+
+#------------------------------------------------------------------------------
+# AUTOVACUUM
+#------------------------------------------------------------------------------
+
+#autovacuum = on			# Enable autovacuum subprocess?  'on'
+					# requires track_counts to also be on.
+#autovacuum_max_workers = 3		# max number of autovacuum subprocesses
+					# (change requires restart)
+#autovacuum_naptime = 1min		# time between autovacuum runs
+#autovacuum_vacuum_threshold = 50	# min number of row updates before
+					# vacuum
+#autovacuum_vacuum_insert_threshold = 1000	# min number of row inserts
+					# before vacuum; -1 disables insert
+					# vacuums
+#autovacuum_analyze_threshold = 50	# min number of row updates before
+					# analyze
+#autovacuum_vacuum_scale_factor = 0.2	# fraction of table size before vacuum
+#autovacuum_vacuum_insert_scale_factor = 0.2	# fraction of inserts over table
+					# size before insert vacuum
+#autovacuum_analyze_scale_factor = 0.1	# fraction of table size before analyze
+#autovacuum_freeze_max_age = 200000000	# maximum XID age before forced vacuum
+					# (change requires restart)
+#autovacuum_multixact_freeze_max_age = 400000000	# maximum multixact age
+					# before forced vacuum
+					# (change requires restart)
+#autovacuum_vacuum_cost_delay = 2ms	# default vacuum cost delay for
+					# autovacuum, in milliseconds;
+					# -1 means use vacuum_cost_delay
+#autovacuum_vacuum_cost_limit = -1	# default vacuum cost limit for
+					# autovacuum, -1 means use
+					# vacuum_cost_limit
+
+
+#------------------------------------------------------------------------------
+# CLIENT CONNECTION DEFAULTS
+#------------------------------------------------------------------------------
+
+# - Statement Behavior -
+
+#client_min_messages = notice		# values in order of decreasing detail:
+					#   debug5
+					#   debug4
+					#   debug3
+					#   debug2
+					#   debug1
+					#   log
+					#   notice
+					#   warning
+					#   error
+#search_path = '"$user", public'	# schema names
+#row_security = on
+#default_table_access_method = 'heap'
+#default_tablespace = ''		# a tablespace name, '' uses the default
+#default_toast_compression = 'pglz'	# 'pglz' or 'lz4'
+#temp_tablespaces = ''			# a list of tablespace names, '' uses
+					# only default tablespace
+#check_function_bodies = on
+#default_transaction_isolation = 'read committed'
+#default_transaction_read_only = off
+#default_transaction_deferrable = off
+#session_replication_role = 'origin'
+#statement_timeout = 0			# in milliseconds, 0 is disabled
+#lock_timeout = 0			# in milliseconds, 0 is disabled
+#idle_in_transaction_session_timeout = 0	# in milliseconds, 0 is disabled
+#idle_session_timeout = 0		# in milliseconds, 0 is disabled
+#vacuum_freeze_table_age = 150000000
+#vacuum_freeze_min_age = 50000000
+#vacuum_failsafe_age = 1600000000
+#vacuum_multixact_freeze_table_age = 150000000
+#vacuum_multixact_freeze_min_age = 5000000
+#vacuum_multixact_failsafe_age = 1600000000
+#bytea_output = 'hex'			# hex, escape
+#xmlbinary = 'base64'
+#xmloption = 'content'
+#gin_pending_list_limit = 4MB
+
+# - Locale and Formatting -
+
+datestyle = 'iso, mdy'
+#intervalstyle = 'postgres'
+timezone = 'Etc/UTC'
+#timezone_abbreviations = 'Default'     # Select the set of available time zone
+					# abbreviations.  Currently, there are
+					#   Default
+					#   Australia (historical usage)
+					#   India
+					# You can create your own file in
+					# share/timezonesets/.
+#extra_float_digits = 1			# min -15, max 3; any value >0 actually
+					# selects precise output mode
+#client_encoding = sql_ascii		# actually, defaults to database
+					# encoding
+
+# These settings are initialized by initdb, but they can be changed.
+lc_messages = 'C.UTF-8'			# locale for system error message
+					# strings
+lc_monetary = 'C.UTF-8'			# locale for monetary formatting
+lc_numeric = 'C.UTF-8'			# locale for number formatting
+lc_time = 'C.UTF-8'				# locale for time formatting
+
+# default configuration for text search
+default_text_search_config = 'pg_catalog.english'
+
+# - Shared Library Preloading -
+
+#local_preload_libraries = ''
+#session_preload_libraries = ''
+#shared_preload_libraries = ''	# (change requires restart)
+#jit_provider = 'llvmjit'		# JIT library to use
+
+# - Other Defaults -
+
+#dynamic_library_path = '$libdir'
+#extension_destdir = ''			# prepend path when loading extensions
+					# and shared objects (added by Debian)
+#gin_fuzzy_search_limit = 0
+
+
+#------------------------------------------------------------------------------
+# LOCK MANAGEMENT
+#------------------------------------------------------------------------------
+
+#deadlock_timeout = 1s
+#max_locks_per_transaction = 64		# min 10
+					# (change requires restart)
+#max_pred_locks_per_transaction = 64	# min 10
+					# (change requires restart)
+#max_pred_locks_per_relation = -2	# negative values mean
+					# (max_pred_locks_per_transaction
+					#  / -max_pred_locks_per_relation) - 1
+#max_pred_locks_per_page = 2            # min 0
+
+
+#------------------------------------------------------------------------------
+# VERSION AND PLATFORM COMPATIBILITY
+#------------------------------------------------------------------------------
+
+# - Previous PostgreSQL Versions -
+
+#array_nulls = on
+#backslash_quote = safe_encoding	# on, off, or safe_encoding
+#escape_string_warning = on
+#lo_compat_privileges = off
+#quote_all_identifiers = off
+#standard_conforming_strings = on
+#synchronize_seqscans = on
+
+# - Other Platforms and Clients -
+
+#transform_null_equals = off
+
+
+#------------------------------------------------------------------------------
+# ERROR HANDLING
+#------------------------------------------------------------------------------
+
+#exit_on_error = off			# terminate session on any error?
+#restart_after_crash = on		# reinitialize after backend crash?
+#data_sync_retry = off			# retry or panic on failure to fsync
+					# data?
+					# (change requires restart)
+#recovery_init_sync_method = fsync	# fsync, syncfs (Linux 5.8+)
+
+
+#------------------------------------------------------------------------------
+# CONFIG FILE INCLUDES
+#------------------------------------------------------------------------------
+
+# These options allow settings to be loaded from files other than the
+# default postgresql.conf.  Note that these are directives, not variable
+# assignments, so they can usefully be given more than once.
+
+include_dir = 'conf.d'			# include files ending in '.conf' from
+					# a directory, e.g., 'conf.d'
+#include_if_exists = '...'		# include file only if it exists
+#include = '...'			# include file
+
+
+#------------------------------------------------------------------------------
+# CUSTOMIZED OPTIONS
+#------------------------------------------------------------------------------
+
+# Add settings for extensions here

accountsdb-plugin-postgres/src/accounts_selector.rs

@@ -0,0 +1,74 @@
+use {log::*, std::collections::HashSet};
+
+#[derive(Debug)]
+pub(crate) struct AccountsSelector {
+    pub accounts: HashSet<Vec<u8>>,
+    pub owners: HashSet<Vec<u8>>,
+    pub select_all_accounts: bool,
+}
+
+impl AccountsSelector {
+    pub fn default() -> Self {
+        AccountsSelector {
+            accounts: HashSet::default(),
+            owners: HashSet::default(),
+            select_all_accounts: true,
+        }
+    }
+
+    pub fn new(accounts: &[String], owners: &[String]) -> Self {
+        info!(
+            "Creating AccountsSelector from accounts: {:?}, owners: {:?}",
+            accounts, owners
+        );
+
+        let select_all_accounts = accounts.iter().any(|key| key == "*");
+        if select_all_accounts {
+            return AccountsSelector {
+                accounts: HashSet::default(),
+                owners: HashSet::default(),
+                select_all_accounts,
+            };
+        }
+        let accounts = accounts
+            .iter()
+            .map(|key| bs58::decode(key).into_vec().unwrap())
+            .collect();
+        let owners = owners
+            .iter()
+            .map(|key| bs58::decode(key).into_vec().unwrap())
+            .collect();
+        AccountsSelector {
+            accounts,
+            owners,
+            select_all_accounts,
+        }
+    }
+
+    pub fn is_account_selected(&self, account: &[u8], owner: &[u8]) -> bool {
+        self.select_all_accounts || self.accounts.contains(account) || self.owners.contains(owner)
+    }
+
+    /// Check if any account is of interested at all
+    pub fn is_enabled(&self) -> bool {
+        self.select_all_accounts || !self.accounts.is_empty() || !self.owners.is_empty()
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::*;
+
+    #[test]
+    fn test_create_accounts_selector() {
+        AccountsSelector::new(
+            &["9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin".to_string()],
+            &[],
+        );
+
+        AccountsSelector::new(
+            &[],
+            &["9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin".to_string()],
+        );
+    }
+}

accountsdb-plugin-postgres/src/accountsdb_plugin_postgres.rs

@@ -0,0 +1,466 @@
+use solana_measure::measure::Measure;
+/// Main entry for the PostgreSQL plugin
+use {
+    crate::{
+        accounts_selector::AccountsSelector,
+        postgres_client::{ParallelPostgresClient, PostgresClientBuilder},
+        transaction_selector::TransactionSelector,
+    },
+    bs58,
+    log::*,
+    serde_derive::{Deserialize, Serialize},
+    serde_json,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        AccountsDbPlugin, AccountsDbPluginError, ReplicaAccountInfoVersions,
+        ReplicaBlockInfoVersions, ReplicaTransactionInfoVersions, Result, SlotStatus,
+    },
+    solana_metrics::*,
+    std::{fs::File, io::Read},
+    thiserror::Error,
+};
+
+#[derive(Default)]
+pub struct AccountsDbPluginPostgres {
+    client: Option<ParallelPostgresClient>,
+    accounts_selector: Option<AccountsSelector>,
+    transaction_selector: Option<TransactionSelector>,
+}
+
+impl std::fmt::Debug for AccountsDbPluginPostgres {
+    fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct AccountsDbPluginPostgresConfig {
+    pub host: Option<String>,
+    pub user: Option<String>,
+    pub port: Option<u16>,
+    pub connection_str: Option<String>,
+    pub threads: Option<usize>,
+    pub batch_size: Option<usize>,
+    pub panic_on_db_errors: Option<bool>,
+    /// Indicates if to store historical data for accounts
+    pub store_account_historical_data: Option<bool>,
+}
+
+#[derive(Error, Debug)]
+pub enum AccountsDbPluginPostgresError {
+    #[error("Error connecting to the backend data store. Error message: ({msg})")]
+    DataStoreConnectionError { msg: String },
+
+    #[error("Error preparing data store schema. Error message: ({msg})")]
+    DataSchemaError { msg: String },
+
+    #[error("Error preparing data store schema. Error message: ({msg})")]
+    ConfigurationError { msg: String },
+}
+
+impl AccountsDbPlugin for AccountsDbPluginPostgres {
+    fn name(&self) -> &'static str {
+        "AccountsDbPluginPostgres"
+    }
+
+    /// Do initialization for the PostgreSQL plugin.
+    ///
+    /// # Format of the config file:
+    /// * The `accounts_selector` section allows the user to controls accounts selections.
+    /// "accounts_selector" : {
+    ///     "accounts" : \["pubkey-1", "pubkey-2", ..., "pubkey-n"\],
+    /// }
+    /// or:
+    /// "accounts_selector" = {
+    ///     "owners" : \["pubkey-1", "pubkey-2", ..., "pubkey-m"\]
+    /// }
+    /// Accounts either satisyfing the accounts condition or owners condition will be selected.
+    /// When only owners is specified,
+    /// all accounts belonging to the owners will be streamed.
+    /// The accounts field supports wildcard to select all accounts:
+    /// "accounts_selector" : {
+    ///     "accounts" : \["*"\],
+    /// }
+    /// * "host", optional, specifies the PostgreSQL server.
+    /// * "user", optional, specifies the PostgreSQL user.
+    /// * "port", optional, specifies the PostgreSQL server's port.
+    /// * "connection_str", optional, the custom PostgreSQL connection string.
+    /// Please refer to https://docs.rs/postgres/0.19.2/postgres/config/struct.Config.html for the connection configuration.
+    /// When `connection_str` is set, the values in "host", "user" and "port" are ignored. If `connection_str` is not given,
+    /// `host` and `user` must be given.
+    /// "store_account_historical_data", optional, set it to 'true', to store historical account data to account_audit
+    /// table.
+    /// * "threads" optional, specifies the number of worker threads for the plugin. A thread
+    /// maintains a PostgreSQL connection to the server. The default is '10'.
+    /// * "batch_size" optional, specifies the batch size of bulk insert when the AccountsDb is created
+    /// from restoring a snapshot. The default is '10'.
+    /// * "panic_on_db_errors", optional, contols if to panic when there are errors replicating data to the
+    /// PostgreSQL database. The default is 'false'.
+    /// * "transaction_selector", optional, controls if and what transaction to store. If this field is missing
+    /// None of the transction is stored.
+    /// "transaction_selector" : {
+    ///     "mentions" : \["pubkey-1", "pubkey-2", ..., "pubkey-n"\],
+    /// }
+    /// The `mentions` field support wildcard to select all transaction or all 'vote' transactions:
+    /// For example, to select all transactions:
+    /// "transaction_selector" : {
+    ///     "mentions" : \["*"\],
+    /// }
+    /// To select all vote transactions:
+    /// "transaction_selector" : {
+    ///     "mentions" : \["all_votes"\],
+    /// }
+    /// # Examples
+    ///
+    /// {
+    ///    "libpath": "/home/solana/target/release/libsolana_accountsdb_plugin_postgres.so",
+    ///    "host": "host_foo",
+    ///    "user": "solana",
+    ///    "threads": 10,
+    ///    "accounts_selector" : {
+    ///       "owners" : ["9oT9R5ZyRovSVnt37QvVoBttGpNqR3J7unkb567NP8k3"]
+    ///    }
+    /// }
+
+    fn on_load(&mut self, config_file: &str) -> Result<()> {
+        solana_logger::setup_with_default("info");
+        info!(
+            "Loading plugin {:?} from config_file {:?}",
+            self.name(),
+            config_file
+        );
+        let mut file = File::open(config_file)?;
+        let mut contents = String::new();
+        file.read_to_string(&mut contents)?;
+
+        let result: serde_json::Value = serde_json::from_str(&contents).unwrap();
+        self.accounts_selector = Some(Self::create_accounts_selector_from_config(&result));
+        self.transaction_selector = Some(Self::create_transaction_selector_from_config(&result));
+
+        let result: serde_json::Result<AccountsDbPluginPostgresConfig> =
+            serde_json::from_str(&contents);
+        match result {
+            Err(err) => {
+                return Err(AccountsDbPluginError::ConfigFileReadError {
+                    msg: format!(
+                        "The config file is not in the JSON format expected: {:?}",
+                        err
+                    ),
+                })
+            }
+            Ok(config) => {
+                let client = PostgresClientBuilder::build_pararallel_postgres_client(&config)?;
+                self.client = Some(client);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn on_unload(&mut self) {
+        info!("Unloading plugin: {:?}", self.name());
+
+        match &mut self.client {
+            None => {}
+            Some(client) => {
+                client.join().unwrap();
+            }
+        }
+    }
+
+    fn update_account(
+        &mut self,
+        account: ReplicaAccountInfoVersions,
+        slot: u64,
+        is_startup: bool,
+    ) -> Result<()> {
+        let mut measure_all = Measure::start("accountsdb-plugin-postgres-update-account-main");
+        match account {
+            ReplicaAccountInfoVersions::V0_0_1(account) => {
+                let mut measure_select =
+                    Measure::start("accountsdb-plugin-postgres-update-account-select");
+                if let Some(accounts_selector) = &self.accounts_selector {
+                    if !accounts_selector.is_account_selected(account.pubkey, account.owner) {
+                        return Ok(());
+                    }
+                } else {
+                    return Ok(());
+                }
+                measure_select.stop();
+                inc_new_counter_debug!(
+                    "accountsdb-plugin-postgres-update-account-select-us",
+                    measure_select.as_us() as usize,
+                    100000,
+                    100000
+                );
+
+                debug!(
+                    "Updating account {:?} with owner {:?} at slot {:?} using account selector {:?}",
+                    bs58::encode(account.pubkey).into_string(),
+                    bs58::encode(account.owner).into_string(),
+                    slot,
+                    self.accounts_selector.as_ref().unwrap()
+                );
+
+                match &mut self.client {
+                    None => {
+                        return Err(AccountsDbPluginError::Custom(Box::new(
+                            AccountsDbPluginPostgresError::DataStoreConnectionError {
+                                msg: "There is no connection to the PostgreSQL database."
+                                    .to_string(),
+                            },
+                        )));
+                    }
+                    Some(client) => {
+                        let mut measure_update =
+                            Measure::start("accountsdb-plugin-postgres-update-account-client");
+                        let result = { client.update_account(account, slot, is_startup) };
+                        measure_update.stop();
+
+                        inc_new_counter_debug!(
+                            "accountsdb-plugin-postgres-update-account-client-us",
+                            measure_update.as_us() as usize,
+                            100000,
+                            100000
+                        );
+
+                        if let Err(err) = result {
+                            return Err(AccountsDbPluginError::AccountsUpdateError {
+                                msg: format!("Failed to persist the update of account to the PostgreSQL database. Error: {:?}", err)
+                            });
+                        }
+                    }
+                }
+            }
+        }
+
+        measure_all.stop();
+
+        inc_new_counter_debug!(
+            "accountsdb-plugin-postgres-update-account-main-us",
+            measure_all.as_us() as usize,
+            100000,
+            100000
+        );
+
+        Ok(())
+    }
+
+    fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<()> {
+        info!("Updating slot {:?} at with status {:?}", slot, status);
+
+        match &mut self.client {
+            None => {
+                return Err(AccountsDbPluginError::Custom(Box::new(
+                    AccountsDbPluginPostgresError::DataStoreConnectionError {
+                        msg: "There is no connection to the PostgreSQL database.".to_string(),
+                    },
+                )));
+            }
+            Some(client) => {
+                let result = client.update_slot_status(slot, parent, status);
+
+                if let Err(err) = result {
+                    return Err(AccountsDbPluginError::SlotStatusUpdateError{
+                        msg: format!("Failed to persist the update of slot to the PostgreSQL database. Error: {:?}", err)
+                    });
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn notify_end_of_startup(&mut self) -> Result<()> {
+        info!("Notifying the end of startup for accounts notifications");
+        match &mut self.client {
+            None => {
+                return Err(AccountsDbPluginError::Custom(Box::new(
+                    AccountsDbPluginPostgresError::DataStoreConnectionError {
+                        msg: "There is no connection to the PostgreSQL database.".to_string(),
+                    },
+                )));
+            }
+            Some(client) => {
+                let result = client.notify_end_of_startup();
+
+                if let Err(err) = result {
+                    return Err(AccountsDbPluginError::SlotStatusUpdateError{
+                        msg: format!("Failed to notify the end of startup for accounts notifications. Error: {:?}", err)
+                    });
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn notify_transaction(
+        &mut self,
+        transaction_info: ReplicaTransactionInfoVersions,
+        slot: u64,
+    ) -> Result<()> {
+        match &mut self.client {
+            None => {
+                return Err(AccountsDbPluginError::Custom(Box::new(
+                    AccountsDbPluginPostgresError::DataStoreConnectionError {
+                        msg: "There is no connection to the PostgreSQL database.".to_string(),
+                    },
+                )));
+            }
+            Some(client) => match transaction_info {
+                ReplicaTransactionInfoVersions::V0_0_1(transaction_info) => {
+                    if let Some(transaction_selector) = &self.transaction_selector {
+                        if !transaction_selector.is_transaction_selected(
+                            transaction_info.is_vote,
+                            transaction_info.transaction.message().account_keys_iter(),
+                        ) {
+                            return Ok(());
+                        }
+                    } else {
+                        return Ok(());
+                    }
+
+                    let result = client.log_transaction_info(transaction_info, slot);
+
+                    if let Err(err) = result {
+                        return Err(AccountsDbPluginError::SlotStatusUpdateError{
+                                msg: format!("Failed to persist the transaction info to the PostgreSQL database. Error: {:?}", err)
+                            });
+                    }
+                }
+            },
+        }
+
+        Ok(())
+    }
+
+    fn notify_block_metadata(&mut self, block_info: ReplicaBlockInfoVersions) -> Result<()> {
+        match &mut self.client {
+            None => {
+                return Err(AccountsDbPluginError::Custom(Box::new(
+                    AccountsDbPluginPostgresError::DataStoreConnectionError {
+                        msg: "There is no connection to the PostgreSQL database.".to_string(),
+                    },
+                )));
+            }
+            Some(client) => match block_info {
+                ReplicaBlockInfoVersions::V0_0_1(block_info) => {
+                    let result = client.update_block_metadata(block_info);
+
+                    if let Err(err) = result {
+                        return Err(AccountsDbPluginError::SlotStatusUpdateError{
+                                msg: format!("Failed to persist the update of block metadata to the PostgreSQL database. Error: {:?}", err)
+                            });
+                    }
+                }
+            },
+        }
+
+        Ok(())
+    }
+
+    /// Check if the plugin is interested in account data
+    /// Default is true -- if the plugin is not interested in
+    /// account data, please return false.
+    fn account_data_notifications_enabled(&self) -> bool {
+        self.accounts_selector
+            .as_ref()
+            .map_or_else(|| false, |selector| selector.is_enabled())
+    }
+
+    /// Check if the plugin is interested in transaction data
+    fn transaction_notifications_enabled(&self) -> bool {
+        self.transaction_selector
+            .as_ref()
+            .map_or_else(|| false, |selector| selector.is_enabled())
+    }
+}
+
+impl AccountsDbPluginPostgres {
+    fn create_accounts_selector_from_config(config: &serde_json::Value) -> AccountsSelector {
+        let accounts_selector = &config["accounts_selector"];
+
+        if accounts_selector.is_null() {
+            AccountsSelector::default()
+        } else {
+            let accounts = &accounts_selector["accounts"];
+            let accounts: Vec<String> = if accounts.is_array() {
+                accounts
+                    .as_array()
+                    .unwrap()
+                    .iter()
+                    .map(|val| val.as_str().unwrap().to_string())
+                    .collect()
+            } else {
+                Vec::default()
+            };
+            let owners = &accounts_selector["owners"];
+            let owners: Vec<String> = if owners.is_array() {
+                owners
+                    .as_array()
+                    .unwrap()
+                    .iter()
+                    .map(|val| val.as_str().unwrap().to_string())
+                    .collect()
+            } else {
+                Vec::default()
+            };
+            AccountsSelector::new(&accounts, &owners)
+        }
+    }
+
+    fn create_transaction_selector_from_config(config: &serde_json::Value) -> TransactionSelector {
+        let transaction_selector = &config["transaction_selector"];
+
+        if transaction_selector.is_null() {
+            TransactionSelector::default()
+        } else {
+            let accounts = &transaction_selector["mentions"];
+            let accounts: Vec<String> = if accounts.is_array() {
+                accounts
+                    .as_array()
+                    .unwrap()
+                    .iter()
+                    .map(|val| val.as_str().unwrap().to_string())
+                    .collect()
+            } else {
+                Vec::default()
+            };
+            TransactionSelector::new(&accounts)
+        }
+    }
+
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+
+#[no_mangle]
+#[allow(improper_ctypes_definitions)]
+/// # Safety
+///
+/// This function returns the AccountsDbPluginPostgres pointer as trait AccountsDbPlugin.
+pub unsafe extern "C" fn _create_plugin() -> *mut dyn AccountsDbPlugin {
+    let plugin = AccountsDbPluginPostgres::new();
+    let plugin: Box<dyn AccountsDbPlugin> = Box::new(plugin);
+    Box::into_raw(plugin)
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use {super::*, serde_json};
+
+    #[test]
+    fn test_accounts_selector_from_config() {
+        let config = "{\"accounts_selector\" : { \
+           \"owners\" : [\"9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin\"] \
+        }}";
+
+        let config: serde_json::Value = serde_json::from_str(config).unwrap();
+        AccountsDbPluginPostgres::create_accounts_selector_from_config(&config);
+    }
+}

accountsdb-plugin-postgres/src/lib.rs

@@ -0,0 +1,4 @@
+pub mod accounts_selector;
+pub mod accountsdb_plugin_postgres;
+pub mod postgres_client;
+pub mod transaction_selector;

accountsdb-plugin-postgres/src/postgres_client/postgres_client_block_metadata.rs

@@ -0,0 +1,97 @@
+use {
+    crate::{
+        accountsdb_plugin_postgres::{
+            AccountsDbPluginPostgresConfig, AccountsDbPluginPostgresError,
+        },
+        postgres_client::{
+            postgres_client_transaction::DbReward, SimplePostgresClient, UpdateBlockMetadataRequest,
+        },
+    },
+    chrono::Utc,
+    log::*,
+    postgres::{Client, Statement},
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        AccountsDbPluginError, ReplicaBlockInfo,
+    },
+};
+
+#[derive(Clone, Debug)]
+pub struct DbBlockInfo {
+    pub slot: i64,
+    pub blockhash: String,
+    pub rewards: Vec<DbReward>,
+    pub block_time: Option<i64>,
+    pub block_height: Option<i64>,
+}
+
+impl<'a> From<&ReplicaBlockInfo<'a>> for DbBlockInfo {
+    fn from(block_info: &ReplicaBlockInfo) -> Self {
+        Self {
+            slot: block_info.slot as i64,
+            blockhash: block_info.blockhash.to_string(),
+            rewards: block_info.rewards.iter().map(DbReward::from).collect(),
+            block_time: block_info.block_time,
+            block_height: block_info
+                .block_height
+                .map(|block_height| block_height as i64),
+        }
+    }
+}
+
+impl SimplePostgresClient {
+    pub(crate) fn build_block_metadata_upsert_statement(
+        client: &mut Client,
+        config: &AccountsDbPluginPostgresConfig,
+    ) -> Result<Statement, AccountsDbPluginError> {
+        let stmt =
+            "INSERT INTO block (slot, blockhash, rewards, block_time, block_height, updated_on) \
+        VALUES ($1, $2, $3, $4, $5, $6)";
+
+        let stmt = client.prepare(stmt);
+
+        match stmt {
+            Err(err) => {
+                return Err(AccountsDbPluginError::Custom(Box::new(AccountsDbPluginPostgresError::DataSchemaError {
+                    msg: format!(
+                        "Error in preparing for the block metadata update PostgreSQL database: ({}) host: {:?} user: {:?} config: {:?}",
+                        err, config.host, config.user, config
+                    ),
+                })));
+            }
+            Ok(stmt) => Ok(stmt),
+        }
+    }
+
+    pub(crate) fn update_block_metadata_impl(
+        &mut self,
+        block_info: UpdateBlockMetadataRequest,
+    ) -> Result<(), AccountsDbPluginError> {
+        let client = self.client.get_mut().unwrap();
+        let statement = &client.update_block_metadata_stmt;
+        let client = &mut client.client;
+        let updated_on = Utc::now().naive_utc();
+
+        let block_info = block_info.block_info;
+        let result = client.query(
+            statement,
+            &[
+                &block_info.slot,
+                &block_info.blockhash,
+                &block_info.rewards,
+                &block_info.block_time,
+                &block_info.block_height,
+                &updated_on,
+            ],
+        );
+
+        if let Err(err) = result {
+            let msg = format!(
+                "Failed to persist the update of block metadata to the PostgreSQL database. Error: {:?}",
+                err);
+            error!("{}", msg);
+            return Err(AccountsDbPluginError::AccountsUpdateError { msg });
+        }
+
+        Ok(())
+    }
+}

accountsdb-plugin-postgres/src/transaction_selector.rs

@@ -0,0 +1,194 @@
+/// The transaction selector is responsible for filtering transactions
+/// in the plugin framework.
+use {log::*, solana_sdk::pubkey::Pubkey, std::collections::HashSet};
+
+pub(crate) struct TransactionSelector {
+    pub mentioned_addresses: HashSet<Vec<u8>>,
+    pub select_all_transactions: bool,
+    pub select_all_vote_transactions: bool,
+}
+
+#[allow(dead_code)]
+impl TransactionSelector {
+    pub fn default() -> Self {
+        Self {
+            mentioned_addresses: HashSet::default(),
+            select_all_transactions: false,
+            select_all_vote_transactions: false,
+        }
+    }
+
+    /// Create a selector based on the mentioned addresses
+    /// To select all transactions use ["*"] or ["all"]
+    /// To select all vote transactions, use ["all_votes"]
+    /// To select transactions mentioning specific addresses use ["<pubkey1>", "<pubkey2>", ...]
+    pub fn new(mentioned_addresses: &[String]) -> Self {
+        info!(
+            "Creating TransactionSelector from addresses: {:?}",
+            mentioned_addresses
+        );
+
+        let select_all_transactions = mentioned_addresses
+            .iter()
+            .any(|key| key == "*" || key == "all");
+        if select_all_transactions {
+            return Self {
+                mentioned_addresses: HashSet::default(),
+                select_all_transactions,
+                select_all_vote_transactions: true,
+            };
+        }
+        let select_all_vote_transactions = mentioned_addresses.iter().any(|key| key == "all_votes");
+        if select_all_vote_transactions {
+            return Self {
+                mentioned_addresses: HashSet::default(),
+                select_all_transactions,
+                select_all_vote_transactions: true,
+            };
+        }
+
+        let mentioned_addresses = mentioned_addresses
+            .iter()
+            .map(|key| bs58::decode(key).into_vec().unwrap())
+            .collect();
+
+        Self {
+            mentioned_addresses,
+            select_all_transactions: false,
+            select_all_vote_transactions: false,
+        }
+    }
+
+    /// Check if a transaction is of interest.
+    pub fn is_transaction_selected(
+        &self,
+        is_vote: bool,
+        mentioned_addresses: Box<dyn Iterator<Item = &Pubkey> + '_>,
+    ) -> bool {
+        if !self.is_enabled() {
+            return false;
+        }
+
+        if self.select_all_transactions || (self.select_all_vote_transactions && is_vote) {
+            return true;
+        }
+        for address in mentioned_addresses {
+            if self.mentioned_addresses.contains(address.as_ref()) {
+                return true;
+            }
+        }
+        false
+    }
+
+    /// Check if any transaction is of interest at all
+    pub fn is_enabled(&self) -> bool {
+        self.select_all_transactions
+            || self.select_all_vote_transactions
+            || !self.mentioned_addresses.is_empty()
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::*;
+
+    #[test]
+    fn test_select_transaction() {
+        let pubkey1 = Pubkey::new_unique();
+        let pubkey2 = Pubkey::new_unique();
+
+        let selector = TransactionSelector::new(&[pubkey1.to_string()]);
+
+        assert!(selector.is_enabled());
+
+        let addresses = [pubkey1];
+
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey2];
+        assert!(!selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey1, pubkey2];
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+    }
+
+    #[test]
+    fn test_select_all_transaction_using_wildcard() {
+        let pubkey1 = Pubkey::new_unique();
+        let pubkey2 = Pubkey::new_unique();
+
+        let selector = TransactionSelector::new(&["*".to_string()]);
+
+        assert!(selector.is_enabled());
+
+        let addresses = [pubkey1];
+
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey2];
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey1, pubkey2];
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+    }
+
+    #[test]
+    fn test_select_all_transaction_all() {
+        let pubkey1 = Pubkey::new_unique();
+        let pubkey2 = Pubkey::new_unique();
+
+        let selector = TransactionSelector::new(&["all".to_string()]);
+
+        assert!(selector.is_enabled());
+
+        let addresses = [pubkey1];
+
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey2];
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey1, pubkey2];
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+    }
+
+    #[test]
+    fn test_select_all_vote_transaction() {
+        let pubkey1 = Pubkey::new_unique();
+        let pubkey2 = Pubkey::new_unique();
+
+        let selector = TransactionSelector::new(&["all_votes".to_string()]);
+
+        assert!(selector.is_enabled());
+
+        let addresses = [pubkey1];
+
+        assert!(!selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey2];
+        assert!(selector.is_transaction_selected(true, Box::new(addresses.iter())));
+
+        let addresses = [pubkey1, pubkey2];
+        assert!(selector.is_transaction_selected(true, Box::new(addresses.iter())));
+    }
+
+    #[test]
+    fn test_select_no_transaction() {
+        let pubkey1 = Pubkey::new_unique();
+        let pubkey2 = Pubkey::new_unique();
+
+        let selector = TransactionSelector::new(&[]);
+
+        assert!(!selector.is_enabled());
+
+        let addresses = [pubkey1];
+
+        assert!(!selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey2];
+        assert!(!selector.is_transaction_selected(true, Box::new(addresses.iter())));
+
+        let addresses = [pubkey1, pubkey2];
+        assert!(!selector.is_transaction_selected(true, Box::new(addresses.iter())));
+    }
+}

banking-bench/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2018"
+edition = "2021"
 name = "solana-banking-bench"
-version = "1.4.0"
+version = "1.9.5"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,20 +10,21 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-crossbeam-channel = "0.4"
-log = "0.4.6"
+crossbeam-channel = "0.5"
+log = "0.4.14"
 rand = "0.7.0"
-rayon = "1.4.0"
-solana-core = { path = "../core", version = "1.4.0" }
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
-solana-streamer = { path = "../streamer", version = "1.4.0" }
-solana-perf = { path = "../perf", version = "1.4.0" }
-solana-ledger = { path = "../ledger", version = "1.4.0" }
-solana-logger = { path = "../logger", version = "1.4.0" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-measure = { path = "../measure", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-version = { path = "../version", version = "1.4.0" }
+rayon = "1.5.1"
+solana-core = { path = "../core", version = "=1.9.5" }
+solana-gossip = { path = "../gossip", version = "=1.9.5" }
+solana-ledger = { path = "../ledger", version = "=1.9.5" }
+solana-logger = { path = "../logger", version = "=1.9.5" }
+solana-measure = { path = "../measure", version = "=1.9.5" }
+solana-perf = { path = "../perf", version = "=1.9.5" }
+solana-poh = { path = "../poh", version = "=1.9.5" }
+solana-runtime = { path = "../runtime", version = "=1.9.5" }
+solana-streamer = { path = "../streamer", version = "=1.9.5" }
+solana-sdk = { path = "../sdk", version = "=1.9.5" }
+solana-version = { path = "../version", version = "=1.9.5" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -1,36 +1,37 @@
-use clap::{crate_description, crate_name, value_t, App, Arg};
-use crossbeam_channel::unbounded;
-use log::*;
-use rand::{thread_rng, Rng};
-use rayon::prelude::*;
-use solana_core::{
-    banking_stage::{create_test_recorder, BankingStage},
-    cluster_info::ClusterInfo,
-    cluster_info::Node,
-    poh_recorder::PohRecorder,
-    poh_recorder::WorkingBankEntry,
-};
-use solana_ledger::{
-    blockstore::Blockstore,
-    genesis_utils::{create_genesis_config, GenesisConfigInfo},
-    get_tmp_ledger_path,
-};
-use solana_measure::measure::Measure;
-use solana_perf::packet::to_packets_chunked;
-use solana_runtime::{bank::Bank, bank_forks::BankForks};
-use solana_sdk::{
-    hash::Hash,
-    pubkey::Pubkey,
-    signature::Keypair,
-    signature::Signature,
-    system_transaction,
-    timing::{duration_as_us, timestamp},
-    transaction::Transaction,
-};
-use std::{
-    sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex},
-    thread::sleep,
-    time::{Duration, Instant},
+#![allow(clippy::integer_arithmetic)]
+use {
+    clap::{crate_description, crate_name, value_t, App, Arg},
+    crossbeam_channel::unbounded,
+    log::*,
+    rand::{thread_rng, Rng},
+    rayon::prelude::*,
+    solana_core::banking_stage::BankingStage,
+    solana_gossip::cluster_info::{ClusterInfo, Node},
+    solana_ledger::{
+        blockstore::Blockstore,
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        get_tmp_ledger_path,
+    },
+    solana_measure::measure::Measure,
+    solana_perf::packet::to_packet_batches,
+    solana_poh::poh_recorder::{create_test_recorder, PohRecorder, WorkingBankEntry},
+    solana_runtime::{
+        accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks,
+        cost_model::CostModel,
+    },
+    solana_sdk::{
+        hash::Hash,
+        signature::{Keypair, Signature},
+        system_transaction,
+        timing::{duration_as_us, timestamp},
+        transaction::Transaction,
+    },
+    solana_streamer::socket::SocketAddrSpace,
+    std::{
+        sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex, RwLock},
+        thread::sleep,
+        time::{Duration, Instant},
+    },
 };
 
 fn check_txs(
@@ -69,18 +70,18 @@ fn make_accounts_txs(
     hash: Hash,
     same_payer: bool,
 ) -> Vec<Transaction> {
-    let to_pubkey = Pubkey::new_rand();
+    let to_pubkey = solana_sdk::pubkey::new_rand();
     let payer_key = Keypair::new();
     let dummy = system_transaction::transfer(&payer_key, &to_pubkey, 1, hash);
     (0..total_num_transactions)
         .into_par_iter()
         .map(|_| {
             let mut new = dummy.clone();
-            let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+            let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
             if !same_payer {
-                new.message.account_keys[0] = Pubkey::new_rand();
+                new.message.account_keys[0] = solana_sdk::pubkey::new_rand();
             }
-            new.message.account_keys[1] = Pubkey::new_rand();
+            new.message.account_keys[1] = solana_sdk::pubkey::new_rand();
             new.signatures = vec![Signature::new(&sig[0..64])];
             new
         })
@@ -167,8 +168,9 @@ fn main() {
 
     let (verified_sender, verified_receiver) = unbounded();
     let (vote_sender, vote_receiver) = unbounded();
+    let (tpu_vote_sender, tpu_vote_receiver) = unbounded();
     let (replay_vote_sender, _replay_vote_receiver) = unbounded();
-    let bank0 = Bank::new(&genesis_config);
+    let bank0 = Bank::new_for_benches(&genesis_config);
     let mut bank_forks = BankForks::new(bank0);
     let mut bank = bank_forks.working_bank();
 
@@ -187,7 +189,7 @@ fn main() {
             genesis_config.hash(),
         );
         // Ignore any pesky duplicate signature errors in the case we are using single-payer
-        let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+        let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
         fund.signatures = vec![Signature::new(&sig[0..64])];
         let x = bank.process_transaction(&fund);
         x.unwrap();
@@ -197,19 +199,20 @@ fn main() {
     if !skip_sanity {
         //sanity check, make sure all the transactions can execute sequentially
         transactions.iter().for_each(|tx| {
-            let res = bank.process_transaction(&tx);
+            let res = bank.process_transaction(tx);
             assert!(res.is_ok(), "sanity test transactions error: {:?}", res);
         });
         bank.clear_signatures();
         //sanity check, make sure all the transactions can execute in parallel
-        let res = bank.process_transactions(&transactions);
+
+        let res = bank.process_transactions(transactions.iter());
         for r in res {
             assert!(r.is_ok(), "sanity parallel execution error: {:?}", r);
         }
         bank.clear_signatures();
     }
 
-    let mut verified: Vec<_> = to_packets_chunked(&transactions, packets_per_chunk);
+    let mut verified: Vec<_> = to_packet_batches(&transactions, packets_per_chunk);
     let ledger_path = get_tmp_ledger_path!();
     {
         let blockstore = Arc::new(
@@ -217,15 +220,21 @@ fn main() {
         );
         let (exit, poh_recorder, poh_service, signal_receiver) =
             create_test_recorder(&bank, &blockstore, None);
-        let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
+        let cluster_info = ClusterInfo::new(
+            Node::new_localhost().info,
+            Arc::new(Keypair::new()),
+            SocketAddrSpace::Unspecified,
+        );
         let cluster_info = Arc::new(cluster_info);
         let banking_stage = BankingStage::new(
             &cluster_info,
             &poh_recorder,
             verified_receiver,
+            tpu_vote_receiver,
             vote_receiver,
             None,
             replay_vote_sender,
+            Arc::new(RwLock::new(CostModel::default())),
         );
         poh_recorder.lock().unwrap().set_bank(&bank);
 
@@ -241,7 +250,7 @@ fn main() {
         let base_tx_count = bank.transaction_count();
         let mut txs_processed = 0;
         let mut root = 1;
-        let collector = Pubkey::new_rand();
+        let collector = solana_sdk::pubkey::new_rand();
         let config = Config {
             packets_per_batch: packets_per_chunk,
             chunk_len,
@@ -305,11 +314,10 @@ fn main() {
                 tx_total_us += duration_as_us(&now.elapsed());
 
                 let mut poh_time = Measure::start("poh_time");
-                poh_recorder.lock().unwrap().reset(
-                    bank.last_blockhash(),
-                    bank.slot(),
-                    Some((bank.slot(), bank.slot() + 1)),
-                );
+                poh_recorder
+                    .lock()
+                    .unwrap()
+                    .reset(bank.clone(), Some((bank.slot(), bank.slot() + 1)));
                 poh_time.stop();
 
                 let mut new_bank_time = Measure::start("new_bank");
@@ -324,7 +332,7 @@ fn main() {
                 poh_recorder.lock().unwrap().set_bank(&bank);
                 assert!(poh_recorder.lock().unwrap().bank().is_some());
                 if bank.slot() > 32 {
-                    bank_forks.set_root(root, &None, None);
+                    bank_forks.set_root(root, &AbsRequestSender::default(), None);
                     root += 1;
                 }
                 debug!(
@@ -353,10 +361,10 @@ fn main() {
             if bank.slot() > 0 && bank.slot() % 16 == 0 {
                 for tx in transactions.iter_mut() {
                     tx.message.recent_blockhash = bank.last_blockhash();
-                    let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+                    let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
                     tx.signatures[0] = Signature::new(&sig[0..64]);
                 }
-                verified = to_packets_chunked(&transactions.clone(), packets_per_chunk);
+                verified = to_packet_batches(&transactions.clone(), packets_per_chunk);
             }
 
             start += chunk_len;
@@ -378,6 +386,7 @@ fn main() {
         );
 
         drop(verified_sender);
+        drop(tpu_vote_sender);
         drop(vote_sender);
         exit.store(true, Ordering::Relaxed);
         banking_stage.join().unwrap();

banks-client/Cargo.toml

@@ -1,26 +1,28 @@
 [package]
 name = "solana-banks-client"
-version = "1.4.0"
+version = "1.9.5"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
-edition = "2018"
+documentation = "https://docs.rs/solana-banks-client"
+edition = "2021"
 
 [dependencies]
-async-trait = "0.1.36"
-bincode = "1.3.1"
+borsh = "0.9.1"
 futures = "0.3"
-solana-banks-interface = { path = "../banks-interface", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-tarpc = { version = "0.22.0", features = ["full"] }
-tokio = "0.2"
-tokio-serde = { version = "0.6", features = ["bincode"] }
+solana-banks-interface = { path = "../banks-interface", version = "=1.9.5" }
+solana-program = { path = "../sdk/program", version = "=1.9.5" }
+solana-sdk = { path = "../sdk", version = "=1.9.5" }
+tarpc = { version = "0.27.2", features = ["full"] }
+thiserror = "1.0"
+tokio = { version = "1", features = ["full"] }
+tokio-serde = { version = "0.8", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-banks-server = { path = "../banks-server", version = "1.4.0" }
+solana-runtime = { path = "../runtime", version = "=1.9.5" }
+solana-banks-server = { path = "../banks-server", version = "=1.9.5" }
 
 [lib]
 crate-type = ["lib"]

banks-client/src/error.rs

@@ -0,0 +1,73 @@
+use {
+    solana_sdk::{transaction::TransactionError, transport::TransportError},
+    std::io,
+    tarpc::client::RpcError,
+    thiserror::Error,
+};
+
+/// Errors from BanksClient
+#[derive(Error, Debug)]
+pub enum BanksClientError {
+    #[error("client error: {0}")]
+    ClientError(&'static str),
+
+    #[error(transparent)]
+    Io(#[from] io::Error),
+
+    #[error(transparent)]
+    RpcError(#[from] RpcError),
+
+    #[error("transport transaction error: {0}")]
+    TransactionError(#[from] TransactionError),
+
+    #[error("simulation error: {err:?}, logs: {logs:?}, units_consumed: {units_consumed:?}")]
+    SimulationError {
+        err: TransactionError,
+        logs: Vec<String>,
+        units_consumed: u64,
+    },
+}
+
+impl BanksClientError {
+    pub fn unwrap(&self) -> TransactionError {
+        match self {
+            BanksClientError::TransactionError(err)
+            | BanksClientError::SimulationError { err, .. } => err.clone(),
+            _ => panic!("unexpected transport error"),
+        }
+    }
+}
+
+impl From<BanksClientError> for io::Error {
+    fn from(err: BanksClientError) -> Self {
+        match err {
+            BanksClientError::ClientError(err) => Self::new(io::ErrorKind::Other, err.to_string()),
+            BanksClientError::Io(err) => err,
+            BanksClientError::RpcError(err) => Self::new(io::ErrorKind::Other, err.to_string()),
+            BanksClientError::TransactionError(err) => {
+                Self::new(io::ErrorKind::Other, err.to_string())
+            }
+            BanksClientError::SimulationError { err, .. } => {
+                Self::new(io::ErrorKind::Other, err.to_string())
+            }
+        }
+    }
+}
+
+impl From<BanksClientError> for TransportError {
+    fn from(err: BanksClientError) -> Self {
+        match err {
+            BanksClientError::ClientError(err) => {
+                Self::IoError(io::Error::new(io::ErrorKind::Other, err.to_string()))
+            }
+            BanksClientError::Io(err) => {
+                Self::IoError(io::Error::new(io::ErrorKind::Other, err.to_string()))
+            }
+            BanksClientError::RpcError(err) => {
+                Self::IoError(io::Error::new(io::ErrorKind::Other, err.to_string()))
+            }
+            BanksClientError::TransactionError(err) => Self::TransactionError(err),
+            BanksClientError::SimulationError { err, .. } => Self::TransactionError(err),
+        }
+    }
+}

banks-client/src/lib.rs

@@ -5,178 +5,398 @@
 //! but they are undocumented, may change over time, and are generally more
 //! cumbersome to use.
 
-use async_trait::async_trait;
-use futures::future::join_all;
-pub use solana_banks_interface::{BanksClient, TransactionStatus};
-use solana_banks_interface::{BanksRequest, BanksResponse};
-use solana_sdk::{
-    account::Account, clock::Slot, commitment_config::CommitmentLevel,
-    fee_calculator::FeeCalculator, hash::Hash, pubkey::Pubkey, signature::Signature,
-    transaction::Transaction, transport,
+pub use solana_banks_interface::{BanksClient as TarpcClient, TransactionStatus};
+use {
+    crate::error::BanksClientError,
+    borsh::BorshDeserialize,
+    futures::{future::join_all, Future, FutureExt, TryFutureExt},
+    solana_banks_interface::{BanksRequest, BanksResponse, BanksTransactionResultWithSimulation},
+    solana_program::{
+        clock::Slot, fee_calculator::FeeCalculator, hash::Hash, program_pack::Pack, pubkey::Pubkey,
+        rent::Rent, sysvar::Sysvar,
+    },
+    solana_sdk::{
+        account::{from_account, Account},
+        commitment_config::CommitmentLevel,
+        message::Message,
+        signature::Signature,
+        transaction::{self, Transaction},
+        transport,
+    },
+    std::io,
+    tarpc::{
+        client::{self, NewClient, RequestDispatch},
+        context::{self, Context},
+        serde_transport::tcp,
+        ClientMessage, Response, Transport,
+    },
+    tokio::{net::ToSocketAddrs, time::Duration},
+    tokio_serde::formats::Bincode,
 };
-use std::io::{self, Error, ErrorKind};
-use tarpc::{
-    client, context,
-    rpc::{transport::channel::UnboundedChannel, ClientMessage, Response},
-    serde_transport::tcp,
-};
-use tokio::{net::ToSocketAddrs, time::Duration};
-use tokio_serde::formats::Bincode;
 
-#[async_trait]
-pub trait BanksClientExt {
+mod error;
+
+// This exists only for backward compatibility
+pub trait BanksClientExt {}
+
+#[derive(Clone)]
+pub struct BanksClient {
+    inner: TarpcClient,
+}
+
+impl BanksClient {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new<C>(
+        config: client::Config,
+        transport: C,
+    ) -> NewClient<TarpcClient, RequestDispatch<BanksRequest, BanksResponse, C>>
+    where
+        C: Transport<ClientMessage<BanksRequest>, Response<BanksResponse>>,
+    {
+        TarpcClient::new(config, transport)
+    }
+
+    pub fn send_transaction_with_context(
+        &mut self,
+        ctx: Context,
+        transaction: Transaction,
+    ) -> impl Future<Output = io::Result<()>> + '_ {
+        self.inner
+            .send_transaction_with_context(ctx, transaction)
+            .map_err(BanksClientError::from) // Remove this when return Err type updated to BanksClientError
+            .map_err(Into::into)
+    }
+
+    #[deprecated(
+        since = "1.9.0",
+        note = "Please use `get_fee_for_message` or `is_blockhash_valid` instead"
+    )]
+    pub fn get_fees_with_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, u64)>> + '_ {
+        #[allow(deprecated)]
+        self.inner
+            .get_fees_with_commitment_and_context(ctx, commitment)
+            .map_err(BanksClientError::from) // Remove this when return Err type updated to BanksClientError
+            .map_err(Into::into)
+    }
+
+    pub fn get_transaction_status_with_context(
+        &mut self,
+        ctx: Context,
+        signature: Signature,
+    ) -> impl Future<Output = io::Result<Option<TransactionStatus>>> + '_ {
+        self.inner
+            .get_transaction_status_with_context(ctx, signature)
+            .map_err(BanksClientError::from) // Remove this when return Err type updated to BanksClientError
+            .map_err(Into::into)
+    }
+
+    pub fn get_slot_with_context(
+        &mut self,
+        ctx: Context,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Slot>> + '_ {
+        self.inner
+            .get_slot_with_context(ctx, commitment)
+            .map_err(BanksClientError::from) // Remove this when return Err type updated to BanksClientError
+            .map_err(Into::into)
+    }
+
+    pub fn get_block_height_with_context(
+        &mut self,
+        ctx: Context,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Slot>> + '_ {
+        self.inner
+            .get_block_height_with_context(ctx, commitment)
+            .map_err(BanksClientError::from) // Remove this when return Err type updated to BanksClientError
+            .map_err(Into::into)
+    }
+
+    pub fn process_transaction_with_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Option<transaction::Result<()>>>> + '_ {
+        self.inner
+            .process_transaction_with_commitment_and_context(ctx, transaction, commitment)
+            .map_err(BanksClientError::from) // Remove this when return Err type updated to BanksClientError
+            .map_err(Into::into)
+    }
+
+    pub fn process_transaction_with_preflight_and_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = Result<BanksTransactionResultWithSimulation, BanksClientError>> + '_
+    {
+        self.inner
+            .process_transaction_with_preflight_and_commitment_and_context(
+                ctx,
+                transaction,
+                commitment,
+            )
+            .map_err(Into::into)
+    }
+
+    pub fn get_account_with_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+        self.inner
+            .get_account_with_commitment_and_context(ctx, address, commitment)
+            .map_err(BanksClientError::from) // Remove this when return Err type updated to BanksClientError
+            .map_err(Into::into)
+    }
+
     /// Send a transaction and return immediately. The server will resend the
     /// transaction until either it is accepted by the cluster or the transaction's
     /// blockhash expires.
-    async fn send_transaction(&mut self, transaction: Transaction) -> io::Result<()>;
-
-    /// Return a recent, rooted blockhash from the server. The cluster will only accept
-    /// transactions with a blockhash that has not yet expired. Use the `get_fees`
-    /// method to get both a blockhash and the blockhash's last valid slot.
-    async fn get_recent_blockhash(&mut self) -> io::Result<Hash>;
+    pub fn send_transaction(
+        &mut self,
+        transaction: Transaction,
+    ) -> impl Future<Output = io::Result<()>> + '_ {
+        self.send_transaction_with_context(context::current(), transaction)
+    }
 
     /// Return the fee parameters associated with a recent, rooted blockhash. The cluster
     /// will use the transaction's blockhash to look up these same fee parameters and
     /// use them to calculate the transaction fee.
-    async fn get_fees(&mut self) -> io::Result<(FeeCalculator, Hash, Slot)>;
+    #[deprecated(
+        since = "1.9.0",
+        note = "Please use `get_fee_for_message` or `is_blockhash_valid` instead"
+    )]
+    pub fn get_fees(
+        &mut self,
+    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, u64)>> + '_ {
+        #[allow(deprecated)]
+        self.get_fees_with_commitment_and_context(context::current(), CommitmentLevel::default())
+    }
+
+    /// Return the cluster Sysvar
+    pub fn get_sysvar<T: Sysvar>(&mut self) -> impl Future<Output = io::Result<T>> + '_ {
+        self.get_account(T::id()).map(|result| {
+            let sysvar = result?
+                .ok_or(BanksClientError::ClientError("Sysvar not present"))
+                .map_err(io::Error::from)?; // Remove this map when return Err type updated to BanksClientError
+            from_account::<T, _>(&sysvar)
+                .ok_or(BanksClientError::ClientError(
+                    "Failed to deserialize sysvar",
+                ))
+                .map_err(Into::into) // Remove this when return Err type updated to BanksClientError
+        })
+    }
+
+    /// Return the cluster rent
+    pub fn get_rent(&mut self) -> impl Future<Output = io::Result<Rent>> + '_ {
+        self.get_sysvar::<Rent>()
+    }
+
+    /// Return a recent, rooted blockhash from the server. The cluster will only accept
+    /// transactions with a blockhash that has not yet expired. Use the `get_fees`
+    /// method to get both a blockhash and the blockhash's last valid slot.
+    #[deprecated(since = "1.9.0", note = "Please use `get_latest_blockhash` instead")]
+    pub fn get_recent_blockhash(&mut self) -> impl Future<Output = io::Result<Hash>> + '_ {
+        #[allow(deprecated)]
+        self.get_fees().map(|result| Ok(result?.1))
+    }
 
     /// Send a transaction and return after the transaction has been rejected or
     /// reached the given level of commitment.
-    async fn process_transaction_with_commitment(
+    pub fn process_transaction_with_commitment(
         &mut self,
         transaction: Transaction,
         commitment: CommitmentLevel,
-    ) -> transport::Result<()>;
+    ) -> impl Future<Output = transport::Result<()>> + '_ {
+        let mut ctx = context::current();
+        ctx.deadline += Duration::from_secs(50);
+        self.process_transaction_with_commitment_and_context(ctx, transaction, commitment)
+            .map(|result| match result? {
+                None => Err(BanksClientError::ClientError(
+                    "invalid blockhash or fee-payer",
+                )),
+                Some(transaction_result) => Ok(transaction_result?),
+            })
+            .map_err(Into::into) // Remove this when return Err type updated to BanksClientError
+    }
 
-    /// Send a transaction and return after the transaction has been finalized or rejected.
-    async fn process_transaction(&mut self, transaction: Transaction) -> transport::Result<()>;
+    /// Send a transaction and return any preflight (sanitization or simulation) errors, or return
+    /// after the transaction has been rejected or reached the given level of commitment.
+    pub fn process_transaction_with_preflight_and_commitment(
+        &mut self,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
+        let mut ctx = context::current();
+        ctx.deadline += Duration::from_secs(50);
+        self.process_transaction_with_preflight_and_commitment_and_context(
+            ctx,
+            transaction,
+            commitment,
+        )
+        .map(|result| match result? {
+            BanksTransactionResultWithSimulation {
+                result: None,
+                simulation_details: _,
+            } => Err(BanksClientError::ClientError(
+                "invalid blockhash or fee-payer",
+            )),
+            BanksTransactionResultWithSimulation {
+                result: Some(Err(err)),
+                simulation_details: Some(simulation_details),
+            } => Err(BanksClientError::SimulationError {
+                err,
+                logs: simulation_details.logs,
+                units_consumed: simulation_details.units_consumed,
+            }),
+            BanksTransactionResultWithSimulation {
+                result: Some(result),
+                simulation_details: _,
+            } => result.map_err(Into::into),
+        })
+    }
+
+    /// Send a transaction and return any preflight (sanitization or simulation) errors, or return
+    /// after the transaction has been finalized or rejected.
+    pub fn process_transaction_with_preflight(
+        &mut self,
+        transaction: Transaction,
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
+        self.process_transaction_with_preflight_and_commitment(
+            transaction,
+            CommitmentLevel::default(),
+        )
+    }
+
+    /// Send a transaction and return until the transaction has been finalized or rejected.
+    pub fn process_transaction(
+        &mut self,
+        transaction: Transaction,
+    ) -> impl Future<Output = transport::Result<()>> + '_ {
+        self.process_transaction_with_commitment(transaction, CommitmentLevel::default())
+    }
+
+    pub async fn process_transactions_with_commitment(
+        &mut self,
+        transactions: Vec<Transaction>,
+        commitment: CommitmentLevel,
+    ) -> transport::Result<()> {
+        let mut clients: Vec<_> = transactions.iter().map(|_| self.clone()).collect();
+        let futures = clients
+            .iter_mut()
+            .zip(transactions)
+            .map(|(client, transaction)| {
+                client.process_transaction_with_commitment(transaction, commitment)
+            });
+        let statuses = join_all(futures).await;
+        statuses.into_iter().collect() // Convert Vec<Result<_, _>> to Result<Vec<_>>
+    }
+
+    /// Send transactions and return until the transaction has been finalized or rejected.
+    pub fn process_transactions(
+        &mut self,
+        transactions: Vec<Transaction>,
+    ) -> impl Future<Output = transport::Result<()>> + '_ {
+        self.process_transactions_with_commitment(transactions, CommitmentLevel::default())
+    }
+
+    /// Return the most recent rooted slot. All transactions at or below this slot
+    /// are said to be finalized. The cluster will not fork to a higher slot.
+    pub fn get_root_slot(&mut self) -> impl Future<Output = io::Result<Slot>> + '_ {
+        self.get_slot_with_context(context::current(), CommitmentLevel::default())
+    }
+
+    /// Return the most recent rooted block height. All transactions at or below this height
+    /// are said to be finalized. The cluster will not fork to a higher block height.
+    pub fn get_root_block_height(&mut self) -> impl Future<Output = io::Result<Slot>> + '_ {
+        self.get_block_height_with_context(context::current(), CommitmentLevel::default())
+    }
+
+    /// Return the account at the given address at the slot corresponding to the given
+    /// commitment level. If the account is not found, None is returned.
+    pub fn get_account_with_commitment(
+        &mut self,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+        self.get_account_with_commitment_and_context(context::current(), address, commitment)
+    }
+
+    /// Return the account at the given address at the time of the most recent root slot.
+    /// If the account is not found, None is returned.
+    pub fn get_account(
+        &mut self,
+        address: Pubkey,
+    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+        self.get_account_with_commitment(address, CommitmentLevel::default())
+    }
+
+    /// Return the unpacked account data at the given address
+    /// If the account is not found, an error is returned
+    pub fn get_packed_account_data<T: Pack>(
+        &mut self,
+        address: Pubkey,
+    ) -> impl Future<Output = io::Result<T>> + '_ {
+        self.get_account(address).map(|result| {
+            let account = result?
+                .ok_or(BanksClientError::ClientError("Account not found"))
+                .map_err(io::Error::from)?; // Remove this map when return Err type updated to BanksClientError
+            T::unpack_from_slice(&account.data)
+                .map_err(|_| BanksClientError::ClientError("Failed to deserialize account"))
+                .map_err(Into::into) // Remove this when return Err type updated to BanksClientError
+        })
+    }
+
+    /// Return the unpacked account data at the given address
+    /// If the account is not found, an error is returned
+    pub fn get_account_data_with_borsh<T: BorshDeserialize>(
+        &mut self,
+        address: Pubkey,
+    ) -> impl Future<Output = io::Result<T>> + '_ {
+        self.get_account(address).map(|result| {
+            let account = result?.ok_or(BanksClientError::ClientError("Account not found"))?;
+            T::try_from_slice(&account.data).map_err(Into::into)
+        })
+    }
+
+    /// Return the balance in lamports of an account at the given address at the slot
+    /// corresponding to the given commitment level.
+    pub fn get_balance_with_commitment(
+        &mut self,
+        address: Pubkey,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<u64>> + '_ {
+        self.get_account_with_commitment_and_context(context::current(), address, commitment)
+            .map(|result| Ok(result?.map(|x| x.lamports).unwrap_or(0)))
+    }
+
+    /// Return the balance in lamports of an account at the given address at the time
+    /// of the most recent root slot.
+    pub fn get_balance(&mut self, address: Pubkey) -> impl Future<Output = io::Result<u64>> + '_ {
+        self.get_balance_with_commitment(address, CommitmentLevel::default())
+    }
 
     /// Return the status of a transaction with a signature matching the transaction's first
     /// signature. Return None if the transaction is not found, which may be because the
     /// blockhash was expired or the fee-paying account had insufficient funds to pay the
     /// transaction fee. Note that servers rarely store the full transaction history. This
     /// method may return None if the transaction status has been discarded.
-    async fn get_transaction_status(
+    pub fn get_transaction_status(
         &mut self,
         signature: Signature,
-    ) -> io::Result<Option<TransactionStatus>>;
+    ) -> impl Future<Output = io::Result<Option<TransactionStatus>>> + '_ {
+        self.get_transaction_status_with_context(context::current(), signature)
+    }
 
     /// Same as get_transaction_status, but for multiple transactions.
-    async fn get_transaction_statuses(
-        &mut self,
-        signatures: Vec<Signature>,
-    ) -> io::Result<Vec<Option<TransactionStatus>>>;
-
-    /// Return the most recent rooted slot height. All transactions at or below this height
-    /// are said to be finalized. The cluster will not fork to a higher slot height.
-    async fn get_root_slot(&mut self) -> io::Result<Slot>;
-
-    /// Return the account at the given address at the slot corresponding to the given
-    /// commitment level. If the account is not found, None is returned.
-    async fn get_account_with_commitment(
-        &mut self,
-        address: Pubkey,
-        commitment: CommitmentLevel,
-    ) -> io::Result<Option<Account>>;
-
-    /// Return the account at the given address at the time of the most recent root slot.
-    /// If the account is not found, None is returned.
-    async fn get_account(&mut self, address: Pubkey) -> io::Result<Option<Account>>;
-
-    /// Return the balance in lamports of an account at the given address at the slot
-    /// corresponding to the given commitment level.
-    async fn get_balance_with_commitment(
-        &mut self,
-        address: Pubkey,
-        commitment: CommitmentLevel,
-    ) -> io::Result<u64>;
-
-    /// Return the balance in lamports of an account at the given address at the time
-    /// of the most recent root slot.
-    async fn get_balance(&mut self, address: Pubkey) -> io::Result<u64>;
-}
-
-#[async_trait]
-impl BanksClientExt for BanksClient {
-    async fn send_transaction(&mut self, transaction: Transaction) -> io::Result<()> {
-        self.send_transaction_with_context(context::current(), transaction)
-            .await
-    }
-
-    async fn get_fees(&mut self) -> io::Result<(FeeCalculator, Hash, Slot)> {
-        self.get_fees_with_commitment_and_context(context::current(), CommitmentLevel::Root)
-            .await
-    }
-
-    async fn get_recent_blockhash(&mut self) -> io::Result<Hash> {
-        Ok(self.get_fees().await?.1)
-    }
-
-    async fn process_transaction_with_commitment(
-        &mut self,
-        transaction: Transaction,
-        commitment: CommitmentLevel,
-    ) -> transport::Result<()> {
-        let mut ctx = context::current();
-        ctx.deadline += Duration::from_secs(50);
-        let result = self
-            .process_transaction_with_commitment_and_context(ctx, transaction, commitment)
-            .await?;
-        match result {
-            None => Err(Error::new(ErrorKind::TimedOut, "invalid blockhash or fee-payer").into()),
-            Some(transaction_result) => Ok(transaction_result?),
-        }
-    }
-
-    async fn process_transaction(&mut self, transaction: Transaction) -> transport::Result<()> {
-        self.process_transaction_with_commitment(transaction, CommitmentLevel::default())
-            .await
-    }
-
-    async fn get_root_slot(&mut self) -> io::Result<Slot> {
-        self.get_slot_with_context(context::current(), CommitmentLevel::Root)
-            .await
-    }
-
-    async fn get_account_with_commitment(
-        &mut self,
-        address: Pubkey,
-        commitment: CommitmentLevel,
-    ) -> io::Result<Option<Account>> {
-        self.get_account_with_commitment_and_context(context::current(), address, commitment)
-            .await
-    }
-
-    async fn get_account(&mut self, address: Pubkey) -> io::Result<Option<Account>> {
-        self.get_account_with_commitment(address, CommitmentLevel::default())
-            .await
-    }
-
-    async fn get_balance_with_commitment(
-        &mut self,
-        address: Pubkey,
-        commitment: CommitmentLevel,
-    ) -> io::Result<u64> {
-        let account = self
-            .get_account_with_commitment_and_context(context::current(), address, commitment)
-            .await?;
-        Ok(account.map(|x| x.lamports).unwrap_or(0))
-    }
-
-    async fn get_balance(&mut self, address: Pubkey) -> io::Result<u64> {
-        self.get_balance_with_commitment(address, CommitmentLevel::default())
-            .await
-    }
-
-    async fn get_transaction_status(
-        &mut self,
-        signature: Signature,
-    ) -> io::Result<Option<TransactionStatus>> {
-        self.get_transaction_status_with_context(context::current(), signature)
-            .await
-    }
-
-    async fn get_transaction_statuses(
+    pub async fn get_transaction_statuses(
         &mut self,
         signatures: Vec<Signature>,
     ) -> io::Result<Vec<Option<TransactionStatus>>> {
@@ -195,28 +415,78 @@ impl BanksClientExt for BanksClient {
         // Convert Vec<Result<_, _>> to Result<Vec<_>>
         statuses.into_iter().collect()
     }
+
+    pub fn get_latest_blockhash(&mut self) -> impl Future<Output = io::Result<Hash>> + '_ {
+        self.get_latest_blockhash_with_commitment(CommitmentLevel::default())
+            .map(|result| {
+                result?
+                    .map(|x| x.0)
+                    .ok_or(BanksClientError::ClientError("valid blockhash not found"))
+                    .map_err(Into::into)
+            })
+    }
+
+    pub fn get_latest_blockhash_with_commitment(
+        &mut self,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Option<(Hash, u64)>>> + '_ {
+        self.get_latest_blockhash_with_commitment_and_context(context::current(), commitment)
+    }
+
+    pub fn get_latest_blockhash_with_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = io::Result<Option<(Hash, u64)>>> + '_ {
+        self.inner
+            .get_latest_blockhash_with_commitment_and_context(ctx, commitment)
+            .map_err(BanksClientError::from) // Remove this when return Err type updated to BanksClientError
+            .map_err(Into::into)
+    }
+
+    pub fn get_fee_for_message_with_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        commitment: CommitmentLevel,
+        message: Message,
+    ) -> impl Future<Output = io::Result<Option<u64>>> + '_ {
+        self.inner
+            .get_fee_for_message_with_commitment_and_context(ctx, commitment, message)
+            .map_err(BanksClientError::from) // Remove this when return Err type updated to BanksClientError
+            .map_err(Into::into)
+    }
 }
 
-pub async fn start_client(
-    transport: UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>>,
-) -> io::Result<BanksClient> {
-    BanksClient::new(client::Config::default(), transport).spawn()
+pub async fn start_client<C>(transport: C) -> io::Result<BanksClient>
+where
+    C: Transport<ClientMessage<BanksRequest>, Response<BanksResponse>> + Send + 'static,
+{
+    Ok(BanksClient {
+        inner: TarpcClient::new(client::Config::default(), transport).spawn(),
+    })
 }
 
 pub async fn start_tcp_client<T: ToSocketAddrs>(addr: T) -> io::Result<BanksClient> {
     let transport = tcp::connect(addr, Bincode::default).await?;
-    BanksClient::new(client::Config::default(), transport).spawn()
+    Ok(BanksClient {
+        inner: TarpcClient::new(client::Config::default(), transport).spawn(),
+    })
 }
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use solana_banks_server::banks_server::start_local_server;
-    use solana_runtime::{bank::Bank, bank_forks::BankForks, genesis_utils::create_genesis_config};
-    use solana_sdk::{message::Message, pubkey::Pubkey, signature::Signer, system_instruction};
-    use std::sync::{Arc, RwLock};
-    use tarpc::transport;
-    use tokio::{runtime::Runtime, time::delay_for};
+    use {
+        super::*,
+        solana_banks_server::banks_server::start_local_server,
+        solana_runtime::{
+            bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache,
+            genesis_utils::create_genesis_config,
+        },
+        solana_sdk::{message::Message, signature::Signer, system_instruction},
+        std::sync::{Arc, RwLock},
+        tarpc::transport,
+        tokio::{runtime::Runtime, time::sleep},
+    };
 
     #[test]
     fn test_banks_client_new() {
@@ -225,27 +495,31 @@ mod tests {
     }
 
     #[test]
-    fn test_banks_server_transfer_via_server() -> io::Result<()> {
+    fn test_banks_server_transfer_via_server() -> Result<(), BanksClientError> {
         // This test shows the preferred way to interact with BanksServer.
         // It creates a runtime explicitly (no globals via tokio macros) and calls
         // `runtime.block_on()` just once, to run all the async code.
 
         let genesis = create_genesis_config(10);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(
-            &genesis.genesis_config,
-        ))));
+        let bank = Bank::new_for_tests(&genesis.genesis_config);
+        let slot = bank.slot();
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
+        ));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
 
-        let bob_pubkey = Pubkey::new_rand();
+        let bob_pubkey = solana_sdk::pubkey::new_rand();
         let mint_pubkey = genesis.mint_keypair.pubkey();
         let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(&bank_forks).await;
-            let mut banks_client =
-                BanksClient::new(client::Config::default(), client_transport).spawn()?;
+            let client_transport =
+                start_local_server(bank_forks, block_commitment_cache, Duration::from_millis(1))
+                    .await;
+            let mut banks_client = start_client(client_transport).await?;
 
-            let recent_blockhash = banks_client.get_recent_blockhash().await?;
+            let recent_blockhash = banks_client.get_latest_blockhash().await?;
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
             banks_client.process_transaction(transaction).await.unwrap();
             assert_eq!(banks_client.get_balance(bob_pubkey).await?, 1);
@@ -254,26 +528,33 @@ mod tests {
     }
 
     #[test]
-    fn test_banks_server_transfer_via_client() -> io::Result<()> {
+    fn test_banks_server_transfer_via_client() -> Result<(), BanksClientError> {
         // The caller may not want to hold the connection open until the transaction
         // is processed (or blockhash expires). In this test, we verify the
         // server-side functionality is available to the client.
 
         let genesis = create_genesis_config(10);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::new(
-            &genesis.genesis_config,
-        ))));
+        let bank = Bank::new_for_tests(&genesis.genesis_config);
+        let slot = bank.slot();
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
+        ));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
 
         let mint_pubkey = &genesis.mint_keypair.pubkey();
-        let bob_pubkey = Pubkey::new_rand();
-        let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
-        let message = Message::new(&[instruction], Some(&mint_pubkey));
+        let bob_pubkey = solana_sdk::pubkey::new_rand();
+        let instruction = system_instruction::transfer(mint_pubkey, &bob_pubkey, 1);
+        let message = Message::new(&[instruction], Some(mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(&bank_forks).await;
-            let mut banks_client =
-                BanksClient::new(client::Config::default(), client_transport).spawn()?;
-            let (_, recent_blockhash, last_valid_slot) = banks_client.get_fees().await?;
+            let client_transport =
+                start_local_server(bank_forks, block_commitment_cache, Duration::from_millis(1))
+                    .await;
+            let mut banks_client = start_client(client_transport).await?;
+            let (recent_blockhash, last_valid_block_height) = banks_client
+                .get_latest_blockhash_with_commitment(CommitmentLevel::default())
+                .await?
+                .unwrap();
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
             let signature = transaction.signatures[0];
             banks_client.send_transaction(transaction).await?;
@@ -281,11 +562,11 @@ mod tests {
             let mut status = banks_client.get_transaction_status(signature).await?;
 
             while status.is_none() {
-                let root_slot = banks_client.get_root_slot().await?;
-                if root_slot > last_valid_slot {
+                let root_block_height = banks_client.get_root_block_height().await?;
+                if root_block_height > last_valid_block_height {
                     break;
                 }
-                delay_for(Duration::from_millis(100)).await;
+                sleep(Duration::from_millis(100)).await;
                 status = banks_client.get_transaction_status(signature).await?;
             }
             assert!(status.unwrap().err.is_none());

banks-interface/Cargo.toml

@@ -1,17 +1,18 @@
 [package]
 name = "solana-banks-interface"
-version = "1.4.0"
+version = "1.9.5"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
-edition = "2018"
+documentation = "https://docs.rs/solana-banks-interface"
+edition = "2021"
 
 [dependencies]
-serde = { version = "1.0.112", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-tarpc = { version = "0.22.0", features = ["full"] }
+serde = { version = "1.0.130", features = ["derive"] }
+solana-sdk = { path = "../sdk", version = "=1.9.5" }
+tarpc = { version = "0.27.2", features = ["full"] }
 
 [lib]
 crate-type = ["lib"]

banks-interface/src/lib.rs

@@ -1,31 +1,66 @@
-use serde::{Deserialize, Serialize};
-use solana_sdk::{
-    account::Account,
-    clock::Slot,
-    commitment_config::CommitmentLevel,
-    fee_calculator::FeeCalculator,
-    hash::Hash,
-    pubkey::Pubkey,
-    signature::Signature,
-    transaction::{self, Transaction, TransactionError},
+#![allow(deprecated)]
+
+use {
+    serde::{Deserialize, Serialize},
+    solana_sdk::{
+        account::Account,
+        clock::Slot,
+        commitment_config::CommitmentLevel,
+        fee_calculator::FeeCalculator,
+        hash::Hash,
+        message::Message,
+        pubkey::Pubkey,
+        signature::Signature,
+        transaction::{self, Transaction, TransactionError},
+    },
 };
 
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub enum TransactionConfirmationStatus {
+    Processed,
+    Confirmed,
+    Finalized,
+}
+
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct TransactionStatus {
     pub slot: Slot,
     pub confirmations: Option<usize>, // None = rooted
     pub err: Option<TransactionError>,
+    pub confirmation_status: Option<TransactionConfirmationStatus>,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TransactionSimulationDetails {
+    pub logs: Vec<String>,
+    pub units_consumed: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct BanksTransactionResultWithSimulation {
+    pub result: Option<transaction::Result<()>>,
+    pub simulation_details: Option<TransactionSimulationDetails>,
 }
 
 #[tarpc::service]
 pub trait Banks {
     async fn send_transaction_with_context(transaction: Transaction);
+    #[deprecated(
+        since = "1.9.0",
+        note = "Please use `get_fee_for_message_with_commitment_and_context` instead"
+    )]
     async fn get_fees_with_commitment_and_context(
         commitment: CommitmentLevel,
     ) -> (FeeCalculator, Hash, Slot);
     async fn get_transaction_status_with_context(signature: Signature)
         -> Option<TransactionStatus>;
     async fn get_slot_with_context(commitment: CommitmentLevel) -> Slot;
+    async fn get_block_height_with_context(commitment: CommitmentLevel) -> u64;
+    async fn process_transaction_with_preflight_and_commitment_and_context(
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> BanksTransactionResultWithSimulation;
     async fn process_transaction_with_commitment_and_context(
         transaction: Transaction,
         commitment: CommitmentLevel,
@@ -34,12 +69,22 @@ pub trait Banks {
         address: Pubkey,
         commitment: CommitmentLevel,
     ) -> Option<Account>;
+    async fn get_latest_blockhash_with_context() -> Hash;
+    async fn get_latest_blockhash_with_commitment_and_context(
+        commitment: CommitmentLevel,
+    ) -> Option<(Hash, u64)>;
+    async fn get_fee_for_message_with_commitment_and_context(
+        commitment: CommitmentLevel,
+        message: Message,
+    ) -> Option<u64>;
 }
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use tarpc::{client, transport};
+    use {
+        super::*,
+        tarpc::{client, transport},
+    };
 
     #[test]
     fn test_banks_client_new() {

banks-server/Cargo.toml

@@ -1,24 +1,25 @@
 [package]
 name = "solana-banks-server"
-version = "1.4.0"
+version = "1.9.5"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
-edition = "2018"
+documentation = "https://docs.rs/solana-banks-server"
+edition = "2021"
 
 [dependencies]
-bincode = "1.3.1"
+bincode = "1.3.3"
 futures = "0.3"
-log = "0.4.8"
-solana-banks-interface = { path = "../banks-interface", version = "1.4.0" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-metrics = { path = "../metrics", version = "1.4.0" }
-tarpc = { version = "0.22.0", features = ["full"] }
-tokio = "0.2"
-tokio-serde = { version = "0.6", features = ["bincode"] }
+solana-banks-interface = { path = "../banks-interface", version = "=1.9.5" }
+solana-runtime = { path = "../runtime", version = "=1.9.5" }
+solana-sdk = { path = "../sdk", version = "=1.9.5" }
+solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.9.5" }
+tarpc = { version = "0.27.2", features = ["full"] }
+tokio = { version = "1", features = ["full"] }
+tokio-serde = { version = "0.8", features = ["bincode"] }
+tokio-stream = "0.1"
 
 [lib]
 crate-type = ["lib"]

banks-server/src/banks_server.rs

@@ -1,51 +1,59 @@
-use crate::send_transaction_service::{SendTransactionService, TransactionInfo};
-use bincode::{deserialize, serialize};
-use futures::{
-    future,
-    prelude::stream::{self, StreamExt},
-};
-use solana_banks_interface::{Banks, BanksRequest, BanksResponse, TransactionStatus};
-use solana_runtime::{
-    bank::Bank,
-    bank_forks::BankForks,
-    commitment::{BlockCommitmentCache, CommitmentSlots},
-};
-use solana_sdk::{
-    account::Account,
-    clock::Slot,
-    commitment_config::CommitmentLevel,
-    fee_calculator::FeeCalculator,
-    hash::Hash,
-    pubkey::Pubkey,
-    signature::Signature,
-    transaction::{self, Transaction},
-};
-use std::{
-    collections::HashMap,
-    io,
-    net::{Ipv4Addr, SocketAddr},
-    sync::{
-        mpsc::{channel, Receiver, Sender},
-        Arc, RwLock,
+use {
+    bincode::{deserialize, serialize},
+    futures::{future, prelude::stream::StreamExt},
+    solana_banks_interface::{
+        Banks, BanksRequest, BanksResponse, BanksTransactionResultWithSimulation,
+        TransactionConfirmationStatus, TransactionSimulationDetails, TransactionStatus,
     },
-    thread::Builder,
-    time::Duration,
+    solana_runtime::{
+        bank::{Bank, TransactionSimulationResult},
+        bank_forks::BankForks,
+        commitment::BlockCommitmentCache,
+    },
+    solana_sdk::{
+        account::Account,
+        clock::Slot,
+        commitment_config::CommitmentLevel,
+        feature_set::FeatureSet,
+        fee_calculator::FeeCalculator,
+        hash::Hash,
+        message::{Message, SanitizedMessage},
+        pubkey::Pubkey,
+        signature::Signature,
+        transaction::{self, SanitizedTransaction, Transaction},
+    },
+    solana_send_transaction_service::{
+        send_transaction_service::{SendTransactionService, TransactionInfo},
+        tpu_info::NullTpuInfo,
+    },
+    std::{
+        convert::TryFrom,
+        io,
+        net::{Ipv4Addr, SocketAddr},
+        sync::{
+            mpsc::{channel, Receiver, Sender},
+            Arc, RwLock,
+        },
+        thread::Builder,
+        time::Duration,
+    },
+    tarpc::{
+        context::Context,
+        serde_transport::tcp,
+        server::{self, incoming::Incoming, Channel},
+        transport::{self, channel::UnboundedChannel},
+        ClientMessage, Response,
+    },
+    tokio::time::sleep,
+    tokio_serde::formats::Bincode,
 };
-use tarpc::{
-    context::Context,
-    rpc::{transport::channel::UnboundedChannel, ClientMessage, Response},
-    serde_transport::tcp,
-    server::{self, Channel, Handler},
-    transport,
-};
-use tokio::time::delay_for;
-use tokio_serde::formats::Bincode;
 
 #[derive(Clone)]
 struct BanksServer {
     bank_forks: Arc<RwLock<BankForks>>,
     block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
     transaction_sender: Sender<TransactionInfo>,
+    poll_signature_status_sleep_duration: Duration,
 }
 
 impl BanksServer {
@@ -57,15 +65,17 @@ impl BanksServer {
         bank_forks: Arc<RwLock<BankForks>>,
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
         transaction_sender: Sender<TransactionInfo>,
+        poll_signature_status_sleep_duration: Duration,
     ) -> Self {
         Self {
             bank_forks,
             block_commitment_cache,
             transaction_sender,
+            poll_signature_status_sleep_duration,
         }
     }
 
-    fn run(bank: &Bank, transaction_receiver: Receiver<TransactionInfo>) {
+    fn run(bank_forks: Arc<RwLock<BankForks>>, transaction_receiver: Receiver<TransactionInfo>) {
         while let Ok(info) = transaction_receiver.recv() {
             let mut transaction_infos = vec![info];
             while let Ok(info) = transaction_receiver.try_recv() {
@@ -75,25 +85,36 @@ impl BanksServer {
                 .into_iter()
                 .map(|info| deserialize(&info.wire_transaction).unwrap())
                 .collect();
-            let _ = bank.process_transactions(&transactions);
+            let bank = bank_forks.read().unwrap().working_bank();
+            let _ = bank.try_process_transactions(transactions.iter());
         }
     }
 
     /// Useful for unit-testing
-    fn new_loopback(bank_forks: Arc<RwLock<BankForks>>) -> Self {
+    fn new_loopback(
+        bank_forks: Arc<RwLock<BankForks>>,
+        block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+        poll_signature_status_sleep_duration: Duration,
+    ) -> Self {
         let (transaction_sender, transaction_receiver) = channel();
         let bank = bank_forks.read().unwrap().working_bank();
         let slot = bank.slot();
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new(
-            HashMap::default(),
-            0,
-            CommitmentSlots::new_from_slot(slot),
-        )));
+        {
+            // ensure that the commitment cache and bank are synced
+            let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
+            w_block_commitment_cache.set_all_slots(slot, slot);
+        }
+        let server_bank_forks = bank_forks.clone();
         Builder::new()
             .name("solana-bank-forks-client".to_string())
-            .spawn(move || Self::run(&bank, transaction_receiver))
+            .spawn(move || Self::run(server_bank_forks, transaction_receiver))
             .unwrap();
-        Self::new(bank_forks, block_commitment_cache, transaction_sender)
+        Self::new(
+            bank_forks,
+            block_commitment_cache,
+            transaction_sender,
+            poll_signature_status_sleep_duration,
+        )
     }
 
     fn slot(&self, commitment: CommitmentLevel) -> Slot {
@@ -109,37 +130,58 @@ impl BanksServer {
 
     async fn poll_signature_status(
         self,
-        signature: Signature,
-        last_valid_slot: Slot,
+        signature: &Signature,
+        blockhash: &Hash,
+        last_valid_block_height: u64,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
-        let mut status = self.bank(commitment).get_signature_status(&signature);
+        let mut status = self
+            .bank(commitment)
+            .get_signature_status_with_blockhash(signature, blockhash);
         while status.is_none() {
-            delay_for(Duration::from_millis(200)).await;
+            sleep(self.poll_signature_status_sleep_duration).await;
             let bank = self.bank(commitment);
-            if bank.slot() > last_valid_slot {
+            if bank.block_height() > last_valid_block_height {
                 break;
             }
-            status = bank.get_signature_status(&signature);
+            status = bank.get_signature_status_with_blockhash(signature, blockhash);
         }
         status
     }
 }
 
+fn verify_transaction(
+    transaction: &Transaction,
+    feature_set: &Arc<FeatureSet>,
+) -> transaction::Result<()> {
+    if let Err(err) = transaction.verify() {
+        Err(err)
+    } else if let Err(err) = transaction.verify_precompiles(feature_set) {
+        Err(err)
+    } else {
+        Ok(())
+    }
+}
+
 #[tarpc::server]
 impl Banks for BanksServer {
     async fn send_transaction_with_context(self, _: Context, transaction: Transaction) {
         let blockhash = &transaction.message.recent_blockhash;
-        let last_valid_slot = self
+        let last_valid_block_height = self
             .bank_forks
             .read()
             .unwrap()
             .root_bank()
-            .get_blockhash_last_valid_slot(&blockhash)
+            .get_blockhash_last_valid_block_height(blockhash)
             .unwrap();
         let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
-        let info =
-            TransactionInfo::new(signature, serialize(&transaction).unwrap(), last_valid_slot);
+        let info = TransactionInfo::new(
+            signature,
+            serialize(&transaction).unwrap(),
+            last_valid_block_height,
+            None,
+            None,
+        );
         self.transaction_sender.send(info).unwrap();
     }
 
@@ -147,11 +189,18 @@ impl Banks for BanksServer {
         self,
         _: Context,
         commitment: CommitmentLevel,
-    ) -> (FeeCalculator, Hash, Slot) {
+    ) -> (FeeCalculator, Hash, u64) {
         let bank = self.bank(commitment);
-        let (blockhash, fee_calculator) = bank.last_blockhash_with_fee_calculator();
-        let last_valid_slot = bank.get_blockhash_last_valid_slot(&blockhash).unwrap();
-        (fee_calculator, blockhash, last_valid_slot)
+        let blockhash = bank.last_blockhash();
+        let lamports_per_signature = bank.get_lamports_per_signature();
+        let last_valid_block_height = bank
+            .get_blockhash_last_valid_block_height(&blockhash)
+            .unwrap();
+        (
+            FeeCalculator::new(lamports_per_signature),
+            blockhash,
+            last_valid_block_height,
+        )
     }
 
     async fn get_transaction_status_with_context(
@@ -159,11 +208,17 @@ impl Banks for BanksServer {
         _: Context,
         signature: Signature,
     ) -> Option<TransactionStatus> {
-        let bank = self.bank(CommitmentLevel::Recent);
+        let bank = self.bank(CommitmentLevel::Processed);
         let (slot, status) = bank.get_signature_status_slot(&signature)?;
         let r_block_commitment_cache = self.block_commitment_cache.read().unwrap();
 
-        let confirmations = if r_block_commitment_cache.root() >= slot {
+        let optimistically_confirmed_bank = self.bank(CommitmentLevel::Confirmed);
+        let optimistically_confirmed =
+            optimistically_confirmed_bank.get_signature_status_slot(&signature);
+
+        let confirmations = if r_block_commitment_cache.root() >= slot
+            && r_block_commitment_cache.highest_confirmed_root() >= slot
+        {
             None
         } else {
             r_block_commitment_cache
@@ -174,6 +229,13 @@ impl Banks for BanksServer {
             slot,
             confirmations,
             err: status.err(),
+            confirmation_status: if confirmations.is_none() {
+                Some(TransactionConfirmationStatus::Finalized)
+            } else if optimistically_confirmed.is_some() {
+                Some(TransactionConfirmationStatus::Confirmed)
+            } else {
+                Some(TransactionConfirmationStatus::Processed)
+            },
         })
     }
 
@@ -181,25 +243,76 @@ impl Banks for BanksServer {
         self.slot(commitment)
     }
 
+    async fn get_block_height_with_context(self, _: Context, commitment: CommitmentLevel) -> u64 {
+        self.bank(commitment).block_height()
+    }
+
+    async fn process_transaction_with_preflight_and_commitment_and_context(
+        self,
+        ctx: Context,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> BanksTransactionResultWithSimulation {
+        let sanitized_transaction =
+            match SanitizedTransaction::try_from_legacy_transaction(transaction.clone()) {
+                Err(err) => {
+                    return BanksTransactionResultWithSimulation {
+                        result: Some(Err(err)),
+                        simulation_details: None,
+                    };
+                }
+                Ok(tx) => tx,
+            };
+        if let TransactionSimulationResult {
+            result: Err(err),
+            logs,
+            post_simulation_accounts: _,
+            units_consumed,
+        } = self
+            .bank(commitment)
+            .simulate_transaction_unchecked(sanitized_transaction)
+        {
+            return BanksTransactionResultWithSimulation {
+                result: Some(Err(err)),
+                simulation_details: Some(TransactionSimulationDetails {
+                    logs,
+                    units_consumed,
+                }),
+            };
+        }
+        BanksTransactionResultWithSimulation {
+            result: self
+                .process_transaction_with_commitment_and_context(ctx, transaction, commitment)
+                .await,
+            simulation_details: None,
+        }
+    }
+
     async fn process_transaction_with_commitment_and_context(
         self,
         _: Context,
         transaction: Transaction,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
+        if let Err(err) = verify_transaction(&transaction, &self.bank(commitment).feature_set) {
+            return Some(Err(err));
+        }
+
         let blockhash = &transaction.message.recent_blockhash;
-        let last_valid_slot = self
-            .bank_forks
-            .read()
-            .unwrap()
-            .root_bank()
-            .get_blockhash_last_valid_slot(&blockhash)
+        let last_valid_block_height = self
+            .bank(commitment)
+            .get_blockhash_last_valid_block_height(blockhash)
             .unwrap();
         let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
-        let info =
-            TransactionInfo::new(signature, serialize(&transaction).unwrap(), last_valid_slot);
+        let info = TransactionInfo::new(
+            signature,
+            serialize(&transaction).unwrap(),
+            last_valid_block_height,
+            None,
+            None,
+        );
         self.transaction_sender.send(info).unwrap();
-        self.poll_signature_status(signature, last_valid_slot, commitment)
+        self.poll_signature_status(&signature, blockhash, last_valid_block_height, commitment)
             .await
     }
 
@@ -210,18 +323,49 @@ impl Banks for BanksServer {
         commitment: CommitmentLevel,
     ) -> Option<Account> {
         let bank = self.bank(commitment);
-        bank.get_account(&address)
+        bank.get_account(&address).map(Account::from)
+    }
+
+    async fn get_latest_blockhash_with_context(self, _: Context) -> Hash {
+        let bank = self.bank(CommitmentLevel::default());
+        bank.last_blockhash()
+    }
+
+    async fn get_latest_blockhash_with_commitment_and_context(
+        self,
+        _: Context,
+        commitment: CommitmentLevel,
+    ) -> Option<(Hash, u64)> {
+        let bank = self.bank(commitment);
+        let blockhash = bank.last_blockhash();
+        let last_valid_block_height = bank.get_blockhash_last_valid_block_height(&blockhash)?;
+        Some((blockhash, last_valid_block_height))
+    }
+
+    async fn get_fee_for_message_with_commitment_and_context(
+        self,
+        _: Context,
+        commitment: CommitmentLevel,
+        message: Message,
+    ) -> Option<u64> {
+        let bank = self.bank(commitment);
+        let sanitized_message = SanitizedMessage::try_from(message).ok()?;
+        bank.get_fee_for_message(&sanitized_message)
     }
 }
 
 pub async fn start_local_server(
-    bank_forks: &Arc<RwLock<BankForks>>,
+    bank_forks: Arc<RwLock<BankForks>>,
+    block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+    poll_signature_status_sleep_duration: Duration,
 ) -> UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>> {
-    let banks_server = BanksServer::new_loopback(bank_forks.clone());
+    let banks_server = BanksServer::new_loopback(
+        bank_forks,
+        block_commitment_cache,
+        poll_signature_status_sleep_duration,
+    );
     let (client_transport, server_transport) = transport::channel::unbounded();
-    let server = server::new(server::Config::default())
-        .incoming(stream::once(future::ready(server_transport)))
-        .respond_with(banks_server.serve());
+    let server = server::BaseChannel::with_defaults(server_transport).execute(banks_server.serve());
     tokio::spawn(server);
     client_transport
 }
@@ -250,11 +394,22 @@ pub async fn start_tcp_server(
         .map(move |chan| {
             let (sender, receiver) = channel();
 
-            SendTransactionService::new(tpu_addr, &bank_forks, receiver);
+            SendTransactionService::new::<NullTpuInfo>(
+                tpu_addr,
+                &bank_forks,
+                None,
+                receiver,
+                5_000,
+                0,
+            );
 
-            let server =
-                BanksServer::new(bank_forks.clone(), block_commitment_cache.clone(), sender);
-            chan.respond_with(server.serve()).execute()
+            let server = BanksServer::new(
+                bank_forks.clone(),
+                block_commitment_cache.clone(),
+                sender,
+                Duration::from_millis(200),
+            );
+            chan.execute(server.serve())
         })
         // Max 10 channels.
         .buffer_unordered(10)

banks-server/src/lib.rs

@@ -1,6 +1,3 @@
+#![allow(clippy::integer_arithmetic)]
 pub mod banks_server;
 pub mod rpc_banks_service;
-pub mod send_transaction_service;
-
-#[macro_use]
-extern crate solana_metrics;

banks-server/src/rpc_banks_service.rs

@@ -1,19 +1,22 @@
 //! The `rpc_banks_service` module implements the Solana Banks RPC API.
 
-use crate::banks_server::start_tcp_server;
-use futures::{future::FutureExt, pin_mut, prelude::stream::StreamExt, select};
-use solana_runtime::{bank_forks::BankForks, commitment::BlockCommitmentCache};
-use std::{
-    net::SocketAddr,
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc, RwLock,
+use {
+    crate::banks_server::start_tcp_server,
+    futures::{future::FutureExt, pin_mut, prelude::stream::StreamExt, select},
+    solana_runtime::{bank_forks::BankForks, commitment::BlockCommitmentCache},
+    std::{
+        net::SocketAddr,
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc, RwLock,
+        },
+        thread::{self, Builder, JoinHandle},
     },
-    thread::{self, Builder, JoinHandle},
-};
-use tokio::{
-    runtime::Runtime,
-    time::{self, Duration},
+    tokio::{
+        runtime::Runtime,
+        time::{self, Duration},
+    },
+    tokio_stream::wrappers::IntervalStream,
 };
 
 pub struct RpcBanksService {
@@ -35,7 +38,7 @@ async fn start_abortable_tcp_server(
         block_commitment_cache.clone(),
     )
     .fuse();
-    let interval = time::interval(Duration::from_millis(100)).fuse();
+    let interval = IntervalStream::new(time::interval(Duration::from_millis(100))).fuse();
     pin_mut!(server, interval);
     loop {
         select! {
@@ -100,12 +103,11 @@ impl RpcBanksService {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use solana_runtime::bank::Bank;
+    use {super::*, solana_runtime::bank::Bank};
 
     #[test]
     fn test_rpc_banks_server_exit() {
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::default())));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::default_for_tests())));
         let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
         let exit = Arc::new(AtomicBool::new(false));
         let addr = "127.0.0.1:0".parse().unwrap();

banks-server/src/send_transaction_service.rs

@@ -1,343 +0,0 @@
-// TODO: Merge this implementation with the one at `core/src/send_transaction_service.rs`
-use log::*;
-use solana_metrics::{datapoint_warn, inc_new_counter_info};
-use solana_runtime::{bank::Bank, bank_forks::BankForks};
-use solana_sdk::{clock::Slot, signature::Signature};
-use std::{
-    collections::HashMap,
-    net::{SocketAddr, UdpSocket},
-    sync::{
-        mpsc::{Receiver, RecvTimeoutError},
-        Arc, RwLock,
-    },
-    thread::{self, Builder, JoinHandle},
-    time::{Duration, Instant},
-};
-
-/// Maximum size of the transaction queue
-const MAX_TRANSACTION_QUEUE_SIZE: usize = 10_000; // This seems like a lot but maybe it needs to be bigger one day
-
-pub struct SendTransactionService {
-    thread: JoinHandle<()>,
-}
-
-pub struct TransactionInfo {
-    pub signature: Signature,
-    pub wire_transaction: Vec<u8>,
-    pub last_valid_slot: Slot,
-}
-
-impl TransactionInfo {
-    pub fn new(signature: Signature, wire_transaction: Vec<u8>, last_valid_slot: Slot) -> Self {
-        Self {
-            signature,
-            wire_transaction,
-            last_valid_slot,
-        }
-    }
-}
-
-#[derive(Default, Debug, PartialEq)]
-struct ProcessTransactionsResult {
-    rooted: u64,
-    expired: u64,
-    retried: u64,
-    failed: u64,
-    retained: u64,
-}
-
-impl SendTransactionService {
-    pub fn new(
-        tpu_address: SocketAddr,
-        bank_forks: &Arc<RwLock<BankForks>>,
-        receiver: Receiver<TransactionInfo>,
-    ) -> Self {
-        let thread = Self::retry_thread(receiver, bank_forks.clone(), tpu_address);
-        Self { thread }
-    }
-
-    fn retry_thread(
-        receiver: Receiver<TransactionInfo>,
-        bank_forks: Arc<RwLock<BankForks>>,
-        tpu_address: SocketAddr,
-    ) -> JoinHandle<()> {
-        let mut last_status_check = Instant::now();
-        let mut transactions = HashMap::new();
-        let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
-
-        Builder::new()
-            .name("send-tx-svc".to_string())
-            .spawn(move || loop {
-                match receiver.recv_timeout(Duration::from_secs(1)) {
-                    Err(RecvTimeoutError::Disconnected) => break,
-                    Err(RecvTimeoutError::Timeout) => {}
-                    Ok(transaction_info) => {
-                        Self::send_transaction(
-                            &send_socket,
-                            &tpu_address,
-                            &transaction_info.wire_transaction,
-                        );
-                        if transactions.len() < MAX_TRANSACTION_QUEUE_SIZE {
-                            transactions.insert(transaction_info.signature, transaction_info);
-                        } else {
-                            datapoint_warn!("send_transaction_service-queue-overflow");
-                        }
-                    }
-                }
-
-                if Instant::now().duration_since(last_status_check).as_secs() >= 5 {
-                    if !transactions.is_empty() {
-                        datapoint_info!(
-                            "send_transaction_service-queue-size",
-                            ("len", transactions.len(), i64)
-                        );
-                        let bank_forks = bank_forks.read().unwrap();
-                        let root_bank = bank_forks.root_bank();
-                        let working_bank = bank_forks.working_bank();
-
-                        let _result = Self::process_transactions(
-                            &working_bank,
-                            &root_bank,
-                            &send_socket,
-                            &tpu_address,
-                            &mut transactions,
-                        );
-                    }
-                    last_status_check = Instant::now();
-                }
-            })
-            .unwrap()
-    }
-
-    fn process_transactions(
-        working_bank: &Arc<Bank>,
-        root_bank: &Arc<Bank>,
-        send_socket: &UdpSocket,
-        tpu_address: &SocketAddr,
-        transactions: &mut HashMap<Signature, TransactionInfo>,
-    ) -> ProcessTransactionsResult {
-        let mut result = ProcessTransactionsResult::default();
-
-        transactions.retain(|signature, transaction_info| {
-            if root_bank.has_signature(signature) {
-                info!("Transaction is rooted: {}", signature);
-                result.rooted += 1;
-                inc_new_counter_info!("send_transaction_service-rooted", 1);
-                false
-            } else if transaction_info.last_valid_slot < root_bank.slot() {
-                info!("Dropping expired transaction: {}", signature);
-                result.expired += 1;
-                inc_new_counter_info!("send_transaction_service-expired", 1);
-                false
-            } else {
-                match working_bank.get_signature_status_slot(signature) {
-                    None => {
-                        // Transaction is unknown to the working bank, it might have been
-                        // dropped or landed in another fork.  Re-send it
-                        info!("Retrying transaction: {}", signature);
-                        result.retried += 1;
-                        inc_new_counter_info!("send_transaction_service-retry", 1);
-                        Self::send_transaction(
-                            &send_socket,
-                            &tpu_address,
-                            &transaction_info.wire_transaction,
-                        );
-                        true
-                    }
-                    Some((_slot, status)) => {
-                        if status.is_err() {
-                            info!("Dropping failed transaction: {}", signature);
-                            result.failed += 1;
-                            inc_new_counter_info!("send_transaction_service-failed", 1);
-                            false
-                        } else {
-                            result.retained += 1;
-                            true
-                        }
-                    }
-                }
-            }
-        });
-
-        result
-    }
-
-    fn send_transaction(
-        send_socket: &UdpSocket,
-        tpu_address: &SocketAddr,
-        wire_transaction: &[u8],
-    ) {
-        if let Err(err) = send_socket.send_to(wire_transaction, tpu_address) {
-            warn!("Failed to send transaction to {}: {:?}", tpu_address, err);
-        }
-    }
-
-    pub fn join(self) -> thread::Result<()> {
-        self.thread.join()
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use solana_sdk::{
-        genesis_config::create_genesis_config, pubkey::Pubkey, signature::Signer,
-        system_transaction,
-    };
-    use std::sync::mpsc::channel;
-
-    #[test]
-    fn service_exit() {
-        let tpu_address = "127.0.0.1:0".parse().unwrap();
-        let bank = Bank::default();
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
-        let (sender, receiver) = channel();
-
-        let send_tranaction_service =
-            SendTransactionService::new(tpu_address, &bank_forks, receiver);
-
-        drop(sender);
-        send_tranaction_service.join().unwrap();
-    }
-
-    #[test]
-    fn process_transactions() {
-        let (genesis_config, mint_keypair) = create_genesis_config(4);
-        let bank = Bank::new(&genesis_config);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
-        let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
-        let tpu_address = "127.0.0.1:0".parse().unwrap();
-
-        let root_bank = Arc::new(Bank::new_from_parent(
-            &bank_forks.read().unwrap().working_bank(),
-            &Pubkey::default(),
-            1,
-        ));
-        let rooted_signature = root_bank
-            .transfer(1, &mint_keypair, &mint_keypair.pubkey())
-            .unwrap();
-
-        let working_bank = Arc::new(Bank::new_from_parent(&root_bank, &Pubkey::default(), 2));
-
-        let non_rooted_signature = working_bank
-            .transfer(2, &mint_keypair, &mint_keypair.pubkey())
-            .unwrap();
-
-        let failed_signature = {
-            let blockhash = working_bank.last_blockhash();
-            let transaction =
-                system_transaction::transfer(&mint_keypair, &Pubkey::default(), 1, blockhash);
-            let signature = transaction.signatures[0];
-            working_bank.process_transaction(&transaction).unwrap_err();
-            signature
-        };
-
-        let mut transactions = HashMap::new();
-
-        info!("Expired transactions are dropped..");
-        transactions.insert(
-            Signature::default(),
-            TransactionInfo::new(Signature::default(), vec![], root_bank.slot() - 1),
-        );
-        let result = SendTransactionService::process_transactions(
-            &working_bank,
-            &root_bank,
-            &send_socket,
-            &tpu_address,
-            &mut transactions,
-        );
-        assert!(transactions.is_empty());
-        assert_eq!(
-            result,
-            ProcessTransactionsResult {
-                expired: 1,
-                ..ProcessTransactionsResult::default()
-            }
-        );
-
-        info!("Rooted transactions are dropped...");
-        transactions.insert(
-            rooted_signature,
-            TransactionInfo::new(rooted_signature, vec![], working_bank.slot()),
-        );
-        let result = SendTransactionService::process_transactions(
-            &working_bank,
-            &root_bank,
-            &send_socket,
-            &tpu_address,
-            &mut transactions,
-        );
-        assert!(transactions.is_empty());
-        assert_eq!(
-            result,
-            ProcessTransactionsResult {
-                rooted: 1,
-                ..ProcessTransactionsResult::default()
-            }
-        );
-
-        info!("Failed transactions are dropped...");
-        transactions.insert(
-            failed_signature,
-            TransactionInfo::new(failed_signature, vec![], working_bank.slot()),
-        );
-        let result = SendTransactionService::process_transactions(
-            &working_bank,
-            &root_bank,
-            &send_socket,
-            &tpu_address,
-            &mut transactions,
-        );
-        assert!(transactions.is_empty());
-        assert_eq!(
-            result,
-            ProcessTransactionsResult {
-                failed: 1,
-                ..ProcessTransactionsResult::default()
-            }
-        );
-
-        info!("Non-rooted transactions are kept...");
-        transactions.insert(
-            non_rooted_signature,
-            TransactionInfo::new(non_rooted_signature, vec![], working_bank.slot()),
-        );
-        let result = SendTransactionService::process_transactions(
-            &working_bank,
-            &root_bank,
-            &send_socket,
-            &tpu_address,
-            &mut transactions,
-        );
-        assert_eq!(transactions.len(), 1);
-        assert_eq!(
-            result,
-            ProcessTransactionsResult {
-                retained: 1,
-                ..ProcessTransactionsResult::default()
-            }
-        );
-        transactions.clear();
-
-        info!("Unknown transactions are retried...");
-        transactions.insert(
-            Signature::default(),
-            TransactionInfo::new(Signature::default(), vec![], working_bank.slot()),
-        );
-        let result = SendTransactionService::process_transactions(
-            &working_bank,
-            &root_bank,
-            &send_socket,
-            &tpu_address,
-            &mut transactions,
-        );
-        assert_eq!(transactions.len(), 1);
-        assert_eq!(
-            result,
-            ProcessTransactionsResult {
-                retried: 1,
-                ..ProcessTransactionsResult::default()
-            }
-        );
-    }
-}

bench-exchange/.gitignore

@@ -1,4 +0,0 @@
-/target/
-/config/
-/config-local/
-/farf/

bench-exchange/Cargo.toml

@@ -1,38 +0,0 @@
-[package]
-authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2018"
-name = "solana-bench-exchange"
-version = "1.4.0"
-repository = "https://github.com/solana-labs/solana"
-license = "Apache-2.0"
-homepage = "https://solana.com/"
-publish = false
-
-[dependencies]
-clap = "2.33.1"
-itertools = "0.9.0"
-log = "0.4.8"
-num-derive = "0.3"
-num-traits = "0.2"
-rand = "0.7.0"
-rayon = "1.4.0"
-serde_json = "1.0.56"
-serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
-solana-core = { path = "../core", version = "1.4.0" }
-solana-genesis = { path = "../genesis", version = "1.4.0" }
-solana-client = { path = "../client", version = "1.4.0" }
-solana-faucet = { path = "../faucet", version = "1.4.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.4.0" }
-solana-logger = { path = "../logger", version = "1.4.0" }
-solana-metrics = { path = "../metrics", version = "1.4.0" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-version = { path = "../version", version = "1.4.0" }
-
-[dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.4.0" }
-
-[package.metadata.docs.rs]
-targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/README.md

@@ -1,479 +0,0 @@
-# token-exchange
-Solana Token Exchange Bench
-
-If you can't wait; jump to [Running the exchange](#Running-the-exchange) to
-learn how to start and interact with the exchange.
-
-### Table of Contents
-[Overview](#Overview)<br>
-[Premise](#Premise)<br>
-[Exchange startup](#Exchange-startup)<br>
-[Order Requests](#Trade-requests)<br>
-[Order Cancellations](#Trade-cancellations)<br>
-[Trade swap](#Trade-swap)<br>
-[Exchange program operations](#Exchange-program-operations)<br>
-[Quotes and OHLCV](#Quotes-and-OHLCV)<br>
-[Investor strategies](#Investor-strategies)<br>
-[Running the exchange](#Running-the-exchange)<br>
-
-## Overview
-
-An exchange is a marketplace where one asset can be traded for another.  This
-demo demonstrates one way to host an exchange on the Solana blockchain by
-emulating a currency exchange.
-
-The assets are virtual tokens held by investors who may post order requests to
-the exchange.  A Matcher monitors the exchange and posts swap requests for
-matching orders.  All the transactions can execute concurrently.
-
-## Premise
-
-- Exchange
-  -  An exchange is a marketplace where one asset can be traded for another.
-     The exchange in this demo is the on-chain program that implements the
-     tokens and the policies for trading those tokens.
-- Token
-  - A virtual asset that can be owned, traded, and holds virtual intrinsic value
-    compared to other assets.  There are four types of tokens in this demo, A,
-    B, C, D.  Each one may be traded for another.
-- Token account
-  - An account owned by the exchange that holds a quantity of one type of token.
-- Account request
-  - A request to create a token account
-- Token request
-  - A request to deposit tokens of a particular type into a token account.
-- Asset pair
-  - A struct with fields Base and Quote, representing the two assets which make up a 
-    trading pair,  which themselves are Tokens. The Base or 'primary' asset is the
-    numerator and the Quote is the denominator for pricing purposes.
-- Order side
-  - Describes which side of the market an investor wants to place a trade on. Options
-    are "Bid" or "Ask", where a bid represents an offer to purchase the Base asset of
-    the AssetPair for a sum of the Quote Asset and an Ask is an offer to sell Base asset
-    for the Quote asset.
-- Price ratio
-  -  An expression of the relative prices of two tokens. Calculated with the Base
-     Asset as the numerator and the Quote Asset as the denominator. Ratios are 
-     represented as fixed point numbers.  The fixed point scaler is defined in
-     [exchange_state.rs](https://github.com/solana-labs/solana/blob/c2fdd1362a029dcf89c8907c562d2079d977df11/programs/exchange_api/src/exchange_state.rs#L7)
-- Order request
-  - A Solana transaction sent by a trader to the exchange to submit an order. 
-    Order requests are made up of the token pair, the order side (bid or ask),
-    quantity of the primary token, the price ratio, and the two token accounts
-    to be credited/deducted.  An example trade request looks like "T AB 5 2" 
-    which reads "Exchange 5 A tokens to B tokens at a price ratio of 1:2"  A fulfilled trade would result in 5 A tokens
-    deducted and 10 B tokens credited to the trade initiator's token accounts.
-    Successful order requests result in an order.
-- Order
-  - The result of a successful order request.  orders are stored in
-    accounts owned by the submitter of the order request.  They can only be
-    canceled by their owner but can be used by anyone in a trade swap.  They
-    contain the same information as the order request.
-- Price spread
-  - The difference between the two matching orders. The spread is the
-    profit of the Matcher initiating the swap request.
-- Match requirements
-  - Policies that result in a successful trade swap.
-- Match request
-  - A request to fill two complementary orders (bid/ask), resulting if successful,
-    in a trade being created.
-- Trade
-  - A successful trade is created from two matching orders that meet
-    swap requirements which are submitted in a Match Request by a Matcher and
-    executed by the exchange. A trade may not wholly satisfy one or both of the
-    orders in which case the orders are adjusted appropriately. Upon execution,
-    tokens are distributed to the traders' accounts and any overlap or 
-    "negative spread" between orders is deposited into the Matcher's profit
-    account.  All successful trades are recorded in the data of a new solana 
-    account for posterity.
-- Investor
-  - Individual investors who hold a number of tokens and wish to trade them on
-    the exchange.  Investors operate as Solana thin clients who own a set of
-    accounts containing tokens and/or order requests.  Investors post
-    transactions to the exchange in order to request tokens and post or cancel
-    order requests.
-- Matcher
-  - An agent who facilitates trading between investors.  Matchers operate as
-    Solana thin clients who monitor all the orders looking for a trade
-    match.  Once found, the Matcher issues a swap request to the exchange.
-    Matchers are the engine of the exchange and are rewarded for their efforts by
-    accumulating the price spreads of the swaps they initiate.  Matchers also
-    provide current bid/ask price and OHLCV (Open, High, Low, Close, Volume)
-    information on demand via a public network port.
-- Transaction fees
-  - Solana transaction fees are paid for by the transaction submitters who are
-    the Investors and Matchers.
-
-## Exchange startup
-
-The exchange is up and running when it reaches a state where it can take
-investors' trades and Matchers' match requests.  To achieve this state the
-following must occur in order:
-
-- Start the Solana blockchain
-- Start the  thin-client
-- The Matcher subscribes to change notifications for all the accounts owned by
-  the exchange program id.  The subscription is managed via Solana's JSON RPC
-  interface.
-- The Matcher starts responding to queries for bid/ask price and OHLCV
-
-The Matcher responding successfully to price and OHLCV requests is the signal to
-the investors that trades submitted after that point will be analyzed.  <!--This
-is not ideal, and instead investors should be able to submit trades at any time,
-and the Matcher could come and go without missing a trade.  One way to achieve
-this is for the Matcher to read the current state of all accounts looking for all
-open orders.-->
-
-Investors will initially query the exchange to discover their current balance
-for each type of token.  If the investor does not already have an account for
-each type of token, they will submit account requests.  Matcher as well will
-request accounts to hold the tokens they earn by initiating trade swaps.
-
-```rust
-/// Supported token types
-pub enum Token {
-    A,
-    B,
-    C,
-    D,
-}
-
-/// Supported token pairs
-pub enum TokenPair {
-    AB,
-    AC,
-    AD,
-    BC,
-    BD,
-    CD,
-}
-
-pub enum ExchangeInstruction {
-    /// New token account
-    /// key 0 - Signer
-    /// key 1 - New token account
-    AccountRequest,
-}
-
-/// Token accounts are populated with this structure
-pub struct TokenAccountInfo {
-    /// Investor who owns this account
-    pub owner: Pubkey,
-    /// Current number of tokens this account holds
-    pub tokens: Tokens,
-}
-```
-
-For this demo investors or Matcher can request more tokens from the exchange at
-any time by submitting token requests. In non-demos, an exchange of this type
-would provide another way to exchange a 3rd party asset into tokens.
-
-To request tokens, investors submit transfer requests:
-
-```rust
-pub enum ExchangeInstruction {
-    /// Transfer tokens between two accounts
-    /// key 0 - Account to transfer tokens to
-    /// key 1 - Account to transfer tokens from.  This can be the exchange program itself,
-    ///         the exchange has a limitless number of tokens it can transfer.
-    TransferRequest(Token, u64),
-}
-```
-
-## Order Requests
-
-When an investor decides to exchange a token of one type for another, they
-submit a transaction to the Solana Blockchain containing an order request, which,
-if successful, is turned into an order.  orders do not expire but are
-cancellable. <!-- orders should have a timestamp to enable trade
-expiration -->  When an order is created, tokens are deducted from a token
-account and the order acts as an escrow.  The tokens are held until the
-order is fulfilled or canceled. If the direction is `To`, then the number
-of `tokens` are deducted from the primary account, if `From` then `tokens`
-multiplied by `price` are deducted from the secondary account.  orders are
-no longer valid when the number of `tokens` goes to zero, at which point they
-can no longer be used. <!-- Could support refilling orders, so order
-accounts are refilled rather than accumulating -->
-
-```rust
-/// Direction of the exchange between two tokens in a pair
-pub enum Direction {
-    /// Trade first token type (primary) in the pair 'To' the second
-    To,
-    /// Trade first token type in the pair 'From' the second (secondary)
-    From,
-}
-
-pub struct OrderRequestInfo {
-    /// Direction of trade
-    pub direction: Direction,
-
-    /// Token pair to trade
-    pub pair: TokenPair,
-
-    /// Number of tokens to exchange; refers to the primary or the secondary depending on the direction
-    pub tokens: u64,
-
-    /// The price ratio the primary price over the secondary price.  The primary price is fixed
-    /// and equal to the variable `SCALER`.
-    pub price: u64,
-
-    /// Token account to deposit tokens on successful swap
-    pub dst_account: Pubkey,
-}
-
-pub enum ExchangeInstruction {
-    /// order request
-    /// key 0 - Signer
-    /// key 1 - Account in which to record the swap
-    /// key 2 - Token account associated with this trade
-    TradeRequest(TradeRequestInfo),
-}
-
-/// Trade accounts are populated with this structure
-pub struct TradeOrderInfo {
-    /// Owner of the order
-    pub owner: Pubkey,
-    /// Direction of the exchange
-    pub direction: Direction,
-    /// Token pair indicating two tokens to exchange, first is primary
-    pub pair: TokenPair,
-    /// Number of tokens to exchange; primary or secondary depending on direction
-    pub tokens: u64,
-    /// Scaled price of the secondary token given the primary is equal to the scale value
-    /// If scale is 1 and price is 2 then ratio is 1:2 or 1 primary token for 2 secondary tokens
-    pub price: u64,
-    /// account which the tokens were source from.  The trade account holds the tokens in escrow
-    /// until either one or more part of a swap or the trade is canceled.
-    pub src_account: Pubkey,
-    /// account which the tokens the tokens will be deposited into on a successful trade
-    pub dst_account: Pubkey,
-}
-```
-
-## Order cancellations
-
-An investor may cancel a trade at anytime, but only trades they own.  If the
-cancellation is successful, any tokens held in escrow are returned to the
-account from which they came.
-
-```rust
-pub enum ExchangeInstruction {
-    /// order cancellation
-    /// key 0 - Signer
-    /// key 1 -order to cancel
-    TradeCancellation,
-}
-```
-
-## Trade swaps
-
-The Matcher is monitoring the accounts assigned to the exchange program and
-building a trade-order table.  The order table is used to identify
-matching orders which could be fulfilled.  When a match is found the
-Matcher should issue a swap request.  Swap requests may not satisfy the entirety
-of either order, but the exchange will greedily fulfill it.  Any leftover tokens
-in either account will keep the order valid for further swap requests in
-the future.
-
-Matching orders are defined by the following swap requirements:
-
-- Opposite polarity (one `To` and one `From`)
-- Operate on the same token pair
-- The price ratio of the `From` order is greater than or equal to the `To` order
-- There are sufficient tokens to perform the trade
-
-Orders can be written in the following format:
-
-`investor direction pair quantity price-ratio`
-
-For example:
-
-- `1 T AB 2 1`
-  - Investor 1 wishes to exchange 2 A tokens to B tokens at a ratio of 1 A to 1
-    B
-- `2 F AC 6 1.2`
-  - Investor 2 wishes to exchange A tokens from 6 B tokens at a ratio of 1 A
-    from 1.2 B
-
-An order table could look something like the following. Notice how the columns
-are sorted low to high and high to low, respectively.  Prices are dramatic and
-whole for clarity.
-
-|Row| To          | From       |
-|---|-------------|------------|
-| 1 | 1 T AB 2 4  | 2 F AB 2 8 |
-| 2 | 1 T AB 1 4  | 2 F AB 2 8 |
-| 3 | 1 T AB 6 6  | 2 F AB 2 7 |
-| 4 | 1 T AB 2 8  | 2 F AB 3 6 |
-| 5 | 1 T AB 2 10 | 2 F AB 1 5 |
-
-As part of a successful swap request, the exchange will credit tokens to the
-Matcher's account equal to the difference in the price ratios or the two orders.
-These tokens are considered the Matcher's profit for initiating the trade.
-
-The Matcher would initiate the following swap on the order table above:
-
-  - Row 1, To:   Investor 1 trades 2 A tokens to 8 B tokens
-  - Row 1, From: Investor 2 trades 2 A tokens from 8 B tokens
-  - Matcher takes 8 B tokens as profit
-
-Both row 1 trades are fully realized, table becomes:
-
-|Row| To          | From       |
-|---|-------------|------------|
-| 1 | 1 T AB 1 4  | 2 F AB 2 8 |
-| 2 | 1 T AB 6 6  | 2 F AB 2 7 |
-| 3 | 1 T AB 2 8  | 2 F AB 3 6 |
-| 4 | 1 T AB 2 10 | 2 F AB 1 5 |
-
-The Matcher would initiate the following swap:
-
-  - Row 1, To:   Investor 1 trades 1 A token to 4 B tokens
-  - Row 1, From: Investor 2 trades 1 A token from 4 B tokens
-  - Matcher takes 4 B tokens as profit
-
-Row 1 From is not fully realized, table becomes:
-
-|Row| To          | From       |
-|---|-------------|------------|
-| 1 | 1 T AB 6 6  | 2 F AB 1 8 |
-| 2 | 1 T AB 2 8  | 2 F AB 2 7 |
-| 3 | 1 T AB 2 10 | 2 F AB 3 6 |
-| 4 |             | 2 F AB 1 5 |
-
-The Matcher would initiate the following swap:
-
-  - Row 1, To:   Investor 1 trades 1 A token to 6 B tokens
-  - Row 1, From: Investor 2 trades 1 A token from 6 B tokens
-  - Matcher takes 2 B tokens as profit
-
-Row 1 To is now fully realized, table becomes:
-
-|Row| To          | From       |
-|---|-------------|------------|
-| 1 | 1 T AB 5 6  | 2 F AB 2 7 |
-| 2 | 1 T AB 2 8  | 2 F AB 3 5 |
-| 3 | 1 T AB 2 10 | 2 F AB 1 5 |
-
-The Matcher would initiate the following last swap:
-
-  - Row 1, To:   Investor 1 trades 2 A token to 12 B tokens
-  - Row 1, From: Investor 2 trades 2 A token from 12 B tokens
-  - Matcher takes 2 B tokens as profit
-
-Table becomes:
-
-|Row| To          | From       |
-|---|-------------|------------|
-| 1 | 1 T AB 3 6  | 2 F AB 3 5 |
-| 2 | 1 T AB 2 8  | 2 F AB 1 5 |
-| 3 | 1 T AB 2 10 |            |
-
-At this point the lowest To's price is larger than the largest From's price so
-no more swaps would be initiated until new orders came in.
-
-```rust
-pub enum ExchangeInstruction {
-    /// Trade swap request
-    /// key 0 - Signer
-    /// key 1 - Account in which to record the swap
-    /// key 2 - 'To' order
-    /// key 3 - `From` order
-    /// key 4 - Token account associated with the To Trade
-    /// key 5 - Token account associated with From trade
-    /// key 6 - Token account in which to deposit the Matcher profit from the swap.
-    SwapRequest,
-}
-
-/// Swap accounts are populated with this structure
-pub struct TradeSwapInfo {
-    /// Pair swapped
-    pub pair: TokenPair,
-    /// `To` order
-    pub to_trade_order: Pubkey,
-    /// `From` order
-    pub from_trade_order: Pubkey,
-    /// Number of primary tokens exchanged
-    pub primary_tokens: u64,
-    /// Price the primary tokens were exchanged for
-    pub primary_price: u64,
-    /// Number of secondary tokens exchanged
-    pub secondary_tokens: u64,
-    /// Price the secondary tokens were exchanged for
-    pub secondary_price: u64,
-}
-```
-
-## Exchange program operations
-
-Putting all the commands together from above, the following operations will be
-supported by the on-chain exchange program:
-
-```rust
-pub enum ExchangeInstruction {
-    /// New token account
-    /// key 0 - Signer
-    /// key 1 - New token account
-    AccountRequest,
-
-    /// Transfer tokens between two accounts
-    /// key 0 - Account to transfer tokens to
-    /// key 1 - Account to transfer tokens from.  This can be the exchange program itself,
-    ///         the exchange has a limitless number of tokens it can transfer.
-    TransferRequest(Token, u64),
-
-    /// order request
-    /// key 0 - Signer
-    /// key 1 - Account in which to record the swap
-    /// key 2 - Token account associated with this trade
-    TradeRequest(TradeRequestInfo),
-
-    /// order cancellation
-    /// key 0 - Signer
-    /// key 1 -order to cancel
-    TradeCancellation,
-
-    /// Trade swap request
-    /// key 0 - Signer
-    /// key 1 - Account in which to record the swap
-    /// key 2 - 'To' order
-    /// key 3 - `From` order
-    /// key 4 - Token account associated with the To Trade
-    /// key 5 - Token account associated with From trade
-    /// key 6 - Token account in which to deposit the Matcher profit from the swap.
-    SwapRequest,
-}
-```
-
-## Quotes and OHLCV
-
-The Matcher will provide current bid/ask price quotes based on trade actively and
-also provide OHLCV based on some time window.  The details of how the bid/ask
-price quotes are calculated are yet to be decided.
-
-## Investor strategies
-
-To make a compelling demo, the investors needs to provide interesting trade
-behavior.  Something as simple as a randomly twiddled baseline would be a
-minimum starting point.
-
-## Running the exchange
-
-The exchange bench posts trades and swaps matches as fast as it can.  
-
-You might want to bump the duration up
-to 60 seconds and the batch size to 1000 for better numbers.  You can modify those
-in client_demo/src/demo.rs::test_exchange_local_cluster.
-
-The following command runs the bench:
-
-```bash
-$ RUST_LOG=solana_bench_exchange=info cargo test --release -- --nocapture test_exchange_local_cluster
-```
-
-To also see the cluster messages:
-
-```bash
-$ RUST_LOG=solana_bench_exchange=info,solana=info cargo test --release -- --nocapture test_exchange_local_cluster
-```

bench-exchange/src/cli.rs

@@ -1,220 +0,0 @@
-use clap::{crate_description, crate_name, value_t, App, Arg, ArgMatches};
-use solana_core::gen_keys::GenKeys;
-use solana_faucet::faucet::FAUCET_PORT;
-use solana_sdk::signature::{read_keypair_file, Keypair};
-use std::net::SocketAddr;
-use std::process::exit;
-use std::time::Duration;
-
-pub struct Config {
-    pub entrypoint_addr: SocketAddr,
-    pub faucet_addr: SocketAddr,
-    pub identity: Keypair,
-    pub threads: usize,
-    pub num_nodes: usize,
-    pub duration: Duration,
-    pub transfer_delay: u64,
-    pub fund_amount: u64,
-    pub batch_size: usize,
-    pub chunk_size: usize,
-    pub account_groups: usize,
-    pub client_ids_and_stake_file: String,
-    pub write_to_client_file: bool,
-    pub read_from_client_file: bool,
-}
-
-impl Default for Config {
-    fn default() -> Self {
-        Self {
-            entrypoint_addr: SocketAddr::from(([127, 0, 0, 1], 8001)),
-            faucet_addr: SocketAddr::from(([127, 0, 0, 1], FAUCET_PORT)),
-            identity: Keypair::new(),
-            num_nodes: 1,
-            threads: 4,
-            duration: Duration::new(u64::max_value(), 0),
-            transfer_delay: 0,
-            fund_amount: 100_000,
-            batch_size: 100,
-            chunk_size: 100,
-            account_groups: 100,
-            client_ids_and_stake_file: String::new(),
-            write_to_client_file: false,
-            read_from_client_file: false,
-        }
-    }
-}
-
-pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
-    App::new(crate_name!())
-        .about(crate_description!())
-        .version(version)
-        .arg(
-            Arg::with_name("entrypoint")
-                .short("n")
-                .long("entrypoint")
-                .value_name("HOST:PORT")
-                .takes_value(true)
-                .required(false)
-                .default_value("127.0.0.1:8001")
-                .help("Cluster entry point; defaults to 127.0.0.1:8001"),
-        )
-        .arg(
-            Arg::with_name("faucet")
-                .short("d")
-                .long("faucet")
-                .value_name("HOST:PORT")
-                .takes_value(true)
-                .required(false)
-                .default_value("127.0.0.1:9900")
-                .help("Location of the faucet; defaults to 127.0.0.1:9900"),
-        )
-        .arg(
-            Arg::with_name("identity")
-                .short("i")
-                .long("identity")
-                .value_name("PATH")
-                .takes_value(true)
-                .help("File containing a client identity (keypair)"),
-        )
-        .arg(
-            Arg::with_name("threads")
-                .long("threads")
-                .value_name("<threads>")
-                .takes_value(true)
-                .required(false)
-                .default_value("1")
-                .help("Number of threads submitting transactions"),
-        )
-        .arg(
-            Arg::with_name("num-nodes")
-                .long("num-nodes")
-                .value_name("NUM")
-                .takes_value(true)
-                .required(false)
-                .default_value("1")
-                .help("Wait for NUM nodes to converge"),
-        )
-        .arg(
-            Arg::with_name("duration")
-                .long("duration")
-                .value_name("SECS")
-                .takes_value(true)
-                .default_value("60")
-                .help("Seconds to run benchmark, then exit; default is forever"),
-        )
-        .arg(
-            Arg::with_name("transfer-delay")
-                .long("transfer-delay")
-                .value_name("<delay>")
-                .takes_value(true)
-                .required(false)
-                .default_value("0")
-                .help("Delay between each chunk"),
-        )
-        .arg(
-            Arg::with_name("fund-amount")
-                .long("fund-amount")
-                .value_name("<fund>")
-                .takes_value(true)
-                .required(false)
-                .default_value("100000")
-                .help("Number of lamports to fund to each signer"),
-        )
-        .arg(
-            Arg::with_name("batch-size")
-                .long("batch-size")
-                .value_name("<batch>")
-                .takes_value(true)
-                .required(false)
-                .default_value("1000")
-                .help("Number of transactions before the signer rolls over"),
-        )
-        .arg(
-            Arg::with_name("chunk-size")
-                .long("chunk-size")
-                .value_name("<cunk>")
-                .takes_value(true)
-                .required(false)
-                .default_value("500")
-                .help("Number of transactions to generate and send at a time"),
-        )
-        .arg(
-            Arg::with_name("account-groups")
-                .long("account-groups")
-                .value_name("<groups>")
-                .takes_value(true)
-                .required(false)
-                .default_value("10")
-                .help("Number of account groups to cycle for each batch"),
-        )
-        .arg(
-            Arg::with_name("write-client-keys")
-                .long("write-client-keys")
-                .value_name("FILENAME")
-                .takes_value(true)
-                .help("Generate client keys and stakes and write the list to YAML file"),
-        )
-        .arg(
-            Arg::with_name("read-client-keys")
-                .long("read-client-keys")
-                .value_name("FILENAME")
-                .takes_value(true)
-                .help("Read client keys and stakes from the YAML file"),
-        )
-}
-
-pub fn extract_args<'a>(matches: &ArgMatches<'a>) -> Config {
-    let mut args = Config::default();
-
-    args.entrypoint_addr = solana_net_utils::parse_host_port(
-        matches.value_of("entrypoint").unwrap(),
-    )
-    .unwrap_or_else(|e| {
-        eprintln!("failed to parse entrypoint address: {}", e);
-        exit(1)
-    });
-
-    args.faucet_addr = solana_net_utils::parse_host_port(matches.value_of("faucet").unwrap())
-        .unwrap_or_else(|e| {
-            eprintln!("failed to parse faucet address: {}", e);
-            exit(1)
-        });
-
-    if matches.is_present("identity") {
-        args.identity = read_keypair_file(matches.value_of("identity").unwrap())
-            .expect("can't read client identity");
-    } else {
-        args.identity = {
-            let seed = [42_u8; 32];
-            let mut rnd = GenKeys::new(seed);
-            rnd.gen_keypair()
-        };
-    }
-    args.threads = value_t!(matches.value_of("threads"), usize).expect("Failed to parse threads");
-    args.num_nodes =
-        value_t!(matches.value_of("num-nodes"), usize).expect("Failed to parse num-nodes");
-    let duration = value_t!(matches.value_of("duration"), u64).expect("Failed to parse duration");
-    args.duration = Duration::from_secs(duration);
-    args.transfer_delay =
-        value_t!(matches.value_of("transfer-delay"), u64).expect("Failed to parse transfer-delay");
-    args.fund_amount =
-        value_t!(matches.value_of("fund-amount"), u64).expect("Failed to parse fund-amount");
-    args.batch_size =
-        value_t!(matches.value_of("batch-size"), usize).expect("Failed to parse batch-size");
-    args.chunk_size =
-        value_t!(matches.value_of("chunk-size"), usize).expect("Failed to parse chunk-size");
-    args.account_groups = value_t!(matches.value_of("account-groups"), usize)
-        .expect("Failed to parse account-groups");
-
-    if let Some(s) = matches.value_of("write-client-keys") {
-        args.write_to_client_file = true;
-        args.client_ids_and_stake_file = s.to_string();
-    }
-
-    if let Some(s) = matches.value_of("read-client-keys") {
-        assert!(!args.write_to_client_file);
-        args.read_from_client_file = true;
-        args.client_ids_and_stake_file = s.to_string();
-    }
-    args
-}

bench-exchange/src/lib.rs

@@ -1,3 +0,0 @@
-pub mod bench;
-pub mod cli;
-mod order_book;

bench-exchange/src/main.rs

@@ -1,82 +0,0 @@
-pub mod bench;
-mod cli;
-pub mod order_book;
-
-use crate::bench::{airdrop_lamports, create_client_accounts_file, do_bench_exchange, Config};
-use log::*;
-use solana_core::gossip_service::{discover_cluster, get_multi_client};
-use solana_sdk::signature::Signer;
-
-fn main() {
-    solana_logger::setup();
-    solana_metrics::set_panic_hook("bench-exchange");
-
-    let matches = cli::build_args(solana_version::version!()).get_matches();
-    let cli_config = cli::extract_args(&matches);
-
-    let cli::Config {
-        entrypoint_addr,
-        faucet_addr,
-        identity,
-        threads,
-        num_nodes,
-        duration,
-        transfer_delay,
-        fund_amount,
-        batch_size,
-        chunk_size,
-        account_groups,
-        client_ids_and_stake_file,
-        write_to_client_file,
-        read_from_client_file,
-        ..
-    } = cli_config;
-
-    let config = Config {
-        identity,
-        threads,
-        duration,
-        transfer_delay,
-        fund_amount,
-        batch_size,
-        chunk_size,
-        account_groups,
-        client_ids_and_stake_file,
-        read_from_client_file,
-    };
-
-    if write_to_client_file {
-        create_client_accounts_file(
-            &config.client_ids_and_stake_file,
-            config.batch_size,
-            config.account_groups,
-            config.fund_amount,
-        );
-    } else {
-        info!("Connecting to the cluster");
-        let nodes = discover_cluster(&entrypoint_addr, num_nodes).unwrap_or_else(|_| {
-            panic!("Failed to discover nodes");
-        });
-
-        let (client, num_clients) = get_multi_client(&nodes);
-
-        info!("{} nodes found", num_clients);
-        if num_clients < num_nodes {
-            panic!("Error: Insufficient nodes discovered");
-        }
-
-        if !read_from_client_file {
-            info!("Funding keypair: {}", config.identity.pubkey());
-
-            let accounts_in_groups = batch_size * account_groups;
-            const NUM_SIGNERS: u64 = 2;
-            airdrop_lamports(
-                &client,
-                &faucet_addr,
-                &config.identity,
-                fund_amount * (accounts_in_groups + 1) as u64 * NUM_SIGNERS,
-            );
-        }
-        do_bench_exchange(vec![client], config);
-    }
-}

bench-exchange/src/order_book.rs

@@ -1,134 +0,0 @@
-use itertools::EitherOrBoth::{Both, Left, Right};
-use itertools::Itertools;
-use log::*;
-use solana_exchange_program::exchange_state::*;
-use solana_sdk::pubkey::Pubkey;
-use std::cmp::Ordering;
-use std::collections::BinaryHeap;
-use std::{error, fmt};
-
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct ToOrder {
-    pub pubkey: Pubkey,
-    pub info: OrderInfo,
-}
-
-impl Ord for ToOrder {
-    fn cmp(&self, other: &Self) -> Ordering {
-        other.info.price.cmp(&self.info.price)
-    }
-}
-impl PartialOrd for ToOrder {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct FromOrder {
-    pub pubkey: Pubkey,
-    pub info: OrderInfo,
-}
-
-impl Ord for FromOrder {
-    fn cmp(&self, other: &Self) -> Ordering {
-        self.info.price.cmp(&other.info.price)
-    }
-}
-impl PartialOrd for FromOrder {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-#[derive(Default)]
-pub struct OrderBook {
-    // TODO scale to x token types
-    to_ab: BinaryHeap<ToOrder>,
-    from_ab: BinaryHeap<FromOrder>,
-}
-impl fmt::Display for OrderBook {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        writeln!(
-            f,
-            "+-Order Book--------------------------+-------------------------------------+"
-        )?;
-        for (i, it) in self
-            .to_ab
-            .iter()
-            .zip_longest(self.from_ab.iter())
-            .enumerate()
-        {
-            match it {
-                Both(to, from) => writeln!(
-                    f,
-                    "| T AB {:8} for {:8}/{:8} | F AB {:8} for {:8}/{:8} |{}",
-                    to.info.tokens,
-                    SCALER,
-                    to.info.price,
-                    from.info.tokens,
-                    SCALER,
-                    from.info.price,
-                    i
-                )?,
-                Left(to) => writeln!(
-                    f,
-                    "| T AB {:8} for {:8}/{:8} |                                     |{}",
-                    to.info.tokens, SCALER, to.info.price, i
-                )?,
-                Right(from) => writeln!(
-                    f,
-                    "|                                     | F AB {:8} for {:8}/{:8} |{}",
-                    from.info.tokens, SCALER, from.info.price, i
-                )?,
-            }
-        }
-        write!(
-            f,
-            "+-------------------------------------+-------------------------------------+"
-        )?;
-        Ok(())
-    }
-}
-
-impl OrderBook {
-    // TODO
-    // pub fn cancel(&mut self, pubkey: Pubkey) -> Result<(), Box<dyn error::Error>> {
-    //     Ok(())
-    // }
-    pub fn push(&mut self, pubkey: Pubkey, info: OrderInfo) -> Result<(), Box<dyn error::Error>> {
-        check_trade(info.side, info.tokens, info.price)?;
-        match info.side {
-            OrderSide::Ask => {
-                self.to_ab.push(ToOrder { pubkey, info });
-            }
-            OrderSide::Bid => {
-                self.from_ab.push(FromOrder { pubkey, info });
-            }
-        }
-        Ok(())
-    }
-    pub fn pop(&mut self) -> Option<(ToOrder, FromOrder)> {
-        if let Some(pair) = Self::pop_pair(&mut self.to_ab, &mut self.from_ab) {
-            return Some(pair);
-        }
-        None
-    }
-    pub fn get_num_outstanding(&self) -> (usize, usize) {
-        (self.to_ab.len(), self.from_ab.len())
-    }
-
-    fn pop_pair(
-        to_ab: &mut BinaryHeap<ToOrder>,
-        from_ab: &mut BinaryHeap<FromOrder>,
-    ) -> Option<(ToOrder, FromOrder)> {
-        let to = to_ab.peek()?;
-        let from = from_ab.peek()?;
-        if from.info.price < to.info.price {
-            debug!("Trade not viable");
-            return None;
-        }
-        let to = to_ab.pop()?;
-        let from = from_ab.pop()?;
-        Some((to, from))
-    }
-}

bench-exchange/tests/bench_exchange.rs

@@ -1,103 +0,0 @@
-use log::*;
-use solana_bench_exchange::bench::{airdrop_lamports, do_bench_exchange, Config};
-use solana_core::gossip_service::{discover_cluster, get_multi_client};
-use solana_core::validator::ValidatorConfig;
-use solana_exchange_program::exchange_processor::process_instruction;
-use solana_exchange_program::id;
-use solana_exchange_program::solana_exchange_program;
-use solana_faucet::faucet::run_local_faucet;
-use solana_local_cluster::local_cluster::{ClusterConfig, LocalCluster};
-use solana_runtime::bank::Bank;
-use solana_runtime::bank_client::BankClient;
-use solana_sdk::genesis_config::create_genesis_config;
-use solana_sdk::signature::{Keypair, Signer};
-use std::process::exit;
-use std::sync::mpsc::channel;
-use std::time::Duration;
-
-#[test]
-#[ignore]
-fn test_exchange_local_cluster() {
-    solana_logger::setup();
-
-    const NUM_NODES: usize = 1;
-
-    let mut config = Config::default();
-    config.identity = Keypair::new();
-    config.duration = Duration::from_secs(1);
-    config.fund_amount = 100_000;
-    config.threads = 1;
-    config.transfer_delay = 20; // 15
-    config.batch_size = 100; // 1000;
-    config.chunk_size = 10; // 200;
-    config.account_groups = 1; // 10;
-    let Config {
-        fund_amount,
-        batch_size,
-        account_groups,
-        ..
-    } = config;
-    let accounts_in_groups = batch_size * account_groups;
-
-    let cluster = LocalCluster::new(&ClusterConfig {
-        node_stakes: vec![100_000; NUM_NODES],
-        cluster_lamports: 100_000_000_000_000,
-        validator_configs: vec![ValidatorConfig::default(); NUM_NODES],
-        native_instruction_processors: [solana_exchange_program!()].to_vec(),
-        ..ClusterConfig::default()
-    });
-
-    let faucet_keypair = Keypair::new();
-    cluster.transfer(
-        &cluster.funding_keypair,
-        &faucet_keypair.pubkey(),
-        2_000_000_000_000,
-    );
-
-    let (addr_sender, addr_receiver) = channel();
-    run_local_faucet(faucet_keypair, addr_sender, Some(1_000_000_000_000));
-    let faucet_addr = addr_receiver.recv_timeout(Duration::from_secs(2)).unwrap();
-
-    info!("Connecting to the cluster");
-    let nodes =
-        discover_cluster(&cluster.entry_point_info.gossip, NUM_NODES).unwrap_or_else(|err| {
-            error!("Failed to discover {} nodes: {:?}", NUM_NODES, err);
-            exit(1);
-        });
-
-    let (client, num_clients) = get_multi_client(&nodes);
-
-    info!("clients: {}", num_clients);
-    assert!(num_clients >= NUM_NODES);
-
-    const NUM_SIGNERS: u64 = 2;
-    airdrop_lamports(
-        &client,
-        &faucet_addr,
-        &config.identity,
-        fund_amount * (accounts_in_groups + 1) as u64 * NUM_SIGNERS,
-    );
-
-    do_bench_exchange(vec![client], config);
-}
-
-#[test]
-fn test_exchange_bank_client() {
-    solana_logger::setup();
-    let (genesis_config, identity) = create_genesis_config(100_000_000_000_000);
-    let mut bank = Bank::new(&genesis_config);
-    bank.add_builtin_program("exchange_program", id(), process_instruction);
-    let clients = vec![BankClient::new(bank)];
-
-    let mut config = Config::default();
-    config.identity = identity;
-    config.duration = Duration::from_secs(1);
-    config.fund_amount = 100_000;
-    config.threads = 1;
-    config.transfer_delay = 20; // 0;
-    config.batch_size = 100; // 1500;
-    config.chunk_size = 10; // 1500;
-    config.account_groups = 1; // 50;
-
-    do_bench_exchange(clients, config);
-}

bench-streamer/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2018"
+edition = "2021"
 name = "solana-bench-streamer"
-version = "1.4.0"
+version = "1.9.5"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
-solana-streamer = { path = "../streamer", version = "1.4.0" }
-solana-logger = { path = "../logger", version = "1.4.0" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
-solana-version = { path = "../version", version = "1.4.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.9.5" }
+solana-streamer = { path = "../streamer", version = "=1.9.5" }
+solana-logger = { path = "../logger", version = "=1.9.5" }
+solana-net-utils = { path = "../net-utils", version = "=1.9.5" }
+solana-version = { path = "../version", version = "=1.9.5" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/src/main.rs

@@ -1,31 +1,38 @@
-use clap::{crate_description, crate_name, App, Arg};
-use solana_streamer::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
-use solana_streamer::streamer::{receiver, PacketReceiver};
-use std::cmp::max;
-use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
-use std::sync::mpsc::channel;
-use std::sync::Arc;
-use std::thread::sleep;
-use std::thread::{spawn, JoinHandle, Result};
-use std::time::Duration;
-use std::time::SystemTime;
+#![allow(clippy::integer_arithmetic)]
+use {
+    clap::{crate_description, crate_name, App, Arg},
+    solana_streamer::{
+        packet::{Packet, PacketBatch, PacketBatchRecycler, PACKET_DATA_SIZE},
+        streamer::{receiver, PacketBatchReceiver},
+    },
+    std::{
+        cmp::max,
+        net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
+        sync::{
+            atomic::{AtomicBool, AtomicUsize, Ordering},
+            mpsc::channel,
+            Arc,
+        },
+        thread::{sleep, spawn, JoinHandle, Result},
+        time::{Duration, SystemTime},
+    },
+};
 
 fn producer(addr: &SocketAddr, exit: Arc<AtomicBool>) -> JoinHandle<()> {
     let send = UdpSocket::bind("0.0.0.0:0").unwrap();
-    let mut msgs = Packets::default();
-    msgs.packets.resize(10, Packet::default());
-    for w in msgs.packets.iter_mut() {
+    let mut packet_batch = PacketBatch::default();
+    packet_batch.packets.resize(10, Packet::default());
+    for w in packet_batch.packets.iter_mut() {
         w.meta.size = PACKET_DATA_SIZE;
-        w.meta.set_addr(&addr);
+        w.meta.set_addr(addr);
     }
-    let msgs = Arc::new(msgs);
+    let packet_batch = Arc::new(packet_batch);
     spawn(move || loop {
         if exit.load(Ordering::Relaxed) {
             return;
         }
         let mut num = 0;
-        for p in &msgs.packets {
+        for p in &packet_batch.packets {
             let a = p.meta.addr();
             assert!(p.meta.size <= PACKET_DATA_SIZE);
             send.send_to(&p.data[..p.meta.size], &a).unwrap();
@@ -35,14 +42,14 @@ fn producer(addr: &SocketAddr, exit: Arc<AtomicBool>) -> JoinHandle<()> {
     })
 }
 
-fn sink(exit: Arc<AtomicBool>, rvs: Arc<AtomicUsize>, r: PacketReceiver) -> JoinHandle<()> {
+fn sink(exit: Arc<AtomicBool>, rvs: Arc<AtomicUsize>, r: PacketBatchReceiver) -> JoinHandle<()> {
     spawn(move || loop {
         if exit.load(Ordering::Relaxed) {
             return;
         }
         let timer = Duration::new(1, 0);
-        if let Ok(msgs) = r.recv_timeout(timer) {
-            rvs.fetch_add(msgs.packets.len(), Ordering::Relaxed);
+        if let Ok(packet_batch) = r.recv_timeout(timer) {
+            rvs.fetch_add(packet_batch.packets.len(), Ordering::Relaxed);
         }
     })
 }
@@ -74,7 +81,7 @@ fn main() -> Result<()> {
 
     let mut read_channels = Vec::new();
     let mut read_threads = Vec::new();
-    let recycler = PacketsRecycler::default();
+    let recycler = PacketBatchRecycler::default();
     for _ in 0..num_sockets {
         let read = solana_net_utils::bind_to(ip_addr, port, false).unwrap();
         read.set_read_timeout(Some(Duration::new(1, 0))).unwrap();
@@ -90,6 +97,8 @@ fn main() -> Result<()> {
             s_reader,
             recycler.clone(),
             "bench-streamer-test",
+            1,
+            true,
         ));
     }
 

bench-tps/Cargo.toml

@@ -1,37 +1,36 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2018"
+edition = "2021"
 name = "solana-bench-tps"
-version = "1.4.0"
+version = "1.9.5"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 publish = false
 
 [dependencies]
-bincode = "1.3.1"
 clap = "2.33.1"
-log = "0.4.8"
-rayon = "1.4.0"
-serde_json = "1.0.56"
-serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
-solana-core = { path = "../core", version = "1.4.0" }
-solana-genesis = { path = "../genesis", version = "1.4.0" }
-solana-client = { path = "../client", version = "1.4.0" }
-solana-faucet = { path = "../faucet", version = "1.4.0" }
-solana-logger = { path = "../logger", version = "1.4.0" }
-solana-metrics = { path = "../metrics", version = "1.4.0" }
-solana-measure = { path = "../measure", version = "1.4.0" }
-solana-net-utils = { path = "../net-utils", version = "1.4.0" }
-solana-runtime = { path = "../runtime", version = "1.4.0" }
-solana-sdk = { path = "../sdk", version = "1.4.0" }
-solana-version = { path = "../version", version = "1.4.0" }
+log = "0.4.14"
+rayon = "1.5.1"
+serde_json = "1.0.72"
+serde_yaml = "0.8.21"
+solana-core = { path = "../core", version = "=1.9.5" }
+solana-genesis = { path = "../genesis", version = "=1.9.5" }
+solana-client = { path = "../client", version = "=1.9.5" }
+solana-faucet = { path = "../faucet", version = "=1.9.5" }
+solana-gossip = { path = "../gossip", version = "=1.9.5" }
+solana-logger = { path = "../logger", version = "=1.9.5" }
+solana-metrics = { path = "../metrics", version = "=1.9.5" }
+solana-measure = { path = "../measure", version = "=1.9.5" }
+solana-net-utils = { path = "../net-utils", version = "=1.9.5" }
+solana-runtime = { path = "../runtime", version = "=1.9.5" }
+solana-sdk = { path = "../sdk", version = "=1.9.5" }
+solana-streamer = { path = "../streamer", version = "=1.9.5" }
+solana-version = { path = "../version", version = "=1.9.5" }
 
 [dev-dependencies]
-serial_test = "0.4.0"
-serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.4.0" }
+serial_test = "0.5.1"
+solana-local-cluster = { path = "../local-cluster", version = "=1.9.5" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/src/bench.rs

@@ -1,39 +1,40 @@
-use crate::cli::Config;
-use log::*;
-use rayon::prelude::*;
-use solana_client::perf_utils::{sample_txs, SampleStats};
-use solana_core::gen_keys::GenKeys;
-use solana_faucet::faucet::request_airdrop_transaction;
-use solana_measure::measure::Measure;
-use solana_metrics::{self, datapoint_info};
-use solana_sdk::{
-    client::Client,
-    clock::{DEFAULT_TICKS_PER_SECOND, DEFAULT_TICKS_PER_SLOT, MAX_PROCESSING_AGE},
-    commitment_config::CommitmentConfig,
-    fee_calculator::FeeCalculator,
-    hash::Hash,
-    message::Message,
-    pubkey::Pubkey,
-    signature::{Keypair, Signer},
-    system_instruction, system_transaction,
-    timing::{duration_as_ms, duration_as_s, duration_as_us, timestamp},
-    transaction::Transaction,
-};
-use std::{
-    collections::{HashSet, VecDeque},
-    net::SocketAddr,
-    process::exit,
-    sync::{
-        atomic::{AtomicBool, AtomicIsize, AtomicUsize, Ordering},
-        Arc, Mutex, RwLock,
+use {
+    crate::cli::Config,
+    log::*,
+    rayon::prelude::*,
+    solana_client::perf_utils::{sample_txs, SampleStats},
+    solana_core::gen_keys::GenKeys,
+    solana_faucet::faucet::request_airdrop_transaction,
+    solana_measure::measure::Measure,
+    solana_metrics::{self, datapoint_info},
+    solana_sdk::{
+        client::Client,
+        clock::{DEFAULT_MS_PER_SLOT, DEFAULT_S_PER_SLOT, MAX_PROCESSING_AGE},
+        commitment_config::CommitmentConfig,
+        hash::Hash,
+        instruction::{AccountMeta, Instruction},
+        message::Message,
+        pubkey::Pubkey,
+        signature::{Keypair, Signer},
+        system_instruction, system_transaction,
+        timing::{duration_as_ms, duration_as_s, duration_as_us, timestamp},
+        transaction::Transaction,
+    },
+    std::{
+        collections::{HashSet, VecDeque},
+        net::SocketAddr,
+        process::exit,
+        sync::{
+            atomic::{AtomicBool, AtomicIsize, AtomicUsize, Ordering},
+            Arc, Mutex, RwLock,
+        },
+        thread::{sleep, Builder, JoinHandle},
+        time::{Duration, Instant},
     },
-    thread::{sleep, Builder, JoinHandle},
-    time::{Duration, Instant},
 };
 
 // The point at which transactions become "too old", in seconds.
-const MAX_TX_QUEUE_AGE: u64 =
-    MAX_PROCESSING_AGE as u64 * DEFAULT_TICKS_PER_SLOT / DEFAULT_TICKS_PER_SECOND;
+const MAX_TX_QUEUE_AGE: u64 = (MAX_PROCESSING_AGE as f64 * DEFAULT_S_PER_SLOT) as u64;
 
 pub const MAX_SPENDS_PER_TX: u64 = 4;
 
@@ -46,14 +47,12 @@ pub type Result<T> = std::result::Result<T, BenchTpsError>;
 
 pub type SharedTransactions = Arc<RwLock<VecDeque<Vec<(Transaction, u64)>>>>;
 
-fn get_recent_blockhash<T: Client>(client: &T) -> (Hash, FeeCalculator) {
+fn get_latest_blockhash<T: Client>(client: &T) -> Hash {
     loop {
-        match client.get_recent_blockhash_with_commitment(CommitmentConfig::recent()) {
-            Ok((blockhash, fee_calculator, _last_valid_slot)) => {
-                return (blockhash, fee_calculator)
-            }
+        match client.get_latest_blockhash_with_commitment(CommitmentConfig::processed()) {
+            Ok((blockhash, _)) => return blockhash,
             Err(err) => {
-                info!("Couldn't get recent blockhash: {:?}", err);
+                info!("Couldn't get last blockhash: {:?}", err);
                 sleep(Duration::from_secs(1));
             }
         };
@@ -240,19 +239,19 @@ where
 
     let shared_txs: SharedTransactions = Arc::new(RwLock::new(VecDeque::new()));
 
-    let recent_blockhash = Arc::new(RwLock::new(get_recent_blockhash(client.as_ref()).0));
+    let blockhash = Arc::new(RwLock::new(get_latest_blockhash(client.as_ref())));
     let shared_tx_active_thread_count = Arc::new(AtomicIsize::new(0));
     let total_tx_sent_count = Arc::new(AtomicUsize::new(0));
 
     let blockhash_thread = {
         let exit_signal = exit_signal.clone();
-        let recent_blockhash = recent_blockhash.clone();
+        let blockhash = blockhash.clone();
         let client = client.clone();
         let id = id.pubkey();
         Builder::new()
             .name("solana-blockhash-poller".to_string())
             .spawn(move || {
-                poll_blockhash(&exit_signal, &recent_blockhash, &client, &id);
+                poll_blockhash(&exit_signal, &blockhash, &client, &id);
             })
             .unwrap()
     };
@@ -272,7 +271,7 @@ where
     let start = Instant::now();
 
     generate_chunked_transfers(
-        recent_blockhash,
+        blockhash,
         &shared_txs,
         shared_tx_active_thread_count,
         source_keypair_chunks,
@@ -392,6 +391,22 @@ fn generate_txs(
     }
 }
 
+fn get_new_latest_blockhash<T: Client>(client: &Arc<T>, blockhash: &Hash) -> Option<Hash> {
+    let start = Instant::now();
+    while start.elapsed().as_secs() < 5 {
+        if let Ok(new_blockhash) = client.get_latest_blockhash() {
+            if new_blockhash != *blockhash {
+                return Some(new_blockhash);
+            }
+        }
+        debug!("Got same blockhash ({:?}), will retry...", blockhash);
+
+        // Retry ~twice during a slot
+        sleep(Duration::from_millis(DEFAULT_MS_PER_SLOT / 2));
+    }
+    None
+}
+
 fn poll_blockhash<T: Client>(
     exit_signal: &Arc<AtomicBool>,
     blockhash: &Arc<RwLock<Hash>>,
@@ -403,7 +418,7 @@ fn poll_blockhash<T: Client>(
     loop {
         let blockhash_updated = {
             let old_blockhash = *blockhash.read().unwrap();
-            if let Ok((new_blockhash, _fee)) = client.get_new_blockhash(&old_blockhash) {
+            if let Some(new_blockhash) = get_new_latest_blockhash(client, &old_blockhash) {
                 *blockhash.write().unwrap() = new_blockhash;
                 blockhash_last_updated = Instant::now();
                 true
@@ -497,7 +512,7 @@ fn do_tx_transfers<T: Client>(
 
 fn verify_funding_transfer<T: Client>(client: &Arc<T>, tx: &Transaction, amount: u64) -> bool {
     for a in &tx.message().account_keys[1..] {
-        match client.get_balance_with_commitment(a, CommitmentConfig::recent()) {
+        match client.get_balance_with_commitment(a, CommitmentConfig::processed()) {
             Ok(balance) => return balance >= amount,
             Err(err) => error!("failed to get balance {:?}", err),
         }
@@ -541,16 +556,16 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
                 self.len(),
             );
 
-            let (blockhash, _fee_calculator) = get_recent_blockhash(client.as_ref());
+            let blockhash = get_latest_blockhash(client.as_ref());
 
             // re-sign retained to_fund_txes with updated blockhash
             self.sign(blockhash);
-            self.send(&client);
+            self.send(client);
 
             // Sleep a few slots to allow transactions to process
             sleep(Duration::from_secs(1));
 
-            self.verify(&client, to_lamports);
+            self.verify(client, to_lamports);
 
             // retry anything that seems to have dropped through cracks
             //  again since these txs are all or nothing, they're fine to
@@ -565,7 +580,7 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
         let to_fund_txs: Vec<(&Keypair, Transaction)> = to_fund
             .par_iter()
             .map(|(k, t)| {
-                let instructions = system_instruction::transfer_many(&k.pubkey(), &t);
+                let instructions = system_instruction::transfer_many(&k.pubkey(), t);
                 let message = Message::new(&instructions, Some(&k.pubkey()));
                 (*k, Transaction::new_unsigned(message))
             })
@@ -618,7 +633,7 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
                         return None;
                     }
 
-                    let verified = if verify_funding_transfer(&client, &tx, to_lamports) {
+                    let verified = if verify_funding_transfer(&client, tx, to_lamports) {
                         verified_txs.fetch_add(1, Ordering::Relaxed);
                         Some(k.pubkey())
                     } else {
@@ -733,8 +748,8 @@ pub fn airdrop_lamports<T: Client>(
             id.pubkey(),
         );
 
-        let (blockhash, _fee_calculator) = get_recent_blockhash(client);
-        match request_airdrop_transaction(&faucet_addr, &id.pubkey(), airdrop_amount, blockhash) {
+        let blockhash = get_latest_blockhash(client);
+        match request_airdrop_transaction(faucet_addr, &id.pubkey(), airdrop_amount, blockhash) {
             Ok(transaction) => {
                 let mut tries = 0;
                 loop {
@@ -762,7 +777,7 @@ pub fn airdrop_lamports<T: Client>(
         };
 
         let current_balance = client
-            .get_balance_with_commitment(&id.pubkey(), CommitmentConfig::recent())
+            .get_balance_with_commitment(&id.pubkey(), CommitmentConfig::processed())
             .unwrap_or_else(|e| {
                 info!("airdrop error {}", e);
                 starting_balance
@@ -891,8 +906,16 @@ pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
     //   pay for the transaction fees in a new run.
     let enough_lamports = 8 * lamports_per_account / 10;
     if first_keypair_balance < enough_lamports || last_keypair_balance < enough_lamports {
-        let fee_rate_governor = client.get_fee_rate_governor().unwrap();
-        let max_fee = fee_rate_governor.max_lamports_per_signature;
+        let single_sig_message = Message::new_with_blockhash(
+            &[Instruction::new_with_bytes(
+                Pubkey::new_unique(),
+                &[],
+                vec![AccountMeta::new(Pubkey::new_unique(), true)],
+            )],
+            None,
+            &client.get_latest_blockhash().unwrap(),
+        );
+        let max_fee = client.get_fee_for_message(&single_sig_message).unwrap();
         let extra_fees = extra * max_fee;
         let total_keypairs = keypairs.len() as u64 + 1; // Add one for funding keypair
         let total = lamports_per_account * total_keypairs + extra_fees;
@@ -925,23 +948,27 @@ pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use solana_runtime::bank::Bank;
-    use solana_runtime::bank_client::BankClient;
-    use solana_sdk::client::SyncClient;
-    use solana_sdk::fee_calculator::FeeRateGovernor;
-    use solana_sdk::genesis_config::create_genesis_config;
+    use {
+        super::*,
+        solana_runtime::{bank::Bank, bank_client::BankClient},
+        solana_sdk::{
+            client::SyncClient, fee_calculator::FeeRateGovernor,
+            genesis_config::create_genesis_config,
+        },
+    };
 
     #[test]
     fn test_bench_tps_bank_client() {
         let (genesis_config, id) = create_genesis_config(10_000);
-        let bank = Bank::new(&genesis_config);
+        let bank = Bank::new_for_tests(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
 
-        let mut config = Config::default();
-        config.id = id;
-        config.tx_count = 10;
-        config.duration = Duration::from_secs(5);
+        let config = Config {
+            id,
+            tx_count: 10,
+            duration: Duration::from_secs(5),
+            ..Config::default()
+        };
 
         let keypair_count = config.tx_count * config.keypair_multiplier;
         let keypairs =
@@ -954,7 +981,7 @@ mod tests {
     #[test]
     fn test_bench_tps_fund_keys() {
         let (genesis_config, id) = create_genesis_config(10_000);
-        let bank = Bank::new(&genesis_config);
+        let bank = Bank::new_for_tests(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
         let keypair_count = 20;
         let lamports = 20;
@@ -965,7 +992,7 @@ mod tests {
         for kp in &keypairs {
             assert_eq!(
                 client
-                    .get_balance_with_commitment(&kp.pubkey(), CommitmentConfig::recent())
+                    .get_balance_with_commitment(&kp.pubkey(), CommitmentConfig::processed())
                     .unwrap(),
                 lamports
             );
@@ -977,7 +1004,7 @@ mod tests {
         let (mut genesis_config, id) = create_genesis_config(10_000);
         let fee_rate_governor = FeeRateGovernor::new(11, 0);
         genesis_config.fee_rate_governor = fee_rate_governor;
-        let bank = Bank::new(&genesis_config);
+        let bank = Bank::new_for_tests(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
         let keypair_count = 20;
         let lamports = 20;

bench-tps/src/cli.rs

@@ -1,11 +1,13 @@
-use clap::{crate_description, crate_name, App, Arg, ArgMatches};
-use solana_faucet::faucet::FAUCET_PORT;
-use solana_sdk::fee_calculator::FeeRateGovernor;
-use solana_sdk::{
-    pubkey::Pubkey,
-    signature::{read_keypair_file, Keypair},
+use {
+    clap::{crate_description, crate_name, App, Arg, ArgMatches},
+    solana_faucet::faucet::FAUCET_PORT,
+    solana_sdk::{
+        fee_calculator::FeeRateGovernor,
+        pubkey::Pubkey,
+        signature::{read_keypair_file, Keypair},
+    },
+    std::{net::SocketAddr, process::exit, time::Duration},
 };
-use std::{net::SocketAddr, process::exit, time::Duration};
 
 const NUM_LAMPORTS_PER_ACCOUNT_DEFAULT: u64 = solana_sdk::native_token::LAMPORTS_PER_SOL;
 
@@ -196,7 +198,7 @@ pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
 /// * `matches` - command line arguments parsed by clap
 /// # Panics
 /// Panics if there is trouble parsing any of the arguments
-pub fn extract_args<'a>(matches: &ArgMatches<'a>) -> Config {
+pub fn extract_args(matches: &ArgMatches) -> Config {
     let mut args = Config::default();
 
     if let Some(addr) = matches.value_of("entrypoint") {

bench-tps/src/lib.rs

@@ -1,2 +1,3 @@
+#![allow(clippy::integer_arithmetic)]
 pub mod bench;
 pub mod cli;

bench-tps/src/main.rs

@@ -1,12 +1,20 @@
-use log::*;
-use solana_bench_tps::bench::{do_bench_tps, generate_and_fund_keypairs, generate_keypairs};
-use solana_bench_tps::cli;
-use solana_core::gossip_service::{discover_cluster, get_client, get_multi_client};
-use solana_genesis::Base64Account;
-use solana_sdk::fee_calculator::FeeRateGovernor;
-use solana_sdk::signature::{Keypair, Signer};
-use solana_sdk::system_program;
-use std::{collections::HashMap, fs::File, io::prelude::*, path::Path, process::exit, sync::Arc};
+#![allow(clippy::integer_arithmetic)]
+use {
+    log::*,
+    solana_bench_tps::{
+        bench::{do_bench_tps, generate_and_fund_keypairs, generate_keypairs},
+        cli,
+    },
+    solana_genesis::Base64Account,
+    solana_gossip::gossip_service::{discover_cluster, get_client, get_multi_client},
+    solana_sdk::{
+        fee_calculator::FeeRateGovernor,
+        signature::{Keypair, Signer},
+        system_program,
+    },
+    solana_streamer::socket::SocketAddrSpace,
+    std::{collections::HashMap, fs::File, io::prelude::*, path::Path, process::exit, sync::Arc},
+};
 
 /// Number of signatures for all transactions in ~1 week at ~100K TPS
 pub const NUM_SIGNATURES_FOR_TXS: u64 = 100_000 * 60 * 60 * 24 * 7;
@@ -38,7 +46,7 @@ fn main() {
     let keypair_count = *tx_count * keypair_multiplier;
     if *write_to_client_file {
         info!("Generating {} keypairs", keypair_count);
-        let (keypairs, _) = generate_keypairs(&id, keypair_count as u64);
+        let (keypairs, _) = generate_keypairs(id, keypair_count as u64);
         let num_accounts = keypairs.len() as u64;
         let max_fee =
             FeeRateGovernor::new(*target_lamports_per_signature, 0).max_lamports_per_signature;
@@ -67,13 +75,14 @@ fn main() {
     }
 
     info!("Connecting to the cluster");
-    let nodes = discover_cluster(&entrypoint_addr, *num_nodes).unwrap_or_else(|err| {
-        eprintln!("Failed to discover {} nodes: {:?}", num_nodes, err);
-        exit(1);
-    });
+    let nodes = discover_cluster(entrypoint_addr, *num_nodes, SocketAddrSpace::Unspecified)
+        .unwrap_or_else(|err| {
+            eprintln!("Failed to discover {} nodes: {:?}", num_nodes, err);
+            exit(1);
+        });
 
     let client = if *multi_client {
-        let (client, num_clients) = get_multi_client(&nodes);
+        let (client, num_clients) = get_multi_client(&nodes, &SocketAddrSpace::Unspecified);
         if nodes.len() < num_clients {
             eprintln!(
                 "Error: Insufficient nodes discovered.  Expecting {} or more",
@@ -87,7 +96,7 @@ fn main() {
         let mut target_client = None;
         for node in nodes {
             if node.id == *target_node {
-                target_client = Some(Arc::new(get_client(&[node])));
+                target_client = Some(Arc::new(get_client(&[node], &SocketAddrSpace::Unspecified)));
                 break;
             }
         }
@@ -96,7 +105,7 @@ fn main() {
             exit(1);
         })
     } else {
-        Arc::new(get_client(&nodes))
+        Arc::new(get_client(&nodes, &SocketAddrSpace::Unspecified))
     };
 
     let keypairs = if *read_from_client_file {
@@ -134,7 +143,7 @@ fn main() {
         generate_and_fund_keypairs(
             client.clone(),
             Some(*faucet_addr),
-            &id,
+            id,
             keypair_count,
             *num_lamports_per_account,
         )

bench-tps/tests/bench_tps.rs

@@ -1,27 +1,44 @@
-use serial_test_derive::serial;
-use solana_bench_tps::bench::{do_bench_tps, generate_and_fund_keypairs};
-use solana_bench_tps::cli::Config;
-use solana_client::thin_client::create_client;
-use solana_core::cluster_info::VALIDATOR_PORT_RANGE;
-use solana_core::validator::ValidatorConfig;
-use solana_faucet::faucet::run_local_faucet;
-use solana_local_cluster::local_cluster::{ClusterConfig, LocalCluster};
-use solana_sdk::signature::{Keypair, Signer};
-use std::sync::{mpsc::channel, Arc};
-use std::time::Duration;
+#![allow(clippy::integer_arithmetic)]
+use {
+    serial_test::serial,
+    solana_bench_tps::{
+        bench::{do_bench_tps, generate_and_fund_keypairs},
+        cli::Config,
+    },
+    solana_client::thin_client::create_client,
+    solana_core::validator::ValidatorConfig,
+    solana_faucet::faucet::run_local_faucet_with_port,
+    solana_gossip::cluster_info::VALIDATOR_PORT_RANGE,
+    solana_local_cluster::{
+        local_cluster::{ClusterConfig, LocalCluster},
+        validator_configs::make_identical_validator_configs,
+    },
+    solana_sdk::signature::{Keypair, Signer},
+    solana_streamer::socket::SocketAddrSpace,
+    std::{
+        sync::{mpsc::channel, Arc},
+        time::Duration,
+    },
+};
 
 fn test_bench_tps_local_cluster(config: Config) {
     let native_instruction_processors = vec![];
 
     solana_logger::setup();
     const NUM_NODES: usize = 1;
-    let cluster = LocalCluster::new(&ClusterConfig {
-        node_stakes: vec![999_990; NUM_NODES],
-        cluster_lamports: 200_000_000,
-        validator_configs: vec![ValidatorConfig::default(); NUM_NODES],
-        native_instruction_processors,
-        ..ClusterConfig::default()
-    });
+    let cluster = LocalCluster::new(
+        &mut ClusterConfig {
+            node_stakes: vec![999_990; NUM_NODES],
+            cluster_lamports: 200_000_000,
+            validator_configs: make_identical_validator_configs(
+                &ValidatorConfig::default(),
+                NUM_NODES,
+            ),
+            native_instruction_processors,
+            ..ClusterConfig::default()
+        },
+        SocketAddrSpace::Unspecified,
+    );
 
     let faucet_keypair = Keypair::new();
     cluster.transfer(
@@ -36,8 +53,11 @@ fn test_bench_tps_local_cluster(config: Config) {
     ));
 
     let (addr_sender, addr_receiver) = channel();
-    run_local_faucet(faucet_keypair, addr_sender, None);
-    let faucet_addr = addr_receiver.recv_timeout(Duration::from_secs(2)).unwrap();
+    run_local_faucet_with_port(faucet_keypair, addr_sender, None, 0);
+    let faucet_addr = addr_receiver
+        .recv_timeout(Duration::from_secs(2))
+        .expect("run_local_faucet")
+        .expect("faucet_addr");
 
     let lamports_per_account = 100;
 
@@ -60,9 +80,9 @@ fn test_bench_tps_local_cluster(config: Config) {
 #[test]
 #[serial]
 fn test_bench_tps_local_cluster_solana() {
-    let mut config = Config::default();
-    config.tx_count = 100;
-    config.duration = Duration::from_secs(10);
-
-    test_bench_tps_local_cluster(config);
+    test_bench_tps_local_cluster(Config {
+        tx_count: 100,
+        duration: Duration::from_secs(10),
+        ..Config::default()
+    });
 }

bloom/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "solana-bloom"
+version = "1.9.5"
+description = "Solana bloom filter"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-bloom"
+edition = "2021"
+
+[dependencies]
+bv = { version = "0.11.1", features = ["serde"] }
+fnv = "1.0.7"
+rand = "0.7.0"
+serde = { version = "1.0.133", features = ["rc"] }
+rayon = "1.5.1"
+serde_derive = "1.0.103"
+solana-frozen-abi = { path = "../frozen-abi", version = "=1.9.5" }
+solana-frozen-abi-macro = { path = "../frozen-abi/macro", version = "=1.9.5" }
+solana-sdk = { path = "../sdk", version = "=1.9.5" }
+log = "0.4.14"
+
+[lib]
+crate-type = ["lib"]
+name = "solana_bloom"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[build-dependencies]
+rustc_version = "0.4"

bloom/benches/bloom.rs

@@ -1,22 +1,23 @@
 #![feature(test)]
 
 extern crate test;
-use bv::BitVec;
-use fnv::FnvHasher;
-use rand::Rng;
-use solana_runtime::bloom::{AtomicBloom, Bloom, BloomHashIndex};
-use solana_sdk::{
-    hash::{hash, Hash},
-    signature::Signature,
+use {
+    bv::BitVec,
+    fnv::FnvHasher,
+    rand::Rng,
+    solana_bloom::bloom::{AtomicBloom, Bloom, BloomHashIndex},
+    solana_sdk::{
+        hash::{hash, Hash},
+        signature::Signature,
+    },
+    std::{collections::HashSet, hash::Hasher},
+    test::Bencher,
 };
-use std::collections::HashSet;
-use std::hash::Hasher;
-use test::Bencher;
 
 #[bench]
 #[ignore]
 fn bench_bits_set(bencher: &mut Bencher) {
-    let mut bits: BitVec<u8> = BitVec::new_fill(false, 38_340_234 as u64);
+    let mut bits: BitVec<u8> = BitVec::new_fill(false, 38_340_234_u64);
     let mut hasher = FnvHasher::default();
 
     bencher.iter(|| {
@@ -31,7 +32,7 @@ fn bench_bits_set(bencher: &mut Bencher) {
 #[bench]
 #[ignore]
 fn bench_bits_set_hasher(bencher: &mut Bencher) {
-    let bits: BitVec<u8> = BitVec::new_fill(false, 38_340_234 as u64);
+    let bits: BitVec<u8> = BitVec::new_fill(false, 38_340_234_u64);
     let mut hasher = FnvHasher::default();
 
     bencher.iter(|| {
@@ -102,7 +103,7 @@ fn bench_sigs_hashmap(bencher: &mut Bencher) {
 #[bench]
 fn bench_add_hash(bencher: &mut Bencher) {
     let mut rng = rand::thread_rng();
-    let hash_values: Vec<_> = std::iter::repeat_with(|| Hash::new_rand(&mut rng))
+    let hash_values: Vec<_> = std::iter::repeat_with(|| solana_sdk::hash::new_rand(&mut rng))
         .take(1200)
         .collect();
     let mut fail = 0;
@@ -122,7 +123,7 @@ fn bench_add_hash(bencher: &mut Bencher) {
 #[bench]
 fn bench_add_hash_atomic(bencher: &mut Bencher) {
     let mut rng = rand::thread_rng();
-    let hash_values: Vec<_> = std::iter::repeat_with(|| Hash::new_rand(&mut rng))
+    let hash_values: Vec<_> = std::iter::repeat_with(|| solana_sdk::hash::new_rand(&mut rng))
         .take(1200)
         .collect();
     let mut fail = 0;
@@ -135,7 +136,6 @@ fn bench_add_hash_atomic(bencher: &mut Bencher) {
         for hash_value in &hash_values {
             bloom.add(hash_value);
         }
-        let bloom: Bloom<_> = bloom.into();
         let index = rng.gen_range(0, hash_values.len());
         if !bloom.contains(&hash_values[index]) {
             fail += 1;

bloom/build.rs

@@ -0,0 +1 @@
+../frozen-abi/build.rs

bloom/src/bloom.rs

@@ -1,11 +1,17 @@
 //! Simple Bloom Filter
-use bv::BitVec;
-use fnv::FnvHasher;
-use rand::{self, Rng};
-use serde::{Deserialize, Serialize};
-use std::fmt;
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::{cmp, hash::Hasher, marker::PhantomData};
+use {
+    bv::BitVec,
+    fnv::FnvHasher,
+    rand::{self, Rng},
+    serde::{Deserialize, Serialize},
+    solana_sdk::sanitize::{Sanitize, SanitizeError},
+    std::{
+        cmp, fmt,
+        hash::Hasher,
+        marker::PhantomData,
+        sync::atomic::{AtomicU64, Ordering},
+    },
+};
 
 /// Generate a stable hash of `self` for each `hash_index`
 /// Best effort can be made for uniqueness of each hash.
@@ -45,7 +51,16 @@ impl<T: BloomHashIndex> fmt::Debug for Bloom<T> {
     }
 }
 
-impl<T: BloomHashIndex> solana_sdk::sanitize::Sanitize for Bloom<T> {}
+impl<T: BloomHashIndex> Sanitize for Bloom<T> {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        // Avoid division by zero in self.pos(...).
+        if self.bits.is_empty() {
+            Err(SanitizeError::InvalidValue)
+        } else {
+            Ok(())
+        }
+    }
+}
 
 impl<T: BloomHashIndex> Bloom<T> {
     pub fn new(num_bits: usize, keys: Vec<u64>) -> Self {
@@ -57,10 +72,12 @@ impl<T: BloomHashIndex> Bloom<T> {
             _phantom: PhantomData::default(),
         }
     }
-    /// create filter optimal for num size given the `FALSE_RATE`
-    /// the keys are randomized for picking data out of a collision resistant hash of size
-    /// `keysize` bytes
-    /// https://hur.st/bloomfilter/
+    /// Create filter optimal for num size given the `FALSE_RATE`.
+    ///
+    /// The keys are randomized for picking data out of a collision resistant hash of size
+    /// `keysize` bytes.
+    ///
+    /// See <https://hur.st/bloomfilter/>.
     pub fn random(num_items: usize, false_rate: f64, max_bits: usize) -> Self {
         let m = Self::num_bits(num_items as f64, false_rate);
         let num_bits = cmp::max(1, cmp::min(m as usize, max_bits));
@@ -84,7 +101,7 @@ impl<T: BloomHashIndex> Bloom<T> {
         }
     }
     fn pos(&self, key: &T, k: u64) -> u64 {
-        key.hash_at_index(k) % self.bits.len()
+        key.hash_at_index(k).wrapping_rem(self.bits.len())
     }
     pub fn clear(&mut self) {
         self.bits = BitVec::new_fill(false, self.bits.len());
@@ -94,7 +111,7 @@ impl<T: BloomHashIndex> Bloom<T> {
         for k in &self.keys {
             let pos = self.pos(key, *k);
             if !self.bits.get(pos) {
-                self.num_bits_set += 1;
+                self.num_bits_set = self.num_bits_set.saturating_add(1);
                 self.bits.set(pos, true);
             }
         }
@@ -146,28 +163,58 @@ impl<T: BloomHashIndex> From<Bloom<T>> for AtomicBloom<T> {
 }
 
 impl<T: BloomHashIndex> AtomicBloom<T> {
+    fn pos(&self, key: &T, hash_index: u64) -> (usize, u64) {
+        let pos = key.hash_at_index(hash_index).wrapping_rem(self.num_bits);
+        // Divide by 64 to figure out which of the
+        // AtomicU64 bit chunks we need to modify.
+        let index = pos.wrapping_shr(6);
+        // (pos & 63) is equivalent to mod 64 so that we can find
+        // the index of the bit within the AtomicU64 to modify.
+        let mask = 1u64.wrapping_shl(u32::try_from(pos & 63).unwrap());
+        (index as usize, mask)
+    }
+
     pub fn add(&self, key: &T) {
         for k in &self.keys {
-            let pos = key.hash_at_index(*k) % self.num_bits;
-            // Divide by 64 to figure out which of the
-            // AtomicU64 bit chunks we need to modify.
-            let index = pos >> 6;
-            // (pos & 63) is equivalent to mod 64 so that we can find
-            // the index of the bit within the AtomicU64 to modify.
-            let bit = 1u64 << (pos & 63);
-            self.bits[index as usize].fetch_or(bit, Ordering::Relaxed);
+            let (index, mask) = self.pos(key, *k);
+            self.bits[index].fetch_or(mask, Ordering::Relaxed);
+        }
+    }
+
+    pub fn contains(&self, key: &T) -> bool {
+        self.keys.iter().all(|k| {
+            let (index, mask) = self.pos(key, *k);
+            let bit = self.bits[index].load(Ordering::Relaxed) & mask;
+            bit != 0u64
+        })
+    }
+
+    // Only for tests and simulations.
+    pub fn mock_clone(&self) -> Self {
+        Self {
+            keys: self.keys.clone(),
+            bits: self
+                .bits
+                .iter()
+                .map(|v| AtomicU64::new(v.load(Ordering::Relaxed)))
+                .collect(),
+            ..*self
         }
     }
 }
 
-impl<T: BloomHashIndex> Into<Bloom<T>> for AtomicBloom<T> {
-    fn into(self) -> Bloom<T> {
-        let bits: Vec<_> = self.bits.into_iter().map(AtomicU64::into_inner).collect();
+impl<T: BloomHashIndex> From<AtomicBloom<T>> for Bloom<T> {
+    fn from(atomic_bloom: AtomicBloom<T>) -> Self {
+        let bits: Vec<_> = atomic_bloom
+            .bits
+            .into_iter()
+            .map(AtomicU64::into_inner)
+            .collect();
         let num_bits_set = bits.iter().map(|x| x.count_ones() as u64).sum();
         let mut bits: BitVec<u64> = bits.into();
-        bits.truncate(self.num_bits);
+        bits.truncate(atomic_bloom.num_bits);
         Bloom {
-            keys: self.keys,
+            keys: atomic_bloom.keys,
             bits,
             num_bits_set,
             _phantom: PhantomData::default(),
@@ -177,9 +224,11 @@ impl<T: BloomHashIndex> Into<Bloom<T>> for AtomicBloom<T> {
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use rayon::prelude::*;
-    use solana_sdk::hash::{hash, Hash};
+    use {
+        super::*,
+        rayon::prelude::*,
+        solana_sdk::hash::{hash, Hash},
+    };
 
     #[test]
     fn test_bloom_filter() {
@@ -218,8 +267,8 @@ mod test {
     fn test_random() {
         let mut b1: Bloom<Hash> = Bloom::random(10, 0.1, 100);
         let mut b2: Bloom<Hash> = Bloom::random(10, 0.1, 100);
-        b1.keys.sort();
-        b2.keys.sort();
+        b1.keys.sort_unstable();
+        b2.keys.sort_unstable();
         assert_ne!(b1.keys, b2.keys);
     }
     // Bloom filter math in python
@@ -264,7 +313,7 @@ mod test {
     #[test]
     fn test_atomic_bloom() {
         let mut rng = rand::thread_rng();
-        let hash_values: Vec<_> = std::iter::repeat_with(|| Hash::new_rand(&mut rng))
+        let hash_values: Vec<_> = std::iter::repeat_with(|| solana_sdk::hash::new_rand(&mut rng))
             .take(1200)
             .collect();
         let bloom: AtomicBloom<_> = Bloom::<Hash>::random(1287, 0.1, 7424).into();
@@ -279,7 +328,7 @@ mod test {
         for hash_value in hash_values {
             assert!(bloom.contains(&hash_value));
         }
-        let false_positive = std::iter::repeat_with(|| Hash::new_rand(&mut rng))
+        let false_positive = std::iter::repeat_with(|| solana_sdk::hash::new_rand(&mut rng))
             .take(10_000)
             .filter(|hash_value| bloom.contains(hash_value))
             .count();
@@ -291,7 +340,7 @@ mod test {
         let mut rng = rand::thread_rng();
         let keys: Vec<_> = std::iter::repeat_with(|| rng.gen()).take(5).collect();
         let mut bloom = Bloom::<Hash>::new(9731, keys.clone());
-        let hash_values: Vec<_> = std::iter::repeat_with(|| Hash::new_rand(&mut rng))
+        let hash_values: Vec<_> = std::iter::repeat_with(|| solana_sdk::hash::new_rand(&mut rng))
             .take(1000)
             .collect();
         for hash_value in &hash_values {
@@ -303,6 +352,9 @@ mod test {
         let bloom: AtomicBloom<_> = bloom.into();
         assert_eq!(bloom.num_bits, 9731);
         assert_eq!(bloom.bits.len(), (9731 + 63) / 64);
+        for hash_value in &hash_values {
+            assert!(bloom.contains(hash_value));
+        }
         let bloom: Bloom<_> = bloom.into();
         assert_eq!(bloom.num_bits_set, num_bits_set);
         for hash_value in &hash_values {
@@ -311,6 +363,9 @@ mod test {
         // Round trip, re-inserting the same hash values.
         let bloom: AtomicBloom<_> = bloom.into();
         hash_values.par_iter().for_each(|v| bloom.add(v));
+        for hash_value in &hash_values {
+            assert!(bloom.contains(hash_value));
+        }
         let bloom: Bloom<_> = bloom.into();
         assert_eq!(bloom.num_bits_set, num_bits_set);
         assert_eq!(bloom.bits.len(), 9731);
@@ -318,13 +373,25 @@ mod test {
             assert!(bloom.contains(hash_value));
         }
         // Round trip, inserting new hash values.
-        let more_hash_values: Vec<_> = std::iter::repeat_with(|| Hash::new_rand(&mut rng))
-            .take(1000)
-            .collect();
+        let more_hash_values: Vec<_> =
+            std::iter::repeat_with(|| solana_sdk::hash::new_rand(&mut rng))
+                .take(1000)
+                .collect();
         let bloom: AtomicBloom<_> = bloom.into();
         assert_eq!(bloom.num_bits, 9731);
         assert_eq!(bloom.bits.len(), (9731 + 63) / 64);
         more_hash_values.par_iter().for_each(|v| bloom.add(v));
+        for hash_value in &hash_values {
+            assert!(bloom.contains(hash_value));
+        }
+        for hash_value in &more_hash_values {
+            assert!(bloom.contains(hash_value));
+        }
+        let false_positive = std::iter::repeat_with(|| solana_sdk::hash::new_rand(&mut rng))
+            .take(10_000)
+            .filter(|hash_value| bloom.contains(hash_value))
+            .count();
+        assert!(false_positive < 2000, "false_positive: {}", false_positive);
         let bloom: Bloom<_> = bloom.into();
         assert_eq!(bloom.bits.len(), 9731);
         assert!(bloom.num_bits_set > num_bits_set);
@@ -339,7 +406,7 @@ mod test {
         for hash_value in &more_hash_values {
             assert!(bloom.contains(hash_value));
         }
-        let false_positive = std::iter::repeat_with(|| Hash::new_rand(&mut rng))
+        let false_positive = std::iter::repeat_with(|| solana_sdk::hash::new_rand(&mut rng))
             .take(10_000)
             .filter(|hash_value| bloom.contains(hash_value))
             .count();

bloom/src/lib.rs

@@ -0,0 +1,5 @@
+#![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(min_specialization))]
+pub mod bloom;
+
+#[macro_use]
+extern crate solana_frozen_abi_macro;

book/src/proposals/handle-duplicate-block.md

@@ -0,0 +1,88 @@
+# Leader Duplicate Block Slashing
+
+This design describes how the cluster slashes leaders that produce duplicate
+blocks.
+
+Leaders that produce multiple blocks for the same slot increase the number of
+potential forks that the cluster has to resolve.
+
+## Primitives
+1. gossip_root: Nodes now gossip their current root
+2. gossip_duplicate_slots: Nodes can gossip up to `N` duplicate slot proofs.
+3. `DUPLICATE_THRESHOLD`: The minimum percentage of stake that needs to vote on a fork with version `X` of a duplicate slot, in order for that fork to become votable.
+
+## Protocol
+1. When WindowStage detects a duplicate slot proof `P`, it checks the new `gossip_root` to see if `<= 1/3` of the nodes have rooted a slot `S >= P`. If so, it pushes a proof to `gossip_duplicate_slots` to gossip. WindowStage then signals ReplayStage about this duplicate slot `S`. These proofs can be purged from gossip once the validator sees > 2/3 of people gossiping roots `R > S`.
+
+2. When ReplayStage receives the signal for a duplicate slot `S` from `1)` above, the validator monitors gossip and replay waiting for`>= DUPLICATE_THRESHOLD` votes for the same hash which implies the same version of the slot. If this conditon is met for some version with hash `H` of slot `S`, this is then known as the `duplicate_confirmed` version of the slot.
+
+Before a duplicate slot `S` is `duplicate_confirmed`, it's first excluded from the vote candidate set in the fork choice rules. In addition, ReplayStage also resets PoH to the *latest* ancestor of the *earliest* `non-duplicate/confirmed_duplicate_slot`, so that block generation can start happening on the earliest known *safe* block.
+
+Some notes about the `DUPLICATE_THRESHOLD`. In the cases below, assume `DUPLICATE_THRESHOLD = 52`:
+
+a) If less than `2 * DUPLICATE_THRESHOLD - 1` percentage of the network is malicious, then there can only be one such `duplicate_confirmed` version of the slot. With `DUPLICATE_THRESHOLD = 52`, this is
+a malcious tolerance of `4%`
+
+b) The liveness of the network is at most `1 - DUPLICATE_THRESHOLD - SWITCH_THRESHOLD`. This is because if you need at least `SWITCH_THRESHOLD` percentage of the stake voting on a different fork in order to switch off of a duplicate fork that has `< DUPLICATE_THRESHOLD` stake voting on it, and is *not* `duplicate_confirmed`. For `DUPLICATE_THRESHOLD = 52` and `DUPLICATE_THRESHOLD = 38`, this implies a liveness tolerance of `10%`.
+
+For example in the situation below, validators that voted on `2` can't vote any further on fork `2` because it's been removed from fork choice. Now slot 6 better have enough stake for a switching proof, or the network halts.
+
+```text
+    |-------- 2 (51% voted, then detected this slot was a duplicate and removed this slot from fork choice)
+0---|
+    |---------- 6 (39%)
+
+```
+
+3. Switching proofs need to be extended to allow including vote hashes from different versions of the same same slot (detected through 1). Right now this is not supported since switching proofs can
+only be built using votes from banks in BankForks, and two different versions of the same slot cannot
+simultaneously exist in BankForks. For instance:
+
+```text
+    |-------- 2
+    |
+0------------- 1 ------ 2'
+    |
+    |---------- 6
+
+```
+
+Imagine each version of slot 2 and 2' have `DUPLICATE_THRESHOLD / 2` of the votes on them, so neither duplicate can be confirmed. At most slot 6 has `1 - DUPLICATE_THRESHOLD / 2` of the votes
+on it, which is less than the switching threshold. Thus, in order for validators voting on `2` or `2'` to switch to slot 6, and make progress, they need to incorporate votes from the other version of the slot into their switching proofs.
+
+
+### The repair problem.
+Now what happens if one of the following occurs:
+
+1) Due to network blips/latencies, some validators fail to observe the gossip votes before they are overwritten by newer votes? Then some validators may conclude a slot `S` is `duplicate_confirmed` while others don't.
+
+2) Due to lockouts, no version of duplicate slot `S` reaches `duplicate_confirmed` status, but one of its descendants may reach `duplicate_confirmed` after those lockouts expire, which by definition, means `S` is also `duplicate_confirmed`.
+
+3) People who are catching up and don't see the votes in gossip encounter a dup block and can't make progress.
+
+We assume that given a network is eventually stable, if at least one correct validator observed `S` is `duplicate_confirmed`, then if `S` is part of the heaviest fork, then eventually all validators will observe some descendant of `S` is duplicate confirmed.
+
+This problem we need to solve is modeled simply by the below scenario:
+
+```text
+1 -> 2 (duplicate) -> 3 -> 4 (duplicate)
+```
+Assume the following:
+
+1. Due to gossiping duplciate proofs, we assume everyone will eventually see duplicate proofs for 2 and 4, so everyone agrees to remove them from fork choice until they are `duplicate_confirmed`.
+
+2. Due to lockouts, `> DUPLICATE_THRESHOLD` of the stake votes on 4, but not 2. This means at least `DUPLICATE_THRESHOLD` of people have the "correct" version of both slots 2 and 4.
+
+3. However, the remaining `1-DUPLICATE_THRESHOLD` of people have wrong version of 2. This means in replay, their slot 3 will be marked dead, *even though the faulty slot is 2*. The goal is to get these people on the right fork again.
+
+Possible solution:
+
+1. Change `EpochSlots` to signal when a bank is frozen, not when a slot is complete. If we see > `DUPLICATE_THRESHOLD` have frozen the dead slot 3, then we attempt recovery. Note this does not mean that all `DUPLICATE_THRESHOLD` have frozen the same version of the bank, it's just a signal to us that something may be wrong with our version of the bank.
+
+2. Recovery takes the form of a special repair request, `RepairDuplicateConfirmed(dead_slot, Vec<(Slot, Hash)>)`, which specifies a  dead slot, and then a vector of `(slot, hash)` of `N` of its latest parents.
+
+3. The repairer sees this request and responds with the correct hash only if any element of the `(slot, hash)` vector is both `duplicate_confirmed` and the hash doesn't match the requester's hash in the vector.
+
+4. Once the requester sees the "correct" hash is different than their frozen hash, they dump the block so that they can accept a new block, and ask the network for the block with the correct hash.
+
+Of course the repairer might lie to you, and you'll get the wrong version of the block, in which case you'll end up with another dead block and repeat the procedure.

bucket_map/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "solana-bucket-map"
+version = "1.9.5"
+description = "solana-bucket-map"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-bucket-map"
+readme = "../README.md"
+repository = "https://github.com/solana-labs/solana"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+license = "Apache-2.0"
+edition = "2021"
+
+[dependencies]
+rayon = "1.5.0"
+solana-logger = { path = "../logger", version = "=1.9.5" }
+solana-sdk = { path = "../sdk", version = "=1.9.5" }
+memmap2 = "0.5.0"
+log = { version = "0.4.11" }
+solana-measure = { path = "../measure", version = "=1.9.5" }
+rand = "0.7.0"
+fs_extra = "1.2.0"
+tempfile = "3.2.0"
+
+[lib]
+crate-type = ["lib"]
+name = "solana_bucket_map"
+
+[[bench]]
+name = "bucket_map"

bucket_map/benches/bucket_map.rs

@@ -0,0 +1,77 @@
+#![feature(test)]
+
+macro_rules! DEFINE_NxM_BENCH {
+    ($i:ident, $n:literal, $m:literal) => {
+        mod $i {
+            use super::*;
+
+            #[bench]
+            fn bench_insert_baseline_hashmap(bencher: &mut Bencher) {
+                do_bench_insert_baseline_hashmap(bencher, $n, $m);
+            }
+
+            #[bench]
+            fn bench_insert_bucket_map(bencher: &mut Bencher) {
+                do_bench_insert_bucket_map(bencher, $n, $m);
+            }
+        }
+    };
+}
+
+extern crate test;
+use {
+    rayon::prelude::*,
+    solana_bucket_map::bucket_map::{BucketMap, BucketMapConfig},
+    solana_sdk::pubkey::Pubkey,
+    std::{collections::hash_map::HashMap, sync::RwLock},
+    test::Bencher,
+};
+
+type IndexValue = u64;
+
+DEFINE_NxM_BENCH!(dim_01x02, 1, 2);
+DEFINE_NxM_BENCH!(dim_02x04, 2, 4);
+DEFINE_NxM_BENCH!(dim_04x08, 4, 8);
+DEFINE_NxM_BENCH!(dim_08x16, 8, 16);
+DEFINE_NxM_BENCH!(dim_16x32, 16, 32);
+DEFINE_NxM_BENCH!(dim_32x64, 32, 64);
+
+/// Benchmark insert with Hashmap as baseline for N threads inserting M keys each
+fn do_bench_insert_baseline_hashmap(bencher: &mut Bencher, n: usize, m: usize) {
+    let index = RwLock::new(HashMap::new());
+    (0..n).into_iter().into_par_iter().for_each(|i| {
+        let key = Pubkey::new_unique();
+        index
+            .write()
+            .unwrap()
+            .insert(key, vec![(i, IndexValue::default())]);
+    });
+    bencher.iter(|| {
+        (0..n).into_iter().into_par_iter().for_each(|_| {
+            for j in 0..m {
+                let key = Pubkey::new_unique();
+                index
+                    .write()
+                    .unwrap()
+                    .insert(key, vec![(j, IndexValue::default())]);
+            }
+        })
+    });
+}
+
+/// Benchmark insert with BucketMap with N buckets for N threads inserting M keys each
+fn do_bench_insert_bucket_map(bencher: &mut Bencher, n: usize, m: usize) {
+    let index = BucketMap::new(BucketMapConfig::new(n));
+    (0..n).into_iter().into_par_iter().for_each(|i| {
+        let key = Pubkey::new_unique();
+        index.update(&key, |_| Some((vec![(i, IndexValue::default())], 0)));
+    });
+    bencher.iter(|| {
+        (0..n).into_iter().into_par_iter().for_each(|_| {
+            for j in 0..m {
+                let key = Pubkey::new_unique();
+                index.update(&key, |_| Some((vec![(j, IndexValue::default())], 0)));
+            }
+        })
+    });
+}

bucket_map/src/bucket.rs

@@ -0,0 +1,491 @@
+use {
+    crate::{
+        bucket_item::BucketItem,
+        bucket_map::BucketMapError,
+        bucket_stats::BucketMapStats,
+        bucket_storage::{BucketStorage, Uid, DEFAULT_CAPACITY_POW2, UID_UNLOCKED},
+        index_entry::IndexEntry,
+        MaxSearch, RefCount,
+    },
+    rand::{thread_rng, Rng},
+    solana_measure::measure::Measure,
+    solana_sdk::pubkey::Pubkey,
+    std::{
+        collections::hash_map::DefaultHasher,
+        hash::{Hash, Hasher},
+        marker::PhantomData,
+        ops::RangeBounds,
+        path::PathBuf,
+        sync::{
+            atomic::{AtomicUsize, Ordering},
+            Arc, Mutex,
+        },
+    },
+};
+
+#[derive(Default)]
+pub struct ReallocatedItems {
+    // Some if the index was reallocated
+    // u64 is random associated with the new index
+    pub index: Option<(u64, BucketStorage)>,
+    // Some for a data bucket reallocation
+    // u64 is data bucket index
+    pub data: Option<(u64, BucketStorage)>,
+}
+
+#[derive(Default)]
+pub struct Reallocated {
+    /// > 0 if reallocations are encoded
+    pub active_reallocations: AtomicUsize,
+
+    /// actual reallocated bucket
+    /// mutex because bucket grow code runs with a read lock
+    pub items: Mutex<ReallocatedItems>,
+}
+
+impl Reallocated {
+    /// specify that a reallocation has occurred
+    pub fn add_reallocation(&self) {
+        assert_eq!(
+            0,
+            self.active_reallocations.fetch_add(1, Ordering::Relaxed),
+            "Only 1 reallocation can occur at a time"
+        );
+    }
+    /// Return true IFF a reallocation has occurred.
+    /// Calling this takes conceptual ownership of the reallocation encoded in the struct.
+    pub fn get_reallocated(&self) -> bool {
+        self.active_reallocations
+            .compare_exchange(1, 0, Ordering::Acquire, Ordering::Relaxed)
+            .is_ok()
+    }
+}
+
+// >= 2 instances of BucketStorage per 'bucket' in the bucket map. 1 for index, >= 1 for data
+pub struct Bucket<T> {
+    drives: Arc<Vec<PathBuf>>,
+    //index
+    pub index: BucketStorage,
+    //random offset for the index
+    random: u64,
+    //storage buckets to store SlotSlice up to a power of 2 in len
+    pub data: Vec<BucketStorage>,
+    _phantom: PhantomData<T>,
+    stats: Arc<BucketMapStats>,
+
+    pub reallocated: Reallocated,
+}
+
+impl<T: Clone + Copy> Bucket<T> {
+    pub fn new(
+        drives: Arc<Vec<PathBuf>>,
+        max_search: MaxSearch,
+        stats: Arc<BucketMapStats>,
+    ) -> Self {
+        let index = BucketStorage::new(
+            Arc::clone(&drives),
+            1,
+            std::mem::size_of::<IndexEntry>() as u64,
+            max_search,
+            Arc::clone(&stats.index),
+        );
+        Self {
+            random: thread_rng().gen(),
+            drives,
+            index,
+            data: vec![],
+            _phantom: PhantomData::default(),
+            stats,
+            reallocated: Reallocated::default(),
+        }
+    }
+
+    pub fn bucket_len(&self) -> u64 {
+        self.index.used.load(Ordering::Relaxed)
+    }
+
+    pub fn keys(&self) -> Vec<Pubkey> {
+        let mut rv = vec![];
+        for i in 0..self.index.capacity() {
+            if self.index.uid(i) == UID_UNLOCKED {
+                continue;
+            }
+            let ix: &IndexEntry = self.index.get(i);
+            rv.push(ix.key);
+        }
+        rv
+    }
+
+    pub fn items_in_range<R>(&self, range: &Option<&R>) -> Vec<BucketItem<T>>
+    where
+        R: RangeBounds<Pubkey>,
+    {
+        let mut result = Vec::with_capacity(self.index.used.load(Ordering::Relaxed) as usize);
+        for i in 0..self.index.capacity() {
+            let ii = i % self.index.capacity();
+            if self.index.uid(ii) == UID_UNLOCKED {
+                continue;
+            }
+            let ix: &IndexEntry = self.index.get(ii);
+            let key = ix.key;
+            if range.map(|r| r.contains(&key)).unwrap_or(true) {
+                let val = ix.read_value(self);
+                result.push(BucketItem {
+                    pubkey: key,
+                    ref_count: ix.ref_count(),
+                    slot_list: val.map(|(v, _ref_count)| v.to_vec()).unwrap_or_default(),
+                });
+            }
+        }
+        result
+    }
+
+    pub fn find_entry(&self, key: &Pubkey) -> Option<(&IndexEntry, u64)> {
+        Self::bucket_find_entry(&self.index, key, self.random)
+    }
+
+    fn find_entry_mut(&self, key: &Pubkey) -> Option<(&mut IndexEntry, u64)> {
+        Self::bucket_find_entry_mut(&self.index, key, self.random)
+    }
+
+    fn bucket_find_entry_mut<'a>(
+        index: &'a BucketStorage,
+        key: &Pubkey,
+        random: u64,
+    ) -> Option<(&'a mut IndexEntry, u64)> {
+        let ix = Self::bucket_index_ix(index, key, random);
+        for i in ix..ix + index.max_search() {
+            let ii = i % index.capacity();
+            if index.uid(ii) == UID_UNLOCKED {
+                continue;
+            }
+            let elem: &mut IndexEntry = index.get_mut(ii);
+            if elem.key == *key {
+                return Some((elem, ii));
+            }
+        }
+        None
+    }
+
+    fn bucket_find_entry<'a>(
+        index: &'a BucketStorage,
+        key: &Pubkey,
+        random: u64,
+    ) -> Option<(&'a IndexEntry, u64)> {
+        let ix = Self::bucket_index_ix(index, key, random);
+        for i in ix..ix + index.max_search() {
+            let ii = i % index.capacity();
+            if index.uid(ii) == UID_UNLOCKED {
+                continue;
+            }
+            let elem: &IndexEntry = index.get(ii);
+            if elem.key == *key {
+                return Some((elem, ii));
+            }
+        }
+        None
+    }
+
+    fn bucket_create_key(
+        index: &BucketStorage,
+        key: &Pubkey,
+        elem_uid: Uid,
+        random: u64,
+    ) -> Result<u64, BucketMapError> {
+        let ix = Self::bucket_index_ix(index, key, random);
+        for i in ix..ix + index.max_search() {
+            let ii = i as u64 % index.capacity();
+            if index.uid(ii) != UID_UNLOCKED {
+                continue;
+            }
+            index.allocate(ii, elem_uid).unwrap();
+            let mut elem: &mut IndexEntry = index.get_mut(ii);
+            elem.key = *key;
+            // These will be overwritten after allocation by callers.
+            // Since this part of the mmapped file could have previously been used by someone else, there can be garbage here.
+            elem.ref_count = 0;
+            elem.storage_offset = 0;
+            elem.storage_capacity_when_created_pow2 = 0;
+            elem.num_slots = 0;
+            //debug!(                "INDEX ALLOC {:?} {} {} {}",                key, ii, index.capacity, elem_uid            );
+            return Ok(ii);
+        }
+        Err(BucketMapError::IndexNoSpace(index.capacity_pow2))
+    }
+
+    pub fn addref(&mut self, key: &Pubkey) -> Option<RefCount> {
+        let (elem, _) = self.find_entry_mut(key)?;
+        elem.ref_count += 1;
+        Some(elem.ref_count)
+    }
+
+    pub fn unref(&mut self, key: &Pubkey) -> Option<RefCount> {
+        let (elem, _) = self.find_entry_mut(key)?;
+        elem.ref_count -= 1;
+        Some(elem.ref_count)
+    }
+
+    fn create_key(&self, key: &Pubkey) -> Result<u64, BucketMapError> {
+        Self::bucket_create_key(&self.index, key, IndexEntry::key_uid(key), self.random)
+    }
+
+    pub fn read_value(&self, key: &Pubkey) -> Option<(&[T], RefCount)> {
+        //debug!("READ_VALUE: {:?}", key);
+        let (elem, _) = self.find_entry(key)?;
+        elem.read_value(self)
+    }
+
+    pub fn try_write(
+        &mut self,
+        key: &Pubkey,
+        data: &[T],
+        ref_count: u64,
+    ) -> Result<(), BucketMapError> {
+        let best_fit_bucket = IndexEntry::data_bucket_from_num_slots(data.len() as u64);
+        if self.data.get(best_fit_bucket as usize).is_none() {
+            // fail early if the data bucket we need doesn't exist - we don't want the index entry partially allocated
+            return Err(BucketMapError::DataNoSpace((best_fit_bucket, 0)));
+        }
+        let index_entry = self.find_entry_mut(key);
+        let (elem, elem_ix) = match index_entry {
+            None => {
+                let ii = self.create_key(key)?;
+                let elem: &mut IndexEntry = self.index.get_mut(ii);
+                (elem, ii)
+            }
+            Some(res) => res,
+        };
+        elem.ref_count = ref_count;
+        let elem_uid = self.index.uid(elem_ix);
+        let bucket_ix = elem.data_bucket_ix();
+        let current_bucket = &self.data[bucket_ix as usize];
+        if best_fit_bucket == bucket_ix && elem.num_slots > 0 {
+            // in place update
+            let elem_loc = elem.data_loc(current_bucket);
+            let slice: &mut [T] = current_bucket.get_mut_cell_slice(elem_loc, data.len() as u64);
+            assert!(current_bucket.uid(elem_loc) == elem_uid);
+            elem.num_slots = data.len() as u64;
+            slice.clone_from_slice(data);
+            Ok(())
+        } else {
+            // need to move the allocation to a best fit spot
+            let best_bucket = &self.data[best_fit_bucket as usize];
+            let cap_power = best_bucket.capacity_pow2;
+            let cap = best_bucket.capacity();
+            let pos = thread_rng().gen_range(0, cap);
+            for i in pos..pos + self.index.max_search() {
+                let ix = i % cap;
+                if best_bucket.uid(ix) == UID_UNLOCKED {
+                    let elem_loc = elem.data_loc(current_bucket);
+                    if elem.num_slots > 0 {
+                        current_bucket.free(elem_loc, elem_uid);
+                    }
+                    elem.storage_offset = ix;
+                    elem.storage_capacity_when_created_pow2 = best_bucket.capacity_pow2;
+                    elem.num_slots = data.len() as u64;
+                    //debug!(                        "DATA ALLOC {:?} {} {} {}",                        key, elem.data_location, best_bucket.capacity, elem_uid                    );
+                    if elem.num_slots > 0 {
+                        best_bucket.allocate(ix, elem_uid).unwrap();
+                        let slice = best_bucket.get_mut_cell_slice(ix, data.len() as u64);
+                        slice.copy_from_slice(data);
+                    }
+                    return Ok(());
+                }
+            }
+            Err(BucketMapError::DataNoSpace((best_fit_bucket, cap_power)))
+        }
+    }
+
+    pub fn delete_key(&mut self, key: &Pubkey) {
+        if let Some((elem, elem_ix)) = self.find_entry(key) {
+            let elem_uid = self.index.uid(elem_ix);
+            if elem.num_slots > 0 {
+                let data_bucket = &self.data[elem.data_bucket_ix() as usize];
+                let loc = elem.data_loc(data_bucket);
+                //debug!(                    "DATA FREE {:?} {} {} {}",                    key, elem.data_location, data_bucket.capacity, elem_uid                );
+                data_bucket.free(loc, elem_uid);
+            }
+            //debug!("INDEX FREE {:?} {}", key, elem_uid);
+            self.index.free(elem_ix, elem_uid);
+        }
+    }
+
+    pub fn grow_index(&self, current_capacity_pow2: u8) {
+        if self.index.capacity_pow2 == current_capacity_pow2 {
+            let mut m = Measure::start("grow_index");
+            //debug!("GROW_INDEX: {}", current_capacity_pow2);
+            let increment = 1;
+            for i in increment.. {
+                //increasing the capacity by ^4 reduces the
+                //likelyhood of a re-index collision of 2^(max_search)^2
+                //1 in 2^32
+                let index = BucketStorage::new_with_capacity(
+                    Arc::clone(&self.drives),
+                    1,
+                    std::mem::size_of::<IndexEntry>() as u64,
+                    // *2 causes rapid growth of index buckets
+                    self.index.capacity_pow2 + i, // * 2,
+                    self.index.max_search,
+                    Arc::clone(&self.stats.index),
+                );
+                let random = thread_rng().gen();
+                let mut valid = true;
+                for ix in 0..self.index.capacity() {
+                    let uid = self.index.uid(ix);
+                    if UID_UNLOCKED != uid {
+                        let elem: &IndexEntry = self.index.get(ix);
+                        let new_ix = Self::bucket_create_key(&index, &elem.key, uid, random);
+                        if new_ix.is_err() {
+                            valid = false;
+                            break;
+                        }
+                        let new_ix = new_ix.unwrap();
+                        let new_elem: &mut IndexEntry = index.get_mut(new_ix);
+                        *new_elem = *elem;
+                        /*
+                        let dbg_elem: IndexEntry = *new_elem;
+                        assert_eq!(
+                            Self::bucket_find_entry(&index, &elem.key, random).unwrap(),
+                            (&dbg_elem, new_ix)
+                        );
+                        */
+                    }
+                }
+                if valid {
+                    let sz = index.capacity();
+                    {
+                        let mut max = self.stats.index.max_size.lock().unwrap();
+                        *max = std::cmp::max(*max, sz);
+                    }
+                    let mut items = self.reallocated.items.lock().unwrap();
+                    items.index = Some((random, index));
+                    self.reallocated.add_reallocation();
+                    break;
+                }
+            }
+            m.stop();
+            self.stats.index.resizes.fetch_add(1, Ordering::Relaxed);
+            self.stats
+                .index
+                .resize_us
+                .fetch_add(m.as_us(), Ordering::Relaxed);
+        }
+    }
+
+    pub fn apply_grow_index(&mut self, random: u64, index: BucketStorage) {
+        self.random = random;
+        self.index = index;
+    }
+
+    fn elem_size() -> u64 {
+        std::mem::size_of::<T>() as u64
+    }
+
+    pub fn apply_grow_data(&mut self, ix: usize, bucket: BucketStorage) {
+        if self.data.get(ix).is_none() {
+            for i in self.data.len()..ix {
+                // insert empty data buckets
+                self.data.push(BucketStorage::new(
+                    Arc::clone(&self.drives),
+                    1 << i,
+                    Self::elem_size(),
+                    self.index.max_search,
+                    Arc::clone(&self.stats.data),
+                ))
+            }
+            self.data.push(bucket);
+        } else {
+            self.data[ix] = bucket;
+        }
+    }
+
+    /// grow a data bucket
+    /// The application of the new bucket is deferred until the next write lock.
+    pub fn grow_data(&self, data_index: u64, current_capacity_pow2: u8) {
+        let new_bucket = BucketStorage::new_resized(
+            &self.drives,
+            self.index.max_search,
+            self.data.get(data_index as usize),
+            std::cmp::max(current_capacity_pow2 + 1, DEFAULT_CAPACITY_POW2),
+            1 << data_index,
+            Self::elem_size(),
+            &self.stats.data,
+        );
+        self.reallocated.add_reallocation();
+        let mut items = self.reallocated.items.lock().unwrap();
+        items.data = Some((data_index, new_bucket));
+    }
+
+    fn bucket_index_ix(index: &BucketStorage, key: &Pubkey, random: u64) -> u64 {
+        let uid = IndexEntry::key_uid(key);
+        let mut s = DefaultHasher::new();
+        uid.hash(&mut s);
+        //the locally generated random will make it hard for an attacker
+        //to deterministically cause all the pubkeys to land in the same
+        //location in any bucket on all validators
+        random.hash(&mut s);
+        let ix = s.finish();
+        ix % index.capacity()
+        //debug!(            "INDEX_IX: {:?} uid:{} loc: {} cap:{}",            key,            uid,            location,            index.capacity()        );
+    }
+
+    /// grow the appropriate piece. Note this takes an immutable ref.
+    /// The actual grow is set into self.reallocated and applied later on a write lock
+    pub fn grow(&self, err: BucketMapError) {
+        match err {
+            BucketMapError::DataNoSpace((data_index, current_capacity_pow2)) => {
+                //debug!("GROWING SPACE {:?}", (data_index, current_capacity_pow2));
+                self.grow_data(data_index, current_capacity_pow2);
+            }
+            BucketMapError::IndexNoSpace(current_capacity_pow2) => {
+                //debug!("GROWING INDEX {}", sz);
+                self.grow_index(current_capacity_pow2);
+            }
+        }
+    }
+
+    /// if a bucket was resized previously with a read lock, then apply that resize now
+    pub fn handle_delayed_grows(&mut self) {
+        if self.reallocated.get_reallocated() {
+            // swap out the bucket that was resized previously with a read lock
+            let mut items = ReallocatedItems::default();
+            std::mem::swap(&mut items, &mut self.reallocated.items.lock().unwrap());
+
+            if let Some((random, bucket)) = items.index.take() {
+                self.apply_grow_index(random, bucket);
+            } else {
+                // data bucket
+                let (i, new_bucket) = items.data.take().unwrap();
+                self.apply_grow_data(i as usize, new_bucket);
+            }
+        }
+    }
+
+    pub fn insert(&mut self, key: &Pubkey, value: (&[T], RefCount)) {
+        let (new, refct) = value;
+        loop {
+            let rv = self.try_write(key, new, refct);
+            match rv {
+                Ok(_) => return,
+                Err(err) => {
+                    self.grow(err);
+                    self.handle_delayed_grows();
+                }
+            }
+        }
+    }
+
+    pub fn update<F>(&mut self, key: &Pubkey, mut updatefn: F)
+    where
+        F: FnMut(Option<(&[T], RefCount)>) -> Option<(Vec<T>, RefCount)>,
+    {
+        let current = self.read_value(key);
+        let new = updatefn(current);
+        if new.is_none() {
+            self.delete_key(key);
+            return;
+        }
+        let (new, refct) = new.unwrap();
+        self.insert(key, (&new, refct));
+    }
+}

bucket_map/src/bucket_api.rs

@@ -0,0 +1,148 @@
+use {
+    crate::{
+        bucket::Bucket, bucket_item::BucketItem, bucket_map::BucketMapError,
+        bucket_stats::BucketMapStats, MaxSearch, RefCount,
+    },
+    solana_sdk::pubkey::Pubkey,
+    std::{
+        ops::RangeBounds,
+        path::PathBuf,
+        sync::{
+            atomic::{AtomicU64, Ordering},
+            Arc, RwLock, RwLockWriteGuard,
+        },
+    },
+};
+
+type LockedBucket<T> = RwLock<Option<Bucket<T>>>;
+
+pub struct BucketApi<T: Clone + Copy> {
+    drives: Arc<Vec<PathBuf>>,
+    max_search: MaxSearch,
+    pub stats: Arc<BucketMapStats>,
+
+    bucket: LockedBucket<T>,
+    count: Arc<AtomicU64>,
+}
+
+impl<T: Clone + Copy> BucketApi<T> {
+    pub fn new(
+        drives: Arc<Vec<PathBuf>>,
+        max_search: MaxSearch,
+        stats: Arc<BucketMapStats>,
+        count: Arc<AtomicU64>,
+    ) -> Self {
+        Self {
+            drives,
+            max_search,
+            stats,
+            bucket: RwLock::default(),
+            count,
+        }
+    }
+
+    /// Get the items for bucket
+    pub fn items_in_range<R>(&self, range: &Option<&R>) -> Vec<BucketItem<T>>
+    where
+        R: RangeBounds<Pubkey>,
+    {
+        self.bucket
+            .read()
+            .unwrap()
+            .as_ref()
+            .map(|bucket| bucket.items_in_range(range))
+            .unwrap_or_default()
+    }
+
+    /// Get the Pubkeys
+    pub fn keys(&self) -> Vec<Pubkey> {
+        self.bucket
+            .read()
+            .unwrap()
+            .as_ref()
+            .map_or_else(Vec::default, |bucket| bucket.keys())
+    }
+
+    /// Get the values for Pubkey `key`
+    pub fn read_value(&self, key: &Pubkey) -> Option<(Vec<T>, RefCount)> {
+        self.bucket.read().unwrap().as_ref().and_then(|bucket| {
+            bucket
+                .read_value(key)
+                .map(|(value, ref_count)| (value.to_vec(), ref_count))
+        })
+    }
+
+    pub fn bucket_len(&self) -> u64 {
+        self.bucket
+            .read()
+            .unwrap()
+            .as_ref()
+            .map(|bucket| bucket.bucket_len())
+            .unwrap_or_default()
+    }
+
+    pub fn delete_key(&self, key: &Pubkey) {
+        let mut bucket = self.get_write_bucket();
+        if let Some(bucket) = bucket.as_mut() {
+            bucket.delete_key(key)
+        }
+    }
+
+    fn get_write_bucket(&self) -> RwLockWriteGuard<Option<Bucket<T>>> {
+        let mut bucket = self.bucket.write().unwrap();
+        if bucket.is_none() {
+            *bucket = Some(Bucket::new(
+                Arc::clone(&self.drives),
+                self.max_search,
+                Arc::clone(&self.stats),
+            ));
+        } else {
+            let write = bucket.as_mut().unwrap();
+            write.handle_delayed_grows();
+            self.count.store(write.bucket_len(), Ordering::Relaxed);
+        }
+        bucket
+    }
+
+    pub fn addref(&self, key: &Pubkey) -> Option<RefCount> {
+        self.get_write_bucket()
+            .as_mut()
+            .and_then(|bucket| bucket.addref(key))
+    }
+
+    pub fn unref(&self, key: &Pubkey) -> Option<RefCount> {
+        self.get_write_bucket()
+            .as_mut()
+            .and_then(|bucket| bucket.unref(key))
+    }
+
+    pub fn insert(&self, pubkey: &Pubkey, value: (&[T], RefCount)) {
+        let mut bucket = self.get_write_bucket();
+        bucket.as_mut().unwrap().insert(pubkey, value)
+    }
+
+    pub fn grow(&self, err: BucketMapError) {
+        // grows are special - they get a read lock and modify 'reallocated'
+        // the grown changes are applied the next time there is a write lock taken
+        if let Some(bucket) = self.bucket.read().unwrap().as_ref() {
+            bucket.grow(err)
+        }
+    }
+
+    pub fn update<F>(&self, key: &Pubkey, updatefn: F)
+    where
+        F: FnMut(Option<(&[T], RefCount)>) -> Option<(Vec<T>, RefCount)>,
+    {
+        let mut bucket = self.get_write_bucket();
+        bucket.as_mut().unwrap().update(key, updatefn)
+    }
+
+    pub fn try_write(
+        &self,
+        pubkey: &Pubkey,
+        value: (&[T], RefCount),
+    ) -> Result<(), BucketMapError> {
+        let mut bucket = self.get_write_bucket();
+        bucket.as_mut().unwrap().try_write(pubkey, value.0, value.1)
+    }
+}

bucket_map/src/bucket_item.rs

@@ -0,0 +1,8 @@
+use {crate::RefCount, solana_sdk::pubkey::Pubkey};
+
+#[derive(Debug, Default, Clone)]
+pub struct BucketItem<T> {
+    pub pubkey: Pubkey,
+    pub ref_count: RefCount,
+    pub slot_list: Vec<T>,
+}

bucket_map/src/bucket_map.rs

@@ -0,0 +1,529 @@
+//! BucketMap is a mostly contention free concurrent map backed by MmapMut
+
+use {
+    crate::{bucket_api::BucketApi, bucket_stats::BucketMapStats, MaxSearch, RefCount},
+    solana_sdk::pubkey::Pubkey,
+    std::{convert::TryInto, fmt::Debug, fs, path::PathBuf, sync::Arc},
+    tempfile::TempDir,
+};
+
+#[derive(Debug, Default, Clone)]
+pub struct BucketMapConfig {
+    pub max_buckets: usize,
+    pub drives: Option<Vec<PathBuf>>,
+    pub max_search: Option<MaxSearch>,
+}
+
+impl BucketMapConfig {
+    /// Create a new BucketMapConfig
+    /// NOTE: BucketMap requires that max_buckets is a power of two
+    pub fn new(max_buckets: usize) -> BucketMapConfig {
+        BucketMapConfig {
+            max_buckets,
+            ..BucketMapConfig::default()
+        }
+    }
+}
+
+pub struct BucketMap<T: Clone + Copy + Debug> {
+    buckets: Vec<Arc<BucketApi<T>>>,
+    drives: Arc<Vec<PathBuf>>,
+    max_buckets_pow2: u8,
+    pub stats: Arc<BucketMapStats>,
+    pub temp_dir: Option<TempDir>,
+}
+
+impl<T: Clone + Copy + Debug> Drop for BucketMap<T> {
+    fn drop(&mut self) {
+        if self.temp_dir.is_none() {
+            BucketMap::<T>::erase_previous_drives(&self.drives);
+        }
+    }
+}
+
+impl<T: Clone + Copy + Debug> std::fmt::Debug for BucketMap<T> {
+    fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+pub enum BucketMapError {
+    /// (bucket_index, current_capacity_pow2)
+    DataNoSpace((u64, u8)),
+    /// current_capacity_pow2
+    IndexNoSpace(u8),
+}
+
+impl<T: Clone + Copy + Debug> BucketMap<T> {
+    pub fn new(config: BucketMapConfig) -> Self {
+        assert_ne!(
+            config.max_buckets, 0,
+            "Max number of buckets must be non-zero"
+        );
+        assert!(
+            config.max_buckets.is_power_of_two(),
+            "Max number of buckets must be a power of two"
+        );
+        // this should be <= 1 << DEFAULT_CAPACITY or we end up searching the same items over and over - probably not a big deal since it is so small anyway
+        const MAX_SEARCH: MaxSearch = 32;
+        let max_search = config.max_search.unwrap_or(MAX_SEARCH);
+
+        if let Some(drives) = config.drives.as_ref() {
+            Self::erase_previous_drives(drives);
+        }
+        let mut temp_dir = None;
+        let drives = config.drives.unwrap_or_else(|| {
+            temp_dir = Some(TempDir::new().unwrap());
+            vec![temp_dir.as_ref().unwrap().path().to_path_buf()]
+        });
+        let drives = Arc::new(drives);
+
+        let mut per_bucket_count = Vec::with_capacity(config.max_buckets);
+        per_bucket_count.resize_with(config.max_buckets, Arc::default);
+        let stats = Arc::new(BucketMapStats {
+            per_bucket_count,
+            ..BucketMapStats::default()
+        });
+        let buckets = stats
+            .per_bucket_count
+            .iter()
+            .map(|per_bucket_count| {
+                Arc::new(BucketApi::new(
+                    Arc::clone(&drives),
+                    max_search,
+                    Arc::clone(&stats),
+                    Arc::clone(per_bucket_count),
+                ))
+            })
+            .collect();
+
+        // A simple log2 function that is correct if x is a power of two
+        let log2 = |x: usize| usize::BITS - x.leading_zeros() - 1;
+
+        Self {
+            buckets,
+            drives,
+            max_buckets_pow2: log2(config.max_buckets) as u8,
+            stats,
+            temp_dir,
+        }
+    }
+
+    fn erase_previous_drives(drives: &[PathBuf]) {
+        drives.iter().for_each(|folder| {
+            let _ = fs::remove_dir_all(&folder);
+            let _ = fs::create_dir_all(&folder);
+        })
+    }
+
+    pub fn num_buckets(&self) -> usize {
+        self.buckets.len()
+    }
+
+    /// Get the values for Pubkey `key`
+    pub fn read_value(&self, key: &Pubkey) -> Option<(Vec<T>, RefCount)> {
+        self.get_bucket(key).read_value(key)
+    }
+
+    /// Delete the Pubkey `key`
+    pub fn delete_key(&self, key: &Pubkey) {
+        self.get_bucket(key).delete_key(key);
+    }
+
+    /// Update Pubkey `key`'s value with 'value'
+    pub fn insert(&self, key: &Pubkey, value: (&[T], RefCount)) {
+        self.get_bucket(key).insert(key, value)
+    }
+
+    /// Update Pubkey `key`'s value with 'value'
+    pub fn try_insert(&self, key: &Pubkey, value: (&[T], RefCount)) -> Result<(), BucketMapError> {
+        self.get_bucket(key).try_write(key, value)
+    }
+
+    /// Update Pubkey `key`'s value with function `updatefn`
+    pub fn update<F>(&self, key: &Pubkey, updatefn: F)
+    where
+        F: FnMut(Option<(&[T], RefCount)>) -> Option<(Vec<T>, RefCount)>,
+    {
+        self.get_bucket(key).update(key, updatefn)
+    }
+
+    pub fn get_bucket(&self, key: &Pubkey) -> &Arc<BucketApi<T>> {
+        self.get_bucket_from_index(self.bucket_ix(key))
+    }
+
+    pub fn get_bucket_from_index(&self, ix: usize) -> &Arc<BucketApi<T>> {
+        &self.buckets[ix]
+    }
+
+    /// Get the bucket index for Pubkey `key`
+    pub fn bucket_ix(&self, key: &Pubkey) -> usize {
+        if self.max_buckets_pow2 > 0 {
+            let location = read_be_u64(key.as_ref());
+            (location >> (u64::BITS - self.max_buckets_pow2 as u32)) as usize
+        } else {
+            0
+        }
+    }
+
+    /// Increment the refcount for Pubkey `key`
+    pub fn addref(&self, key: &Pubkey) -> Option<RefCount> {
+        let ix = self.bucket_ix(key);
+        let bucket = &self.buckets[ix];
+        bucket.addref(key)
+    }
+
+    /// Decrement the refcount for Pubkey `key`
+    pub fn unref(&self, key: &Pubkey) -> Option<RefCount> {
+        let ix = self.bucket_ix(key);
+        let bucket = &self.buckets[ix];
+        bucket.unref(key)
+    }
+}
+
+/// Look at the first 8 bytes of the input and reinterpret them as a u64
+fn read_be_u64(input: &[u8]) -> u64 {
+    assert!(input.len() >= std::mem::size_of::<u64>());
+    u64::from_be_bytes(input[0..std::mem::size_of::<u64>()].try_into().unwrap())
+}
+
+#[cfg(test)]
+mod tests {
+    use {
+        super::*,
+        rand::{thread_rng, Rng},
+        std::{collections::HashMap, sync::RwLock},
+    };
+
+    #[test]
+    fn bucket_map_test_insert() {
+        let key = Pubkey::new_unique();
+        let config = BucketMapConfig::new(1 << 1);
+        let index = BucketMap::new(config);
+        index.update(&key, |_| Some((vec![0], 0)));
+        assert_eq!(index.read_value(&key), Some((vec![0], 0)));
+    }
+
+    #[test]
+    fn bucket_map_test_insert2() {
+        for pass in 0..3 {
+            let key = Pubkey::new_unique();
+            let config = BucketMapConfig::new(1 << 1);
+            let index = BucketMap::new(config);
+            let bucket = index.get_bucket(&key);
+            if pass == 0 {
+                index.insert(&key, (&[0], 0));
+            } else {
+                let result = index.try_insert(&key, (&[0], 0));
+                assert!(result.is_err());
+                assert_eq!(index.read_value(&key), None);
+                if pass == 2 {
+                    // another call to try insert again - should still return an error
+                    let result = index.try_insert(&key, (&[0], 0));
+                    assert!(result.is_err());
+                    assert_eq!(index.read_value(&key), None);
+                }
+                bucket.grow(result.unwrap_err());
+                let result = index.try_insert(&key, (&[0], 0));
+                assert!(result.is_ok());
+            }
+            assert_eq!(index.read_value(&key), Some((vec![0], 0)));
+        }
+    }
+
+    #[test]
+    fn bucket_map_test_update2() {
+        let key = Pubkey::new_unique();
+        let config = BucketMapConfig::new(1 << 1);
+        let index = BucketMap::new(config);
+        index.insert(&key, (&[0], 0));
+        assert_eq!(index.read_value(&key), Some((vec![0], 0)));
+        index.insert(&key, (&[1], 0));
+        assert_eq!(index.read_value(&key), Some((vec![1], 0)));
+    }
+
+    #[test]
+    fn bucket_map_test_update() {
+        let key = Pubkey::new_unique();
+        let config = BucketMapConfig::new(1 << 1);
+        let index = BucketMap::new(config);
+        index.update(&key, |_| Some((vec![0], 0)));
+        assert_eq!(index.read_value(&key), Some((vec![0], 0)));
+        index.update(&key, |_| Some((vec![1], 0)));
+        assert_eq!(index.read_value(&key), Some((vec![1], 0)));
+    }
+
+    #[test]
+    fn bucket_map_test_update_to_0_len() {
+        solana_logger::setup();
+        let key = Pubkey::new_unique();
+        let config = BucketMapConfig::new(1 << 1);
+        let index = BucketMap::new(config);
+        index.update(&key, |_| Some((vec![0], 1)));
+        assert_eq!(index.read_value(&key), Some((vec![0], 1)));
+        // sets len to 0, updates in place
+        index.update(&key, |_| Some((vec![], 1)));
+        assert_eq!(index.read_value(&key), Some((vec![], 1)));
+        // sets len to 0, doesn't update in place - finds a new place, which causes us to no longer have an allocation in data
+        index.update(&key, |_| Some((vec![], 2)));
+        assert_eq!(index.read_value(&key), Some((vec![], 2)));
+        // sets len to 1, doesn't update in place - finds a new place
+        index.update(&key, |_| Some((vec![1], 2)));
+        assert_eq!(index.read_value(&key), Some((vec![1], 2)));
+    }
+
+    #[test]
+    fn bucket_map_test_delete() {
+        let config = BucketMapConfig::new(1 << 1);
+        let index = BucketMap::new(config);
+        for i in 0..10 {
+            let key = Pubkey::new_unique();
+            assert_eq!(index.read_value(&key), None);
+
+            index.update(&key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
+
+            index.delete_key(&key);
+            assert_eq!(index.read_value(&key), None);
+
+            index.update(&key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
+            index.delete_key(&key);
+        }
+    }
+
+    #[test]
+    fn bucket_map_test_delete_2() {
+        let config = BucketMapConfig::new(1 << 2);
+        let index = BucketMap::new(config);
+        for i in 0..100 {
+            let key = Pubkey::new_unique();
+            assert_eq!(index.read_value(&key), None);
+
+            index.update(&key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
+
+            index.delete_key(&key);
+            assert_eq!(index.read_value(&key), None);
+
+            index.update(&key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
+            index.delete_key(&key);
+        }
+    }
+
+    #[test]
+    fn bucket_map_test_n_drives() {
+        let config = BucketMapConfig::new(1 << 2);
+        let index = BucketMap::new(config);
+        for i in 0..100 {
+            let key = Pubkey::new_unique();
+            index.update(&key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
+        }
+    }
+    #[test]
+    fn bucket_map_test_grow_read() {
+        let config = BucketMapConfig::new(1 << 2);
+        let index = BucketMap::new(config);
+        let keys: Vec<Pubkey> = (0..100).into_iter().map(|_| Pubkey::new_unique()).collect();
+        for k in 0..keys.len() {
+            let key = &keys[k];
+            let i = read_be_u64(key.as_ref());
+            index.update(key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(key), Some((vec![i], 0)));
+            for (ix, key) in keys.iter().enumerate() {
+                let i = read_be_u64(key.as_ref());
+                //debug!("READ: {:?} {}", key, i);
+                let expected = if ix <= k { Some((vec![i], 0)) } else { None };
+                assert_eq!(index.read_value(key), expected);
+            }
+        }
+    }
+
+    #[test]
+    fn bucket_map_test_n_delete() {
+        let config = BucketMapConfig::new(1 << 2);
+        let index = BucketMap::new(config);
+        let keys: Vec<Pubkey> = (0..20).into_iter().map(|_| Pubkey::new_unique()).collect();
+        for key in keys.iter() {
+            let i = read_be_u64(key.as_ref());
+            index.update(key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(key), Some((vec![i], 0)));
+        }
+        for key in keys.iter() {
+            let i = read_be_u64(key.as_ref());
+            //debug!("READ: {:?} {}", key, i);
+            assert_eq!(index.read_value(key), Some((vec![i], 0)));
+        }
+        for k in 0..keys.len() {
+            let key = &keys[k];
+            index.delete_key(key);
+            assert_eq!(index.read_value(key), None);
+            for key in keys.iter().skip(k + 1) {
+                let i = read_be_u64(key.as_ref());
+                assert_eq!(index.read_value(key), Some((vec![i], 0)));
+            }
+        }
+    }
+
+    #[test]
+    fn hashmap_compare() {
+        use std::sync::Mutex;
+        solana_logger::setup();
+        let maps = (0..2)
+            .into_iter()
+            .map(|max_buckets_pow2| {
+                let config = BucketMapConfig::new(1 << max_buckets_pow2);
+                BucketMap::new(config)
+            })
+            .collect::<Vec<_>>();
+        let hash_map = RwLock::new(HashMap::<Pubkey, (Vec<(usize, usize)>, RefCount)>::new());
+        let max_slot_list_len = 3;
+        let all_keys = Mutex::new(vec![]);
+
+        let gen_rand_value = || {
+            let count = thread_rng().gen_range(0, max_slot_list_len);
+            let v = (0..count)
+                .into_iter()
+                .map(|x| (x as usize, x as usize /*thread_rng().gen::<usize>()*/))
+                .collect::<Vec<_>>();
+            let rc = thread_rng().gen::<RefCount>();
+            (v, rc)
+        };
+
+        let get_key = || {
+            let mut keys = all_keys.lock().unwrap();
+            if keys.is_empty() {
+                return None;
+            }
+            let len = keys.len();
+            Some(keys.remove(thread_rng().gen_range(0, len)))
+        };
+        let return_key = |key| {
+            let mut keys = all_keys.lock().unwrap();
+            keys.push(key);
+        };
+
+        let verify = || {
+            let mut maps = maps
+                .iter()
+                .map(|map| {
+                    let mut r = vec![];
+                    for bin in 0..map.num_buckets() {
+                        r.append(
+                            &mut map.buckets[bin]
+                                .items_in_range(&None::<&std::ops::RangeInclusive<Pubkey>>),
+                        );
+                    }
+                    r
+                })
+                .collect::<Vec<_>>();
+            let hm = hash_map.read().unwrap();
+            for (k, v) in hm.iter() {
+                for map in maps.iter_mut() {
+                    for i in 0..map.len() {
+                        if k == &map[i].pubkey {
+                            assert_eq!(map[i].slot_list, v.0);
+                            assert_eq!(map[i].ref_count, v.1);
+                            map.remove(i);
+                            break;
+                        }
+                    }
+                }
+            }
+            for map in maps.iter() {
+                assert!(map.is_empty());
+            }
+        };
+        let mut initial = 100; // put this many items in to start
+
+        // do random operations: insert, update, delete, add/unref in random order
+        // verify consistency between hashmap and all bucket maps
+        for i in 0..10000 {
+            if initial > 0 {
+                initial -= 1;
+            }
+            if initial > 0 || thread_rng().gen_range(0, 5) == 0 {
+                // insert
+                let k = solana_sdk::pubkey::new_rand();
+                let v = gen_rand_value();
+                hash_map.write().unwrap().insert(k, v.clone());
+                let insert = thread_rng().gen_range(0, 2) == 0;
+                maps.iter().for_each(|map| {
+                    if insert {
+                        map.insert(&k, (&v.0, v.1))
+                    } else {
+                        map.update(&k, |current| {
+                            assert!(current.is_none());
+                            Some(v.clone())
+                        })
+                    }
+                });
+                return_key(k);
+            }
+            if thread_rng().gen_range(0, 10) == 0 {
+                // update
+                if let Some(k) = get_key() {
+                    let hm = hash_map.read().unwrap();
+                    let (v, rc) = gen_rand_value();
+                    let v_old = hm.get(&k);
+                    let insert = thread_rng().gen_range(0, 2) == 0;
+                    maps.iter().for_each(|map| {
+                        if insert {
+                            map.insert(&k, (&v, rc))
+                        } else {
+                            map.update(&k, |current| {
+                                assert_eq!(current, v_old.map(|(v, rc)| (&v[..], *rc)), "{}", k);
+                                Some((v.clone(), rc))
+                            })
+                        }
+                    });
+                    drop(hm);
+                    hash_map.write().unwrap().insert(k, (v, rc));
+                    return_key(k);
+                }
+            }
+            if thread_rng().gen_range(0, 20) == 0 {
+                // delete
+                if let Some(k) = get_key() {
+                    let mut hm = hash_map.write().unwrap();
+                    hm.remove(&k);
+                    maps.iter().for_each(|map| {
+                        map.delete_key(&k);
+                    });
+                }
+            }
+            if thread_rng().gen_range(0, 10) == 0 {
+                // add/unref
+                if let Some(k) = get_key() {
+                    let mut inc = thread_rng().gen_range(0, 2) == 0;
+                    let mut hm = hash_map.write().unwrap();
+                    let (v, mut rc) = hm.get(&k).map(|(v, rc)| (v.to_vec(), *rc)).unwrap();
+                    if !inc && rc == 0 {
+                        // can't decrement rc=0
+                        inc = true;
+                    }
+                    rc = if inc { rc + 1 } else { rc - 1 };
+                    hm.insert(k, (v.to_vec(), rc));
+                    maps.iter().for_each(|map| {
+                        if thread_rng().gen_range(0, 2) == 0 {
+                            map.update(&k, |current| Some((current.unwrap().0.to_vec(), rc)))
+                        } else if inc {
+                            map.addref(&k);
+                        } else {
+                            map.unref(&k);
+                        }
+                    });
+
+                    return_key(k);
+                }
+            }
+            if i % 1000 == 0 {
+                verify();
+            }
+        }
+        verify();
+    }
+}

bucket_map/src/bucket_stats.rs

@@ -0,0 +1,18 @@
+use std::sync::{atomic::AtomicU64, Arc, Mutex};
+
+#[derive(Debug, Default)]
+pub struct BucketStats {
+    pub resizes: AtomicU64,
+    pub max_size: Mutex<u64>,
+    pub resize_us: AtomicU64,
+    pub new_file_us: AtomicU64,
+    pub flush_file_us: AtomicU64,
+    pub mmap_us: AtomicU64,
+}
+
+#[derive(Debug, Default)]
+pub struct BucketMapStats {
+    pub index: Arc<BucketStats>,
+    pub data: Arc<BucketStats>,
+    pub per_bucket_count: Vec<Arc<AtomicU64>>,
+}

bucket_map/src/bucket_storage.rs

@@ -0,0 +1,343 @@
+use {
+    crate::{bucket_stats::BucketStats, MaxSearch},
+    memmap2::MmapMut,
+    rand::{thread_rng, Rng},
+    solana_measure::measure::Measure,
+    std::{
+        fs::{remove_file, OpenOptions},
+        io::{Seek, SeekFrom, Write},
+        path::PathBuf,
+        sync::{
+            atomic::{AtomicU64, Ordering},
+            Arc,
+        },
+    },
+};
+
+/*
+1	2
+2	4
+3	8
+4	16
+5	32
+6	64
+7	128
+8	256
+9	512
+10	1,024
+11	2,048
+12	4,096
+13	8,192
+14	16,384
+23  8,388,608
+24  16,777,216
+*/
+pub const DEFAULT_CAPACITY_POW2: u8 = 5;
+
+/// A Header UID of 0 indicates that the header is unlocked
+pub(crate) const UID_UNLOCKED: Uid = 0;
+
+pub(crate) type Uid = u64;
+
+#[repr(C)]
+struct Header {
+    lock: AtomicU64,
+}
+
+impl Header {
+    fn try_lock(&self, uid: Uid) -> bool {
+        Ok(UID_UNLOCKED)
+            == self
+                .lock
+                .compare_exchange(UID_UNLOCKED, uid, Ordering::AcqRel, Ordering::Relaxed)
+    }
+    fn unlock(&self) -> Uid {
+        self.lock.swap(UID_UNLOCKED, Ordering::Release)
+    }
+    fn uid(&self) -> Uid {
+        self.lock.load(Ordering::Acquire)
+    }
+}
+
+pub struct BucketStorage {
+    path: PathBuf,
+    mmap: MmapMut,
+    pub cell_size: u64,
+    pub capacity_pow2: u8,
+    pub used: AtomicU64,
+    pub stats: Arc<BucketStats>,
+    pub max_search: MaxSearch,
+}
+
+#[derive(Debug)]
+pub enum BucketStorageError {
+    AlreadyAllocated,
+}
+
+impl Drop for BucketStorage {
+    fn drop(&mut self) {
+        let _ = remove_file(&self.path);
+    }
+}
+
+impl BucketStorage {
+    pub fn new_with_capacity(
+        drives: Arc<Vec<PathBuf>>,
+        num_elems: u64,
+        elem_size: u64,
+        capacity_pow2: u8,
+        max_search: MaxSearch,
+        stats: Arc<BucketStats>,
+    ) -> Self {
+        let cell_size = elem_size * num_elems + std::mem::size_of::<Header>() as u64;
+        let (mmap, path) = Self::new_map(&drives, cell_size as usize, capacity_pow2, &stats);
+        Self {
+            path,
+            mmap,
+            cell_size,
+            used: AtomicU64::new(0),
+            capacity_pow2,
+            stats,
+            max_search,
+        }
+    }
+
+    pub fn max_search(&self) -> u64 {
+        self.max_search as u64
+    }
+
+    pub fn new(
+        drives: Arc<Vec<PathBuf>>,
+        num_elems: u64,
+        elem_size: u64,
+        max_search: MaxSearch,
+        stats: Arc<BucketStats>,
+    ) -> Self {
+        Self::new_with_capacity(
+            drives,
+            num_elems,
+            elem_size,
+            DEFAULT_CAPACITY_POW2,
+            max_search,
+            stats,
+        )
+    }
+
+    pub fn uid(&self, ix: u64) -> Uid {
+        assert!(ix < self.capacity(), "bad index size");
+        let ix = (ix * self.cell_size) as usize;
+        let hdr_slice: &[u8] = &self.mmap[ix..ix + std::mem::size_of::<Header>()];
+        unsafe {
+            let hdr = hdr_slice.as_ptr() as *const Header;
+            return hdr.as_ref().unwrap().uid();
+        }
+    }
+
+    pub fn allocate(&self, ix: u64, uid: Uid) -> Result<(), BucketStorageError> {
+        assert!(ix < self.capacity(), "allocate: bad index size");
+        assert!(UID_UNLOCKED != uid, "allocate: bad uid");
+        let mut e = Err(BucketStorageError::AlreadyAllocated);
+        let ix = (ix * self.cell_size) as usize;
+        //debug!("ALLOC {} {}", ix, uid);
+        let hdr_slice: &[u8] = &self.mmap[ix..ix + std::mem::size_of::<Header>()];
+        unsafe {
+            let hdr = hdr_slice.as_ptr() as *const Header;
+            if hdr.as_ref().unwrap().try_lock(uid) {
+                e = Ok(());
+                self.used.fetch_add(1, Ordering::Relaxed);
+            }
+        };
+        e
+    }
+
+    pub fn free(&self, ix: u64, uid: Uid) {
+        assert!(ix < self.capacity(), "bad index size");
+        assert!(UID_UNLOCKED != uid, "free: bad uid");
+        let ix = (ix * self.cell_size) as usize;
+        //debug!("FREE {} {}", ix, uid);
+        let hdr_slice: &[u8] = &self.mmap[ix..ix + std::mem::size_of::<Header>()];
+        unsafe {
+            let hdr = hdr_slice.as_ptr() as *const Header;
+            //debug!("FREE uid: {}", hdr.as_ref().unwrap().uid());
+            let previous_uid = hdr.as_ref().unwrap().unlock();
+            assert_eq!(
+                previous_uid, uid,
+                "free: unlocked a header with a differet uid: {}",
+                previous_uid
+            );
+            self.used.fetch_sub(1, Ordering::Relaxed);
+        }
+    }
+
+    pub fn get<T: Sized>(&self, ix: u64) -> &T {
+        assert!(ix < self.capacity(), "bad index size");
+        let start = (ix * self.cell_size) as usize + std::mem::size_of::<Header>();
+        let end = start + std::mem::size_of::<T>();
+        let item_slice: &[u8] = &self.mmap[start..end];
+        unsafe {
+            let item = item_slice.as_ptr() as *const T;
+            &*item
+        }
+    }
+
+    pub fn get_empty_cell_slice<T: Sized>(&self) -> &[T] {
+        let len = 0;
+        let item_slice: &[u8] = &self.mmap[0..0];
+        unsafe {
+            let item = item_slice.as_ptr() as *const T;
+            std::slice::from_raw_parts(item, len as usize)
+        }
+    }
+
+    pub fn get_cell_slice<T: Sized>(&self, ix: u64, len: u64) -> &[T] {
+        assert!(ix < self.capacity(), "bad index size");
+        let ix = self.cell_size * ix;
+        let start = ix as usize + std::mem::size_of::<Header>();
+        let end = start + std::mem::size_of::<T>() * len as usize;
+        //debug!("GET slice {} {}", start, end);
+        let item_slice: &[u8] = &self.mmap[start..end];
+        unsafe {
+            let item = item_slice.as_ptr() as *const T;
+            std::slice::from_raw_parts(item, len as usize)
+        }
+    }
+
+    #[allow(clippy::mut_from_ref)]
+    pub fn get_mut<T: Sized>(&self, ix: u64) -> &mut T {
+        assert!(ix < self.capacity(), "bad index size");
+        let start = (ix * self.cell_size) as usize + std::mem::size_of::<Header>();
+        let end = start + std::mem::size_of::<T>();
+        let item_slice: &[u8] = &self.mmap[start..end];
+        unsafe {
+            let item = item_slice.as_ptr() as *mut T;
+            &mut *item
+        }
+    }
+
+    #[allow(clippy::mut_from_ref)]
+    pub fn get_mut_cell_slice<T: Sized>(&self, ix: u64, len: u64) -> &mut [T] {
+        assert!(ix < self.capacity(), "bad index size");
+        let ix = self.cell_size * ix;
+        let start = ix as usize + std::mem::size_of::<Header>();
+        let end = start + std::mem::size_of::<T>() * len as usize;
+        //debug!("GET mut slice {} {}", start, end);
+        let item_slice: &[u8] = &self.mmap[start..end];
+        unsafe {
+            let item = item_slice.as_ptr() as *mut T;
+            std::slice::from_raw_parts_mut(item, len as usize)
+        }
+    }
+
+    fn new_map(
+        drives: &[PathBuf],
+        cell_size: usize,
+        capacity_pow2: u8,
+        stats: &BucketStats,
+    ) -> (MmapMut, PathBuf) {
+        let mut measure_new_file = Measure::start("measure_new_file");
+        let capacity = 1u64 << capacity_pow2;
+        let r = thread_rng().gen_range(0, drives.len());
+        let drive = &drives[r];
+        let pos = format!("{}", thread_rng().gen_range(0, u128::MAX),);
+        let file = drive.join(pos);
+        let mut data = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .open(file.clone())
+            .map_err(|e| {
+                panic!(
+                    "Unable to create data file {} in current dir({:?}): {:?}",
+                    file.display(),
+                    std::env::current_dir(),
+                    e
+                );
+            })
+            .unwrap();
+
+        // Theoretical performance optimization: write a zero to the end of
+        // the file so that we won't have to resize it later, which may be
+        // expensive.
+        //debug!("GROWING file {}", capacity * cell_size as u64);
+        data.seek(SeekFrom::Start(capacity * cell_size as u64 - 1))
+            .unwrap();
+        data.write_all(&[0]).unwrap();
+        data.seek(SeekFrom::Start(0)).unwrap();
+        measure_new_file.stop();
+        let mut measure_flush = Measure::start("measure_flush");
+        data.flush().unwrap(); // can we skip this?
+        measure_flush.stop();
+        let mut measure_mmap = Measure::start("measure_mmap");
+        let res = (unsafe { MmapMut::map_mut(&data).unwrap() }, file);
+        measure_mmap.stop();
+        stats
+            .new_file_us
+            .fetch_add(measure_new_file.as_us(), Ordering::Relaxed);
+        stats
+            .flush_file_us
+            .fetch_add(measure_flush.as_us(), Ordering::Relaxed);
+        stats
+            .mmap_us
+            .fetch_add(measure_mmap.as_us(), Ordering::Relaxed);
+        res
+    }
+
+    /// copy contents from 'old_bucket' to 'self'
+    fn copy_contents(&mut self, old_bucket: &Self) {
+        let mut m = Measure::start("grow");
+        let old_cap = old_bucket.capacity();
+        let old_map = &old_bucket.mmap;
+
+        let increment = self.capacity_pow2 - old_bucket.capacity_pow2;
+        let index_grow = 1 << increment;
+        (0..old_cap as usize).into_iter().for_each(|i| {
+            let old_ix = i * old_bucket.cell_size as usize;
+            let new_ix = old_ix * index_grow;
+            let dst_slice: &[u8] = &self.mmap[new_ix..new_ix + old_bucket.cell_size as usize];
+            let src_slice: &[u8] = &old_map[old_ix..old_ix + old_bucket.cell_size as usize];
+
+            unsafe {
+                let dst = dst_slice.as_ptr() as *mut u8;
+                let src = src_slice.as_ptr() as *const u8;
+                std::ptr::copy_nonoverlapping(src, dst, old_bucket.cell_size as usize);
+            };
+        });
+        m.stop();
+        self.stats.resizes.fetch_add(1, Ordering::Relaxed);
+        self.stats.resize_us.fetch_add(m.as_us(), Ordering::Relaxed);
+    }
+
+    /// allocate a new bucket, copying data from 'bucket'
+    pub fn new_resized(
+        drives: &Arc<Vec<PathBuf>>,
+        max_search: MaxSearch,
+        bucket: Option<&Self>,
+        capacity_pow_2: u8,
+        num_elems: u64,
+        elem_size: u64,
+        stats: &Arc<BucketStats>,
+    ) -> Self {
+        let mut new_bucket = Self::new_with_capacity(
+            Arc::clone(drives),
+            num_elems,
+            elem_size,
+            capacity_pow_2,
+            max_search,
+            Arc::clone(stats),
+        );
+        if let Some(bucket) = bucket {
+            new_bucket.copy_contents(bucket);
+        }
+        let sz = new_bucket.capacity();
+        {
+            let mut max = new_bucket.stats.max_size.lock().unwrap();
+            *max = std::cmp::max(*max, sz);
+        }
+        new_bucket
+    }
+
+    /// Return the number of cells currently allocated
+    pub fn capacity(&self) -> u64 {
+        1 << self.capacity_pow2
+    }
+}

bucket_map/src/index_entry.rs

@@ -0,0 +1,67 @@
+use {
+    crate::{
+        bucket::Bucket,
+        bucket_storage::{BucketStorage, Uid},
+        RefCount,
+    },
+    solana_sdk::{clock::Slot, pubkey::Pubkey},
+    std::{
+        collections::hash_map::DefaultHasher,
+        fmt::Debug,
+        hash::{Hash, Hasher},
+    },
+};
+
+#[repr(C)]
+#[derive(Debug, Copy, Clone, PartialEq)]
+// one instance of this per item in the index
+// stored in the index bucket
+pub struct IndexEntry {
+    pub key: Pubkey, // can this be smaller if we have reduced the keys into buckets already?
+    pub ref_count: RefCount, // can this be smaller? Do we ever need more than 4B refcounts?
+    pub storage_offset: u64, // smaller? since these are variably sized, this could get tricky. well, actually accountinfo is not variable sized...
+    // if the bucket doubled, the index can be recomputed using create_bucket_capacity_pow2
+    pub storage_capacity_when_created_pow2: u8, // see data_location
+    pub num_slots: Slot, // can this be smaller? epoch size should ~ be the max len. this is the num elements in the slot list
+}
+
+impl IndexEntry {
+    pub fn data_bucket_from_num_slots(num_slots: Slot) -> u64 {
+        (num_slots as f64).log2().ceil() as u64 // use int log here?
+    }
+
+    pub fn data_bucket_ix(&self) -> u64 {
+        Self::data_bucket_from_num_slots(self.num_slots)
+    }
+
+    pub fn ref_count(&self) -> RefCount {
+        self.ref_count
+    }
+
+    // This function maps the original data location into an index in the current bucket storage.
+    // This is coupled with how we resize bucket storages.
+    pub fn data_loc(&self, storage: &BucketStorage) -> u64 {
+        self.storage_offset << (storage.capacity_pow2 - self.storage_capacity_when_created_pow2)
+    }
+
+    pub fn read_value<'a, T>(&self, bucket: &'a Bucket<T>) -> Option<(&'a [T], RefCount)> {
+        let data_bucket_ix = self.data_bucket_ix();
+        let data_bucket = &bucket.data[data_bucket_ix as usize];
+        let slice = if self.num_slots > 0 {
+            let loc = self.data_loc(data_bucket);
+            let uid = Self::key_uid(&self.key);
+            assert_eq!(uid, bucket.data[data_bucket_ix as usize].uid(loc));
+            bucket.data[data_bucket_ix as usize].get_cell_slice(loc, self.num_slots)
+        } else {
+            // num_slots is 0. This means we don't have an actual allocation.
+            // can we trust that the data_bucket is even safe?
+            bucket.data[data_bucket_ix as usize].get_empty_cell_slice()
+        };
+        Some((slice, self.ref_count))
+    }
+    pub fn key_uid(key: &Pubkey) -> Uid {
+        let mut s = DefaultHasher::new();
+        key.hash(&mut s);
+        s.finish().max(1u64)
+    }
+}

bucket_map/src/lib.rs

@@ -0,0 +1,11 @@
+#![allow(clippy::integer_arithmetic)]
+mod bucket;
+pub mod bucket_api;
+mod bucket_item;
+pub mod bucket_map;
+mod bucket_stats;
+mod bucket_storage;
+mod index_entry;
+
+pub type MaxSearch = u8;
+pub type RefCount = u64;