feat: add error types for each sigma protocol

* 1.made load_credentials accept credential path as a parameter. 2.partial implement bigtable comparasion function

* finding missing blocks in bigtables in a specified range

* refactor compare-blocks,add unit test for missing_blocks and fmt

* compare-block fix last block bug

* refactor compare-block and improve wording

* Update ledger-tool/src/bigtable.rs

Co-authored-by: Tyera Eulberg <teulberg@gmail.com>

* update compare-block command-line description

* style:improve wording/naming/code style

Co-authored-by: Tyera Eulberg <teulberg@gmail.com>

.github/workflows/explorer.yml

@@ -0,0 +1,28 @@
+name: Explorer_build&test_on_PR
+on:
+  pull_request:
+    branches:
+     - master
+    paths:
+      - 'explorer/**'
+jobs:
+  check-explorer:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: explorer
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          cache: 'npm'
+          cache-dependency-path: explorer/package-lock.json
+      - run: npm i -g npm@7
+      - run: npm ci
+      - run: npm run format
+      - run: npm run build
+      - run: npm run test

.github/workflows/explorer_preview.yml

@@ -0,0 +1,22 @@
+name : explorer_preview
+on:
+  workflow_run:
+    workflows: ["Explorer_build&test_on_PR"]
+    types: 
+      - completed
+jobs:
+   explorer_preview:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: amondnet/vercel-action@v20
+        with:
+          vercel-token: ${{ secrets.VERCEL_TOKEN }} # Required
+          github-token: ${{ secrets.GITHUB_TOKEN }} #Optional 
+          vercel-org-id: ${{ secrets.ORG_ID}}  #Required
+          vercel-project-id: ${{ secrets.PROJECT_ID}} #Required 
+          working-directory: ./explorer
+          scope: ${{ secrets.TEAM_ID }}

.github/workflows/explorer_production.yml

@@ -0,0 +1,46 @@
+name: Explorer_production_build&test
+on:
+  push:
+   branches: [master]
+   paths:
+      - 'explorer/**'
+jobs:
+    Explorer_production_build_test:
+      runs-on: ubuntu-latest
+      defaults:
+        run:
+          working-directory: explorer
+      steps:
+       - uses: actions/checkout@v2
+         with:
+          ref: ${{ github.event.pull_request.head.sha }}
+       - uses: actions/setup-node@v2
+         with:
+           node-version: '14'
+           cache: 'npm'
+           cache-dependency-path: explorer/package-lock.json
+       - run: npm i -g npm@7
+       - run: npm ci
+       - run: npm run format
+       - run: npm run build
+       - run: npm run test
+       
+    Explorer_production_deploy:
+      needs: Explorer_production_build_test
+      runs-on: ubuntu-latest
+      defaults:
+        run:
+          working-directory: explorer
+      steps:
+       - uses: actions/checkout@v2
+         with:
+           ref: ${{ github.event.pull_request.head.sha }}
+       - uses: amondnet/vercel-action@v20
+         with:
+           vercel-token: ${{ secrets.VERCEL_TOKEN }} # Required
+           github-token: ${{ secrets.GITHUB_TOKEN }} #Optional 
+           vercel-args: '--prod' #for production
+           vercel-org-id: ${{ secrets.ORG_ID}}  #Required
+           vercel-project-id: ${{ secrets.PROJECT_ID}} #Required 
+           working-directory: ./explorer
+           scope: ${{ secrets.TEAM_ID }}

.github/workflows/solana-action.yml.txt

@@ -0,0 +1,208 @@
+name : minimal 
+
+on:
+  push:
+    branches:  [master]
+  pull_request:
+    branches:  [master]
+
+jobs:
+  Export_Github_Repositories:
+    runs-on: ubuntu-latest
+    env: 
+       VERCEL_TOKEN: ${{secrets.VERCEL_TOKEN}}
+       GITHUB_TOKEN: ${{secrets.PAT_ANM}} 
+       COMMIT_RANGE: ${{ github.event.before}}...${{ github.event.after}}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
+      - run: echo "COMMIT_DIFF_RANGE=$(echo $COMMIT_RANGE)" >> $GITHUB_ENV
+#       - run: echo "$COMMIT_DIFF_RANGE"
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          GITHUB_TOKEN: ${{secrets.PAT_ANM}}
+        if: ${{ github.event_name == 'push' && 'cron'&& github.ref == 'refs/heads/master'}}
+        
+      - name: cmd
+        run : |
+            .travis/export-github-repo.sh web3.js/ solana-web3.js
+            
+  macos-artifacts:
+    needs: [Export_Github_Repositories]
+    strategy:
+      fail-fast: false
+    runs-on: macos-latest
+    if : ${{ github.event_name == 'api' && 'cron' || 'push' || startsWith(github.ref, 'refs/tags/v')}}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Setup | Rust
+        uses: ATiltedTree/setup-rust@v1
+        with:
+          rust-version: stable
+      - name:  release artifact
+        run: |
+           source ci/rust-version.sh
+           brew install coreutils
+           export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
+           greadlink -f .
+           source ci/env.sh
+           rustup set profile default
+           ci/publish-tarball.sh
+        shell: bash
+        
+      - name: Cache modules
+        uses: actions/cache@master
+        id: yarn-cache
+        with:
+          path: node_modules
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: ${{ runner.os }}-yarn-
+          
+
+# - To stop from uploading on the production
+#      - uses: ochanje210/simple-s3-upload-action@master
+#        with:
+#          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_KEY_ID }}
+#          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+#          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+#          SOURCE_DIR: 'travis-s3-upload1'
+#          DEST_DIR: 'giitsol'
+
+#      - uses: ochanje210/simple-s3-upload-action@master
+#        with:
+#          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_KEY_ID }}
+#          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
+#          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+#          SOURCE_DIR: './docs/'
+#          DEST_DIR: 'giitsol'
+
+  
+  windows-artifact:
+    needs: [Export_Github_Repositories]
+    strategy:
+      fail-fast: false 
+    runs-on: windows-latest
+    if : ${{ github.event_name == 'api' && 'cron' || 'push' || startsWith(github.ref, 'refs/tags/v')}}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Setup | Rust
+        uses: ATiltedTree/setup-rust@v1
+        with:
+          rust-version: stable
+  release-artifact:
+    needs: windows-artifact
+    runs-on: windows-latest
+    if : ${{ github.event_name == 'api' && 'cron' || github.ref == 'refs/heads/master'}}
+    steps:
+      - name: release artifact 
+        run: |
+             git clone git://git.openssl.org/openssl.git
+             cd openssl
+             make
+             make test
+             make install
+             openssl version
+#            choco install openssl 
+#            vcpkg integrate install
+#            refreshenv
+
+      - name: Checkout repository
+        uses: actions/checkout@v2 
+      - uses: actions/checkout@v2
+      - run: choco install msys2
+      - uses: actions/checkout@v2
+      - run: |
+           openssl version
+           bash ci/rust-version.sh
+           readlink -f .
+           bash ci/env.sh
+           rustup set profile default
+           bash ci/publish-tarball.sh
+        shell: bash
+        
+      - name: Cache modules
+        uses: actions/cache@v1
+        id: yarn-cache
+        with:
+          path: node_modules
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: ${{ runner.os }}-yarn-
+
+# - To stop from uploading on the production      
+#      - name: Config. aws cred
+#        uses: aws-actions/configure-aws-credentials@v1
+#        with:
+#          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+#          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+#          aws-region: us-east-2
+#      - name: Deploy
+#        uses: shallwefootball/s3-upload-action@master
+#        with:
+#            folder: build
+#            aws_bucket: ${{ secrets.AWS_S3_BUCKET }}
+#            aws_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+#            aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+#            destination_dir: /
+#            bucket-region: us-east-2
+#            delete-removed: true
+#            no-cache: true
+#            private: true
+  
+#  Docs:
+#    needs: [windows-artifact,release-artifact]
+#    runs-on: ubuntu-latest
+#    env: 
+#       GITHUB_TOKEN: ${{secrets.PAT_NEW}} 
+#       GITHUB_EVENT_BEFORE: ${{ github.event.before }}
+#       GITHUB_EVENT_AFTER: ${{ github.event.after }}
+#       COMMIT_RANGE: ${{ github.event.before}}...${{ github.event.after}}
+#    steps:
+#      - name: Checkout repo
+#        uses: actions/checkout@v2
+#        with:
+#          fetch-depth: 2
+#      - name: docs
+#        if: ${{github.event_name == 'pull_request' || startsWith(github.ref, 'refs/tags/v')}}
+#        run: |
+#             touch .env
+#             echo "COMMIT_RANGE=($COMMIT_RANGE)" > .env
+#             source ci/env.sh
+#             .travis/channel_restriction.sh edge beta || exit 0
+#             .travis/affects.sh docs/ .travis || exit 0
+#             cd docs/
+#             source .travis/before_install.sh
+#             source .travis/script.sh
+#      - name: setup-node
+#        uses: actions/checkout@v2
+#      - name: setup-node
+#        uses: actions/setup-node@v2
+#        with:
+#          node-version: 'lts/*'
+#      - name: Cache
+#        uses: actions/cache@v1
+#        with:
+#          path: ~/.npm
+#          key: ${{ runner.OS }}-npm-cache-${{ hashFiles('**/package-lock.json') }}
+#          restore-keys: |
+#            ${{ runner.OS }}-npm-cache-2  
+
+#  auto_bump:
+#     needs: [windows-artifact,release-artifact,Docs]
+#     runs-on: ubuntu-latest
+#     steps:
+#       - name : checkout repo
+#         uses: actions/checkout@v2
+#         with:
+#          fetch-depth: '0'
+#       - name: Bump version and push tag
+#         uses: anothrNick/github-tag-action@1.26.0
+#         env:
+#           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+#           WITH_V: true
+#           DEFAULT_BUMP: patch

.github/workflows/web3.yml

@@ -0,0 +1,84 @@
+name: Web3
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - "web3.js/**"
+  pull_request:
+    branches: [ master ]
+    paths:
+      - "web3.js/**"
+
+jobs:
+  # needed for grouping check-web3 strategies into one check for mergify
+  all-web3-checks:
+    runs-on: ubuntu-latest
+    needs:
+      - check-web3
+    steps:
+      - run: echo "Done"
+
+  web3-commit-lint:
+    runs-on: ubuntu-latest
+    # Set to true in order to avoid cancelling other workflow jobs.
+    # Mergify will still require web3-commit-lint for automerge
+    continue-on-error: true
+
+    defaults:
+      run:
+        working-directory: web3.js
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # maybe needed for base sha below
+          fetch-depth: 0
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+          cache: 'npm'
+          cache-dependency-path: web3.js/package-lock.json
+      - run: npm ci
+      - name: commit-lint
+        if: ${{ github.event_name == 'pull_request' }}
+        run: bash commitlint.sh
+        env:
+          COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
+
+  check-web3:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: web3.js
+
+    strategy:
+      matrix:
+        node: [ '12', '14', '16' ]
+
+    name: Node ${{ matrix.node }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node }}
+          cache: 'npm'
+          cache-dependency-path: web3.js/package-lock.json
+      - run: npm i -g npm@7
+      - run: npm ci
+      - run: npm run lint
+      - run: |
+          npm run build
+          ls -l lib
+          test -r lib/index.iife.js
+          test -r lib/index.cjs.js
+          test -r lib/index.esm.js
+      - run: npm run doc
+      - run: npm run codecov
+      - run: |
+          sh -c "$(curl -sSfL https://release.solana.com/edge/install)"
+          echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH
+          PATH="$HOME/.local/share/solana/install/active_release/bin:$PATH"
+          solana --version
+      - run: npm run test:live-with-test-validator

.gitignore

@@ -4,6 +4,7 @@
 /solana-metrics/
 /solana-metrics.tar.bz2
 /target/
+/test-ledger/
 
 **/*.rs.bk
 .cargo
@@ -27,3 +28,5 @@ log-*/
 /spl_*.so
 
 .DS_Store
+# scripts that may be generated by cargo *-bpf commands
+**/cargo-*-bpf-child-script-*.sh

.mergify.yml

@@ -4,24 +4,71 @@
 #
 # https://doc.mergify.io/
 pull_request_rules:
+  - name: label changes from community
+    conditions:
+      - author≠@core-contributors
+      - author≠mergify[bot]
+      - author≠dependabot[bot]
+    actions:
+      label:
+        add:
+          - community
+  - name: request review for community changes
+    conditions:
+      - author≠@core-contributors
+      - author≠mergify[bot]
+      - author≠dependabot[bot]
+      # Only request reviews from the pr subscribers group if no one
+      # has reviewed the community PR yet. These checks only match
+      # reviewers with admin, write or maintain permission on the repository.
+      - "#approved-reviews-by=0"
+      - "#commented-reviews-by=0"
+      - "#changes-requested-reviews-by=0"
+    actions:
+      request_reviews:
+        teams:
+          - "@solana-labs/community-pr-subscribers"
   - name: automatic merge (squash) on CI success
     conditions:
-      - status-success=buildkite/solana
-      - status-success=Travis CI - Pull Request
-      - status-success=ci-gate
-      - label=automerge
-      - author≠@dont-squash-my-commits
+      - and:
+        - status-success=buildkite/solana
+        - status-success=ci-gate
+        - label=automerge
+        - author≠@dont-squash-my-commits
+        - or:
+          # only require travis success if docs files changed
+          - status-success=Travis CI - Pull Request
+          - -files~=^docs/
+        - or:
+          # only require explorer checks if explorer files changed
+          - status-success=check-explorer
+          - -files~=^explorer/
+        - or:
+          - and:
+            - status-success=all-web3-checks
+            - status-success=web3-commit-lint
+          # only require web3 checks if web3.js files changed
+          - -files~=^web3.js/
     actions:
       merge:
         method: squash
   # Join the dont-squash-my-commits group if you won't like your commits squashed
   - name: automatic merge (rebase) on CI success
     conditions:
-      - status-success=buildkite/solana
-      - status-success=Travis CI - Pull Request
-      - status-success=ci-gate
-      - label=automerge
-      - author=@dont-squash-my-commits
+      - and:
+        - status-success=buildkite/solana
+        - status-success=Travis CI - Pull Request
+        - status-success=ci-gate
+        - label=automerge
+        - author=@dont-squash-my-commits
+        - or:
+          # only require explorer checks if explorer files changed
+          - status-success=check-explorer
+          - -files~=^explorer/
+        - or:
+          # only require web3 checks if web3.js files changed
+          - status-success=all-web3-checks
+          - -files~=^web3.js/
     actions:
       merge:
         method: rebase
@@ -50,35 +97,19 @@ pull_request_rules:
       label:
         add:
           - automerge
-  - name: v1.4 backport
+  - name: v1.8 backport
     conditions:
-      - label=v1.4
+      - label=v1.8
     actions:
       backport:
         ignore_conflicts: true
         branches:
-          - v1.4
-  - name: v1.5 backport
+          - v1.8
+  - name: v1.9 backport
     conditions:
-      - label=v1.5
+      - label=v1.9
     actions:
       backport:
         ignore_conflicts: true
         branches:
-          - v1.5
-  - name: v1.6 backport
-    conditions:
-      - label=v1.6
-    actions:
-      backport:
-        ignore_conflicts: true
-        branches:
-          - v1.6
-  - name: v1.7 backport
-    conditions:
-      - label=v1.7
-    actions:
-      backport:
-        ignore_conflicts: true
-        branches:
-          - v1.7
+          - v1.9

.travis.yml

@@ -23,12 +23,12 @@ jobs:
         depth: false
       script:
         - .travis/export-github-repo.sh web3.js/ solana-web3.js
-        - .travis/export-github-repo.sh explorer/ explorer
 
     - &release-artifacts
       if: type IN (api, cron) OR tag IS present
       name: "macOS release artifacts"
       os: osx
+      osx_image: xcode12
       language: rust
       rust:
         - stable
@@ -36,8 +36,12 @@ jobs:
         - source ci/rust-version.sh
         - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
         - readlink -f .
+        - brew install gnu-tar
+        - PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"
+        - tar --version
       script:
         - source ci/env.sh
+        - rustup set profile default
         - ci/publish-tarball.sh
       deploy:
         - provider: s3
@@ -60,6 +64,12 @@ jobs:
     - <<: *release-artifacts
       name: "Windows release artifacts"
       os: windows
+      install:
+        - choco install openssl
+        - export OPENSSL_DIR="C:\Program Files\OpenSSL-Win64"
+        - source ci/rust-version.sh
+        - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
+        - readlink -f .
     #  Linux release artifacts are still built by ci/buildkite-secondary.yml
     #- <<: *release-artifacts
     #  name: "Linux release artifacts"
@@ -67,56 +77,13 @@ jobs:
     #  before_install:
     #    - sudo apt-get install libssl-dev libudev-dev
 
-    # explorer pull request
-    - name: "explorer"
-      if: type = pull_request AND branch = master
-
-      language: node_js
-      node_js:
-        - "lts/*"
-
-      cache:
-        directories:
-          - ~/.npm
-
-      before_install:
-        - .travis/affects.sh explorer/ .travis || travis_terminate 0
-        - cd explorer
-
-      script:
-        - npm run build
-        - npm run format
-
-    # web3.js pull request
-    - name: "web3.js"
-      if: type = pull_request AND branch = master
-
-      language: node_js
-      node_js:
-        - "lts/*"
-
-      services:
-        - docker
-
-      cache:
-        directories:
-          - ~/.npm
-
-      before_install:
-        - .travis/affects.sh web3.js/ .travis || travis_terminate 0
-        - cd web3.js/
-        - source .travis/before_install.sh
-
-      script:
-        - ../.travis/commitlint.sh
-        - source .travis/script.sh
-
     # docs pull request
+   
     - name: "docs"
       if: type IN (push, pull_request) OR tag IS present
       language: node_js
       node_js:
-        - "node"
+        - "lts/*"
 
       services:
         - docker

CONTRIBUTING.md

@@ -74,7 +74,7 @@ minutes to execute. Use that time to write a detailed problem description. Once
 the description is written and CI succeeds, click the "Ready to Review" button
 and add reviewers. Adding reviewers before CI succeeds is a fast path to losing
 reviewer engagement. Not only will they be notified and see the PR is not yet
-ready for them, they will also be bombarded them with additional notifications
+ready for them, they will also be bombarded with additional notifications
 each time you push a commit to get past CI or until they "mute" the PR. Once
 muted, you'll need to reach out over some other medium, such as Discord, to
 request they have another look. When you use draft PRs, no notifications are

Cargo.toml

@@ -1,7 +1,9 @@
 [workspace]
 members = [
+    "accountsdb-plugin-interface",
+    "accountsdb-plugin-manager",
+    "accountsdb-plugin-postgres",
     "accounts-cluster-bench",
-    "bench-exchange",
     "bench-streamer",
     "bench-tps",
     "accounts-bench",
@@ -9,6 +11,7 @@ members = [
     "banks-client",
     "banks-interface",
     "banks-server",
+    "bucket_map",
     "clap-utils",
     "cli-config",
     "cli-output",
@@ -16,6 +19,7 @@ members = [
     "core",
     "dos",
     "download-utils",
+    "entry",
     "faucet",
     "frozen-abi",
     "perf",
@@ -39,29 +43,30 @@ members = [
     "metrics",
     "net-shaper",
     "notifier",
+    "poh",
     "poh-bench",
     "program-test",
-    "programs/secp256k1",
+    "programs/address-lookup-table",
+    "programs/address-lookup-table-tests",
     "programs/bpf_loader",
-    "programs/budget",
+    "programs/bpf_loader/gen-syscall-list",
+    "programs/compute-budget",
     "programs/config",
-    "programs/exchange",
-    "programs/failure",
-    "programs/noop",
-    "programs/ownable",
     "programs/stake",
     "programs/vote",
+    "rbpf-cli",
     "remote-wallet",
+    "rpc",
     "runtime",
     "runtime/store-tool",
     "sdk",
     "sdk/cargo-build-bpf",
     "sdk/cargo-test-bpf",
-    "scripts",
+    "send-transaction-service",
     "stake-accounts",
-    "stake-monitor",
     "sys-tuner",
     "tokens",
+    "transaction-dos",
     "transaction-status",
     "account-decoder",
     "upload-perf",
@@ -70,11 +75,15 @@ members = [
     "cli",
     "rayon-threadlimit",
     "watchtower",
+    "replica-node",
+    "replica-lib",
+    "test-validator",
+    "rpc-test",
+    "client-test",
+    "zk-token-sdk",
+    "programs/zk-token-proof",
 ]
 
 exclude = [
     "programs/bpf",
 ]
-
-[profile.dev]
-split-debuginfo = "unpacked"

README.md

@@ -1,6 +1,6 @@
 <p align="center">
   <a href="https://solana.com">
-    <img alt="Solana" src="https://i.imgur.com/uBVzyX3.png" width="250" />
+    <img alt="Solana" src="https://i.imgur.com/IKyzQ6T.png" width="250" />
   </a>
 </p>
 
@@ -19,12 +19,18 @@ $ source $HOME/.cargo/env
 $ rustup component add rustfmt
 ```
 
-Please sure you are always using the latest stable rust version by running:
+When building the master branch, please make sure you are using the latest stable rust version by running:
 
 ```bash
 $ rustup update
 ```
 
+When building a specific release branch, you should check the rust version in `ci/rust-version.sh` and if necessary, install that version by running:
+```bash
+$ rustup install VERSION
+```
+Note that if this is not the latest rust version on your machine, cargo commands may require an [override](https://rust-lang.github.io/rustup/overrides.html) in order to use the correct version.
+
 On Linux systems you may need to install libssl-dev, pkg-config, zlib1g-dev, etc.  On Ubuntu:
 
 ```bash
@@ -32,12 +38,6 @@ $ sudo apt-get update
 $ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang make
 ```
 
-On Mac M1s, make sure you set up your terminal & homebrew [to use](https://5balloons.info/correct-way-to-install-and-use-homebrew-on-m1-macs/) Rosetta. You can install it with:
-
-```bash
-$ softwareupdate --install-rosetta
-```
-
 ## **2. Download the source code.**
 
 ```bash
@@ -51,11 +51,6 @@ $ cd solana
 $ cargo build
 ```
 
-## **4. Run a minimal local cluster.**
-```bash
-$ ./run.sh
-```
-
 # Testing
 
 **Run the test suite:**
@@ -73,7 +68,7 @@ devnet.solana.com. Runs 24/7. Learn more about the [public clusters](https://doc
 
 # Benchmarking
 
-First install the nightly build of rustc. `cargo bench` requires use of the
+First, install the nightly build of rustc. `cargo bench` requires the use of the
 unstable features only available in the nightly build.
 
 ```bash
@@ -115,17 +110,17 @@ send us that patch!
 
 All claims, content, designs, algorithms, estimates, roadmaps,
 specifications, and performance measurements described in this project
-are done with the Solana Foundation's ("SF") best efforts. It is up to
+are done with the Solana Foundation's ("SF") good faith efforts. It is up to
 the reader to check and validate their accuracy and truthfulness.
-Furthermore nothing in this project constitutes a solicitation for
+Furthermore, nothing in this project constitutes a solicitation for
 investment.
 
-Any content produced by SF or developer resources that SF provides, are
-for educational and inspiration purposes only. SF does not encourage,
+Any content produced by SF or developer resources that SF provides are
+for educational and inspirational purposes only. SF does not encourage,
 induce or sanction the deployment, integration or use of any such
 applications (including the code comprising the Solana blockchain
 protocol) in violation of applicable laws or regulations and hereby
-prohibits any such deployment, integration or use. This includes use of
+prohibits any such deployment, integration or use. This includes the use of
 any such applications by the reader (a) in violation of export control
 or sanctions laws of the United States or any other applicable
 jurisdiction, (b) if the reader is located in or ordinarily resident in
@@ -138,7 +133,7 @@ prohibitions.
 The reader should be aware that U.S. export control and sanctions laws
 prohibit U.S. persons (and other persons that are subject to such laws)
 from transacting with persons in certain countries and territories or
-that are on the SDN list. As a project based primarily on open-source
+that are on the SDN list. As a project-based primarily on open-source
 software, it is possible that such sanctioned persons may nevertheless
 bypass prohibitions, obtain the code comprising the Solana blockchain
 protocol (or other project code or applications) and deploy, integrate,

RELEASE.md

@@ -94,7 +94,7 @@ Alternatively use the Github UI.
     ```
 1. Confirm that your freshly cut release branch is shown as `BETA_CHANNEL` and the previous release branch as `STABLE_CHANNEL`:
     ```
-    ci/channel_info.sh
+    ci/channel-info.sh
     ```
 
 ## Steps to Create a Release
@@ -152,5 +152,5 @@ appearing.  To check for progress:
 [Crates.io](https://crates.io/crates/solana) should have an updated Solana version.  This can take 2-3 hours, and sometimes fails in the `solana-secondary` job.
 If this happens and the error is non-fatal, click "Retry" on the "publish crate" job
 
-### Update software on devnet.solana.com/testnet.solana.com/mainnet-beta.solana.com
-See the documentation at https://github.com/solana-labs/cluster-ops/
+### Update software on testnet.solana.com
+See the documentation at https://github.com/solana-labs/cluster-ops/. devnet.solana.com and mainnet-beta.solana.com run stable releases that have been tested on testnet. Do not update devnet or mainnet-beta with a beta release.

SECURITY.md

@@ -18,24 +18,24 @@ Expect a response as fast as possible, within one business day at the latest.
 We offer bounties for critical security issues. Please see below for more details.
 
 Loss of Funds:
-$500,000 USD in locked SOL tokens (locked for 12 months)
+$2,000,000 USD in locked SOL tokens (locked for 12 months)
 * Theft of funds without users signature from any account
 * Theft of funds without users interaction in system, token, stake, vote programs
 * Theft of funds that requires users signature - creating a vote program that drains the delegated stakes.
 
 Consensus/Safety Violations:
-$250,000 USD in locked SOL tokens (locked for 12 months)
+$1,000,000 USD in locked SOL tokens (locked for 12 months)
 * Consensus safety violation
 * Tricking a validator to accept an optimistic confirmation or rooted slot without a double vote, etc..
 
 Other Attacks:
- $100,000 USD in locked SOL tokens (locked for 12 months)
+ $400,000 USD in locked SOL tokens (locked for 12 months)
 * Protocol liveness attacks,
 * Eclipse attacks,
 * Remote attacks that partition the network,
 
 DoS Attacks:
-$25,000 USD in locked SOL tokens (locked for 12 months)
+$100,000 USD in locked SOL tokens (locked for 12 months)
 * Remote resource exaustion via Non-RPC protocols
 
 RPC DoS/Crashes:

account-decoder/Cargo.toml

@@ -1,31 +1,30 @@
 [package]
 name = "solana-account-decoder"
-version = "1.7.0"
+version = "1.10.0"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-account-decoder"
 license = "Apache-2.0"
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 base64 = "0.12.3"
-bincode = "1.3.1"
-bs58 = "0.3.1"
+bincode = "1.3.3"
+bs58 = "0.4.0"
 bv = "0.11.1"
 Inflector = "0.11.4"
 lazy_static = "1.4.0"
-serde = "1.0.122"
+serde = "1.0.133"
 serde_derive = "1.0.103"
-serde_json = "1.0.56"
-solana-config-program = { path = "../programs/config", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-stake-program = { path = "../programs/stake", version = "=1.7.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.7.0" }
-spl-token-v2-0 = { package = "spl-token", version = "=3.1.0", features = ["no-entrypoint"] }
+serde_json = "1.0.74"
+solana-config-program = { path = "../programs/config", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-vote-program = { path = "../programs/vote", version = "=1.10.0" }
+spl-token = { version = "=3.2.0", features = ["no-entrypoint"] }
 thiserror = "1.0"
-zstd = "0.5.1"
+zstd = "0.9.1"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

account-decoder/src/lib.rs

@@ -17,8 +17,10 @@ pub mod validator_info;
 use {
     crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount},
     solana_sdk::{
-        account::ReadableAccount, account::WritableAccount, clock::Epoch,
-        fee_calculator::FeeCalculator, pubkey::Pubkey,
+        account::{ReadableAccount, WritableAccount},
+        clock::Epoch,
+        fee_calculator::FeeCalculator,
+        pubkey::Pubkey,
     },
     std::{
         io::{Read, Write},
@@ -28,6 +30,7 @@ use {
 
 pub type StringAmount = String;
 pub type StringDecimals = String;
+pub const MAX_BASE58_BYTES: usize = 128;
 
 /// A duplicate representation of an Account for pretty JSON serialization
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -48,7 +51,7 @@ pub enum UiAccountData {
     Binary(String, UiAccountEncoding),
 }
 
-#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq)]
+#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq, Hash)]
 #[serde(rename_all = "camelCase")]
 pub enum UiAccountEncoding {
     Binary, // Legacy. Retained for RPC backwards compatibility
@@ -60,6 +63,17 @@ pub enum UiAccountEncoding {
 }
 
 impl UiAccount {
+    fn encode_bs58<T: ReadableAccount>(
+        account: &T,
+        data_slice_config: Option<UiDataSliceConfig>,
+    ) -> String {
+        if account.data().len() <= MAX_BASE58_BYTES {
+            bs58::encode(slice_data(account.data(), data_slice_config)).into_string()
+        } else {
+            "error: data too large for bs58 encoding".to_string()
+        }
+    }
+
     pub fn encode<T: ReadableAccount>(
         pubkey: &Pubkey,
         account: &T,
@@ -68,33 +82,34 @@ impl UiAccount {
         data_slice_config: Option<UiDataSliceConfig>,
     ) -> Self {
         let data = match encoding {
-            UiAccountEncoding::Binary => UiAccountData::LegacyBinary(
-                bs58::encode(slice_data(&account.data(), data_slice_config)).into_string(),
-            ),
-            UiAccountEncoding::Base58 => UiAccountData::Binary(
-                bs58::encode(slice_data(&account.data(), data_slice_config)).into_string(),
-                encoding,
-            ),
+            UiAccountEncoding::Binary => {
+                let data = Self::encode_bs58(account, data_slice_config);
+                UiAccountData::LegacyBinary(data)
+            }
+            UiAccountEncoding::Base58 => {
+                let data = Self::encode_bs58(account, data_slice_config);
+                UiAccountData::Binary(data, encoding)
+            }
             UiAccountEncoding::Base64 => UiAccountData::Binary(
-                base64::encode(slice_data(&account.data(), data_slice_config)),
+                base64::encode(slice_data(account.data(), data_slice_config)),
                 encoding,
             ),
             UiAccountEncoding::Base64Zstd => {
                 let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 0).unwrap();
                 match encoder
-                    .write_all(slice_data(&account.data(), data_slice_config))
+                    .write_all(slice_data(account.data(), data_slice_config))
                     .and_then(|()| encoder.finish())
                 {
                     Ok(zstd_data) => UiAccountData::Binary(base64::encode(zstd_data), encoding),
                     Err(_) => UiAccountData::Binary(
-                        base64::encode(slice_data(&account.data(), data_slice_config)),
+                        base64::encode(slice_data(account.data(), data_slice_config)),
                         UiAccountEncoding::Base64,
                     ),
                 }
             }
             UiAccountEncoding::JsonParsed => {
                 if let Ok(parsed_data) =
-                    parse_account_data(pubkey, &account.owner(), &account.data(), additional_data)
+                    parse_account_data(pubkey, account.owner(), account.data(), additional_data)
                 {
                     UiAccountData::Json(parsed_data)
                 } else {
@@ -166,7 +181,7 @@ impl Default for UiFeeCalculator {
     }
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct UiDataSliceConfig {
     pub offset: usize,
@@ -189,8 +204,10 @@ fn slice_data(data: &[u8], data_slice_config: Option<UiDataSliceConfig>) -> &[u8
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use solana_sdk::account::{Account, AccountSharedData};
+    use {
+        super::*,
+        solana_sdk::account::{Account, AccountSharedData},
+    };
 
     #[test]
     fn test_slice_data() {

account-decoder/src/parse_account_data.rs

@@ -1,25 +1,27 @@
-use crate::{
-    parse_bpf_loader::parse_bpf_upgradeable_loader,
-    parse_config::parse_config,
-    parse_nonce::parse_nonce,
-    parse_stake::parse_stake,
-    parse_sysvar::parse_sysvar,
-    parse_token::{parse_token, spl_token_id_v2_0},
-    parse_vote::parse_vote,
+use {
+    crate::{
+        parse_bpf_loader::parse_bpf_upgradeable_loader,
+        parse_config::parse_config,
+        parse_nonce::parse_nonce,
+        parse_stake::parse_stake,
+        parse_sysvar::parse_sysvar,
+        parse_token::{parse_token, spl_token_id},
+        parse_vote::parse_vote,
+    },
+    inflector::Inflector,
+    serde_json::Value,
+    solana_sdk::{instruction::InstructionError, pubkey::Pubkey, stake, system_program, sysvar},
+    std::collections::HashMap,
+    thiserror::Error,
 };
-use inflector::Inflector;
-use serde_json::Value;
-use solana_sdk::{instruction::InstructionError, pubkey::Pubkey, system_program, sysvar};
-use std::collections::HashMap;
-use thiserror::Error;
 
 lazy_static! {
     static ref BPF_UPGRADEABLE_LOADER_PROGRAM_ID: Pubkey = solana_sdk::bpf_loader_upgradeable::id();
     static ref CONFIG_PROGRAM_ID: Pubkey = solana_config_program::id();
-    static ref STAKE_PROGRAM_ID: Pubkey = solana_stake_program::id();
+    static ref STAKE_PROGRAM_ID: Pubkey = stake::program::id();
     static ref SYSTEM_PROGRAM_ID: Pubkey = system_program::id();
     static ref SYSVAR_PROGRAM_ID: Pubkey = sysvar::id();
-    static ref TOKEN_PROGRAM_ID: Pubkey = spl_token_id_v2_0();
+    static ref TOKEN_PROGRAM_ID: Pubkey = spl_token_id();
     static ref VOTE_PROGRAM_ID: Pubkey = solana_vote_program::id();
     pub static ref PARSABLE_PROGRAM_IDS: HashMap<Pubkey, ParsableAccount> = {
         let mut m = HashMap::new();
@@ -112,12 +114,14 @@ pub fn parse_account_data(
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use solana_sdk::nonce::{
-        state::{Data, Versions},
-        State,
+    use {
+        super::*,
+        solana_sdk::nonce::{
+            state::{Data, Versions},
+            State,
+        },
+        solana_vote_program::vote_state::{VoteState, VoteStateVersions},
     };
-    use solana_vote_program::vote_state::{VoteState, VoteStateVersions};
 
     #[test]
     fn test_parse_account_data() {

account-decoder/src/parse_bpf_loader.rs

@@ -1,9 +1,11 @@
-use crate::{
-    parse_account_data::{ParsableAccount, ParseAccountError},
-    UiAccountData, UiAccountEncoding,
+use {
+    crate::{
+        parse_account_data::{ParsableAccount, ParseAccountError},
+        UiAccountData, UiAccountEncoding,
+    },
+    bincode::{deserialize, serialized_size},
+    solana_sdk::{bpf_loader_upgradeable::UpgradeableLoaderState, pubkey::Pubkey},
 };
-use bincode::{deserialize, serialized_size};
-use solana_sdk::{bpf_loader_upgradeable::UpgradeableLoaderState, pubkey::Pubkey};
 
 pub fn parse_bpf_upgradeable_loader(
     data: &[u8],
@@ -90,9 +92,7 @@ pub struct UiProgramData {
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use bincode::serialize;
-    use solana_sdk::pubkey::Pubkey;
+    use {super::*, bincode::serialize, solana_sdk::pubkey::Pubkey};
 
     #[test]
     fn test_parse_bpf_upgradeable_loader_accounts() {

account-decoder/src/parse_config.rs

@@ -1,15 +1,19 @@
-use crate::{
-    parse_account_data::{ParsableAccount, ParseAccountError},
-    validator_info,
+use {
+    crate::{
+        parse_account_data::{ParsableAccount, ParseAccountError},
+        validator_info,
+    },
+    bincode::deserialize,
+    serde_json::Value,
+    solana_config_program::{get_config_data, ConfigKeys},
+    solana_sdk::{
+        pubkey::Pubkey,
+        stake::config::{self as stake_config, Config as StakeConfig},
+    },
 };
-use bincode::deserialize;
-use serde_json::Value;
-use solana_config_program::{get_config_data, ConfigKeys};
-use solana_sdk::pubkey::Pubkey;
-use solana_stake_program::config::Config as StakeConfig;
 
 pub fn parse_config(data: &[u8], pubkey: &Pubkey) -> Result<ConfigAccountType, ParseAccountError> {
-    let parsed_account = if pubkey == &solana_stake_program::config::id() {
+    let parsed_account = if pubkey == &stake_config::id() {
         get_config_data(data)
             .ok()
             .and_then(|data| deserialize::<StakeConfig>(data).ok())
@@ -37,7 +41,7 @@ fn parse_config_data<T>(data: &[u8], keys: Vec<(Pubkey, bool)>) -> Option<UiConf
 where
     T: serde::de::DeserializeOwned,
 {
-    let config_data: T = deserialize(&get_config_data(data).ok()?).ok()?;
+    let config_data: T = deserialize(get_config_data(data).ok()?).ok()?;
     let keys = keys
         .iter()
         .map(|key| UiConfigKey {
@@ -87,11 +91,10 @@ pub struct UiConfig<T> {
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use crate::validator_info::ValidatorInfo;
-    use serde_json::json;
-    use solana_config_program::create_config_account;
-    use solana_sdk::account::ReadableAccount;
+    use {
+        super::*, crate::validator_info::ValidatorInfo, serde_json::json,
+        solana_config_program::create_config_account, solana_sdk::account::ReadableAccount,
+    };
 
     #[test]
     fn test_parse_config() {
@@ -101,11 +104,7 @@ mod test {
         };
         let stake_config_account = create_config_account(vec![], &stake_config, 10);
         assert_eq!(
-            parse_config(
-                &stake_config_account.data(),
-                &solana_stake_program::config::id()
-            )
-            .unwrap(),
+            parse_config(stake_config_account.data(), &stake_config::id()).unwrap(),
             ConfigAccountType::StakeConfig(UiStakeConfig {
                 warmup_cooldown_rate: 0.25,
                 slash_penalty: 50,
@@ -125,7 +124,7 @@ mod test {
             10,
         );
         assert_eq!(
-            parse_config(&validator_info_config_account.data(), &info_pubkey).unwrap(),
+            parse_config(validator_info_config_account.data(), &info_pubkey).unwrap(),
             ConfigAccountType::ValidatorInfo(UiConfig {
                 keys: vec![
                     UiConfigKey {

account-decoder/src/parse_nonce.rs

@@ -1,7 +1,9 @@
-use crate::{parse_account_data::ParseAccountError, UiFeeCalculator};
-use solana_sdk::{
-    instruction::InstructionError,
-    nonce::{state::Versions, State},
+use {
+    crate::{parse_account_data::ParseAccountError, UiFeeCalculator},
+    solana_sdk::{
+        instruction::InstructionError,
+        nonce::{state::Versions, State},
+    },
 };
 
 pub fn parse_nonce(data: &[u8]) -> Result<UiNonceState, ParseAccountError> {
@@ -42,14 +44,16 @@ pub struct UiNonceData {
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use solana_sdk::{
-        hash::Hash,
-        nonce::{
-            state::{Data, Versions},
-            State,
+    use {
+        super::*,
+        solana_sdk::{
+            hash::Hash,
+            nonce::{
+                state::{Data, Versions},
+                State,
+            },
+            pubkey::Pubkey,
         },
-        pubkey::Pubkey,
     };
 
     #[test]

account-decoder/src/parse_stake.rs

@@ -1,10 +1,14 @@
-use crate::{
-    parse_account_data::{ParsableAccount, ParseAccountError},
-    StringAmount,
+use {
+    crate::{
+        parse_account_data::{ParsableAccount, ParseAccountError},
+        StringAmount,
+    },
+    bincode::deserialize,
+    solana_sdk::{
+        clock::{Epoch, UnixTimestamp},
+        stake::state::{Authorized, Delegation, Lockup, Meta, Stake, StakeState},
+    },
 };
-use bincode::deserialize;
-use solana_sdk::clock::{Epoch, UnixTimestamp};
-use solana_stake_program::stake_state::{Authorized, Delegation, Lockup, Meta, Stake, StakeState};
 
 pub fn parse_stake(data: &[u8]) -> Result<StakeAccountType, ParseAccountError> {
     let stake_state: StakeState = deserialize(data)
@@ -132,8 +136,7 @@ impl From<Delegation> for UiDelegation {
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use bincode::serialize;
+    use {super::*, bincode::serialize};
 
     #[test]
     fn test_parse_stake() {

account-decoder/src/parse_sysvar.rs

@@ -1,21 +1,26 @@
-use crate::{
-    parse_account_data::{ParsableAccount, ParseAccountError},
-    StringAmount, UiFeeCalculator,
-};
-use bincode::deserialize;
-use bv::BitVec;
-use solana_sdk::{
-    clock::{Clock, Epoch, Slot, UnixTimestamp},
-    epoch_schedule::EpochSchedule,
-    pubkey::Pubkey,
-    rent::Rent,
-    slot_hashes::SlotHashes,
-    slot_history::{self, SlotHistory},
-    stake_history::{StakeHistory, StakeHistoryEntry},
-    sysvar::{self, fees::Fees, recent_blockhashes::RecentBlockhashes, rewards::Rewards},
+#[allow(deprecated)]
+use solana_sdk::sysvar::{fees::Fees, recent_blockhashes::RecentBlockhashes};
+use {
+    crate::{
+        parse_account_data::{ParsableAccount, ParseAccountError},
+        StringAmount, UiFeeCalculator,
+    },
+    bincode::deserialize,
+    bv::BitVec,
+    solana_sdk::{
+        clock::{Clock, Epoch, Slot, UnixTimestamp},
+        epoch_schedule::EpochSchedule,
+        pubkey::Pubkey,
+        rent::Rent,
+        slot_hashes::SlotHashes,
+        slot_history::{self, SlotHistory},
+        stake_history::{StakeHistory, StakeHistoryEntry},
+        sysvar::{self, rewards::Rewards},
+    },
 };
 
 pub fn parse_sysvar(data: &[u8], pubkey: &Pubkey) -> Result<SysvarAccountType, ParseAccountError> {
+    #[allow(deprecated)]
     let parsed_account = {
         if pubkey == &sysvar::clock::id() {
             deserialize::<Clock>(data)
@@ -91,7 +96,9 @@ pub fn parse_sysvar(data: &[u8], pubkey: &Pubkey) -> Result<SysvarAccountType, P
 pub enum SysvarAccountType {
     Clock(UiClock),
     EpochSchedule(EpochSchedule),
+    #[allow(deprecated)]
     Fees(UiFees),
+    #[allow(deprecated)]
     RecentBlockhashes(Vec<UiRecentBlockhashesEntry>),
     Rent(UiRent),
     Rewards(UiRewards),
@@ -127,6 +134,7 @@ impl From<Clock> for UiClock {
 pub struct UiFees {
     pub fee_calculator: UiFeeCalculator,
 }
+#[allow(deprecated)]
 impl From<Fees> for UiFees {
     fn from(fees: Fees) -> Self {
         Self {
@@ -212,14 +220,17 @@ pub struct UiStakeHistoryEntry {
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use solana_sdk::{
-        account::create_account_for_test, fee_calculator::FeeCalculator, hash::Hash,
-        sysvar::recent_blockhashes::IterItem,
+    #[allow(deprecated)]
+    use solana_sdk::sysvar::recent_blockhashes::IterItem;
+    use {
+        super::*,
+        solana_sdk::{account::create_account_for_test, fee_calculator::FeeCalculator, hash::Hash},
     };
 
     #[test]
     fn test_parse_sysvars() {
+        let hash = Hash::new(&[1; 32]);
+
         let clock_sysvar = create_account_for_test(&Clock::default());
         assert_eq!(
             parse_sysvar(&clock_sysvar.data, &sysvar::clock::id()).unwrap(),
@@ -239,31 +250,29 @@ mod test {
             SysvarAccountType::EpochSchedule(epoch_schedule),
         );
 
-        let fees_sysvar = create_account_for_test(&Fees::default());
-        assert_eq!(
-            parse_sysvar(&fees_sysvar.data, &sysvar::fees::id()).unwrap(),
-            SysvarAccountType::Fees(UiFees::default()),
-        );
+        #[allow(deprecated)]
+        {
+            let fees_sysvar = create_account_for_test(&Fees::default());
+            assert_eq!(
+                parse_sysvar(&fees_sysvar.data, &sysvar::fees::id()).unwrap(),
+                SysvarAccountType::Fees(UiFees::default()),
+            );
 
-        let hash = Hash::new(&[1; 32]);
-        let fee_calculator = FeeCalculator {
-            lamports_per_signature: 10,
-        };
-        let recent_blockhashes: RecentBlockhashes = vec![IterItem(0, &hash, &fee_calculator)]
-            .into_iter()
-            .collect();
-        let recent_blockhashes_sysvar = create_account_for_test(&recent_blockhashes);
-        assert_eq!(
-            parse_sysvar(
-                &recent_blockhashes_sysvar.data,
-                &sysvar::recent_blockhashes::id()
-            )
-            .unwrap(),
-            SysvarAccountType::RecentBlockhashes(vec![UiRecentBlockhashesEntry {
-                blockhash: hash.to_string(),
-                fee_calculator: fee_calculator.into(),
-            }]),
-        );
+            let recent_blockhashes: RecentBlockhashes =
+                vec![IterItem(0, &hash, 10)].into_iter().collect();
+            let recent_blockhashes_sysvar = create_account_for_test(&recent_blockhashes);
+            assert_eq!(
+                parse_sysvar(
+                    &recent_blockhashes_sysvar.data,
+                    &sysvar::recent_blockhashes::id()
+                )
+                .unwrap(),
+                SysvarAccountType::RecentBlockhashes(vec![UiRecentBlockhashesEntry {
+                    blockhash: hash.to_string(),
+                    fee_calculator: FeeCalculator::new(10).into(),
+                }]),
+            );
+        }
 
         let rent = Rent {
             lamports_per_byte_year: 10,

account-decoder/src/parse_token.rs

@@ -1,35 +1,37 @@
-use crate::{
-    parse_account_data::{ParsableAccount, ParseAccountError},
-    StringAmount, StringDecimals,
-};
-use solana_sdk::pubkey::Pubkey;
-use spl_token_v2_0::{
-    solana_program::{
-        program_option::COption, program_pack::Pack, pubkey::Pubkey as SplTokenPubkey,
+use {
+    crate::{
+        parse_account_data::{ParsableAccount, ParseAccountError},
+        StringAmount, StringDecimals,
     },
-    state::{Account, AccountState, Mint, Multisig},
+    solana_sdk::pubkey::Pubkey,
+    spl_token::{
+        solana_program::{
+            program_option::COption, program_pack::Pack, pubkey::Pubkey as SplTokenPubkey,
+        },
+        state::{Account, AccountState, Mint, Multisig},
+    },
+    std::str::FromStr,
 };
-use std::str::FromStr;
 
-// A helper function to convert spl_token_v2_0::id() as spl_sdk::pubkey::Pubkey to
+// A helper function to convert spl_token::id() as spl_sdk::pubkey::Pubkey to
 // solana_sdk::pubkey::Pubkey
-pub fn spl_token_id_v2_0() -> Pubkey {
-    Pubkey::new_from_array(spl_token_v2_0::id().to_bytes())
+pub fn spl_token_id() -> Pubkey {
+    Pubkey::new_from_array(spl_token::id().to_bytes())
 }
 
-// A helper function to convert spl_token_v2_0::native_mint::id() as spl_sdk::pubkey::Pubkey to
+// A helper function to convert spl_token::native_mint::id() as spl_sdk::pubkey::Pubkey to
 // solana_sdk::pubkey::Pubkey
-pub fn spl_token_v2_0_native_mint() -> Pubkey {
-    Pubkey::new_from_array(spl_token_v2_0::native_mint::id().to_bytes())
+pub fn spl_token_native_mint() -> Pubkey {
+    Pubkey::new_from_array(spl_token::native_mint::id().to_bytes())
 }
 
 // A helper function to convert a solana_sdk::pubkey::Pubkey to spl_sdk::pubkey::Pubkey
-pub fn spl_token_v2_0_pubkey(pubkey: &Pubkey) -> SplTokenPubkey {
+pub fn spl_token_pubkey(pubkey: &Pubkey) -> SplTokenPubkey {
     SplTokenPubkey::new_from_array(pubkey.to_bytes())
 }
 
 // A helper function to convert a spl_sdk::pubkey::Pubkey to solana_sdk::pubkey::Pubkey
-pub fn pubkey_from_spl_token_v2_0(pubkey: &SplTokenPubkey) -> Pubkey {
+pub fn pubkey_from_spl_token(pubkey: &SplTokenPubkey) -> Pubkey {
     Pubkey::new_from_array(pubkey.to_bytes())
 }
 
@@ -116,6 +118,7 @@ pub fn parse_token(
 
 #[derive(Debug, Serialize, Deserialize, PartialEq)]
 #[serde(rename_all = "camelCase", tag = "type", content = "info")]
+#[allow(clippy::large_enum_variant)]
 pub enum TokenAccountType {
     Account(UiTokenAccount),
     Mint(UiMint),

account-decoder/src/parse_vote.rs

@@ -1,9 +1,11 @@
-use crate::{parse_account_data::ParseAccountError, StringAmount};
-use solana_sdk::{
-    clock::{Epoch, Slot},
-    pubkey::Pubkey,
+use {
+    crate::{parse_account_data::ParseAccountError, StringAmount},
+    solana_sdk::{
+        clock::{Epoch, Slot},
+        pubkey::Pubkey,
+    },
+    solana_vote_program::vote_state::{BlockTimestamp, Lockout, VoteState},
 };
-use solana_vote_program::vote_state::{BlockTimestamp, Lockout, VoteState};
 
 pub fn parse_vote(data: &[u8]) -> Result<VoteAccountType, ParseAccountError> {
     let mut vote_state = VoteState::deserialize(data).map_err(ParseAccountError::from)?;
@@ -121,8 +123,7 @@ struct UiEpochCredits {
 
 #[cfg(test)]
 mod test {
-    use super::*;
-    use solana_vote_program::vote_state::VoteStateVersions;
+    use {super::*, solana_vote_program::vote_state::VoteStateVersions};
 
     #[test]
     fn test_parse_vote() {

accounts-bench/Cargo.toml

@@ -1,24 +1,22 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2018"
+edition = "2021"
 name = "solana-accounts-bench"
-version = "1.7.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 publish = false
 
 [dependencies]
-log = "0.4.11"
-rayon = "1.5.0"
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
-rand = "0.7.0"
+log = "0.4.14"
+rayon = "1.5.1"
+solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
 clap = "2.33.1"
-crossbeam-channel = "0.4"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accounts-bench/src/main.rs

@@ -1,16 +1,19 @@
 #![allow(clippy::integer_arithmetic)]
 #[macro_use]
 extern crate log;
-use clap::{crate_description, crate_name, value_t, App, Arg};
-use rayon::prelude::*;
-use solana_measure::measure::Measure;
-use solana_runtime::{
-    accounts::{create_test_accounts, update_accounts_bench, Accounts},
-    accounts_index::AccountSecondaryIndexes,
-    ancestors::Ancestors,
+use {
+    clap::{crate_description, crate_name, value_t, App, Arg},
+    rayon::prelude::*,
+    solana_measure::measure::Measure,
+    solana_runtime::{
+        accounts::{create_test_accounts, update_accounts_bench, Accounts},
+        accounts_db::AccountShrinkThreshold,
+        accounts_index::AccountSecondaryIndexes,
+        ancestors::Ancestors,
+    },
+    solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey},
+    std::{env, fs, path::PathBuf},
 };
-use solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey};
-use std::{env, fs, path::PathBuf};
 
 fn main() {
     solana_logger::setup();
@@ -59,11 +62,12 @@ fn main() {
     if fs::remove_dir_all(path.clone()).is_err() {
         println!("Warning: Couldn't remove {:?}", path);
     }
-    let accounts = Accounts::new_with_config(
+    let accounts = Accounts::new_with_config_for_benches(
         vec![path],
         &ClusterType::Testnet,
         AccountSecondaryIndexes::default(),
         false,
+        AccountShrinkThreshold::default(),
     );
     println!("Creating {} accounts", num_accounts);
     let mut create_time = Measure::start("create accounts");
@@ -100,7 +104,7 @@ fn main() {
     for x in 0..iterations {
         if clean {
             let mut time = Measure::start("clean");
-            accounts.accounts_db.clean_accounts(None, false);
+            accounts.accounts_db.clean_accounts(None, false, None);
             time.stop();
             println!("{}", time);
             for slot in 0..num_slots {
@@ -119,6 +123,9 @@ fn main() {
                 solana_sdk::clock::Slot::default(),
                 &ancestors,
                 None,
+                false,
+                None,
+                false,
             );
             time_store.stop();
             if results != results_store {

accounts-cluster-bench/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2018"
+edition = "2021"
 name = "solana-accounts-cluster-bench"
-version = "1.7.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,26 +10,28 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-log = "0.4.11"
+log = "0.4.14"
 rand = "0.7.0"
-rayon = "1.4.1"
-solana-account-decoder = { path = "../account-decoder", version = "=1.7.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-client = { path = "../client", version = "=1.7.0" }
-solana-core = { path = "../core", version = "=1.7.0" }
-solana-faucet = { path = "../faucet", version = "=1.7.0" }
-solana-gossip = { path = "../gossip", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
-spl-token-v2-0 = { package = "spl-token", version = "=3.1.0", features = ["no-entrypoint"] }
+rayon = "1.5.1"
+solana-account-decoder = { path = "../account-decoder", version = "=1.10.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.10.0" }
+solana-client = { path = "../client", version = "=1.10.0" }
+solana-faucet = { path = "../faucet", version = "=1.10.0" }
+solana-gossip = { path = "../gossip", version = "=1.10.0" }
+solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-streamer = { path = "../streamer", version = "=1.10.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
+spl-token = { version = "=3.2.0", features = ["no-entrypoint"] }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "=1.7.0" }
+solana-core = { path = "../core", version = "=1.10.0" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.10.0" }
+solana-test-validator = { path = "../test-validator", version = "=1.10.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accounts-cluster-bench/src/main.rs

@@ -1,41 +1,40 @@
 #![allow(clippy::integer_arithmetic)]
-use clap::{crate_description, crate_name, value_t, values_t_or_exit, App, Arg};
-use log::*;
-use rand::{thread_rng, Rng};
-use rayon::prelude::*;
-use solana_account_decoder::parse_token::spl_token_v2_0_pubkey;
-use solana_clap_utils::input_parsers::pubkey_of;
-use solana_client::rpc_client::RpcClient;
-use solana_faucet::faucet::{request_airdrop_transaction, FAUCET_PORT};
-use solana_gossip::gossip_service::discover;
-use solana_measure::measure::Measure;
-use solana_runtime::inline_spl_token_v2_0;
-use solana_sdk::{
-    commitment_config::CommitmentConfig,
-    message::Message,
-    pubkey::Pubkey,
-    rpc_port::DEFAULT_RPC_PORT,
-    signature::{read_keypair_file, Keypair, Signature, Signer},
-    system_instruction, system_program,
-    timing::timestamp,
-    transaction::Transaction,
-};
-use solana_transaction_status::parse_token::spl_token_v2_0_instruction;
-use std::{
-    net::SocketAddr,
-    process::exit,
-    sync::{
-        atomic::{AtomicBool, AtomicU64, Ordering},
-        Arc, RwLock,
+use {
+    clap::{crate_description, crate_name, value_t, values_t_or_exit, App, Arg},
+    log::*,
+    rand::{thread_rng, Rng},
+    rayon::prelude::*,
+    solana_account_decoder::parse_token::spl_token_pubkey,
+    solana_clap_utils::input_parsers::pubkey_of,
+    solana_client::{rpc_client::RpcClient, transaction_executor::TransactionExecutor},
+    solana_faucet::faucet::{request_airdrop_transaction, FAUCET_PORT},
+    solana_gossip::gossip_service::discover,
+    solana_runtime::inline_spl_token,
+    solana_sdk::{
+        commitment_config::CommitmentConfig,
+        instruction::{AccountMeta, Instruction},
+        message::Message,
+        pubkey::Pubkey,
+        rpc_port::DEFAULT_RPC_PORT,
+        signature::{read_keypair_file, Keypair, Signer},
+        system_instruction, system_program,
+        transaction::Transaction,
+    },
+    solana_streamer::socket::SocketAddrSpace,
+    solana_transaction_status::parse_token::spl_token_instruction,
+    std::{
+        cmp::min,
+        net::SocketAddr,
+        process::exit,
+        sync::{
+            atomic::{AtomicU64, Ordering},
+            Arc,
+        },
+        thread::sleep,
+        time::{Duration, Instant},
     },
-    thread::{sleep, Builder, JoinHandle},
-    time::{Duration, Instant},
 };
 
-// Create and close messages both require 2 signatures; if transaction construction changes, update
-// this magic number
-const NUM_SIGNATURES: u64 = 2;
-
 pub fn airdrop_lamports(
     client: &RpcClient,
     faucet_addr: &SocketAddr,
@@ -54,8 +53,8 @@ pub fn airdrop_lamports(
             id.pubkey(),
         );
 
-        let (blockhash, _fee_calculator) = client.get_recent_blockhash().unwrap();
-        match request_airdrop_transaction(&faucet_addr, &id.pubkey(), airdrop_amount, blockhash) {
+        let blockhash = client.get_latest_blockhash().unwrap();
+        match request_airdrop_transaction(faucet_addr, &id.pubkey(), airdrop_amount, blockhash) {
             Ok(transaction) => {
                 let mut tries = 0;
                 loop {
@@ -99,160 +98,6 @@ pub fn airdrop_lamports(
     true
 }
 
-// signature, timestamp, id
-type PendingQueue = Vec<(Signature, u64, u64)>;
-
-struct TransactionExecutor {
-    sig_clear_t: JoinHandle<()>,
-    sigs: Arc<RwLock<PendingQueue>>,
-    cleared: Arc<RwLock<Vec<u64>>>,
-    exit: Arc<AtomicBool>,
-    counter: AtomicU64,
-    client: RpcClient,
-}
-
-impl TransactionExecutor {
-    fn new(entrypoint_addr: SocketAddr) -> Self {
-        let sigs = Arc::new(RwLock::new(Vec::new()));
-        let cleared = Arc::new(RwLock::new(Vec::new()));
-        let exit = Arc::new(AtomicBool::new(false));
-        let sig_clear_t = Self::start_sig_clear_thread(&exit, &sigs, &cleared, entrypoint_addr);
-        let client =
-            RpcClient::new_socket_with_commitment(entrypoint_addr, CommitmentConfig::confirmed());
-        Self {
-            sigs,
-            cleared,
-            sig_clear_t,
-            exit,
-            counter: AtomicU64::new(0),
-            client,
-        }
-    }
-
-    fn num_outstanding(&self) -> usize {
-        self.sigs.read().unwrap().len()
-    }
-
-    fn push_transactions(&self, txs: Vec<Transaction>) -> Vec<u64> {
-        let mut ids = vec![];
-        let new_sigs = txs.into_iter().filter_map(|tx| {
-            let id = self.counter.fetch_add(1, Ordering::Relaxed);
-            ids.push(id);
-            match self.client.send_transaction(&tx) {
-                Ok(sig) => {
-                    return Some((sig, timestamp(), id));
-                }
-                Err(e) => {
-                    info!("error: {:#?}", e);
-                }
-            }
-            None
-        });
-        let mut sigs_w = self.sigs.write().unwrap();
-        sigs_w.extend(new_sigs);
-        ids
-    }
-
-    fn drain_cleared(&self) -> Vec<u64> {
-        std::mem::take(&mut *self.cleared.write().unwrap())
-    }
-
-    fn close(self) {
-        self.exit.store(true, Ordering::Relaxed);
-        self.sig_clear_t.join().unwrap();
-    }
-
-    fn start_sig_clear_thread(
-        exit: &Arc<AtomicBool>,
-        sigs: &Arc<RwLock<PendingQueue>>,
-        cleared: &Arc<RwLock<Vec<u64>>>,
-        entrypoint_addr: SocketAddr,
-    ) -> JoinHandle<()> {
-        let sigs = sigs.clone();
-        let exit = exit.clone();
-        let cleared = cleared.clone();
-        Builder::new()
-            .name("sig_clear".to_string())
-            .spawn(move || {
-                let client = RpcClient::new_socket_with_commitment(
-                    entrypoint_addr,
-                    CommitmentConfig::confirmed(),
-                );
-                let mut success = 0;
-                let mut error_count = 0;
-                let mut timed_out = 0;
-                let mut last_log = Instant::now();
-                while !exit.load(Ordering::Relaxed) {
-                    let sigs_len = sigs.read().unwrap().len();
-                    if sigs_len > 0 {
-                        let mut sigs_w = sigs.write().unwrap();
-                        let mut start = Measure::start("sig_status");
-                        let statuses: Vec<_> = sigs_w
-                            .chunks(200)
-                            .flat_map(|sig_chunk| {
-                                let only_sigs: Vec<_> = sig_chunk.iter().map(|s| s.0).collect();
-                                client
-                                    .get_signature_statuses(&only_sigs)
-                                    .expect("status fail")
-                                    .value
-                            })
-                            .collect();
-                        let mut num_cleared = 0;
-                        let start_len = sigs_w.len();
-                        let now = timestamp();
-                        let mut new_ids = vec![];
-                        let mut i = 0;
-                        let mut j = 0;
-                        while i != sigs_w.len() {
-                            let mut retain = true;
-                            let sent_ts = sigs_w[i].1;
-                            if let Some(e) = &statuses[j] {
-                                debug!("error: {:?}", e);
-                                if e.status.is_ok() {
-                                    success += 1;
-                                } else {
-                                    error_count += 1;
-                                }
-                                num_cleared += 1;
-                                retain = false;
-                            } else if now - sent_ts > 30_000 {
-                                retain = false;
-                                timed_out += 1;
-                            }
-                            if !retain {
-                                new_ids.push(sigs_w.remove(i).2);
-                            } else {
-                                i += 1;
-                            }
-                            j += 1;
-                        }
-                        let final_sigs_len = sigs_w.len();
-                        drop(sigs_w);
-                        cleared.write().unwrap().extend(new_ids);
-                        start.stop();
-                        debug!(
-                            "sigs len: {:?} success: {} took: {}ms cleared: {}/{}",
-                            final_sigs_len,
-                            success,
-                            start.as_ms(),
-                            num_cleared,
-                            start_len,
-                        );
-                        if last_log.elapsed().as_millis() > 5000 {
-                            info!(
-                                "success: {} error: {} timed_out: {}",
-                                success, error_count, timed_out,
-                            );
-                            last_log = Instant::now();
-                        }
-                    }
-                    sleep(Duration::from_millis(200));
-                }
-            })
-            .unwrap()
-    }
-}
-
 struct SeedTracker {
     max_created: Arc<AtomicU64>,
     max_closed: Arc<AtomicU64>,
@@ -273,7 +118,7 @@ fn make_create_message(
         .into_iter()
         .map(|_| {
             let program_id = if mint.is_some() {
-                inline_spl_token_v2_0::id()
+                inline_spl_token::id()
             } else {
                 system_program::id()
             };
@@ -290,12 +135,12 @@ fn make_create_message(
                 &program_id,
             )];
             if let Some(mint_address) = mint {
-                instructions.push(spl_token_v2_0_instruction(
-                    spl_token_v2_0::instruction::initialize_account(
-                        &spl_token_v2_0::id(),
-                        &spl_token_v2_0_pubkey(&to_pubkey),
-                        &spl_token_v2_0_pubkey(&mint_address),
-                        &spl_token_v2_0_pubkey(&base_keypair.pubkey()),
+                instructions.push(spl_token_instruction(
+                    spl_token::instruction::initialize_account(
+                        &spl_token::id(),
+                        &spl_token_pubkey(&to_pubkey),
+                        &spl_token_pubkey(&mint_address),
+                        &spl_token_pubkey(&base_keypair.pubkey()),
                     )
                     .unwrap(),
                 ));
@@ -303,8 +148,8 @@ fn make_create_message(
 
             instructions
         })
+        .flatten()
         .collect();
-    let instructions: Vec<_> = instructions.into_iter().flatten().collect();
 
     Message::new(&instructions, Some(&keypair.pubkey()))
 }
@@ -312,42 +157,48 @@ fn make_create_message(
 fn make_close_message(
     keypair: &Keypair,
     base_keypair: &Keypair,
-    max_closed_seed: Arc<AtomicU64>,
+    max_created: Arc<AtomicU64>,
+    max_closed: Arc<AtomicU64>,
     num_instructions: usize,
     balance: u64,
     spl_token: bool,
 ) -> Message {
     let instructions: Vec<_> = (0..num_instructions)
         .into_iter()
-        .map(|_| {
+        .filter_map(|_| {
             let program_id = if spl_token {
-                inline_spl_token_v2_0::id()
+                inline_spl_token::id()
             } else {
                 system_program::id()
             };
-            let seed = max_closed_seed.fetch_add(1, Ordering::Relaxed).to_string();
+            let max_created_seed = max_created.load(Ordering::Relaxed);
+            let max_closed_seed = max_closed.load(Ordering::Relaxed);
+            if max_closed_seed >= max_created_seed {
+                return None;
+            }
+            let seed = max_closed.fetch_add(1, Ordering::Relaxed).to_string();
             let address =
                 Pubkey::create_with_seed(&base_keypair.pubkey(), &seed, &program_id).unwrap();
             if spl_token {
-                spl_token_v2_0_instruction(
-                    spl_token_v2_0::instruction::close_account(
-                        &spl_token_v2_0::id(),
-                        &spl_token_v2_0_pubkey(&address),
-                        &spl_token_v2_0_pubkey(&keypair.pubkey()),
-                        &spl_token_v2_0_pubkey(&base_keypair.pubkey()),
+                Some(spl_token_instruction(
+                    spl_token::instruction::close_account(
+                        &spl_token::id(),
+                        &spl_token_pubkey(&address),
+                        &spl_token_pubkey(&keypair.pubkey()),
+                        &spl_token_pubkey(&base_keypair.pubkey()),
                         &[],
                     )
                     .unwrap(),
-                )
+                ))
             } else {
-                system_instruction::transfer_with_seed(
+                Some(system_instruction::transfer_with_seed(
                     &address,
                     &base_keypair.pubkey(),
                     seed,
                     &program_id,
                     &keypair.pubkey(),
                     balance,
-                )
+                ))
             }
         })
         .collect();
@@ -363,10 +214,11 @@ fn run_accounts_bench(
     iterations: usize,
     maybe_space: Option<u64>,
     batch_size: usize,
-    close_nth: u64,
+    close_nth_batch: u64,
     maybe_lamports: Option<u64>,
     num_instructions: usize,
     mint: Option<Pubkey>,
+    reclaim_accounts: bool,
 ) {
     assert!(num_instructions > 0);
     let client =
@@ -374,10 +226,10 @@ fn run_accounts_bench(
 
     info!("Targeting {}", entrypoint_addr);
 
-    let mut last_blockhash = Instant::now();
+    let mut latest_blockhash = Instant::now();
     let mut last_log = Instant::now();
     let mut count = 0;
-    let mut recent_blockhash = client.get_recent_blockhash().expect("blockhash");
+    let mut blockhash = client.get_latest_blockhash().expect("blockhash");
     let mut tx_sent_count = 0;
     let mut total_accounts_created = 0;
     let mut total_accounts_closed = 0;
@@ -405,16 +257,33 @@ fn run_accounts_bench(
 
     let executor = TransactionExecutor::new(entrypoint_addr);
 
+    // Create and close messages both require 2 signatures, fake a 2 signature message to calculate fees
+    let mut message = Message::new(
+        &[
+            Instruction::new_with_bytes(
+                Pubkey::new_unique(),
+                &[],
+                vec![AccountMeta::new(Pubkey::new_unique(), true)],
+            ),
+            Instruction::new_with_bytes(
+                Pubkey::new_unique(),
+                &[],
+                vec![AccountMeta::new(Pubkey::new_unique(), true)],
+            ),
+        ],
+        None,
+    );
+
     loop {
-        if last_blockhash.elapsed().as_millis() > 10_000 {
-            recent_blockhash = client.get_recent_blockhash().expect("blockhash");
-            last_blockhash = Instant::now();
+        if latest_blockhash.elapsed().as_millis() > 10_000 {
+            blockhash = client.get_latest_blockhash().expect("blockhash");
+            latest_blockhash = Instant::now();
         }
 
-        let fee = recent_blockhash
-            .1
-            .lamports_per_signature
-            .saturating_mul(NUM_SIGNATURES);
+        message.recent_blockhash = blockhash;
+        let fee = client
+            .get_fee_for_message(&message)
+            .expect("get_fee_for_message");
         let lamports = min_balance + fee;
 
         for (i, balance) in balances.iter_mut().enumerate() {
@@ -431,7 +300,7 @@ fn run_accounts_bench(
                     if !airdrop_lamports(
                         &client,
                         &faucet_addr,
-                        &payer_keypairs[i],
+                        payer_keypairs[i],
                         lamports * 100_000,
                     ) {
                         warn!("failed airdrop, exiting");
@@ -441,6 +310,7 @@ fn run_accounts_bench(
             }
         }
 
+        // Create accounts
         let sigs_len = executor.num_outstanding();
         if sigs_len < batch_size {
             let num_to_create = batch_size - sigs_len;
@@ -462,7 +332,7 @@ fn run_accounts_bench(
                                     mint,
                                 );
                                 let signers: Vec<&Keypair> = vec![keypair, &base_keypair];
-                                Transaction::new(&signers, message, recent_blockhash.0)
+                                Transaction::new(&signers, message, blockhash)
                             })
                             .collect();
                         balances[i] = balances[i].saturating_sub(lamports * txs.len() as u64);
@@ -475,22 +345,27 @@ fn run_accounts_bench(
                 }
             }
 
-            if close_nth > 0 {
-                let expected_closed = total_accounts_created as u64 / close_nth;
-                if expected_closed > total_accounts_closed {
-                    let txs: Vec<_> = (0..expected_closed - total_accounts_closed)
+            if close_nth_batch > 0 {
+                let num_batches_to_close =
+                    total_accounts_created as u64 / (close_nth_batch * batch_size as u64);
+                let expected_closed = num_batches_to_close * batch_size as u64;
+                let max_closed_seed = seed_tracker.max_closed.load(Ordering::Relaxed);
+                // Close every account we've created with seed between max_closed_seed..expected_closed
+                if max_closed_seed < expected_closed {
+                    let txs: Vec<_> = (0..expected_closed - max_closed_seed)
                         .into_par_iter()
                         .map(|_| {
                             let message = make_close_message(
-                                &payer_keypairs[0],
+                                payer_keypairs[0],
                                 &base_keypair,
+                                seed_tracker.max_created.clone(),
                                 seed_tracker.max_closed.clone(),
                                 1,
                                 min_balance,
                                 mint.is_some(),
                             );
-                            let signers: Vec<&Keypair> = vec![&payer_keypairs[0], &base_keypair];
-                            Transaction::new(&signers, message, recent_blockhash.0)
+                            let signers: Vec<&Keypair> = vec![payer_keypairs[0], &base_keypair];
+                            Transaction::new(&signers, message, blockhash)
                         })
                         .collect();
                     balances[0] = balances[0].saturating_sub(fee * txs.len() as u64);
@@ -506,7 +381,7 @@ fn run_accounts_bench(
         }
 
         count += 1;
-        if last_log.elapsed().as_millis() > 3000 {
+        if last_log.elapsed().as_millis() > 3000 || count >= iterations {
             info!(
                 "total_accounts_created: {} total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance(s): {:?}",
                 total_accounts_created, total_accounts_closed, tx_sent_count, count, balances
@@ -521,6 +396,83 @@ fn run_accounts_bench(
         }
     }
     executor.close();
+
+    if reclaim_accounts {
+        let executor = TransactionExecutor::new(entrypoint_addr);
+        loop {
+            let max_closed_seed = seed_tracker.max_closed.load(Ordering::Relaxed);
+            let max_created_seed = seed_tracker.max_created.load(Ordering::Relaxed);
+
+            if latest_blockhash.elapsed().as_millis() > 10_000 {
+                blockhash = client.get_latest_blockhash().expect("blockhash");
+                latest_blockhash = Instant::now();
+            }
+            message.recent_blockhash = blockhash;
+            let fee = client
+                .get_fee_for_message(&message)
+                .expect("get_fee_for_message");
+
+            let sigs_len = executor.num_outstanding();
+            if sigs_len < batch_size && max_closed_seed < max_created_seed {
+                let num_to_close = min(
+                    batch_size - sigs_len,
+                    (max_created_seed - max_closed_seed) as usize,
+                );
+                if num_to_close >= payer_keypairs.len() {
+                    info!("closing {} accounts", num_to_close);
+                    let chunk_size = num_to_close / payer_keypairs.len();
+                    info!("{:?} chunk_size", chunk_size);
+                    if chunk_size > 0 {
+                        for (i, keypair) in payer_keypairs.iter().enumerate() {
+                            let txs: Vec<_> = (0..chunk_size)
+                                .into_par_iter()
+                                .filter_map(|_| {
+                                    let message = make_close_message(
+                                        keypair,
+                                        &base_keypair,
+                                        seed_tracker.max_created.clone(),
+                                        seed_tracker.max_closed.clone(),
+                                        num_instructions,
+                                        min_balance,
+                                        mint.is_some(),
+                                    );
+                                    if message.instructions.is_empty() {
+                                        return None;
+                                    }
+                                    let signers: Vec<&Keypair> = vec![keypair, &base_keypair];
+                                    Some(Transaction::new(&signers, message, blockhash))
+                                })
+                                .collect();
+                            balances[i] = balances[i].saturating_sub(fee * txs.len() as u64);
+                            info!("close txs: {}", txs.len());
+                            let new_ids = executor.push_transactions(txs);
+                            info!("close ids: {}", new_ids.len());
+                            tx_sent_count += new_ids.len();
+                            total_accounts_closed += (num_instructions * new_ids.len()) as u64;
+                        }
+                    }
+                }
+            } else {
+                let _ = executor.drain_cleared();
+            }
+            count += 1;
+            if last_log.elapsed().as_millis() > 3000 || max_closed_seed >= max_created_seed {
+                info!(
+                    "total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance(s): {:?}",
+                    total_accounts_closed, tx_sent_count, count, balances
+                );
+                last_log = Instant::now();
+            }
+
+            if max_closed_seed >= max_created_seed {
+                break;
+            }
+            if executor.num_outstanding() >= batch_size {
+                sleep(Duration::from_millis(500));
+            }
+        }
+        executor.close();
+    }
 }
 
 fn main() {
@@ -572,14 +524,14 @@ fn main() {
                 .help("Number of transactions to send per batch"),
         )
         .arg(
-            Arg::with_name("close_nth")
+            Arg::with_name("close_nth_batch")
                 .long("close-frequency")
                 .takes_value(true)
                 .value_name("BYTES")
                 .help(
-                    "Send close transactions after this many accounts created. \
-                    Note: a `close-frequency` value near or below `batch-size` \
-                    may result in transaction-simulation errors, as the close \
+                    "Every `n` batches, create a batch of close transactions for
+                    the earliest remaining batch of accounts created.
+                    Note: Should be > 1 to avoid situations where the close \
                     transactions will be submitted before the corresponding \
                     create transactions have been confirmed",
                 ),
@@ -596,7 +548,7 @@ fn main() {
                 .long("iterations")
                 .takes_value(true)
                 .value_name("NUM")
-                .help("Number of iterations to make"),
+                .help("Number of iterations to make. 0 = unlimited iterations."),
         )
         .arg(
             Arg::with_name("check_gossip")
@@ -609,6 +561,12 @@ fn main() {
                 .takes_value(true)
                 .help("Mint address to initialize account"),
         )
+        .arg(
+            Arg::with_name("reclaim_accounts")
+                .long("reclaim-accounts")
+                .takes_value(false)
+                .help("Reclaim accounts after session ends; incompatible with --iterations 0"),
+        )
         .get_matches();
 
     let skip_gossip = !matches.is_present("check_gossip");
@@ -632,7 +590,7 @@ fn main() {
     let space = value_t!(matches, "space", u64).ok();
     let lamports = value_t!(matches, "lamports", u64).ok();
     let batch_size = value_t!(matches, "batch_size", usize).unwrap_or(4);
-    let close_nth = value_t!(matches, "close_nth", u64).unwrap_or(0);
+    let close_nth_batch = value_t!(matches, "close_nth_batch", u64).unwrap_or(0);
     let iterations = value_t!(matches, "iterations", usize).unwrap_or(10);
     let num_instructions = value_t!(matches, "num_instructions", usize).unwrap_or(1);
     if num_instructions == 0 || num_instructions > 500 {
@@ -665,6 +623,7 @@ fn main() {
             Some(&entrypoint_addr),  // find_node_by_gossip_addr
             None,                    // my_gossip_addr
             0,                       // my_shred_version
+            SocketAddrSpace::Unspecified,
         )
         .unwrap_or_else(|err| {
             eprintln!("Failed to discover {} node: {:?}", entrypoint_addr, err);
@@ -685,22 +644,32 @@ fn main() {
         iterations,
         space,
         batch_size,
-        close_nth,
+        close_nth_batch,
         lamports,
         num_instructions,
         mint,
+        matches.is_present("reclaim_accounts"),
     );
 }
 
 #[cfg(test)]
 pub mod test {
-    use super::*;
-    use solana_core::validator::ValidatorConfig;
-    use solana_local_cluster::{
-        local_cluster::{ClusterConfig, LocalCluster},
-        validator_configs::make_identical_validator_configs,
+    use {
+        super::*,
+        solana_core::validator::ValidatorConfig,
+        solana_faucet::faucet::run_local_faucet,
+        solana_local_cluster::{
+            local_cluster::{ClusterConfig, LocalCluster},
+            validator_configs::make_identical_validator_configs,
+        },
+        solana_measure::measure::Measure,
+        solana_sdk::{native_token::sol_to_lamports, poh_config::PohConfig},
+        solana_test_validator::TestValidator,
+        spl_token::{
+            solana_program::program_pack::Pack,
+            state::{Account, Mint},
+        },
     };
-    use solana_sdk::poh_config::PohConfig;
 
     #[test]
     fn test_accounts_cluster_bench() {
@@ -716,11 +685,11 @@ pub mod test {
         };
 
         let faucet_addr = SocketAddr::from(([127, 0, 0, 1], 9900));
-        let cluster = LocalCluster::new(&mut config);
+        let cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
         let iterations = 10;
         let maybe_space = None;
         let batch_size = 100;
-        let close_nth = 100;
+        let close_nth_batch = 100;
         let maybe_lamports = None;
         let num_instructions = 2;
         let mut start = Measure::start("total accounts run");
@@ -731,10 +700,112 @@ pub mod test {
             iterations,
             maybe_space,
             batch_size,
-            close_nth,
+            close_nth_batch,
             maybe_lamports,
             num_instructions,
             None,
+            false,
+        );
+        start.stop();
+        info!("{}", start);
+    }
+
+    #[test]
+    fn test_create_then_reclaim_spl_token_accounts() {
+        solana_logger::setup();
+        let mint_keypair = Keypair::new();
+        let mint_pubkey = mint_keypair.pubkey();
+        let faucet_addr = run_local_faucet(mint_keypair, None);
+        let test_validator = TestValidator::with_custom_fees(
+            mint_pubkey,
+            1,
+            Some(faucet_addr),
+            SocketAddrSpace::Unspecified,
+        );
+        let rpc_client =
+            RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
+
+        // Created funder
+        let funder = Keypair::new();
+        let latest_blockhash = rpc_client.get_latest_blockhash().unwrap();
+        let signature = rpc_client
+            .request_airdrop_with_blockhash(
+                &funder.pubkey(),
+                sol_to_lamports(1.0),
+                &latest_blockhash,
+            )
+            .unwrap();
+        rpc_client
+            .confirm_transaction_with_spinner(
+                &signature,
+                &latest_blockhash,
+                CommitmentConfig::confirmed(),
+            )
+            .unwrap();
+
+        // Create Mint
+        let spl_mint_keypair = Keypair::new();
+        let spl_mint_len = Mint::get_packed_len();
+        let spl_mint_rent = rpc_client
+            .get_minimum_balance_for_rent_exemption(spl_mint_len)
+            .unwrap();
+        let transaction = Transaction::new_signed_with_payer(
+            &[
+                system_instruction::create_account(
+                    &funder.pubkey(),
+                    &spl_mint_keypair.pubkey(),
+                    spl_mint_rent,
+                    spl_mint_len as u64,
+                    &inline_spl_token::id(),
+                ),
+                spl_token_instruction(
+                    spl_token::instruction::initialize_mint(
+                        &spl_token::id(),
+                        &spl_token_pubkey(&spl_mint_keypair.pubkey()),
+                        &spl_token_pubkey(&spl_mint_keypair.pubkey()),
+                        None,
+                        2,
+                    )
+                    .unwrap(),
+                ),
+            ],
+            Some(&funder.pubkey()),
+            &[&funder, &spl_mint_keypair],
+            latest_blockhash,
+        );
+        let _sig = rpc_client
+            .send_and_confirm_transaction(&transaction)
+            .unwrap();
+
+        let account_len = Account::get_packed_len();
+        let minimum_balance = rpc_client
+            .get_minimum_balance_for_rent_exemption(account_len)
+            .unwrap();
+
+        let iterations = 5;
+        let batch_size = 100;
+        let close_nth_batch = 0;
+        let num_instructions = 4;
+        let mut start = Measure::start("total accounts run");
+        let keypair0 = Keypair::new();
+        let keypair1 = Keypair::new();
+        let keypair2 = Keypair::new();
+        run_accounts_bench(
+            test_validator
+                .rpc_url()
+                .replace("http://", "")
+                .parse()
+                .unwrap(),
+            faucet_addr,
+            &[&keypair0, &keypair1, &keypair2],
+            iterations,
+            Some(account_len as u64),
+            batch_size,
+            close_nth_batch,
+            Some(minimum_balance),
+            num_instructions,
+            Some(spl_mint_keypair.pubkey()),
+            true,
         );
         start.stop();
         info!("{}", start);

accountsdb-plugin-interface/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2021"
+name = "solana-accountsdb-plugin-interface"
+description = "The Solana AccountsDb plugin interface."
+version = "1.10.0"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-accountsdb-plugin-interface"
+
+[dependencies]
+log = "0.4.11"
+thiserror = "1.0.30"
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-interface/README.md

@@ -0,0 +1,20 @@
+<p align="center">
+  <a href="https://solana.com">
+    <img alt="Solana" src="https://i.imgur.com/IKyzQ6T.png" width="250" />
+  </a>
+</p>
+
+# Solana AccountsDb Plugin Interface
+
+This crate enables an AccountsDb plugin to be plugged into the Solana Validator runtime to take actions
+at the time of each account update; for example, saving the account state to an external database. The plugin must implement the `AccountsDbPlugin` trait. Please see the detail of the `accountsdb_plugin_interface.rs` for the interface definition.
+
+The plugin should produce a `cdylib` dynamic library, which must expose a `C` function `_create_plugin()` that
+instantiates the implementation of the interface.
+
+The `solana-accountsdb-plugin-postgres` crate provides an example of how to create a plugin which saves the accounts data into an
+external PostgreSQL databases.
+
+More information about Solana is available in the [Solana documentation](https://docs.solana.com/).
+
+Still have questions?  Ask us on [Discord](https://discordapp.com/invite/pquxPsq)

accountsdb-plugin-interface/src/accountsdb_plugin_interface.rs

@@ -0,0 +1,220 @@
+/// The interface for AccountsDb plugins. A plugin must implement
+/// the AccountsDbPlugin trait to work with the runtime.
+/// In addition, the dynamic library must export a "C" function _create_plugin which
+/// creates the implementation of the plugin.
+use {
+    solana_sdk::{clock::UnixTimestamp, signature::Signature, transaction::SanitizedTransaction},
+    solana_transaction_status::{Reward, TransactionStatusMeta},
+    std::{any::Any, error, io},
+    thiserror::Error,
+};
+
+impl Eq for ReplicaAccountInfo<'_> {}
+
+#[derive(Clone, PartialEq, Debug)]
+/// Information about an account being updated
+pub struct ReplicaAccountInfo<'a> {
+    /// The Pubkey for the account
+    pub pubkey: &'a [u8],
+
+    /// The lamports for the account
+    pub lamports: u64,
+
+    /// The Pubkey of the owner program account
+    pub owner: &'a [u8],
+
+    /// This account's data contains a loaded program (and is now read-only)
+    pub executable: bool,
+
+    /// The epoch at which this account will next owe rent
+    pub rent_epoch: u64,
+
+    /// The data held in this account.
+    pub data: &'a [u8],
+
+    /// A global monotonically increasing atomic number, which can be used
+    /// to tell the order of the account update. For example, when an
+    /// account is updated in the same slot multiple times, the update
+    /// with higher write_version should supersede the one with lower
+    /// write_version.
+    pub write_version: u64,
+}
+
+/// A wrapper to future-proof ReplicaAccountInfo handling.
+/// If there were a change to the structure of ReplicaAccountInfo,
+/// there would be new enum entry for the newer version, forcing
+/// plugin implementations to handle the change.
+pub enum ReplicaAccountInfoVersions<'a> {
+    V0_0_1(&'a ReplicaAccountInfo<'a>),
+}
+
+/// Information about a transaction
+#[derive(Clone, Debug)]
+pub struct ReplicaTransactionInfo<'a> {
+    /// The first signature of the transaction, used for identifying the transaction.
+    pub signature: &'a Signature,
+
+    /// Indicates if the transaction is a simple vote transaction.
+    pub is_vote: bool,
+
+    /// The sanitized transaction.
+    pub transaction: &'a SanitizedTransaction,
+
+    /// Metadata of the transaction status.
+    pub transaction_status_meta: &'a TransactionStatusMeta,
+}
+
+/// A wrapper to future-proof ReplicaTransactionInfo handling.
+/// If there were a change to the structure of ReplicaTransactionInfo,
+/// there would be new enum entry for the newer version, forcing
+/// plugin implementations to handle the change.
+pub enum ReplicaTransactionInfoVersions<'a> {
+    V0_0_1(&'a ReplicaTransactionInfo<'a>),
+}
+
+#[derive(Clone, Debug)]
+pub struct ReplicaBlockInfo<'a> {
+    pub slot: u64,
+    pub blockhash: &'a str,
+    pub rewards: &'a [Reward],
+    pub block_time: Option<UnixTimestamp>,
+    pub block_height: Option<u64>,
+}
+
+pub enum ReplicaBlockInfoVersions<'a> {
+    V0_0_1(&'a ReplicaBlockInfo<'a>),
+}
+
+/// Errors returned by plugin calls
+#[derive(Error, Debug)]
+pub enum AccountsDbPluginError {
+    /// Error opening the configuration file; for example, when the file
+    /// is not found or when the validator process has no permission to read it.
+    #[error("Error opening config file. Error detail: ({0}).")]
+    ConfigFileOpenError(#[from] io::Error),
+
+    /// Error in reading the content of the config file or the content
+    /// is not in the expected format.
+    #[error("Error reading config file. Error message: ({msg})")]
+    ConfigFileReadError { msg: String },
+
+    /// Error when updating the account.
+    #[error("Error updating account. Error message: ({msg})")]
+    AccountsUpdateError { msg: String },
+
+    /// Error when updating the slot status
+    #[error("Error updating slot status. Error message: ({msg})")]
+    SlotStatusUpdateError { msg: String },
+
+    /// Any custom error defined by the plugin.
+    #[error("Plugin-defined custom error. Error message: ({0})")]
+    Custom(Box<dyn error::Error + Send + Sync>),
+}
+
+/// The current status of a slot
+#[derive(Debug, Clone)]
+pub enum SlotStatus {
+    /// The highest slot of the heaviest fork processed by the node. Ledger state at this slot is
+    /// not derived from a confirmed or finalized block, but if multiple forks are present, is from
+    /// the fork the validator believes is most likely to finalize.
+    Processed,
+
+    /// The highest slot having reached max vote lockout.
+    Rooted,
+
+    /// The highest slot that has been voted on by supermajority of the cluster, ie. is confirmed.
+    Confirmed,
+}
+
+impl SlotStatus {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            SlotStatus::Confirmed => "confirmed",
+            SlotStatus::Processed => "processed",
+            SlotStatus::Rooted => "rooted",
+        }
+    }
+}
+
+pub type Result<T> = std::result::Result<T, AccountsDbPluginError>;
+
+/// Defines an AccountsDb plugin, to stream data from the runtime.
+/// AccountsDb plugins must describe desired behavior for load and unload,
+/// as well as how they will handle streamed data.
+pub trait AccountsDbPlugin: Any + Send + Sync + std::fmt::Debug {
+    fn name(&self) -> &'static str;
+
+    /// The callback called when a plugin is loaded by the system,
+    /// used for doing whatever initialization is required by the plugin.
+    /// The _config_file contains the name of the
+    /// of the config file. The config must be in JSON format and
+    /// include a field "libpath" indicating the full path
+    /// name of the shared library implementing this interface.
+    fn on_load(&mut self, _config_file: &str) -> Result<()> {
+        Ok(())
+    }
+
+    /// The callback called right before a plugin is unloaded by the system
+    /// Used for doing cleanup before unload.
+    fn on_unload(&mut self) {}
+
+    /// Called when an account is updated at a slot.
+    /// When `is_startup` is true, it indicates the account is loaded from
+    /// snapshots when the validator starts up. When `is_startup` is false,
+    /// the account is updated during transaction processing.
+    #[allow(unused_variables)]
+    fn update_account(
+        &mut self,
+        account: ReplicaAccountInfoVersions,
+        slot: u64,
+        is_startup: bool,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    /// Called when all accounts are notified of during startup.
+    fn notify_end_of_startup(&mut self) -> Result<()> {
+        Ok(())
+    }
+
+    /// Called when a slot status is updated
+    #[allow(unused_variables)]
+    fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    /// Called when a transaction is updated at a slot.
+    #[allow(unused_variables)]
+    fn notify_transaction(
+        &mut self,
+        transaction: ReplicaTransactionInfoVersions,
+        slot: u64,
+    ) -> Result<()> {
+        Ok(())
+    }
+
+    /// Called when block's metadata is updated.
+    #[allow(unused_variables)]
+    fn notify_block_metadata(&mut self, blockinfo: ReplicaBlockInfoVersions) -> Result<()> {
+        Ok(())
+    }
+
+    /// Check if the plugin is interested in account data
+    /// Default is true -- if the plugin is not interested in
+    /// account data, please return false.
+    fn account_data_notifications_enabled(&self) -> bool {
+        true
+    }
+
+    /// Check if the plugin is interested in transaction data
+    /// Default is false -- if the plugin is not interested in
+    /// transaction data, please return false.
+    fn transaction_notifications_enabled(&self) -> bool {
+        false
+    }
+}

accountsdb-plugin-interface/src/lib.rs

@@ -0,0 +1 @@
+pub mod accountsdb_plugin_interface;

accountsdb-plugin-manager/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2021"
+name = "solana-accountsdb-plugin-manager"
+description = "The Solana AccountsDb plugin manager."
+version = "1.10.0"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-validator"
+
+[dependencies]
+bs58 = "0.4.0"
+crossbeam-channel = "0.5"
+libloading = "0.7.2"
+log = "0.4.11"
+serde = "1.0.133"
+serde_derive = "1.0.103"
+serde_json = "1.0.74"
+solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-metrics = { path = "../metrics", version = "=1.10.0" }
+solana-rpc = { path = "../rpc", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
+thiserror = "1.0.30"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-manager/src/accounts_update_notifier.rs

@@ -0,0 +1,180 @@
+/// Module responsible for notifying plugins of account updates
+use {
+    crate::accountsdb_plugin_manager::AccountsDbPluginManager,
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        ReplicaAccountInfo, ReplicaAccountInfoVersions,
+    },
+    solana_measure::measure::Measure,
+    solana_metrics::*,
+    solana_runtime::{
+        accounts_update_notifier_interface::AccountsUpdateNotifierInterface,
+        append_vec::{StoredAccountMeta, StoredMeta},
+    },
+    solana_sdk::{
+        account::{AccountSharedData, ReadableAccount},
+        clock::Slot,
+    },
+    std::sync::{Arc, RwLock},
+};
+#[derive(Debug)]
+pub(crate) struct AccountsUpdateNotifierImpl {
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+}
+
+impl AccountsUpdateNotifierInterface for AccountsUpdateNotifierImpl {
+    fn notify_account_update(&self, slot: Slot, meta: &StoredMeta, account: &AccountSharedData) {
+        if let Some(account_info) = self.accountinfo_from_shared_account_data(meta, account) {
+            self.notify_plugins_of_account_update(account_info, slot, false);
+        }
+    }
+
+    fn notify_account_restore_from_snapshot(&self, slot: Slot, account: &StoredAccountMeta) {
+        let mut measure_all = Measure::start("accountsdb-plugin-notify-account-restore-all");
+        let mut measure_copy = Measure::start("accountsdb-plugin-copy-stored-account-info");
+
+        let account = self.accountinfo_from_stored_account_meta(account);
+        measure_copy.stop();
+
+        inc_new_counter_debug!(
+            "accountsdb-plugin-copy-stored-account-info-us",
+            measure_copy.as_us() as usize,
+            100000,
+            100000
+        );
+
+        if let Some(account_info) = account {
+            self.notify_plugins_of_account_update(account_info, slot, true);
+        }
+        measure_all.stop();
+
+        inc_new_counter_debug!(
+            "accountsdb-plugin-notify-account-restore-all-us",
+            measure_all.as_us() as usize,
+            100000,
+            100000
+        );
+    }
+
+    fn notify_end_of_restore_from_snapshot(&self) {
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-end-of-restore-from-snapshot");
+            match plugin.notify_end_of_startup() {
+                Err(err) => {
+                    error!(
+                        "Failed to notify the end of restore from snapshot, error: {} to plugin {}",
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully notified the end of restore from snapshot to plugin {}",
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-end-of-restore-from-snapshot",
+                measure.as_us() as usize
+            );
+        }
+    }
+}
+
+impl AccountsUpdateNotifierImpl {
+    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
+        AccountsUpdateNotifierImpl { plugin_manager }
+    }
+
+    fn accountinfo_from_shared_account_data<'a>(
+        &self,
+        meta: &'a StoredMeta,
+        account: &'a AccountSharedData,
+    ) -> Option<ReplicaAccountInfo<'a>> {
+        Some(ReplicaAccountInfo {
+            pubkey: meta.pubkey.as_ref(),
+            lamports: account.lamports(),
+            owner: account.owner().as_ref(),
+            executable: account.executable(),
+            rent_epoch: account.rent_epoch(),
+            data: account.data(),
+            write_version: meta.write_version,
+        })
+    }
+
+    fn accountinfo_from_stored_account_meta<'a>(
+        &self,
+        stored_account_meta: &'a StoredAccountMeta,
+    ) -> Option<ReplicaAccountInfo<'a>> {
+        Some(ReplicaAccountInfo {
+            pubkey: stored_account_meta.meta.pubkey.as_ref(),
+            lamports: stored_account_meta.account_meta.lamports,
+            owner: stored_account_meta.account_meta.owner.as_ref(),
+            executable: stored_account_meta.account_meta.executable,
+            rent_epoch: stored_account_meta.account_meta.rent_epoch,
+            data: stored_account_meta.data,
+            write_version: stored_account_meta.meta.write_version,
+        })
+    }
+
+    fn notify_plugins_of_account_update(
+        &self,
+        account: ReplicaAccountInfo,
+        slot: Slot,
+        is_startup: bool,
+    ) {
+        let mut measure2 = Measure::start("accountsdb-plugin-notify_plugins_of_account_update");
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-update-account");
+            match plugin.update_account(
+                ReplicaAccountInfoVersions::V0_0_1(&account),
+                slot,
+                is_startup,
+            ) {
+                Err(err) => {
+                    error!(
+                        "Failed to update account {} at slot {}, error: {} to plugin {}",
+                        bs58::encode(account.pubkey).into_string(),
+                        slot,
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully updated account {} at slot {} to plugin {}",
+                        bs58::encode(account.pubkey).into_string(),
+                        slot,
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-update-account-us",
+                measure.as_us() as usize,
+                100000,
+                100000
+            );
+        }
+        measure2.stop();
+        inc_new_counter_debug!(
+            "accountsdb-plugin-notify_plugins_of_account_update-us",
+            measure2.as_us() as usize,
+            100000,
+            100000
+        );
+    }
+}

accountsdb-plugin-manager/src/accountsdb_plugin_manager.rs

@@ -0,0 +1,75 @@
+/// Managing the AccountsDb plugins
+use {
+    libloading::{Library, Symbol},
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::AccountsDbPlugin,
+    std::error::Error,
+};
+
+#[derive(Default, Debug)]
+pub struct AccountsDbPluginManager {
+    pub plugins: Vec<Box<dyn AccountsDbPlugin>>,
+    libs: Vec<Library>,
+}
+
+impl AccountsDbPluginManager {
+    pub fn new() -> Self {
+        AccountsDbPluginManager {
+            plugins: Vec::default(),
+            libs: Vec::default(),
+        }
+    }
+
+    /// # Safety
+    ///
+    /// This function loads the dynamically linked library specified in the path. The library
+    /// must do necessary initializations.
+    pub unsafe fn load_plugin(
+        &mut self,
+        libpath: &str,
+        config_file: &str,
+    ) -> Result<(), Box<dyn Error>> {
+        type PluginConstructor = unsafe fn() -> *mut dyn AccountsDbPlugin;
+        let lib = Library::new(libpath)?;
+        let constructor: Symbol<PluginConstructor> = lib.get(b"_create_plugin")?;
+        let plugin_raw = constructor();
+        let mut plugin = Box::from_raw(plugin_raw);
+        plugin.on_load(config_file)?;
+        self.plugins.push(plugin);
+        self.libs.push(lib);
+        Ok(())
+    }
+
+    /// Unload all plugins and loaded plugin libraries, making sure to fire
+    /// their `on_plugin_unload()` methods so they can do any necessary cleanup.
+    pub fn unload(&mut self) {
+        for mut plugin in self.plugins.drain(..) {
+            info!("Unloading plugin for {:?}", plugin.name());
+            plugin.on_unload();
+        }
+
+        for lib in self.libs.drain(..) {
+            drop(lib);
+        }
+    }
+
+    /// Check if there is any plugin interested in account data
+    pub fn account_data_notifications_enabled(&self) -> bool {
+        for plugin in &self.plugins {
+            if plugin.account_data_notifications_enabled() {
+                return true;
+            }
+        }
+        false
+    }
+
+    /// Check if there is any plugin interested in transaction data
+    pub fn transaction_notifications_enabled(&self) -> bool {
+        for plugin in &self.plugins {
+            if plugin.transaction_notifications_enabled() {
+                return true;
+            }
+        }
+        false
+    }
+}

accountsdb-plugin-manager/src/accountsdb_plugin_service.rs

@@ -0,0 +1,211 @@
+use {
+    crate::{
+        accounts_update_notifier::AccountsUpdateNotifierImpl,
+        accountsdb_plugin_manager::AccountsDbPluginManager,
+        block_metadata_notifier::BlockMetadataNotifierImpl,
+        block_metadata_notifier_interface::BlockMetadataNotifierLock,
+        slot_status_notifier::SlotStatusNotifierImpl, slot_status_observer::SlotStatusObserver,
+        transaction_notifier::TransactionNotifierImpl,
+    },
+    crossbeam_channel::Receiver,
+    log::*,
+    serde_json,
+    solana_rpc::{
+        optimistically_confirmed_bank_tracker::BankNotification,
+        transaction_notifier_interface::TransactionNotifierLock,
+    },
+    solana_runtime::accounts_update_notifier_interface::AccountsUpdateNotifier,
+    std::{
+        fs::File,
+        io::Read,
+        path::{Path, PathBuf},
+        sync::{Arc, RwLock},
+        thread,
+    },
+    thiserror::Error,
+};
+
+#[derive(Error, Debug)]
+pub enum AccountsdbPluginServiceError {
+    #[error("Cannot open the the plugin config file")]
+    CannotOpenConfigFile(String),
+
+    #[error("Cannot read the the plugin config file")]
+    CannotReadConfigFile(String),
+
+    #[error("The config file is not in a valid Json format")]
+    InvalidConfigFileFormat(String),
+
+    #[error("Plugin library path is not specified in the config file")]
+    LibPathNotSet,
+
+    #[error("Invalid plugin path")]
+    InvalidPluginPath,
+
+    #[error("Cannot load plugin shared library")]
+    PluginLoadError(String),
+}
+
+/// The service managing the AccountsDb plugin workflow.
+pub struct AccountsDbPluginService {
+    slot_status_observer: Option<SlotStatusObserver>,
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+    accounts_update_notifier: Option<AccountsUpdateNotifier>,
+    transaction_notifier: Option<TransactionNotifierLock>,
+    block_metadata_notifier: Option<BlockMetadataNotifierLock>,
+}
+
+impl AccountsDbPluginService {
+    /// Creates and returns the AccountsDbPluginService.
+    /// # Arguments
+    /// * `confirmed_bank_receiver` - The receiver for confirmed bank notification
+    /// * `accountsdb_plugin_config_file` - The config file path for the plugin. The
+    ///    config file controls the plugin responsible
+    ///    for transporting the data to external data stores. It is defined in JSON format.
+    ///    The `libpath` field should be pointed to the full path of the dynamic shared library
+    ///    (.so file) to be loaded. The shared library must implement the `AccountsDbPlugin`
+    ///    trait. And the shared library shall export a `C` function `_create_plugin` which
+    ///    shall create the implementation of `AccountsDbPlugin` and returns to the caller.
+    ///    The rest of the JSON fields' definition is up to to the concrete plugin implementation
+    ///    It is usually used to configure the connection information for the external data store.
+
+    pub fn new(
+        confirmed_bank_receiver: Receiver<BankNotification>,
+        accountsdb_plugin_config_files: &[PathBuf],
+    ) -> Result<Self, AccountsdbPluginServiceError> {
+        info!(
+            "Starting AccountsDbPluginService from config files: {:?}",
+            accountsdb_plugin_config_files
+        );
+        let mut plugin_manager = AccountsDbPluginManager::new();
+
+        for accountsdb_plugin_config_file in accountsdb_plugin_config_files {
+            Self::load_plugin(&mut plugin_manager, accountsdb_plugin_config_file)?;
+        }
+        let account_data_notifications_enabled =
+            plugin_manager.account_data_notifications_enabled();
+        let transaction_notifications_enabled = plugin_manager.transaction_notifications_enabled();
+
+        let plugin_manager = Arc::new(RwLock::new(plugin_manager));
+
+        let accounts_update_notifier: Option<AccountsUpdateNotifier> =
+            if account_data_notifications_enabled {
+                let accounts_update_notifier =
+                    AccountsUpdateNotifierImpl::new(plugin_manager.clone());
+                Some(Arc::new(RwLock::new(accounts_update_notifier)))
+            } else {
+                None
+            };
+
+        let transaction_notifier: Option<TransactionNotifierLock> =
+            if transaction_notifications_enabled {
+                let transaction_notifier = TransactionNotifierImpl::new(plugin_manager.clone());
+                Some(Arc::new(RwLock::new(transaction_notifier)))
+            } else {
+                None
+            };
+
+        let (slot_status_observer, block_metadata_notifier): (
+            Option<SlotStatusObserver>,
+            Option<BlockMetadataNotifierLock>,
+        ) = if account_data_notifications_enabled || transaction_notifications_enabled {
+            let slot_status_notifier = SlotStatusNotifierImpl::new(plugin_manager.clone());
+            let slot_status_notifier = Arc::new(RwLock::new(slot_status_notifier));
+            (
+                Some(SlotStatusObserver::new(
+                    confirmed_bank_receiver,
+                    slot_status_notifier,
+                )),
+                Some(Arc::new(RwLock::new(BlockMetadataNotifierImpl::new(
+                    plugin_manager.clone(),
+                )))),
+            )
+        } else {
+            (None, None)
+        };
+
+        info!("Started AccountsDbPluginService");
+        Ok(AccountsDbPluginService {
+            slot_status_observer,
+            plugin_manager,
+            accounts_update_notifier,
+            transaction_notifier,
+            block_metadata_notifier,
+        })
+    }
+
+    fn load_plugin(
+        plugin_manager: &mut AccountsDbPluginManager,
+        accountsdb_plugin_config_file: &Path,
+    ) -> Result<(), AccountsdbPluginServiceError> {
+        let mut file = match File::open(accountsdb_plugin_config_file) {
+            Ok(file) => file,
+            Err(err) => {
+                return Err(AccountsdbPluginServiceError::CannotOpenConfigFile(format!(
+                    "Failed to open the plugin config file {:?}, error: {:?}",
+                    accountsdb_plugin_config_file, err
+                )));
+            }
+        };
+
+        let mut contents = String::new();
+        if let Err(err) = file.read_to_string(&mut contents) {
+            return Err(AccountsdbPluginServiceError::CannotReadConfigFile(format!(
+                "Failed to read the plugin config file {:?}, error: {:?}",
+                accountsdb_plugin_config_file, err
+            )));
+        }
+
+        let result: serde_json::Value = match serde_json::from_str(&contents) {
+            Ok(value) => value,
+            Err(err) => {
+                return Err(AccountsdbPluginServiceError::InvalidConfigFileFormat(
+                    format!(
+                        "The config file {:?} is not in a valid Json format, error: {:?}",
+                        accountsdb_plugin_config_file, err
+                    ),
+                ));
+            }
+        };
+
+        let libpath = result["libpath"]
+            .as_str()
+            .ok_or(AccountsdbPluginServiceError::LibPathNotSet)?;
+        let config_file = accountsdb_plugin_config_file
+            .as_os_str()
+            .to_str()
+            .ok_or(AccountsdbPluginServiceError::InvalidPluginPath)?;
+
+        unsafe {
+            let result = plugin_manager.load_plugin(libpath, config_file);
+            if let Err(err) = result {
+                let msg = format!(
+                    "Failed to load the plugin library: {:?}, error: {:?}",
+                    libpath, err
+                );
+                return Err(AccountsdbPluginServiceError::PluginLoadError(msg));
+            }
+        }
+        Ok(())
+    }
+
+    pub fn get_accounts_update_notifier(&self) -> Option<AccountsUpdateNotifier> {
+        self.accounts_update_notifier.clone()
+    }
+
+    pub fn get_transaction_notifier(&self) -> Option<TransactionNotifierLock> {
+        self.transaction_notifier.clone()
+    }
+
+    pub fn get_block_metadata_notifier(&self) -> Option<BlockMetadataNotifierLock> {
+        self.block_metadata_notifier.clone()
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        if let Some(mut slot_status_observer) = self.slot_status_observer {
+            slot_status_observer.join()?;
+        }
+        self.plugin_manager.write().unwrap().unload();
+        Ok(())
+    }
+}

accountsdb-plugin-manager/src/block_metadata_notifier.rs

@@ -0,0 +1,105 @@
+use {
+    crate::{
+        accountsdb_plugin_manager::AccountsDbPluginManager,
+        block_metadata_notifier_interface::BlockMetadataNotifier,
+    },
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        ReplicaBlockInfo, ReplicaBlockInfoVersions,
+    },
+    solana_measure::measure::Measure,
+    solana_metrics::*,
+    solana_runtime::bank::RewardInfo,
+    solana_sdk::{clock::UnixTimestamp, pubkey::Pubkey},
+    solana_transaction_status::{Reward, Rewards},
+    std::sync::{Arc, RwLock},
+};
+
+pub(crate) struct BlockMetadataNotifierImpl {
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+}
+
+impl BlockMetadataNotifier for BlockMetadataNotifierImpl {
+    /// Notify the block metadata
+    fn notify_block_metadata(
+        &self,
+        slot: u64,
+        blockhash: &str,
+        rewards: &RwLock<Vec<(Pubkey, RewardInfo)>>,
+        block_time: Option<UnixTimestamp>,
+        block_height: Option<u64>,
+    ) {
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+        let rewards = Self::build_rewards(rewards);
+
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-update-slot");
+            let block_info =
+                Self::build_replica_block_info(slot, blockhash, &rewards, block_time, block_height);
+            let block_info = ReplicaBlockInfoVersions::V0_0_1(&block_info);
+            match plugin.notify_block_metadata(block_info) {
+                Err(err) => {
+                    error!(
+                        "Failed to update block metadata at slot {}, error: {} to plugin {}",
+                        slot,
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully updated block metadata at slot {} to plugin {}",
+                        slot,
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-update-block-metadata-us",
+                measure.as_us() as usize,
+                1000,
+                1000
+            );
+        }
+    }
+}
+
+impl BlockMetadataNotifierImpl {
+    fn build_rewards(rewards: &RwLock<Vec<(Pubkey, RewardInfo)>>) -> Rewards {
+        let rewards = rewards.read().unwrap();
+        rewards
+            .iter()
+            .map(|(pubkey, reward)| Reward {
+                pubkey: pubkey.to_string(),
+                lamports: reward.lamports,
+                post_balance: reward.post_balance,
+                reward_type: Some(reward.reward_type),
+                commission: reward.commission,
+            })
+            .collect()
+    }
+
+    fn build_replica_block_info<'a>(
+        slot: u64,
+        blockhash: &'a str,
+        rewards: &'a [Reward],
+        block_time: Option<UnixTimestamp>,
+        block_height: Option<u64>,
+    ) -> ReplicaBlockInfo<'a> {
+        ReplicaBlockInfo {
+            slot,
+            blockhash,
+            rewards,
+            block_time,
+            block_height,
+        }
+    }
+
+    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
+        Self { plugin_manager }
+    }
+}

accountsdb-plugin-manager/src/block_metadata_notifier_interface.rs

@@ -0,0 +1,20 @@
+use {
+    solana_runtime::bank::RewardInfo,
+    solana_sdk::{clock::UnixTimestamp, pubkey::Pubkey},
+    std::sync::{Arc, RwLock},
+};
+
+/// Interface for notifying block metadata changes
+pub trait BlockMetadataNotifier {
+    /// Notify the block metadata
+    fn notify_block_metadata(
+        &self,
+        slot: u64,
+        blockhash: &str,
+        rewards: &RwLock<Vec<(Pubkey, RewardInfo)>>,
+        block_time: Option<UnixTimestamp>,
+        block_height: Option<u64>,
+    );
+}
+
+pub type BlockMetadataNotifierLock = Arc<RwLock<dyn BlockMetadataNotifier + Sync + Send>>;

accountsdb-plugin-manager/src/lib.rs

@@ -0,0 +1,8 @@
+pub mod accounts_update_notifier;
+pub mod accountsdb_plugin_manager;
+pub mod accountsdb_plugin_service;
+pub mod block_metadata_notifier;
+pub mod block_metadata_notifier_interface;
+pub mod slot_status_notifier;
+pub mod slot_status_observer;
+pub mod transaction_notifier;

accountsdb-plugin-manager/src/slot_status_notifier.rs

@@ -0,0 +1,81 @@
+use {
+    crate::accountsdb_plugin_manager::AccountsDbPluginManager,
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::SlotStatus,
+    solana_measure::measure::Measure,
+    solana_metrics::*,
+    solana_sdk::clock::Slot,
+    std::sync::{Arc, RwLock},
+};
+
+pub trait SlotStatusNotifierInterface {
+    /// Notified when a slot is optimistically confirmed
+    fn notify_slot_confirmed(&self, slot: Slot, parent: Option<Slot>);
+
+    /// Notified when a slot is marked frozen.
+    fn notify_slot_processed(&self, slot: Slot, parent: Option<Slot>);
+
+    /// Notified when a slot is rooted.
+    fn notify_slot_rooted(&self, slot: Slot, parent: Option<Slot>);
+}
+
+pub type SlotStatusNotifier = Arc<RwLock<dyn SlotStatusNotifierInterface + Sync + Send>>;
+
+pub struct SlotStatusNotifierImpl {
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+}
+
+impl SlotStatusNotifierInterface for SlotStatusNotifierImpl {
+    fn notify_slot_confirmed(&self, slot: Slot, parent: Option<Slot>) {
+        self.notify_slot_status(slot, parent, SlotStatus::Confirmed);
+    }
+
+    fn notify_slot_processed(&self, slot: Slot, parent: Option<Slot>) {
+        self.notify_slot_status(slot, parent, SlotStatus::Processed);
+    }
+
+    fn notify_slot_rooted(&self, slot: Slot, parent: Option<Slot>) {
+        self.notify_slot_status(slot, parent, SlotStatus::Rooted);
+    }
+}
+
+impl SlotStatusNotifierImpl {
+    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
+        Self { plugin_manager }
+    }
+
+    pub fn notify_slot_status(&self, slot: Slot, parent: Option<Slot>, slot_status: SlotStatus) {
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+
+        for plugin in plugin_manager.plugins.iter_mut() {
+            let mut measure = Measure::start("accountsdb-plugin-update-slot");
+            match plugin.update_slot_status(slot, parent, slot_status.clone()) {
+                Err(err) => {
+                    error!(
+                        "Failed to update slot status at slot {}, error: {} to plugin {}",
+                        slot,
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully updated slot status at slot {} to plugin {}",
+                        slot,
+                        plugin.name()
+                    );
+                }
+            }
+            measure.stop();
+            inc_new_counter_debug!(
+                "accountsdb-plugin-update-slot-us",
+                measure.as_us() as usize,
+                1000,
+                1000
+            );
+        }
+    }
+}

accountsdb-plugin-manager/src/slot_status_observer.rs

@@ -0,0 +1,80 @@
+use {
+    crate::slot_status_notifier::SlotStatusNotifier,
+    crossbeam_channel::Receiver,
+    solana_rpc::optimistically_confirmed_bank_tracker::BankNotification,
+    std::{
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc,
+        },
+        thread::{self, Builder, JoinHandle},
+    },
+};
+
+#[derive(Debug)]
+pub(crate) struct SlotStatusObserver {
+    bank_notification_receiver_service: Option<JoinHandle<()>>,
+    exit_updated_slot_server: Arc<AtomicBool>,
+}
+
+impl SlotStatusObserver {
+    pub fn new(
+        bank_notification_receiver: Receiver<BankNotification>,
+        slot_status_notifier: SlotStatusNotifier,
+    ) -> Self {
+        let exit_updated_slot_server = Arc::new(AtomicBool::new(false));
+
+        Self {
+            bank_notification_receiver_service: Some(Self::run_bank_notification_receiver(
+                bank_notification_receiver,
+                exit_updated_slot_server.clone(),
+                slot_status_notifier,
+            )),
+            exit_updated_slot_server,
+        }
+    }
+
+    pub fn join(&mut self) -> thread::Result<()> {
+        self.exit_updated_slot_server.store(true, Ordering::Relaxed);
+        self.bank_notification_receiver_service
+            .take()
+            .map(JoinHandle::join)
+            .unwrap()
+    }
+
+    fn run_bank_notification_receiver(
+        bank_notification_receiver: Receiver<BankNotification>,
+        exit: Arc<AtomicBool>,
+        slot_status_notifier: SlotStatusNotifier,
+    ) -> JoinHandle<()> {
+        Builder::new()
+            .name("bank_notification_receiver".to_string())
+            .spawn(move || {
+                while !exit.load(Ordering::Relaxed) {
+                    if let Ok(slot) = bank_notification_receiver.recv() {
+                        match slot {
+                            BankNotification::OptimisticallyConfirmed(slot) => {
+                                slot_status_notifier
+                                    .read()
+                                    .unwrap()
+                                    .notify_slot_confirmed(slot, None);
+                            }
+                            BankNotification::Frozen(bank) => {
+                                slot_status_notifier
+                                    .read()
+                                    .unwrap()
+                                    .notify_slot_processed(bank.slot(), Some(bank.parent_slot()));
+                            }
+                            BankNotification::Root(bank) => {
+                                slot_status_notifier
+                                    .read()
+                                    .unwrap()
+                                    .notify_slot_rooted(bank.slot(), Some(bank.parent_slot()));
+                            }
+                        }
+                    }
+                }
+            })
+            .unwrap()
+    }
+}

accountsdb-plugin-manager/src/transaction_notifier.rs

@@ -0,0 +1,93 @@
+/// Module responsible for notifying plugins of transactions
+use {
+    crate::accountsdb_plugin_manager::AccountsDbPluginManager,
+    log::*,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        ReplicaTransactionInfo, ReplicaTransactionInfoVersions,
+    },
+    solana_measure::measure::Measure,
+    solana_metrics::*,
+    solana_rpc::transaction_notifier_interface::TransactionNotifier,
+    solana_runtime::bank,
+    solana_sdk::{clock::Slot, signature::Signature, transaction::SanitizedTransaction},
+    solana_transaction_status::TransactionStatusMeta,
+    std::sync::{Arc, RwLock},
+};
+
+/// This implementation of TransactionNotifier is passed to the rpc's TransactionStatusService
+/// at the validator startup. TransactionStatusService invokes the notify_transaction method
+/// for new transactions. The implementation in turn invokes the notify_transaction of each
+/// plugin enabled with transaction notification managed by the AccountsDbPluginManager.
+pub(crate) struct TransactionNotifierImpl {
+    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
+}
+
+impl TransactionNotifier for TransactionNotifierImpl {
+    fn notify_transaction(
+        &self,
+        slot: Slot,
+        signature: &Signature,
+        transaction_status_meta: &TransactionStatusMeta,
+        transaction: &SanitizedTransaction,
+    ) {
+        let mut measure = Measure::start("accountsdb-plugin-notify_plugins_of_transaction_info");
+        let transaction_log_info =
+            Self::build_replica_transaction_info(signature, transaction_status_meta, transaction);
+
+        let mut plugin_manager = self.plugin_manager.write().unwrap();
+
+        if plugin_manager.plugins.is_empty() {
+            return;
+        }
+
+        for plugin in plugin_manager.plugins.iter_mut() {
+            if !plugin.transaction_notifications_enabled() {
+                continue;
+            }
+            match plugin.notify_transaction(
+                ReplicaTransactionInfoVersions::V0_0_1(&transaction_log_info),
+                slot,
+            ) {
+                Err(err) => {
+                    error!(
+                        "Failed to notify transaction, error: ({}) to plugin {}",
+                        err,
+                        plugin.name()
+                    )
+                }
+                Ok(_) => {
+                    trace!(
+                        "Successfully notified transaction to plugin {}",
+                        plugin.name()
+                    );
+                }
+            }
+        }
+        measure.stop();
+        inc_new_counter_debug!(
+            "accountsdb-plugin-notify_plugins_of_transaction_info-us",
+            measure.as_us() as usize,
+            10000,
+            10000
+        );
+    }
+}
+
+impl TransactionNotifierImpl {
+    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
+        Self { plugin_manager }
+    }
+
+    fn build_replica_transaction_info<'a>(
+        signature: &'a Signature,
+        transaction_status_meta: &'a TransactionStatusMeta,
+        transaction: &'a SanitizedTransaction,
+    ) -> ReplicaTransactionInfo<'a> {
+        ReplicaTransactionInfo {
+            signature,
+            is_vote: bank::is_simple_vote_transaction(transaction),
+            transaction,
+            transaction_status_meta,
+        }
+    }
+}

accountsdb-plugin-postgres/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2021"
+name = "solana-accountsdb-plugin-postgres"
+description = "The Solana AccountsDb plugin for PostgreSQL database."
+version = "1.10.0"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-validator"
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+bs58 = "0.4.0"
+chrono = { version = "0.4.11", features = ["serde"] }
+crossbeam-channel = "0.5"
+log = "0.4.14"
+postgres = { version = "0.19.2", features = ["with-chrono-0_4"] }
+postgres-types = { version = "0.2.2", features = ["derive"] }
+serde = "1.0.133"
+serde_derive = "1.0.103"
+serde_json = "1.0.74"
+solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.10.0" }
+solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-metrics = { path = "../metrics", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
+thiserror = "1.0.30"
+tokio-postgres = "0.7.4"
+
+[dev-dependencies]
+solana-account-decoder = { path = "../account-decoder", version = "=1.10.0" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-postgres/README.md

@@ -0,0 +1,5 @@
+This is an example implementing the AccountsDb plugin for PostgreSQL database.
+Please see the `src/accountsdb_plugin_postgres.rs` for the format of the plugin's configuration file.
+
+To create the schema objects for the database, please use `scripts/create_schema.sql`.
+`scripts/drop_schema.sql` can be used to tear down the schema objects.

accountsdb-plugin-postgres/scripts/create_schema.sql

@@ -0,0 +1,195 @@
+/**
+ * This plugin implementation for PostgreSQL requires the following tables
+ */
+-- The table storing accounts
+
+
+CREATE TABLE account (
+    pubkey BYTEA PRIMARY KEY,
+    owner BYTEA,
+    lamports BIGINT NOT NULL,
+    slot BIGINT NOT NULL,
+    executable BOOL NOT NULL,
+    rent_epoch BIGINT NOT NULL,
+    data BYTEA,
+    write_version BIGINT NOT NULL,
+    updated_on TIMESTAMP NOT NULL
+);
+
+-- The table storing slot information
+CREATE TABLE slot (
+    slot BIGINT PRIMARY KEY,
+    parent BIGINT,
+    status VARCHAR(16) NOT NULL,
+    updated_on TIMESTAMP NOT NULL
+);
+
+-- Types for Transactions
+
+Create TYPE "TransactionErrorCode" AS ENUM (
+    'AccountInUse',
+    'AccountLoadedTwice',
+    'AccountNotFound',
+    'ProgramAccountNotFound',
+    'InsufficientFundsForFee',
+    'InvalidAccountForFee',
+    'AlreadyProcessed',
+    'BlockhashNotFound',
+    'InstructionError',
+    'CallChainTooDeep',
+    'MissingSignatureForFee',
+    'InvalidAccountIndex',
+    'SignatureFailure',
+    'InvalidProgramForExecution',
+    'SanitizeFailure',
+    'ClusterMaintenance',
+    'AccountBorrowOutstanding',
+    'WouldExceedMaxAccountCostLimit',
+    'WouldExceedMaxBlockCostLimit',
+    'UnsupportedVersion',
+    'InvalidWritableAccount',
+    'WouldExceedMaxAccountDataCostLimit'
+);
+
+CREATE TYPE "TransactionError" AS (
+    error_code "TransactionErrorCode",
+    error_detail VARCHAR(256)
+);
+
+CREATE TYPE "CompiledInstruction" AS (
+    program_id_index SMALLINT,
+    accounts SMALLINT[],
+    data BYTEA
+);
+
+CREATE TYPE "InnerInstructions" AS (
+    index SMALLINT,
+    instructions "CompiledInstruction"[]
+);
+
+CREATE TYPE "TransactionTokenBalance" AS (
+    account_index SMALLINT,
+    mint VARCHAR(44),
+    ui_token_amount DOUBLE PRECISION,
+    owner VARCHAR(44)
+);
+
+Create TYPE "RewardType" AS ENUM (
+    'Fee',
+    'Rent',
+    'Staking',
+    'Voting'
+);
+
+CREATE TYPE "Reward" AS (
+    pubkey VARCHAR(44),
+    lamports BIGINT,
+    post_balance BIGINT,
+    reward_type "RewardType",
+    commission SMALLINT
+);
+
+CREATE TYPE "TransactionStatusMeta" AS (
+    error "TransactionError",
+    fee BIGINT,
+    pre_balances BIGINT[],
+    post_balances BIGINT[],
+    inner_instructions "InnerInstructions"[],
+    log_messages TEXT[],
+    pre_token_balances "TransactionTokenBalance"[],
+    post_token_balances "TransactionTokenBalance"[],
+    rewards "Reward"[]
+);
+
+CREATE TYPE "TransactionMessageHeader" AS (
+    num_required_signatures SMALLINT,
+    num_readonly_signed_accounts SMALLINT,
+    num_readonly_unsigned_accounts SMALLINT
+);
+
+CREATE TYPE "TransactionMessage" AS (
+    header "TransactionMessageHeader",
+    account_keys BYTEA[],
+    recent_blockhash BYTEA,
+    instructions "CompiledInstruction"[]
+);
+
+CREATE TYPE "TransactionMessageAddressTableLookup" AS (
+    account_key BYTEA,
+    writable_indexes SMALLINT[],
+    readonly_indexes SMALLINT[]
+);
+
+CREATE TYPE "TransactionMessageV0" AS (
+    header "TransactionMessageHeader",
+    account_keys BYTEA[],
+    recent_blockhash BYTEA,
+    instructions "CompiledInstruction"[],
+    address_table_lookups "TransactionMessageAddressTableLookup"[]
+);
+
+CREATE TYPE "LoadedAddresses" AS (
+    writable BYTEA[],
+    readonly BYTEA[]
+);
+
+CREATE TYPE "LoadedMessageV0" AS (
+    message "TransactionMessageV0",
+    loaded_addresses "LoadedAddresses"
+);
+
+-- The table storing transactions
+CREATE TABLE transaction (
+    slot BIGINT NOT NULL,
+    signature BYTEA NOT NULL,
+    is_vote BOOL NOT NULL,
+    message_type SMALLINT, -- 0: legacy, 1: v0 message
+    legacy_message "TransactionMessage",
+    v0_loaded_message "LoadedMessageV0",
+    signatures BYTEA[],
+    message_hash BYTEA,
+    meta "TransactionStatusMeta",
+    updated_on TIMESTAMP NOT NULL,
+    CONSTRAINT transaction_pk PRIMARY KEY (slot, signature)
+);
+
+-- The table storing block metadata
+CREATE TABLE block (
+    slot BIGINT PRIMARY KEY,
+    blockhash VARCHAR(44),
+    rewards "Reward"[],
+    block_time BIGINT,
+    block_height BIGINT,
+    updated_on TIMESTAMP NOT NULL
+);
+
+/**
+ * The following is for keeping historical data for accounts and is not required for plugin to work.
+ */
+-- The table storing historical data for accounts
+CREATE TABLE account_audit (
+    pubkey BYTEA,
+    owner BYTEA,
+    lamports BIGINT NOT NULL,
+    slot BIGINT NOT NULL,
+    executable BOOL NOT NULL,
+    rent_epoch BIGINT NOT NULL,
+    data BYTEA,
+    write_version BIGINT NOT NULL,
+    updated_on TIMESTAMP NOT NULL
+);
+
+CREATE INDEX account_audit_account_key ON  account_audit (pubkey, write_version);
+
+CREATE FUNCTION audit_account_update() RETURNS trigger AS $audit_account_update$
+    BEGIN
+		INSERT INTO account_audit (pubkey, owner, lamports, slot, executable, rent_epoch, data, write_version, updated_on)
+            VALUES (OLD.pubkey, OLD.owner, OLD.lamports, OLD.slot,
+                    OLD.executable, OLD.rent_epoch, OLD.data, OLD.write_version, OLD.updated_on);
+        RETURN NEW;
+    END;
+
+$audit_account_update$ LANGUAGE plpgsql;
+
+CREATE TRIGGER account_update_trigger AFTER UPDATE OR DELETE ON account
+    FOR EACH ROW EXECUTE PROCEDURE audit_account_update();

accountsdb-plugin-postgres/scripts/drop_schema.sql

@@ -0,0 +1,26 @@
+/**
+ * Script for cleaning up the schema for PostgreSQL used for the AccountsDb plugin.
+ */
+
+DROP TRIGGER account_update_trigger ON account;
+DROP FUNCTION audit_account_update;
+DROP TABLE account_audit;
+DROP TABLE account;
+DROP TABLE slot;
+DROP TABLE transaction;
+DROP TABLE block;
+
+DROP TYPE "TransactionError" CASCADE;
+DROP TYPE "TransactionErrorCode" CASCADE;
+DROP TYPE "LoadedMessageV0" CASCADE;
+DROP TYPE "LoadedAddresses" CASCADE;
+DROP TYPE "TransactionMessageV0" CASCADE;
+DROP TYPE "TransactionMessage" CASCADE;
+DROP TYPE "TransactionMessageHeader" CASCADE;
+DROP TYPE "TransactionMessageAddressTableLookup" CASCADE;
+DROP TYPE "TransactionStatusMeta" CASCADE;
+DROP TYPE "RewardType" CASCADE;
+DROP TYPE "Reward" CASCADE;
+DROP TYPE "TransactionTokenBalance" CASCADE;
+DROP TYPE "InnerInstructions" CASCADE;
+DROP TYPE "CompiledInstruction" CASCADE;

accountsdb-plugin-postgres/scripts/postgresql.conf

@@ -0,0 +1,802 @@
+# This a reference configuration file for the PostgreSQL database version 14.
+
+# -----------------------------
+# PostgreSQL configuration file
+# -----------------------------
+#
+# This file consists of lines of the form:
+#
+#   name = value
+#
+# (The "=" is optional.)  Whitespace may be used.  Comments are introduced with
+# "#" anywhere on a line.  The complete list of parameter names and allowed
+# values can be found in the PostgreSQL documentation.
+#
+# The commented-out settings shown in this file represent the default values.
+# Re-commenting a setting is NOT sufficient to revert it to the default value;
+# you need to reload the server.
+#
+# This file is read on server startup and when the server receives a SIGHUP
+# signal.  If you edit the file on a running system, you have to SIGHUP the
+# server for the changes to take effect, run "pg_ctl reload", or execute
+# "SELECT pg_reload_conf()".  Some parameters, which are marked below,
+# require a server shutdown and restart to take effect.
+#
+# Any parameter can also be given as a command-line option to the server, e.g.,
+# "postgres -c log_connections=on".  Some parameters can be changed at run time
+# with the "SET" SQL command.
+#
+# Memory units:  B  = bytes            Time units:  us  = microseconds
+#                kB = kilobytes                     ms  = milliseconds
+#                MB = megabytes                     s   = seconds
+#                GB = gigabytes                     min = minutes
+#                TB = terabytes                     h   = hours
+#                                                   d   = days
+
+
+#------------------------------------------------------------------------------
+# FILE LOCATIONS
+#------------------------------------------------------------------------------
+
+# The default values of these variables are driven from the -D command-line
+# option or PGDATA environment variable, represented here as ConfigDir.
+
+data_directory = '/var/lib/postgresql/14/main'		# use data in another directory
+					# (change requires restart)
+
+hba_file = '/etc/postgresql/14/main/pg_hba.conf'	# host-based authentication file
+					# (change requires restart)
+ident_file = '/etc/postgresql/14/main/pg_ident.conf'	# ident configuration file
+					# (change requires restart)
+
+# If external_pid_file is not explicitly set, no extra PID file is written.
+external_pid_file = '/var/run/postgresql/14-main.pid'			# write an extra PID file
+					# (change requires restart)
+
+
+#------------------------------------------------------------------------------
+# CONNECTIONS AND AUTHENTICATION
+#------------------------------------------------------------------------------
+
+# - Connection Settings -
+
+#listen_addresses = 'localhost'		# what IP address(es) to listen on;
+					# comma-separated list of addresses;
+					# defaults to 'localhost'; use '*' for all
+					# (change requires restart)
+listen_addresses = '*'
+port = 5433				# (change requires restart)
+max_connections = 200			# (change requires restart)
+#superuser_reserved_connections = 3	# (change requires restart)
+unix_socket_directories = '/var/run/postgresql'	# comma-separated list of directories
+					# (change requires restart)
+#unix_socket_group = ''			# (change requires restart)
+#unix_socket_permissions = 0777		# begin with 0 to use octal notation
+					# (change requires restart)
+#bonjour = off				# advertise server via Bonjour
+					# (change requires restart)
+#bonjour_name = ''			# defaults to the computer name
+					# (change requires restart)
+
+# - TCP settings -
+# see "man tcp" for details
+
+#tcp_keepalives_idle = 0		# TCP_KEEPIDLE, in seconds;
+					# 0 selects the system default
+#tcp_keepalives_interval = 0		# TCP_KEEPINTVL, in seconds;
+					# 0 selects the system default
+#tcp_keepalives_count = 0		# TCP_KEEPCNT;
+					# 0 selects the system default
+#tcp_user_timeout = 0			# TCP_USER_TIMEOUT, in milliseconds;
+					# 0 selects the system default
+
+#client_connection_check_interval = 0	# time between checks for client
+					# disconnection while running queries;
+					# 0 for never
+
+# - Authentication -
+
+#authentication_timeout = 1min		# 1s-600s
+#password_encryption = scram-sha-256	# scram-sha-256 or md5
+#db_user_namespace = off
+
+# GSSAPI using Kerberos
+#krb_server_keyfile = 'FILE:${sysconfdir}/krb5.keytab'
+#krb_caseins_users = off
+
+# - SSL -
+
+ssl = on
+#ssl_ca_file = ''
+ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem'
+#ssl_crl_file = ''
+#ssl_crl_dir = ''
+ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key'
+#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers
+#ssl_prefer_server_ciphers = on
+#ssl_ecdh_curve = 'prime256v1'
+#ssl_min_protocol_version = 'TLSv1.2'
+#ssl_max_protocol_version = ''
+#ssl_dh_params_file = ''
+#ssl_passphrase_command = ''
+#ssl_passphrase_command_supports_reload = off
+
+
+#------------------------------------------------------------------------------
+# RESOURCE USAGE (except WAL)
+#------------------------------------------------------------------------------
+
+# - Memory -
+
+shared_buffers = 1GB			# min 128kB
+					# (change requires restart)
+#huge_pages = try			# on, off, or try
+					# (change requires restart)
+#huge_page_size = 0			# zero for system default
+					# (change requires restart)
+#temp_buffers = 8MB			# min 800kB
+#max_prepared_transactions = 0		# zero disables the feature
+					# (change requires restart)
+# Caution: it is not advisable to set max_prepared_transactions nonzero unless
+# you actively intend to use prepared transactions.
+#work_mem = 4MB				# min 64kB
+#hash_mem_multiplier = 1.0		# 1-1000.0 multiplier on hash table work_mem
+#maintenance_work_mem = 64MB		# min 1MB
+#autovacuum_work_mem = -1		# min 1MB, or -1 to use maintenance_work_mem
+#logical_decoding_work_mem = 64MB	# min 64kB
+#max_stack_depth = 2MB			# min 100kB
+#shared_memory_type = mmap		# the default is the first option
+					# supported by the operating system:
+					#   mmap
+					#   sysv
+					#   windows
+					# (change requires restart)
+dynamic_shared_memory_type = posix	# the default is the first option
+					# supported by the operating system:
+					#   posix
+					#   sysv
+					#   windows
+					#   mmap
+					# (change requires restart)
+#min_dynamic_shared_memory = 0MB	# (change requires restart)
+
+# - Disk -
+
+#temp_file_limit = -1			# limits per-process temp file space
+					# in kilobytes, or -1 for no limit
+
+# - Kernel Resources -
+
+#max_files_per_process = 1000		# min 64
+					# (change requires restart)
+
+# - Cost-Based Vacuum Delay -
+
+#vacuum_cost_delay = 0			# 0-100 milliseconds (0 disables)
+#vacuum_cost_page_hit = 1		# 0-10000 credits
+#vacuum_cost_page_miss = 2		# 0-10000 credits
+#vacuum_cost_page_dirty = 20		# 0-10000 credits
+#vacuum_cost_limit = 200		# 1-10000 credits
+
+# - Background Writer -
+
+#bgwriter_delay = 200ms			# 10-10000ms between rounds
+#bgwriter_lru_maxpages = 100		# max buffers written/round, 0 disables
+#bgwriter_lru_multiplier = 2.0		# 0-10.0 multiplier on buffers scanned/round
+#bgwriter_flush_after = 512kB		# measured in pages, 0 disables
+
+# - Asynchronous Behavior -
+
+#backend_flush_after = 0		# measured in pages, 0 disables
+effective_io_concurrency = 1000		# 1-1000; 0 disables prefetching
+#maintenance_io_concurrency = 10	# 1-1000; 0 disables prefetching
+#max_worker_processes = 8		# (change requires restart)
+#max_parallel_workers_per_gather = 2	# taken from max_parallel_workers
+#max_parallel_maintenance_workers = 2	# taken from max_parallel_workers
+#max_parallel_workers = 8		# maximum number of max_worker_processes that
+					# can be used in parallel operations
+#parallel_leader_participation = on
+#old_snapshot_threshold = -1		# 1min-60d; -1 disables; 0 is immediate
+					# (change requires restart)
+
+
+#------------------------------------------------------------------------------
+# WRITE-AHEAD LOG
+#------------------------------------------------------------------------------
+
+# - Settings -
+
+wal_level = minimal 			# minimal, replica, or logical
+					# (change requires restart)
+fsync = off				# flush data to disk for crash safety
+					# (turning this off can cause
+					# unrecoverable data corruption)
+synchronous_commit = off		# synchronization level;
+					# off, local, remote_write, remote_apply, or on
+#wal_sync_method = fsync		# the default is the first option
+					# supported by the operating system:
+					#   open_datasync
+					#   fdatasync (default on Linux and FreeBSD)
+					#   fsync
+					#   fsync_writethrough
+					#   open_sync
+full_page_writes = off			# recover from partial page writes
+#wal_log_hints = off			# also do full page writes of non-critical updates
+					# (change requires restart)
+#wal_compression = off			# enable compression of full-page writes
+#wal_init_zero = on			# zero-fill new WAL files
+#wal_recycle = on			# recycle WAL files
+#wal_buffers = -1			# min 32kB, -1 sets based on shared_buffers
+					# (change requires restart)
+#wal_writer_delay = 200ms		# 1-10000 milliseconds
+#wal_writer_flush_after = 1MB		# measured in pages, 0 disables
+#wal_skip_threshold = 2MB
+
+#commit_delay = 0			# range 0-100000, in microseconds
+#commit_siblings = 5			# range 1-1000
+
+# - Checkpoints -
+
+#checkpoint_timeout = 5min		# range 30s-1d
+#checkpoint_completion_target = 0.9	# checkpoint target duration, 0.0 - 1.0
+#checkpoint_flush_after = 256kB		# measured in pages, 0 disables
+#checkpoint_warning = 30s		# 0 disables
+max_wal_size = 1GB
+min_wal_size = 80MB
+
+# - Archiving -
+
+#archive_mode = off		# enables archiving; off, on, or always
+				# (change requires restart)
+#archive_command = ''		# command to use to archive a logfile segment
+				# placeholders: %p = path of file to archive
+				#               %f = file name only
+				# e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f'
+#archive_timeout = 0		# force a logfile segment switch after this
+				# number of seconds; 0 disables
+
+# - Archive Recovery -
+
+# These are only used in recovery mode.
+
+#restore_command = ''		# command to use to restore an archived logfile segment
+				# placeholders: %p = path of file to restore
+				#               %f = file name only
+				# e.g. 'cp /mnt/server/archivedir/%f %p'
+#archive_cleanup_command = ''	# command to execute at every restartpoint
+#recovery_end_command = ''	# command to execute at completion of recovery
+
+# - Recovery Target -
+
+# Set these only when performing a targeted recovery.
+
+#recovery_target = ''		# 'immediate' to end recovery as soon as a
+                                # consistent state is reached
+				# (change requires restart)
+#recovery_target_name = ''	# the named restore point to which recovery will proceed
+				# (change requires restart)
+#recovery_target_time = ''	# the time stamp up to which recovery will proceed
+				# (change requires restart)
+#recovery_target_xid = ''	# the transaction ID up to which recovery will proceed
+				# (change requires restart)
+#recovery_target_lsn = ''	# the WAL LSN up to which recovery will proceed
+				# (change requires restart)
+#recovery_target_inclusive = on # Specifies whether to stop:
+				# just after the specified recovery target (on)
+				# just before the recovery target (off)
+				# (change requires restart)
+#recovery_target_timeline = 'latest'	# 'current', 'latest', or timeline ID
+				# (change requires restart)
+#recovery_target_action = 'pause'	# 'pause', 'promote', 'shutdown'
+				# (change requires restart)
+
+
+#------------------------------------------------------------------------------
+# REPLICATION
+#------------------------------------------------------------------------------
+
+# - Sending Servers -
+
+# Set these on the primary and on any standby that will send replication data.
+
+max_wal_senders = 0		# max number of walsender processes
+				# (change requires restart)
+#max_replication_slots = 10	# max number of replication slots
+				# (change requires restart)
+#wal_keep_size = 0		# in megabytes; 0 disables
+#max_slot_wal_keep_size = -1	# in megabytes; -1 disables
+#wal_sender_timeout = 60s	# in milliseconds; 0 disables
+#track_commit_timestamp = off	# collect timestamp of transaction commit
+				# (change requires restart)
+
+# - Primary Server -
+
+# These settings are ignored on a standby server.
+
+#synchronous_standby_names = ''	# standby servers that provide sync rep
+				# method to choose sync standbys, number of sync standbys,
+				# and comma-separated list of application_name
+				# from standby(s); '*' = all
+#vacuum_defer_cleanup_age = 0	# number of xacts by which cleanup is delayed
+
+# - Standby Servers -
+
+# These settings are ignored on a primary server.
+
+#primary_conninfo = ''			# connection string to sending server
+#primary_slot_name = ''			# replication slot on sending server
+#promote_trigger_file = ''		# file name whose presence ends recovery
+#hot_standby = on			# "off" disallows queries during recovery
+					# (change requires restart)
+#max_standby_archive_delay = 30s	# max delay before canceling queries
+					# when reading WAL from archive;
+					# -1 allows indefinite delay
+#max_standby_streaming_delay = 30s	# max delay before canceling queries
+					# when reading streaming WAL;
+					# -1 allows indefinite delay
+#wal_receiver_create_temp_slot = off	# create temp slot if primary_slot_name
+					# is not set
+#wal_receiver_status_interval = 10s	# send replies at least this often
+					# 0 disables
+#hot_standby_feedback = off		# send info from standby to prevent
+					# query conflicts
+#wal_receiver_timeout = 60s		# time that receiver waits for
+					# communication from primary
+					# in milliseconds; 0 disables
+#wal_retrieve_retry_interval = 5s	# time to wait before retrying to
+					# retrieve WAL after a failed attempt
+#recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
+
+# - Subscribers -
+
+# These settings are ignored on a publisher.
+
+#max_logical_replication_workers = 4	# taken from max_worker_processes
+					# (change requires restart)
+#max_sync_workers_per_subscription = 2	# taken from max_logical_replication_workers
+
+
+#------------------------------------------------------------------------------
+# QUERY TUNING
+#------------------------------------------------------------------------------
+
+# - Planner Method Configuration -
+
+#enable_async_append = on
+#enable_bitmapscan = on
+#enable_gathermerge = on
+#enable_hashagg = on
+#enable_hashjoin = on
+#enable_incremental_sort = on
+#enable_indexscan = on
+#enable_indexonlyscan = on
+#enable_material = on
+#enable_memoize = on
+#enable_mergejoin = on
+#enable_nestloop = on
+#enable_parallel_append = on
+#enable_parallel_hash = on
+#enable_partition_pruning = on
+#enable_partitionwise_join = off
+#enable_partitionwise_aggregate = off
+#enable_seqscan = on
+#enable_sort = on
+#enable_tidscan = on
+
+# - Planner Cost Constants -
+
+#seq_page_cost = 1.0			# measured on an arbitrary scale
+#random_page_cost = 4.0			# same scale as above
+#cpu_tuple_cost = 0.01			# same scale as above
+#cpu_index_tuple_cost = 0.005		# same scale as above
+#cpu_operator_cost = 0.0025		# same scale as above
+#parallel_setup_cost = 1000.0	# same scale as above
+#parallel_tuple_cost = 0.1		# same scale as above
+#min_parallel_table_scan_size = 8MB
+#min_parallel_index_scan_size = 512kB
+#effective_cache_size = 4GB
+
+#jit_above_cost = 100000		# perform JIT compilation if available
+					# and query more expensive than this;
+					# -1 disables
+#jit_inline_above_cost = 500000		# inline small functions if query is
+					# more expensive than this; -1 disables
+#jit_optimize_above_cost = 500000	# use expensive JIT optimizations if
+					# query is more expensive than this;
+					# -1 disables
+
+# - Genetic Query Optimizer -
+
+#geqo = on
+#geqo_threshold = 12
+#geqo_effort = 5			# range 1-10
+#geqo_pool_size = 0			# selects default based on effort
+#geqo_generations = 0			# selects default based on effort
+#geqo_selection_bias = 2.0		# range 1.5-2.0
+#geqo_seed = 0.0			# range 0.0-1.0
+
+# - Other Planner Options -
+
+#default_statistics_target = 100	# range 1-10000
+#constraint_exclusion = partition	# on, off, or partition
+#cursor_tuple_fraction = 0.1		# range 0.0-1.0
+#from_collapse_limit = 8
+#jit = on				# allow JIT compilation
+#join_collapse_limit = 8		# 1 disables collapsing of explicit
+					# JOIN clauses
+#plan_cache_mode = auto			# auto, force_generic_plan or
+					# force_custom_plan
+
+
+#------------------------------------------------------------------------------
+# REPORTING AND LOGGING
+#------------------------------------------------------------------------------
+
+# - Where to Log -
+
+#log_destination = 'stderr'		# Valid values are combinations of
+					# stderr, csvlog, syslog, and eventlog,
+					# depending on platform.  csvlog
+					# requires logging_collector to be on.
+
+# This is used when logging to stderr:
+#logging_collector = off		# Enable capturing of stderr and csvlog
+					# into log files. Required to be on for
+					# csvlogs.
+					# (change requires restart)
+
+# These are only used if logging_collector is on:
+#log_directory = 'log'			# directory where log files are written,
+					# can be absolute or relative to PGDATA
+#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'	# log file name pattern,
+					# can include strftime() escapes
+#log_file_mode = 0600			# creation mode for log files,
+					# begin with 0 to use octal notation
+#log_rotation_age = 1d			# Automatic rotation of logfiles will
+					# happen after that time.  0 disables.
+#log_rotation_size = 10MB		# Automatic rotation of logfiles will
+					# happen after that much log output.
+					# 0 disables.
+#log_truncate_on_rotation = off		# If on, an existing log file with the
+					# same name as the new log file will be
+					# truncated rather than appended to.
+					# But such truncation only occurs on
+					# time-driven rotation, not on restarts
+					# or size-driven rotation.  Default is
+					# off, meaning append to existing files
+					# in all cases.
+
+# These are relevant when logging to syslog:
+#syslog_facility = 'LOCAL0'
+#syslog_ident = 'postgres'
+#syslog_sequence_numbers = on
+#syslog_split_messages = on
+
+# This is only relevant when logging to eventlog (Windows):
+# (change requires restart)
+#event_source = 'PostgreSQL'
+
+# - When to Log -
+
+#log_min_messages = warning		# values in order of decreasing detail:
+					#   debug5
+					#   debug4
+					#   debug3
+					#   debug2
+					#   debug1
+					#   info
+					#   notice
+					#   warning
+					#   error
+					#   log
+					#   fatal
+					#   panic
+
+#log_min_error_statement = error	# values in order of decreasing detail:
+					#   debug5
+					#   debug4
+					#   debug3
+					#   debug2
+					#   debug1
+					#   info
+					#   notice
+					#   warning
+					#   error
+					#   log
+					#   fatal
+					#   panic (effectively off)
+
+#log_min_duration_statement = -1	# -1 is disabled, 0 logs all statements
+					# and their durations, > 0 logs only
+					# statements running at least this number
+					# of milliseconds
+
+#log_min_duration_sample = -1		# -1 is disabled, 0 logs a sample of statements
+					# and their durations, > 0 logs only a sample of
+					# statements running at least this number
+					# of milliseconds;
+					# sample fraction is determined by log_statement_sample_rate
+
+#log_statement_sample_rate = 1.0	# fraction of logged statements exceeding
+					# log_min_duration_sample to be logged;
+					# 1.0 logs all such statements, 0.0 never logs
+
+
+#log_transaction_sample_rate = 0.0	# fraction of transactions whose statements
+					# are logged regardless of their duration; 1.0 logs all
+					# statements from all transactions, 0.0 never logs
+
+# - What to Log -
+
+#debug_print_parse = off
+#debug_print_rewritten = off
+#debug_print_plan = off
+#debug_pretty_print = on
+#log_autovacuum_min_duration = -1	# log autovacuum activity;
+					# -1 disables, 0 logs all actions and
+					# their durations, > 0 logs only
+					# actions running at least this number
+					# of milliseconds.
+#log_checkpoints = off
+#log_connections = off
+#log_disconnections = off
+#log_duration = off
+#log_error_verbosity = default		# terse, default, or verbose messages
+#log_hostname = off
+log_line_prefix = '%m [%p] %q%u@%d '		# special values:
+					#   %a = application name
+					#   %u = user name
+					#   %d = database name
+					#   %r = remote host and port
+					#   %h = remote host
+					#   %b = backend type
+					#   %p = process ID
+					#   %P = process ID of parallel group leader
+					#   %t = timestamp without milliseconds
+					#   %m = timestamp with milliseconds
+					#   %n = timestamp with milliseconds (as a Unix epoch)
+					#   %Q = query ID (0 if none or not computed)
+					#   %i = command tag
+					#   %e = SQL state
+					#   %c = session ID
+					#   %l = session line number
+					#   %s = session start timestamp
+					#   %v = virtual transaction ID
+					#   %x = transaction ID (0 if none)
+					#   %q = stop here in non-session
+					#        processes
+					#   %% = '%'
+					# e.g. '<%u%%%d> '
+#log_lock_waits = off			# log lock waits >= deadlock_timeout
+#log_recovery_conflict_waits = off	# log standby recovery conflict waits
+					# >= deadlock_timeout
+#log_parameter_max_length = -1		# when logging statements, limit logged
+					# bind-parameter values to N bytes;
+					# -1 means print in full, 0 disables
+#log_parameter_max_length_on_error = 0	# when logging an error, limit logged
+					# bind-parameter values to N bytes;
+					# -1 means print in full, 0 disables
+#log_statement = 'none'			# none, ddl, mod, all
+#log_replication_commands = off
+#log_temp_files = -1			# log temporary files equal or larger
+					# than the specified size in kilobytes;
+					# -1 disables, 0 logs all temp files
+log_timezone = 'Etc/UTC'
+
+
+#------------------------------------------------------------------------------
+# PROCESS TITLE
+#------------------------------------------------------------------------------
+
+cluster_name = '14/main'			# added to process titles if nonempty
+					# (change requires restart)
+#update_process_title = on
+
+
+#------------------------------------------------------------------------------
+# STATISTICS
+#------------------------------------------------------------------------------
+
+# - Query and Index Statistics Collector -
+
+#track_activities = on
+#track_activity_query_size = 1024	# (change requires restart)
+#track_counts = on
+#track_io_timing = off
+#track_wal_io_timing = off
+#track_functions = none			# none, pl, all
+stats_temp_directory = '/var/run/postgresql/14-main.pg_stat_tmp'
+
+
+# - Monitoring -
+
+#compute_query_id = auto
+#log_statement_stats = off
+#log_parser_stats = off
+#log_planner_stats = off
+#log_executor_stats = off
+
+
+#------------------------------------------------------------------------------
+# AUTOVACUUM
+#------------------------------------------------------------------------------
+
+#autovacuum = on			# Enable autovacuum subprocess?  'on'
+					# requires track_counts to also be on.
+#autovacuum_max_workers = 3		# max number of autovacuum subprocesses
+					# (change requires restart)
+#autovacuum_naptime = 1min		# time between autovacuum runs
+#autovacuum_vacuum_threshold = 50	# min number of row updates before
+					# vacuum
+#autovacuum_vacuum_insert_threshold = 1000	# min number of row inserts
+					# before vacuum; -1 disables insert
+					# vacuums
+#autovacuum_analyze_threshold = 50	# min number of row updates before
+					# analyze
+#autovacuum_vacuum_scale_factor = 0.2	# fraction of table size before vacuum
+#autovacuum_vacuum_insert_scale_factor = 0.2	# fraction of inserts over table
+					# size before insert vacuum
+#autovacuum_analyze_scale_factor = 0.1	# fraction of table size before analyze
+#autovacuum_freeze_max_age = 200000000	# maximum XID age before forced vacuum
+					# (change requires restart)
+#autovacuum_multixact_freeze_max_age = 400000000	# maximum multixact age
+					# before forced vacuum
+					# (change requires restart)
+#autovacuum_vacuum_cost_delay = 2ms	# default vacuum cost delay for
+					# autovacuum, in milliseconds;
+					# -1 means use vacuum_cost_delay
+#autovacuum_vacuum_cost_limit = -1	# default vacuum cost limit for
+					# autovacuum, -1 means use
+					# vacuum_cost_limit
+
+
+#------------------------------------------------------------------------------
+# CLIENT CONNECTION DEFAULTS
+#------------------------------------------------------------------------------
+
+# - Statement Behavior -
+
+#client_min_messages = notice		# values in order of decreasing detail:
+					#   debug5
+					#   debug4
+					#   debug3
+					#   debug2
+					#   debug1
+					#   log
+					#   notice
+					#   warning
+					#   error
+#search_path = '"$user", public'	# schema names
+#row_security = on
+#default_table_access_method = 'heap'
+#default_tablespace = ''		# a tablespace name, '' uses the default
+#default_toast_compression = 'pglz'	# 'pglz' or 'lz4'
+#temp_tablespaces = ''			# a list of tablespace names, '' uses
+					# only default tablespace
+#check_function_bodies = on
+#default_transaction_isolation = 'read committed'
+#default_transaction_read_only = off
+#default_transaction_deferrable = off
+#session_replication_role = 'origin'
+#statement_timeout = 0			# in milliseconds, 0 is disabled
+#lock_timeout = 0			# in milliseconds, 0 is disabled
+#idle_in_transaction_session_timeout = 0	# in milliseconds, 0 is disabled
+#idle_session_timeout = 0		# in milliseconds, 0 is disabled
+#vacuum_freeze_table_age = 150000000
+#vacuum_freeze_min_age = 50000000
+#vacuum_failsafe_age = 1600000000
+#vacuum_multixact_freeze_table_age = 150000000
+#vacuum_multixact_freeze_min_age = 5000000
+#vacuum_multixact_failsafe_age = 1600000000
+#bytea_output = 'hex'			# hex, escape
+#xmlbinary = 'base64'
+#xmloption = 'content'
+#gin_pending_list_limit = 4MB
+
+# - Locale and Formatting -
+
+datestyle = 'iso, mdy'
+#intervalstyle = 'postgres'
+timezone = 'Etc/UTC'
+#timezone_abbreviations = 'Default'     # Select the set of available time zone
+					# abbreviations.  Currently, there are
+					#   Default
+					#   Australia (historical usage)
+					#   India
+					# You can create your own file in
+					# share/timezonesets/.
+#extra_float_digits = 1			# min -15, max 3; any value >0 actually
+					# selects precise output mode
+#client_encoding = sql_ascii		# actually, defaults to database
+					# encoding
+
+# These settings are initialized by initdb, but they can be changed.
+lc_messages = 'C.UTF-8'			# locale for system error message
+					# strings
+lc_monetary = 'C.UTF-8'			# locale for monetary formatting
+lc_numeric = 'C.UTF-8'			# locale for number formatting
+lc_time = 'C.UTF-8'				# locale for time formatting
+
+# default configuration for text search
+default_text_search_config = 'pg_catalog.english'
+
+# - Shared Library Preloading -
+
+#local_preload_libraries = ''
+#session_preload_libraries = ''
+#shared_preload_libraries = ''	# (change requires restart)
+#jit_provider = 'llvmjit'		# JIT library to use
+
+# - Other Defaults -
+
+#dynamic_library_path = '$libdir'
+#extension_destdir = ''			# prepend path when loading extensions
+					# and shared objects (added by Debian)
+#gin_fuzzy_search_limit = 0
+
+
+#------------------------------------------------------------------------------
+# LOCK MANAGEMENT
+#------------------------------------------------------------------------------
+
+#deadlock_timeout = 1s
+#max_locks_per_transaction = 64		# min 10
+					# (change requires restart)
+#max_pred_locks_per_transaction = 64	# min 10
+					# (change requires restart)
+#max_pred_locks_per_relation = -2	# negative values mean
+					# (max_pred_locks_per_transaction
+					#  / -max_pred_locks_per_relation) - 1
+#max_pred_locks_per_page = 2            # min 0
+
+
+#------------------------------------------------------------------------------
+# VERSION AND PLATFORM COMPATIBILITY
+#------------------------------------------------------------------------------
+
+# - Previous PostgreSQL Versions -
+
+#array_nulls = on
+#backslash_quote = safe_encoding	# on, off, or safe_encoding
+#escape_string_warning = on
+#lo_compat_privileges = off
+#quote_all_identifiers = off
+#standard_conforming_strings = on
+#synchronize_seqscans = on
+
+# - Other Platforms and Clients -
+
+#transform_null_equals = off
+
+
+#------------------------------------------------------------------------------
+# ERROR HANDLING
+#------------------------------------------------------------------------------
+
+#exit_on_error = off			# terminate session on any error?
+#restart_after_crash = on		# reinitialize after backend crash?
+#data_sync_retry = off			# retry or panic on failure to fsync
+					# data?
+					# (change requires restart)
+#recovery_init_sync_method = fsync	# fsync, syncfs (Linux 5.8+)
+
+
+#------------------------------------------------------------------------------
+# CONFIG FILE INCLUDES
+#------------------------------------------------------------------------------
+
+# These options allow settings to be loaded from files other than the
+# default postgresql.conf.  Note that these are directives, not variable
+# assignments, so they can usefully be given more than once.
+
+include_dir = 'conf.d'			# include files ending in '.conf' from
+					# a directory, e.g., 'conf.d'
+#include_if_exists = '...'		# include file only if it exists
+#include = '...'			# include file
+
+
+#------------------------------------------------------------------------------
+# CUSTOMIZED OPTIONS
+#------------------------------------------------------------------------------
+
+# Add settings for extensions here

accountsdb-plugin-postgres/src/accounts_selector.rs

@@ -0,0 +1,74 @@
+use {log::*, std::collections::HashSet};
+
+#[derive(Debug)]
+pub(crate) struct AccountsSelector {
+    pub accounts: HashSet<Vec<u8>>,
+    pub owners: HashSet<Vec<u8>>,
+    pub select_all_accounts: bool,
+}
+
+impl AccountsSelector {
+    pub fn default() -> Self {
+        AccountsSelector {
+            accounts: HashSet::default(),
+            owners: HashSet::default(),
+            select_all_accounts: true,
+        }
+    }
+
+    pub fn new(accounts: &[String], owners: &[String]) -> Self {
+        info!(
+            "Creating AccountsSelector from accounts: {:?}, owners: {:?}",
+            accounts, owners
+        );
+
+        let select_all_accounts = accounts.iter().any(|key| key == "*");
+        if select_all_accounts {
+            return AccountsSelector {
+                accounts: HashSet::default(),
+                owners: HashSet::default(),
+                select_all_accounts,
+            };
+        }
+        let accounts = accounts
+            .iter()
+            .map(|key| bs58::decode(key).into_vec().unwrap())
+            .collect();
+        let owners = owners
+            .iter()
+            .map(|key| bs58::decode(key).into_vec().unwrap())
+            .collect();
+        AccountsSelector {
+            accounts,
+            owners,
+            select_all_accounts,
+        }
+    }
+
+    pub fn is_account_selected(&self, account: &[u8], owner: &[u8]) -> bool {
+        self.select_all_accounts || self.accounts.contains(account) || self.owners.contains(owner)
+    }
+
+    /// Check if any account is of interested at all
+    pub fn is_enabled(&self) -> bool {
+        self.select_all_accounts || !self.accounts.is_empty() || !self.owners.is_empty()
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::*;
+
+    #[test]
+    fn test_create_accounts_selector() {
+        AccountsSelector::new(
+            &["9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin".to_string()],
+            &[],
+        );
+
+        AccountsSelector::new(
+            &[],
+            &["9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin".to_string()],
+        );
+    }
+}

accountsdb-plugin-postgres/src/accountsdb_plugin_postgres.rs

@@ -0,0 +1,466 @@
+use solana_measure::measure::Measure;
+/// Main entry for the PostgreSQL plugin
+use {
+    crate::{
+        accounts_selector::AccountsSelector,
+        postgres_client::{ParallelPostgresClient, PostgresClientBuilder},
+        transaction_selector::TransactionSelector,
+    },
+    bs58,
+    log::*,
+    serde_derive::{Deserialize, Serialize},
+    serde_json,
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        AccountsDbPlugin, AccountsDbPluginError, ReplicaAccountInfoVersions,
+        ReplicaBlockInfoVersions, ReplicaTransactionInfoVersions, Result, SlotStatus,
+    },
+    solana_metrics::*,
+    std::{fs::File, io::Read},
+    thiserror::Error,
+};
+
+#[derive(Default)]
+pub struct AccountsDbPluginPostgres {
+    client: Option<ParallelPostgresClient>,
+    accounts_selector: Option<AccountsSelector>,
+    transaction_selector: Option<TransactionSelector>,
+}
+
+impl std::fmt::Debug for AccountsDbPluginPostgres {
+    fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct AccountsDbPluginPostgresConfig {
+    pub host: Option<String>,
+    pub user: Option<String>,
+    pub port: Option<u16>,
+    pub connection_str: Option<String>,
+    pub threads: Option<usize>,
+    pub batch_size: Option<usize>,
+    pub panic_on_db_errors: Option<bool>,
+    /// Indicates if to store historical data for accounts
+    pub store_account_historical_data: Option<bool>,
+}
+
+#[derive(Error, Debug)]
+pub enum AccountsDbPluginPostgresError {
+    #[error("Error connecting to the backend data store. Error message: ({msg})")]
+    DataStoreConnectionError { msg: String },
+
+    #[error("Error preparing data store schema. Error message: ({msg})")]
+    DataSchemaError { msg: String },
+
+    #[error("Error preparing data store schema. Error message: ({msg})")]
+    ConfigurationError { msg: String },
+}
+
+impl AccountsDbPlugin for AccountsDbPluginPostgres {
+    fn name(&self) -> &'static str {
+        "AccountsDbPluginPostgres"
+    }
+
+    /// Do initialization for the PostgreSQL plugin.
+    ///
+    /// # Format of the config file:
+    /// * The `accounts_selector` section allows the user to controls accounts selections.
+    /// "accounts_selector" : {
+    ///     "accounts" : \["pubkey-1", "pubkey-2", ..., "pubkey-n"\],
+    /// }
+    /// or:
+    /// "accounts_selector" = {
+    ///     "owners" : \["pubkey-1", "pubkey-2", ..., "pubkey-m"\]
+    /// }
+    /// Accounts either satisyfing the accounts condition or owners condition will be selected.
+    /// When only owners is specified,
+    /// all accounts belonging to the owners will be streamed.
+    /// The accounts field supports wildcard to select all accounts:
+    /// "accounts_selector" : {
+    ///     "accounts" : \["*"\],
+    /// }
+    /// * "host", optional, specifies the PostgreSQL server.
+    /// * "user", optional, specifies the PostgreSQL user.
+    /// * "port", optional, specifies the PostgreSQL server's port.
+    /// * "connection_str", optional, the custom PostgreSQL connection string.
+    /// Please refer to https://docs.rs/postgres/0.19.2/postgres/config/struct.Config.html for the connection configuration.
+    /// When `connection_str` is set, the values in "host", "user" and "port" are ignored. If `connection_str` is not given,
+    /// `host` and `user` must be given.
+    /// "store_account_historical_data", optional, set it to 'true', to store historical account data to account_audit
+    /// table.
+    /// * "threads" optional, specifies the number of worker threads for the plugin. A thread
+    /// maintains a PostgreSQL connection to the server. The default is '10'.
+    /// * "batch_size" optional, specifies the batch size of bulk insert when the AccountsDb is created
+    /// from restoring a snapshot. The default is '10'.
+    /// * "panic_on_db_errors", optional, contols if to panic when there are errors replicating data to the
+    /// PostgreSQL database. The default is 'false'.
+    /// * "transaction_selector", optional, controls if and what transaction to store. If this field is missing
+    /// None of the transction is stored.
+    /// "transaction_selector" : {
+    ///     "mentions" : \["pubkey-1", "pubkey-2", ..., "pubkey-n"\],
+    /// }
+    /// The `mentions` field support wildcard to select all transaction or all 'vote' transactions:
+    /// For example, to select all transactions:
+    /// "transaction_selector" : {
+    ///     "mentions" : \["*"\],
+    /// }
+    /// To select all vote transactions:
+    /// "transaction_selector" : {
+    ///     "mentions" : \["all_votes"\],
+    /// }
+    /// # Examples
+    ///
+    /// {
+    ///    "libpath": "/home/solana/target/release/libsolana_accountsdb_plugin_postgres.so",
+    ///    "host": "host_foo",
+    ///    "user": "solana",
+    ///    "threads": 10,
+    ///    "accounts_selector" : {
+    ///       "owners" : ["9oT9R5ZyRovSVnt37QvVoBttGpNqR3J7unkb567NP8k3"]
+    ///    }
+    /// }
+
+    fn on_load(&mut self, config_file: &str) -> Result<()> {
+        solana_logger::setup_with_default("info");
+        info!(
+            "Loading plugin {:?} from config_file {:?}",
+            self.name(),
+            config_file
+        );
+        let mut file = File::open(config_file)?;
+        let mut contents = String::new();
+        file.read_to_string(&mut contents)?;
+
+        let result: serde_json::Value = serde_json::from_str(&contents).unwrap();
+        self.accounts_selector = Some(Self::create_accounts_selector_from_config(&result));
+        self.transaction_selector = Some(Self::create_transaction_selector_from_config(&result));
+
+        let result: serde_json::Result<AccountsDbPluginPostgresConfig> =
+            serde_json::from_str(&contents);
+        match result {
+            Err(err) => {
+                return Err(AccountsDbPluginError::ConfigFileReadError {
+                    msg: format!(
+                        "The config file is not in the JSON format expected: {:?}",
+                        err
+                    ),
+                })
+            }
+            Ok(config) => {
+                let client = PostgresClientBuilder::build_pararallel_postgres_client(&config)?;
+                self.client = Some(client);
+            }
+        }
+
+        Ok(())
+    }
+
+    fn on_unload(&mut self) {
+        info!("Unloading plugin: {:?}", self.name());
+
+        match &mut self.client {
+            None => {}
+            Some(client) => {
+                client.join().unwrap();
+            }
+        }
+    }
+
+    fn update_account(
+        &mut self,
+        account: ReplicaAccountInfoVersions,
+        slot: u64,
+        is_startup: bool,
+    ) -> Result<()> {
+        let mut measure_all = Measure::start("accountsdb-plugin-postgres-update-account-main");
+        match account {
+            ReplicaAccountInfoVersions::V0_0_1(account) => {
+                let mut measure_select =
+                    Measure::start("accountsdb-plugin-postgres-update-account-select");
+                if let Some(accounts_selector) = &self.accounts_selector {
+                    if !accounts_selector.is_account_selected(account.pubkey, account.owner) {
+                        return Ok(());
+                    }
+                } else {
+                    return Ok(());
+                }
+                measure_select.stop();
+                inc_new_counter_debug!(
+                    "accountsdb-plugin-postgres-update-account-select-us",
+                    measure_select.as_us() as usize,
+                    100000,
+                    100000
+                );
+
+                debug!(
+                    "Updating account {:?} with owner {:?} at slot {:?} using account selector {:?}",
+                    bs58::encode(account.pubkey).into_string(),
+                    bs58::encode(account.owner).into_string(),
+                    slot,
+                    self.accounts_selector.as_ref().unwrap()
+                );
+
+                match &mut self.client {
+                    None => {
+                        return Err(AccountsDbPluginError::Custom(Box::new(
+                            AccountsDbPluginPostgresError::DataStoreConnectionError {
+                                msg: "There is no connection to the PostgreSQL database."
+                                    .to_string(),
+                            },
+                        )));
+                    }
+                    Some(client) => {
+                        let mut measure_update =
+                            Measure::start("accountsdb-plugin-postgres-update-account-client");
+                        let result = { client.update_account(account, slot, is_startup) };
+                        measure_update.stop();
+
+                        inc_new_counter_debug!(
+                            "accountsdb-plugin-postgres-update-account-client-us",
+                            measure_update.as_us() as usize,
+                            100000,
+                            100000
+                        );
+
+                        if let Err(err) = result {
+                            return Err(AccountsDbPluginError::AccountsUpdateError {
+                                msg: format!("Failed to persist the update of account to the PostgreSQL database. Error: {:?}", err)
+                            });
+                        }
+                    }
+                }
+            }
+        }
+
+        measure_all.stop();
+
+        inc_new_counter_debug!(
+            "accountsdb-plugin-postgres-update-account-main-us",
+            measure_all.as_us() as usize,
+            100000,
+            100000
+        );
+
+        Ok(())
+    }
+
+    fn update_slot_status(
+        &mut self,
+        slot: u64,
+        parent: Option<u64>,
+        status: SlotStatus,
+    ) -> Result<()> {
+        info!("Updating slot {:?} at with status {:?}", slot, status);
+
+        match &mut self.client {
+            None => {
+                return Err(AccountsDbPluginError::Custom(Box::new(
+                    AccountsDbPluginPostgresError::DataStoreConnectionError {
+                        msg: "There is no connection to the PostgreSQL database.".to_string(),
+                    },
+                )));
+            }
+            Some(client) => {
+                let result = client.update_slot_status(slot, parent, status);
+
+                if let Err(err) = result {
+                    return Err(AccountsDbPluginError::SlotStatusUpdateError{
+                        msg: format!("Failed to persist the update of slot to the PostgreSQL database. Error: {:?}", err)
+                    });
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn notify_end_of_startup(&mut self) -> Result<()> {
+        info!("Notifying the end of startup for accounts notifications");
+        match &mut self.client {
+            None => {
+                return Err(AccountsDbPluginError::Custom(Box::new(
+                    AccountsDbPluginPostgresError::DataStoreConnectionError {
+                        msg: "There is no connection to the PostgreSQL database.".to_string(),
+                    },
+                )));
+            }
+            Some(client) => {
+                let result = client.notify_end_of_startup();
+
+                if let Err(err) = result {
+                    return Err(AccountsDbPluginError::SlotStatusUpdateError{
+                        msg: format!("Failed to notify the end of startup for accounts notifications. Error: {:?}", err)
+                    });
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn notify_transaction(
+        &mut self,
+        transaction_info: ReplicaTransactionInfoVersions,
+        slot: u64,
+    ) -> Result<()> {
+        match &mut self.client {
+            None => {
+                return Err(AccountsDbPluginError::Custom(Box::new(
+                    AccountsDbPluginPostgresError::DataStoreConnectionError {
+                        msg: "There is no connection to the PostgreSQL database.".to_string(),
+                    },
+                )));
+            }
+            Some(client) => match transaction_info {
+                ReplicaTransactionInfoVersions::V0_0_1(transaction_info) => {
+                    if let Some(transaction_selector) = &self.transaction_selector {
+                        if !transaction_selector.is_transaction_selected(
+                            transaction_info.is_vote,
+                            transaction_info.transaction.message().account_keys_iter(),
+                        ) {
+                            return Ok(());
+                        }
+                    } else {
+                        return Ok(());
+                    }
+
+                    let result = client.log_transaction_info(transaction_info, slot);
+
+                    if let Err(err) = result {
+                        return Err(AccountsDbPluginError::SlotStatusUpdateError{
+                                msg: format!("Failed to persist the transaction info to the PostgreSQL database. Error: {:?}", err)
+                            });
+                    }
+                }
+            },
+        }
+
+        Ok(())
+    }
+
+    fn notify_block_metadata(&mut self, block_info: ReplicaBlockInfoVersions) -> Result<()> {
+        match &mut self.client {
+            None => {
+                return Err(AccountsDbPluginError::Custom(Box::new(
+                    AccountsDbPluginPostgresError::DataStoreConnectionError {
+                        msg: "There is no connection to the PostgreSQL database.".to_string(),
+                    },
+                )));
+            }
+            Some(client) => match block_info {
+                ReplicaBlockInfoVersions::V0_0_1(block_info) => {
+                    let result = client.update_block_metadata(block_info);
+
+                    if let Err(err) = result {
+                        return Err(AccountsDbPluginError::SlotStatusUpdateError{
+                                msg: format!("Failed to persist the update of block metadata to the PostgreSQL database. Error: {:?}", err)
+                            });
+                    }
+                }
+            },
+        }
+
+        Ok(())
+    }
+
+    /// Check if the plugin is interested in account data
+    /// Default is true -- if the plugin is not interested in
+    /// account data, please return false.
+    fn account_data_notifications_enabled(&self) -> bool {
+        self.accounts_selector
+            .as_ref()
+            .map_or_else(|| false, |selector| selector.is_enabled())
+    }
+
+    /// Check if the plugin is interested in transaction data
+    fn transaction_notifications_enabled(&self) -> bool {
+        self.transaction_selector
+            .as_ref()
+            .map_or_else(|| false, |selector| selector.is_enabled())
+    }
+}
+
+impl AccountsDbPluginPostgres {
+    fn create_accounts_selector_from_config(config: &serde_json::Value) -> AccountsSelector {
+        let accounts_selector = &config["accounts_selector"];
+
+        if accounts_selector.is_null() {
+            AccountsSelector::default()
+        } else {
+            let accounts = &accounts_selector["accounts"];
+            let accounts: Vec<String> = if accounts.is_array() {
+                accounts
+                    .as_array()
+                    .unwrap()
+                    .iter()
+                    .map(|val| val.as_str().unwrap().to_string())
+                    .collect()
+            } else {
+                Vec::default()
+            };
+            let owners = &accounts_selector["owners"];
+            let owners: Vec<String> = if owners.is_array() {
+                owners
+                    .as_array()
+                    .unwrap()
+                    .iter()
+                    .map(|val| val.as_str().unwrap().to_string())
+                    .collect()
+            } else {
+                Vec::default()
+            };
+            AccountsSelector::new(&accounts, &owners)
+        }
+    }
+
+    fn create_transaction_selector_from_config(config: &serde_json::Value) -> TransactionSelector {
+        let transaction_selector = &config["transaction_selector"];
+
+        if transaction_selector.is_null() {
+            TransactionSelector::default()
+        } else {
+            let accounts = &transaction_selector["mentions"];
+            let accounts: Vec<String> = if accounts.is_array() {
+                accounts
+                    .as_array()
+                    .unwrap()
+                    .iter()
+                    .map(|val| val.as_str().unwrap().to_string())
+                    .collect()
+            } else {
+                Vec::default()
+            };
+            TransactionSelector::new(&accounts)
+        }
+    }
+
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+
+#[no_mangle]
+#[allow(improper_ctypes_definitions)]
+/// # Safety
+///
+/// This function returns the AccountsDbPluginPostgres pointer as trait AccountsDbPlugin.
+pub unsafe extern "C" fn _create_plugin() -> *mut dyn AccountsDbPlugin {
+    let plugin = AccountsDbPluginPostgres::new();
+    let plugin: Box<dyn AccountsDbPlugin> = Box::new(plugin);
+    Box::into_raw(plugin)
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use {super::*, serde_json};
+
+    #[test]
+    fn test_accounts_selector_from_config() {
+        let config = "{\"accounts_selector\" : { \
+           \"owners\" : [\"9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin\"] \
+        }}";
+
+        let config: serde_json::Value = serde_json::from_str(config).unwrap();
+        AccountsDbPluginPostgres::create_accounts_selector_from_config(&config);
+    }
+}

accountsdb-plugin-postgres/src/lib.rs

@@ -0,0 +1,4 @@
+pub mod accounts_selector;
+pub mod accountsdb_plugin_postgres;
+pub mod postgres_client;
+pub mod transaction_selector;

accountsdb-plugin-postgres/src/postgres_client/postgres_client_block_metadata.rs

@@ -0,0 +1,97 @@
+use {
+    crate::{
+        accountsdb_plugin_postgres::{
+            AccountsDbPluginPostgresConfig, AccountsDbPluginPostgresError,
+        },
+        postgres_client::{
+            postgres_client_transaction::DbReward, SimplePostgresClient, UpdateBlockMetadataRequest,
+        },
+    },
+    chrono::Utc,
+    log::*,
+    postgres::{Client, Statement},
+    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
+        AccountsDbPluginError, ReplicaBlockInfo,
+    },
+};
+
+#[derive(Clone, Debug)]
+pub struct DbBlockInfo {
+    pub slot: i64,
+    pub blockhash: String,
+    pub rewards: Vec<DbReward>,
+    pub block_time: Option<i64>,
+    pub block_height: Option<i64>,
+}
+
+impl<'a> From<&ReplicaBlockInfo<'a>> for DbBlockInfo {
+    fn from(block_info: &ReplicaBlockInfo) -> Self {
+        Self {
+            slot: block_info.slot as i64,
+            blockhash: block_info.blockhash.to_string(),
+            rewards: block_info.rewards.iter().map(DbReward::from).collect(),
+            block_time: block_info.block_time,
+            block_height: block_info
+                .block_height
+                .map(|block_height| block_height as i64),
+        }
+    }
+}
+
+impl SimplePostgresClient {
+    pub(crate) fn build_block_metadata_upsert_statement(
+        client: &mut Client,
+        config: &AccountsDbPluginPostgresConfig,
+    ) -> Result<Statement, AccountsDbPluginError> {
+        let stmt =
+            "INSERT INTO block (slot, blockhash, rewards, block_time, block_height, updated_on) \
+        VALUES ($1, $2, $3, $4, $5, $6)";
+
+        let stmt = client.prepare(stmt);
+
+        match stmt {
+            Err(err) => {
+                return Err(AccountsDbPluginError::Custom(Box::new(AccountsDbPluginPostgresError::DataSchemaError {
+                    msg: format!(
+                        "Error in preparing for the block metadata update PostgreSQL database: ({}) host: {:?} user: {:?} config: {:?}",
+                        err, config.host, config.user, config
+                    ),
+                })));
+            }
+            Ok(stmt) => Ok(stmt),
+        }
+    }
+
+    pub(crate) fn update_block_metadata_impl(
+        &mut self,
+        block_info: UpdateBlockMetadataRequest,
+    ) -> Result<(), AccountsDbPluginError> {
+        let client = self.client.get_mut().unwrap();
+        let statement = &client.update_block_metadata_stmt;
+        let client = &mut client.client;
+        let updated_on = Utc::now().naive_utc();
+
+        let block_info = block_info.block_info;
+        let result = client.query(
+            statement,
+            &[
+                &block_info.slot,
+                &block_info.blockhash,
+                &block_info.rewards,
+                &block_info.block_time,
+                &block_info.block_height,
+                &updated_on,
+            ],
+        );
+
+        if let Err(err) = result {
+            let msg = format!(
+                "Failed to persist the update of block metadata to the PostgreSQL database. Error: {:?}",
+                err);
+            error!("{}", msg);
+            return Err(AccountsDbPluginError::AccountsUpdateError { msg });
+        }
+
+        Ok(())
+    }
+}

accountsdb-plugin-postgres/src/transaction_selector.rs

@@ -0,0 +1,194 @@
+/// The transaction selector is responsible for filtering transactions
+/// in the plugin framework.
+use {log::*, solana_sdk::pubkey::Pubkey, std::collections::HashSet};
+
+pub(crate) struct TransactionSelector {
+    pub mentioned_addresses: HashSet<Vec<u8>>,
+    pub select_all_transactions: bool,
+    pub select_all_vote_transactions: bool,
+}
+
+#[allow(dead_code)]
+impl TransactionSelector {
+    pub fn default() -> Self {
+        Self {
+            mentioned_addresses: HashSet::default(),
+            select_all_transactions: false,
+            select_all_vote_transactions: false,
+        }
+    }
+
+    /// Create a selector based on the mentioned addresses
+    /// To select all transactions use ["*"] or ["all"]
+    /// To select all vote transactions, use ["all_votes"]
+    /// To select transactions mentioning specific addresses use ["<pubkey1>", "<pubkey2>", ...]
+    pub fn new(mentioned_addresses: &[String]) -> Self {
+        info!(
+            "Creating TransactionSelector from addresses: {:?}",
+            mentioned_addresses
+        );
+
+        let select_all_transactions = mentioned_addresses
+            .iter()
+            .any(|key| key == "*" || key == "all");
+        if select_all_transactions {
+            return Self {
+                mentioned_addresses: HashSet::default(),
+                select_all_transactions,
+                select_all_vote_transactions: true,
+            };
+        }
+        let select_all_vote_transactions = mentioned_addresses.iter().any(|key| key == "all_votes");
+        if select_all_vote_transactions {
+            return Self {
+                mentioned_addresses: HashSet::default(),
+                select_all_transactions,
+                select_all_vote_transactions: true,
+            };
+        }
+
+        let mentioned_addresses = mentioned_addresses
+            .iter()
+            .map(|key| bs58::decode(key).into_vec().unwrap())
+            .collect();
+
+        Self {
+            mentioned_addresses,
+            select_all_transactions: false,
+            select_all_vote_transactions: false,
+        }
+    }
+
+    /// Check if a transaction is of interest.
+    pub fn is_transaction_selected(
+        &self,
+        is_vote: bool,
+        mentioned_addresses: Box<dyn Iterator<Item = &Pubkey> + '_>,
+    ) -> bool {
+        if !self.is_enabled() {
+            return false;
+        }
+
+        if self.select_all_transactions || (self.select_all_vote_transactions && is_vote) {
+            return true;
+        }
+        for address in mentioned_addresses {
+            if self.mentioned_addresses.contains(address.as_ref()) {
+                return true;
+            }
+        }
+        false
+    }
+
+    /// Check if any transaction is of interest at all
+    pub fn is_enabled(&self) -> bool {
+        self.select_all_transactions
+            || self.select_all_vote_transactions
+            || !self.mentioned_addresses.is_empty()
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    use super::*;
+
+    #[test]
+    fn test_select_transaction() {
+        let pubkey1 = Pubkey::new_unique();
+        let pubkey2 = Pubkey::new_unique();
+
+        let selector = TransactionSelector::new(&[pubkey1.to_string()]);
+
+        assert!(selector.is_enabled());
+
+        let addresses = [pubkey1];
+
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey2];
+        assert!(!selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey1, pubkey2];
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+    }
+
+    #[test]
+    fn test_select_all_transaction_using_wildcard() {
+        let pubkey1 = Pubkey::new_unique();
+        let pubkey2 = Pubkey::new_unique();
+
+        let selector = TransactionSelector::new(&["*".to_string()]);
+
+        assert!(selector.is_enabled());
+
+        let addresses = [pubkey1];
+
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey2];
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey1, pubkey2];
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+    }
+
+    #[test]
+    fn test_select_all_transaction_all() {
+        let pubkey1 = Pubkey::new_unique();
+        let pubkey2 = Pubkey::new_unique();
+
+        let selector = TransactionSelector::new(&["all".to_string()]);
+
+        assert!(selector.is_enabled());
+
+        let addresses = [pubkey1];
+
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey2];
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey1, pubkey2];
+        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
+    }
+
+    #[test]
+    fn test_select_all_vote_transaction() {
+        let pubkey1 = Pubkey::new_unique();
+        let pubkey2 = Pubkey::new_unique();
+
+        let selector = TransactionSelector::new(&["all_votes".to_string()]);
+
+        assert!(selector.is_enabled());
+
+        let addresses = [pubkey1];
+
+        assert!(!selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey2];
+        assert!(selector.is_transaction_selected(true, Box::new(addresses.iter())));
+
+        let addresses = [pubkey1, pubkey2];
+        assert!(selector.is_transaction_selected(true, Box::new(addresses.iter())));
+    }
+
+    #[test]
+    fn test_select_no_transaction() {
+        let pubkey1 = Pubkey::new_unique();
+        let pubkey2 = Pubkey::new_unique();
+
+        let selector = TransactionSelector::new(&[]);
+
+        assert!(!selector.is_enabled());
+
+        let addresses = [pubkey1];
+
+        assert!(!selector.is_transaction_selected(false, Box::new(addresses.iter())));
+
+        let addresses = [pubkey2];
+        assert!(!selector.is_transaction_selected(true, Box::new(addresses.iter())));
+
+        let addresses = [pubkey1, pubkey2];
+        assert!(!selector.is_transaction_selected(true, Box::new(addresses.iter())));
+    }
+}

banking-bench/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2018"
+edition = "2021"
 name = "solana-banking-bench"
-version = "1.7.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,21 +10,21 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-crossbeam-channel = "0.4"
-log = "0.4.11"
+crossbeam-channel = "0.5"
+log = "0.4.14"
 rand = "0.7.0"
-rayon = "1.5.0"
-solana-core = { path = "../core", version = "=1.7.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-gossip = { path = "../gossip", version = "=1.7.0" }
-solana-ledger = { path = "../ledger", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-perf = { path = "../perf", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-streamer = { path = "../streamer", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+rayon = "1.5.1"
+solana-core = { path = "../core", version = "=1.10.0" }
+solana-gossip = { path = "../gossip", version = "=1.10.0" }
+solana-ledger = { path = "../ledger", version = "=1.10.0" }
+solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-perf = { path = "../perf", version = "=1.10.0" }
+solana-poh = { path = "../poh", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-streamer = { path = "../streamer", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -1,37 +1,37 @@
 #![allow(clippy::integer_arithmetic)]
-use clap::{crate_description, crate_name, value_t, App, Arg};
-use crossbeam_channel::unbounded;
-use log::*;
-use rand::{thread_rng, Rng};
-use rayon::prelude::*;
-use solana_core::{
-    banking_stage::{create_test_recorder, BankingStage},
-    poh_recorder::PohRecorder,
-    poh_recorder::WorkingBankEntry,
-};
-use solana_gossip::{cluster_info::ClusterInfo, cluster_info::Node};
-use solana_ledger::{
-    blockstore::Blockstore,
-    genesis_utils::{create_genesis_config, GenesisConfigInfo},
-    get_tmp_ledger_path,
-};
-use solana_measure::measure::Measure;
-use solana_perf::packet::to_packets_chunked;
-use solana_runtime::{
-    accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks,
-};
-use solana_sdk::{
-    hash::Hash,
-    signature::Keypair,
-    signature::Signature,
-    system_transaction,
-    timing::{duration_as_us, timestamp},
-    transaction::Transaction,
-};
-use std::{
-    sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex},
-    thread::sleep,
-    time::{Duration, Instant},
+use {
+    clap::{crate_description, crate_name, value_t, App, Arg},
+    crossbeam_channel::unbounded,
+    log::*,
+    rand::{thread_rng, Rng},
+    rayon::prelude::*,
+    solana_core::{banking_stage::BankingStage, packet_deduper::PacketDeduper},
+    solana_gossip::cluster_info::{ClusterInfo, Node},
+    solana_ledger::{
+        blockstore::Blockstore,
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+        get_tmp_ledger_path,
+    },
+    solana_measure::measure::Measure,
+    solana_perf::packet::to_packet_batches,
+    solana_poh::poh_recorder::{create_test_recorder, PohRecorder, WorkingBankEntry},
+    solana_runtime::{
+        accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks,
+        cost_model::CostModel,
+    },
+    solana_sdk::{
+        hash::Hash,
+        signature::{Keypair, Signature},
+        system_transaction,
+        timing::{duration_as_us, timestamp},
+        transaction::Transaction,
+    },
+    solana_streamer::socket::SocketAddrSpace,
+    std::{
+        sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex, RwLock},
+        thread::sleep,
+        time::{Duration, Instant},
+    },
 };
 
 fn check_txs(
@@ -77,7 +77,7 @@ fn make_accounts_txs(
         .into_par_iter()
         .map(|_| {
             let mut new = dummy.clone();
-            let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+            let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
             if !same_payer {
                 new.message.account_keys[0] = solana_sdk::pubkey::new_rand();
             }
@@ -168,8 +168,9 @@ fn main() {
 
     let (verified_sender, verified_receiver) = unbounded();
     let (vote_sender, vote_receiver) = unbounded();
+    let (tpu_vote_sender, tpu_vote_receiver) = unbounded();
     let (replay_vote_sender, _replay_vote_receiver) = unbounded();
-    let bank0 = Bank::new(&genesis_config);
+    let bank0 = Bank::new_for_benches(&genesis_config);
     let mut bank_forks = BankForks::new(bank0);
     let mut bank = bank_forks.working_bank();
 
@@ -188,7 +189,7 @@ fn main() {
             genesis_config.hash(),
         );
         // Ignore any pesky duplicate signature errors in the case we are using single-payer
-        let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+        let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
         fund.signatures = vec![Signature::new(&sig[0..64])];
         let x = bank.process_transaction(&fund);
         x.unwrap();
@@ -198,19 +199,20 @@ fn main() {
     if !skip_sanity {
         //sanity check, make sure all the transactions can execute sequentially
         transactions.iter().for_each(|tx| {
-            let res = bank.process_transaction(&tx);
+            let res = bank.process_transaction(tx);
             assert!(res.is_ok(), "sanity test transactions error: {:?}", res);
         });
         bank.clear_signatures();
         //sanity check, make sure all the transactions can execute in parallel
-        let res = bank.process_transactions(&transactions);
+
+        let res = bank.process_transactions(transactions.iter());
         for r in res {
             assert!(r.is_ok(), "sanity parallel execution error: {:?}", r);
         }
         bank.clear_signatures();
     }
 
-    let mut verified: Vec<_> = to_packets_chunked(&transactions, packets_per_chunk);
+    let mut verified: Vec<_> = to_packet_batches(&transactions, packets_per_chunk);
     let ledger_path = get_tmp_ledger_path!();
     {
         let blockstore = Arc::new(
@@ -218,15 +220,23 @@ fn main() {
         );
         let (exit, poh_recorder, poh_service, signal_receiver) =
             create_test_recorder(&bank, &blockstore, None);
-        let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
+        let cluster_info = ClusterInfo::new(
+            Node::new_localhost().info,
+            Arc::new(Keypair::new()),
+            SocketAddrSpace::Unspecified,
+        );
         let cluster_info = Arc::new(cluster_info);
+        let packet_deduper = PacketDeduper::default();
         let banking_stage = BankingStage::new(
             &cluster_info,
             &poh_recorder,
             verified_receiver,
+            tpu_vote_receiver,
             vote_receiver,
             None,
             replay_vote_sender,
+            Arc::new(RwLock::new(CostModel::default())),
+            packet_deduper.clone(),
         );
         poh_recorder.lock().unwrap().set_bank(&bank);
 
@@ -306,11 +316,10 @@ fn main() {
                 tx_total_us += duration_as_us(&now.elapsed());
 
                 let mut poh_time = Measure::start("poh_time");
-                poh_recorder.lock().unwrap().reset(
-                    bank.last_blockhash(),
-                    bank.slot(),
-                    Some((bank.slot(), bank.slot() + 1)),
-                );
+                poh_recorder
+                    .lock()
+                    .unwrap()
+                    .reset(bank.clone(), Some((bank.slot(), bank.slot() + 1)));
                 poh_time.stop();
 
                 let mut new_bank_time = Measure::start("new_bank");
@@ -342,6 +351,7 @@ fn main() {
             // in this chunk, but since we rotate between CHUNKS then
             // we should clear them by the time we come around again to re-use that chunk.
             bank.clear_signatures();
+            packet_deduper.reset();
             total_us += duration_as_us(&now.elapsed());
             debug!(
                 "time: {} us checked: {} sent: {}",
@@ -354,10 +364,10 @@ fn main() {
             if bank.slot() > 0 && bank.slot() % 16 == 0 {
                 for tx in transactions.iter_mut() {
                     tx.message.recent_blockhash = bank.last_blockhash();
-                    let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+                    let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
                     tx.signatures[0] = Signature::new(&sig[0..64]);
                 }
-                verified = to_packets_chunked(&transactions.clone(), packets_per_chunk);
+                verified = to_packet_batches(&transactions.clone(), packets_per_chunk);
             }
 
             start += chunk_len;
@@ -379,6 +389,7 @@ fn main() {
         );
 
         drop(verified_sender);
+        drop(tpu_vote_sender);
         drop(vote_sender);
         exit.store(true, Ordering::Relaxed);
         banking_stage.join().unwrap();

banks-client/Cargo.toml

@@ -1,30 +1,28 @@
 [package]
 name = "solana-banks-client"
-version = "1.7.0"
+version = "1.10.0"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-banks-client"
-edition = "2018"
+edition = "2021"
 
 [dependencies]
-bincode = "1.3.1"
-borsh = "0.8.1"
-borsh-derive = "0.8.1"
+borsh = "0.9.1"
 futures = "0.3"
-mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "=1.7.0" }
-solana-program = { path = "../sdk/program", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-tarpc = { version = "0.24.1", features = ["full"] }
+solana-banks-interface = { path = "../banks-interface", version = "=1.10.0" }
+solana-program = { path = "../sdk/program", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+tarpc = { version = "0.27.2", features = ["full"] }
+thiserror = "1.0"
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-banks-server = { path = "../banks-server", version = "=1.7.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-banks-server = { path = "../banks-server", version = "=1.10.0" }
 
 [lib]
 crate-type = ["lib"]

banks-client/src/error.rs

@@ -0,0 +1,73 @@
+use {
+    solana_sdk::{transaction::TransactionError, transport::TransportError},
+    std::io,
+    tarpc::client::RpcError,
+    thiserror::Error,
+};
+
+/// Errors from BanksClient
+#[derive(Error, Debug)]
+pub enum BanksClientError {
+    #[error("client error: {0}")]
+    ClientError(&'static str),
+
+    #[error(transparent)]
+    Io(#[from] io::Error),
+
+    #[error(transparent)]
+    RpcError(#[from] RpcError),
+
+    #[error("transport transaction error: {0}")]
+    TransactionError(#[from] TransactionError),
+
+    #[error("simulation error: {err:?}, logs: {logs:?}, units_consumed: {units_consumed:?}")]
+    SimulationError {
+        err: TransactionError,
+        logs: Vec<String>,
+        units_consumed: u64,
+    },
+}
+
+impl BanksClientError {
+    pub fn unwrap(&self) -> TransactionError {
+        match self {
+            BanksClientError::TransactionError(err)
+            | BanksClientError::SimulationError { err, .. } => err.clone(),
+            _ => panic!("unexpected transport error"),
+        }
+    }
+}
+
+impl From<BanksClientError> for io::Error {
+    fn from(err: BanksClientError) -> Self {
+        match err {
+            BanksClientError::ClientError(err) => Self::new(io::ErrorKind::Other, err.to_string()),
+            BanksClientError::Io(err) => err,
+            BanksClientError::RpcError(err) => Self::new(io::ErrorKind::Other, err.to_string()),
+            BanksClientError::TransactionError(err) => {
+                Self::new(io::ErrorKind::Other, err.to_string())
+            }
+            BanksClientError::SimulationError { err, .. } => {
+                Self::new(io::ErrorKind::Other, err.to_string())
+            }
+        }
+    }
+}
+
+impl From<BanksClientError> for TransportError {
+    fn from(err: BanksClientError) -> Self {
+        match err {
+            BanksClientError::ClientError(err) => {
+                Self::IoError(io::Error::new(io::ErrorKind::Other, err.to_string()))
+            }
+            BanksClientError::Io(err) => {
+                Self::IoError(io::Error::new(io::ErrorKind::Other, err.to_string()))
+            }
+            BanksClientError::RpcError(err) => {
+                Self::IoError(io::Error::new(io::ErrorKind::Other, err.to_string()))
+            }
+            BanksClientError::TransactionError(err) => Self::TransactionError(err),
+            BanksClientError::SimulationError { err, .. } => Self::TransactionError(err),
+        }
+    }
+}

banks-client/src/lib.rs

@@ -5,31 +5,34 @@
 //! but they are undocumented, may change over time, and are generally more
 //! cumbersome to use.
 
-use borsh::BorshDeserialize;
-use futures::{future::join_all, Future, FutureExt};
 pub use solana_banks_interface::{BanksClient as TarpcClient, TransactionStatus};
-use solana_banks_interface::{BanksRequest, BanksResponse};
-use solana_program::{
-    clock::Slot, fee_calculator::FeeCalculator, hash::Hash, program_pack::Pack, pubkey::Pubkey,
-    rent::Rent, sysvar,
+use {
+    crate::error::BanksClientError,
+    borsh::BorshDeserialize,
+    futures::{future::join_all, Future, FutureExt, TryFutureExt},
+    solana_banks_interface::{BanksRequest, BanksResponse, BanksTransactionResultWithSimulation},
+    solana_program::{
+        clock::Slot, fee_calculator::FeeCalculator, hash::Hash, program_pack::Pack, pubkey::Pubkey,
+        rent::Rent, sysvar::Sysvar,
+    },
+    solana_sdk::{
+        account::{from_account, Account},
+        commitment_config::CommitmentLevel,
+        message::Message,
+        signature::Signature,
+        transaction::{self, Transaction},
+    },
+    tarpc::{
+        client::{self, NewClient, RequestDispatch},
+        context::{self, Context},
+        serde_transport::tcp,
+        ClientMessage, Response, Transport,
+    },
+    tokio::{net::ToSocketAddrs, time::Duration},
+    tokio_serde::formats::Bincode,
 };
-use solana_sdk::{
-    account::{from_account, Account},
-    commitment_config::CommitmentLevel,
-    signature::Signature,
-    transaction::{self, Transaction},
-    transport,
-};
-use std::io::{self, Error, ErrorKind};
-use tarpc::{
-    client::{self, channel::RequestDispatch, NewClient},
-    context::{self, Context},
-    rpc::{ClientMessage, Response},
-    serde_transport::tcp,
-    Transport,
-};
-use tokio::{net::ToSocketAddrs, time::Duration};
-use tokio_serde::formats::Bincode;
+
+mod error;
 
 // This exists only for backward compatibility
 pub trait BanksClientExt {}
@@ -55,34 +58,55 @@ impl BanksClient {
         &mut self,
         ctx: Context,
         transaction: Transaction,
-    ) -> impl Future<Output = io::Result<()>> + '_ {
-        self.inner.send_transaction_with_context(ctx, transaction)
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
+        self.inner
+            .send_transaction_with_context(ctx, transaction)
+            .map_err(Into::into)
     }
 
+    #[deprecated(
+        since = "1.9.0",
+        note = "Please use `get_fee_for_message` or `is_blockhash_valid` instead"
+    )]
     pub fn get_fees_with_commitment_and_context(
         &mut self,
         ctx: Context,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, Slot)>> + '_ {
+    ) -> impl Future<Output = Result<(FeeCalculator, Hash, u64), BanksClientError>> + '_ {
+        #[allow(deprecated)]
         self.inner
             .get_fees_with_commitment_and_context(ctx, commitment)
+            .map_err(Into::into)
     }
 
     pub fn get_transaction_status_with_context(
         &mut self,
         ctx: Context,
         signature: Signature,
-    ) -> impl Future<Output = io::Result<Option<TransactionStatus>>> + '_ {
+    ) -> impl Future<Output = Result<Option<TransactionStatus>, BanksClientError>> + '_ {
         self.inner
             .get_transaction_status_with_context(ctx, signature)
+            .map_err(Into::into)
     }
 
     pub fn get_slot_with_context(
         &mut self,
         ctx: Context,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Slot>> + '_ {
-        self.inner.get_slot_with_context(ctx, commitment)
+    ) -> impl Future<Output = Result<Slot, BanksClientError>> + '_ {
+        self.inner
+            .get_slot_with_context(ctx, commitment)
+            .map_err(Into::into)
+    }
+
+    pub fn get_block_height_with_context(
+        &mut self,
+        ctx: Context,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = Result<Slot, BanksClientError>> + '_ {
+        self.inner
+            .get_block_height_with_context(ctx, commitment)
+            .map_err(Into::into)
     }
 
     pub fn process_transaction_with_commitment_and_context(
@@ -90,9 +114,26 @@ impl BanksClient {
         ctx: Context,
         transaction: Transaction,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Option<transaction::Result<()>>>> + '_ {
+    ) -> impl Future<Output = Result<Option<transaction::Result<()>>, BanksClientError>> + '_ {
         self.inner
             .process_transaction_with_commitment_and_context(ctx, transaction, commitment)
+            .map_err(Into::into)
+    }
+
+    pub fn process_transaction_with_preflight_and_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = Result<BanksTransactionResultWithSimulation, BanksClientError>> + '_
+    {
+        self.inner
+            .process_transaction_with_preflight_and_commitment_and_context(
+                ctx,
+                transaction,
+                commitment,
+            )
+            .map_err(Into::into)
     }
 
     pub fn get_account_with_commitment_and_context(
@@ -100,9 +141,10 @@ impl BanksClient {
         ctx: Context,
         address: Pubkey,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+    ) -> impl Future<Output = Result<Option<Account>, BanksClientError>> + '_ {
         self.inner
             .get_account_with_commitment_and_context(ctx, address, commitment)
+            .map_err(Into::into)
     }
 
     /// Send a transaction and return immediately. The server will resend the
@@ -111,34 +153,49 @@ impl BanksClient {
     pub fn send_transaction(
         &mut self,
         transaction: Transaction,
-    ) -> impl Future<Output = io::Result<()>> + '_ {
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
         self.send_transaction_with_context(context::current(), transaction)
     }
 
     /// Return the fee parameters associated with a recent, rooted blockhash. The cluster
     /// will use the transaction's blockhash to look up these same fee parameters and
     /// use them to calculate the transaction fee.
+    #[deprecated(
+        since = "1.9.0",
+        note = "Please use `get_fee_for_message` or `is_blockhash_valid` instead"
+    )]
     pub fn get_fees(
         &mut self,
-    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, Slot)>> + '_ {
+    ) -> impl Future<Output = Result<(FeeCalculator, Hash, u64), BanksClientError>> + '_ {
+        #[allow(deprecated)]
         self.get_fees_with_commitment_and_context(context::current(), CommitmentLevel::default())
     }
 
-    /// Return the cluster rent
-    pub fn get_rent(&mut self) -> impl Future<Output = io::Result<Rent>> + '_ {
-        self.get_account(sysvar::rent::id()).map(|result| {
-            let rent_sysvar = result?
-                .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Rent sysvar not present"))?;
-            from_account::<Rent, _>(&rent_sysvar).ok_or_else(|| {
-                io::Error::new(io::ErrorKind::Other, "Failed to deserialize Rent sysvar")
-            })
+    /// Return the cluster Sysvar
+    pub fn get_sysvar<T: Sysvar>(
+        &mut self,
+    ) -> impl Future<Output = Result<T, BanksClientError>> + '_ {
+        self.get_account(T::id()).map(|result| {
+            let sysvar = result?.ok_or(BanksClientError::ClientError("Sysvar not present"))?;
+            from_account::<T, _>(&sysvar).ok_or(BanksClientError::ClientError(
+                "Failed to deserialize sysvar",
+            ))
         })
     }
 
+    /// Return the cluster rent
+    pub fn get_rent(&mut self) -> impl Future<Output = Result<Rent, BanksClientError>> + '_ {
+        self.get_sysvar::<Rent>()
+    }
+
     /// Return a recent, rooted blockhash from the server. The cluster will only accept
     /// transactions with a blockhash that has not yet expired. Use the `get_fees`
     /// method to get both a blockhash and the blockhash's last valid slot.
-    pub fn get_recent_blockhash(&mut self) -> impl Future<Output = io::Result<Hash>> + '_ {
+    #[deprecated(since = "1.9.0", note = "Please use `get_latest_blockhash` instead")]
+    pub fn get_recent_blockhash(
+        &mut self,
+    ) -> impl Future<Output = Result<Hash, BanksClientError>> + '_ {
+        #[allow(deprecated)]
         self.get_fees().map(|result| Ok(result?.1))
     }
 
@@ -148,23 +205,71 @@ impl BanksClient {
         &mut self,
         transaction: Transaction,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = transport::Result<()>> + '_ {
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
         let mut ctx = context::current();
         ctx.deadline += Duration::from_secs(50);
         self.process_transaction_with_commitment_and_context(ctx, transaction, commitment)
             .map(|result| match result? {
-                None => {
-                    Err(Error::new(ErrorKind::TimedOut, "invalid blockhash or fee-payer").into())
-                }
+                None => Err(BanksClientError::ClientError(
+                    "invalid blockhash or fee-payer",
+                )),
                 Some(transaction_result) => Ok(transaction_result?),
             })
     }
 
+    /// Send a transaction and return any preflight (sanitization or simulation) errors, or return
+    /// after the transaction has been rejected or reached the given level of commitment.
+    pub fn process_transaction_with_preflight_and_commitment(
+        &mut self,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
+        let mut ctx = context::current();
+        ctx.deadline += Duration::from_secs(50);
+        self.process_transaction_with_preflight_and_commitment_and_context(
+            ctx,
+            transaction,
+            commitment,
+        )
+        .map(|result| match result? {
+            BanksTransactionResultWithSimulation {
+                result: None,
+                simulation_details: _,
+            } => Err(BanksClientError::ClientError(
+                "invalid blockhash or fee-payer",
+            )),
+            BanksTransactionResultWithSimulation {
+                result: Some(Err(err)),
+                simulation_details: Some(simulation_details),
+            } => Err(BanksClientError::SimulationError {
+                err,
+                logs: simulation_details.logs,
+                units_consumed: simulation_details.units_consumed,
+            }),
+            BanksTransactionResultWithSimulation {
+                result: Some(result),
+                simulation_details: _,
+            } => result.map_err(Into::into),
+        })
+    }
+
+    /// Send a transaction and return any preflight (sanitization or simulation) errors, or return
+    /// after the transaction has been finalized or rejected.
+    pub fn process_transaction_with_preflight(
+        &mut self,
+        transaction: Transaction,
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
+        self.process_transaction_with_preflight_and_commitment(
+            transaction,
+            CommitmentLevel::default(),
+        )
+    }
+
     /// Send a transaction and return until the transaction has been finalized or rejected.
     pub fn process_transaction(
         &mut self,
         transaction: Transaction,
-    ) -> impl Future<Output = transport::Result<()>> + '_ {
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
         self.process_transaction_with_commitment(transaction, CommitmentLevel::default())
     }
 
@@ -172,7 +277,7 @@ impl BanksClient {
         &mut self,
         transactions: Vec<Transaction>,
         commitment: CommitmentLevel,
-    ) -> transport::Result<()> {
+    ) -> Result<(), BanksClientError> {
         let mut clients: Vec<_> = transactions.iter().map(|_| self.clone()).collect();
         let futures = clients
             .iter_mut()
@@ -188,23 +293,31 @@ impl BanksClient {
     pub fn process_transactions(
         &mut self,
         transactions: Vec<Transaction>,
-    ) -> impl Future<Output = transport::Result<()>> + '_ {
+    ) -> impl Future<Output = Result<(), BanksClientError>> + '_ {
         self.process_transactions_with_commitment(transactions, CommitmentLevel::default())
     }
 
-    /// Return the most recent rooted slot height. All transactions at or below this height
-    /// are said to be finalized. The cluster will not fork to a higher slot height.
-    pub fn get_root_slot(&mut self) -> impl Future<Output = io::Result<Slot>> + '_ {
+    /// Return the most recent rooted slot. All transactions at or below this slot
+    /// are said to be finalized. The cluster will not fork to a higher slot.
+    pub fn get_root_slot(&mut self) -> impl Future<Output = Result<Slot, BanksClientError>> + '_ {
         self.get_slot_with_context(context::current(), CommitmentLevel::default())
     }
 
+    /// Return the most recent rooted block height. All transactions at or below this height
+    /// are said to be finalized. The cluster will not fork to a higher block height.
+    pub fn get_root_block_height(
+        &mut self,
+    ) -> impl Future<Output = Result<Slot, BanksClientError>> + '_ {
+        self.get_block_height_with_context(context::current(), CommitmentLevel::default())
+    }
+
     /// Return the account at the given address at the slot corresponding to the given
     /// commitment level. If the account is not found, None is returned.
     pub fn get_account_with_commitment(
         &mut self,
         address: Pubkey,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+    ) -> impl Future<Output = Result<Option<Account>, BanksClientError>> + '_ {
         self.get_account_with_commitment_and_context(context::current(), address, commitment)
     }
 
@@ -213,7 +326,7 @@ impl BanksClient {
     pub fn get_account(
         &mut self,
         address: Pubkey,
-    ) -> impl Future<Output = io::Result<Option<Account>>> + '_ {
+    ) -> impl Future<Output = Result<Option<Account>, BanksClientError>> + '_ {
         self.get_account_with_commitment(address, CommitmentLevel::default())
     }
 
@@ -222,12 +335,11 @@ impl BanksClient {
     pub fn get_packed_account_data<T: Pack>(
         &mut self,
         address: Pubkey,
-    ) -> impl Future<Output = io::Result<T>> + '_ {
+    ) -> impl Future<Output = Result<T, BanksClientError>> + '_ {
         self.get_account(address).map(|result| {
-            let account =
-                result?.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Account not found"))?;
+            let account = result?.ok_or(BanksClientError::ClientError("Account not found"))?;
             T::unpack_from_slice(&account.data)
-                .map_err(|_| io::Error::new(io::ErrorKind::Other, "Failed to deserialize account"))
+                .map_err(|_| BanksClientError::ClientError("Failed to deserialize account"))
         })
     }
 
@@ -236,11 +348,10 @@ impl BanksClient {
     pub fn get_account_data_with_borsh<T: BorshDeserialize>(
         &mut self,
         address: Pubkey,
-    ) -> impl Future<Output = io::Result<T>> + '_ {
+    ) -> impl Future<Output = Result<T, BanksClientError>> + '_ {
         self.get_account(address).map(|result| {
-            let account =
-                result?.ok_or_else(|| io::Error::new(io::ErrorKind::Other, "account not found"))?;
-            T::try_from_slice(&account.data)
+            let account = result?.ok_or(BanksClientError::ClientError("Account not found"))?;
+            T::try_from_slice(&account.data).map_err(Into::into)
         })
     }
 
@@ -250,14 +361,17 @@ impl BanksClient {
         &mut self,
         address: Pubkey,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<u64>> + '_ {
+    ) -> impl Future<Output = Result<u64, BanksClientError>> + '_ {
         self.get_account_with_commitment_and_context(context::current(), address, commitment)
             .map(|result| Ok(result?.map(|x| x.lamports).unwrap_or(0)))
     }
 
     /// Return the balance in lamports of an account at the given address at the time
     /// of the most recent root slot.
-    pub fn get_balance(&mut self, address: Pubkey) -> impl Future<Output = io::Result<u64>> + '_ {
+    pub fn get_balance(
+        &mut self,
+        address: Pubkey,
+    ) -> impl Future<Output = Result<u64, BanksClientError>> + '_ {
         self.get_balance_with_commitment(address, CommitmentLevel::default())
     }
 
@@ -269,7 +383,7 @@ impl BanksClient {
     pub fn get_transaction_status(
         &mut self,
         signature: Signature,
-    ) -> impl Future<Output = io::Result<Option<TransactionStatus>>> + '_ {
+    ) -> impl Future<Output = Result<Option<TransactionStatus>, BanksClientError>> + '_ {
         self.get_transaction_status_with_context(context::current(), signature)
     }
 
@@ -277,7 +391,7 @@ impl BanksClient {
     pub async fn get_transaction_statuses(
         &mut self,
         signatures: Vec<Signature>,
-    ) -> io::Result<Vec<Option<TransactionStatus>>> {
+    ) -> Result<Vec<Option<TransactionStatus>>, BanksClientError> {
         // tarpc futures oddly hold a mutable reference back to the client so clone the client upfront
         let mut clients_and_signatures: Vec<_> = signatures
             .into_iter()
@@ -293,36 +407,78 @@ impl BanksClient {
         // Convert Vec<Result<_, _>> to Result<Vec<_>>
         statuses.into_iter().collect()
     }
+
+    pub fn get_latest_blockhash(
+        &mut self,
+    ) -> impl Future<Output = Result<Hash, BanksClientError>> + '_ {
+        self.get_latest_blockhash_with_commitment(CommitmentLevel::default())
+            .map(|result| {
+                result?
+                    .map(|x| x.0)
+                    .ok_or(BanksClientError::ClientError("valid blockhash not found"))
+                    .map_err(Into::into)
+            })
+    }
+
+    pub fn get_latest_blockhash_with_commitment(
+        &mut self,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = Result<Option<(Hash, u64)>, BanksClientError>> + '_ {
+        self.get_latest_blockhash_with_commitment_and_context(context::current(), commitment)
+    }
+
+    pub fn get_latest_blockhash_with_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        commitment: CommitmentLevel,
+    ) -> impl Future<Output = Result<Option<(Hash, u64)>, BanksClientError>> + '_ {
+        self.inner
+            .get_latest_blockhash_with_commitment_and_context(ctx, commitment)
+            .map_err(Into::into)
+    }
+
+    pub fn get_fee_for_message_with_commitment_and_context(
+        &mut self,
+        ctx: Context,
+        commitment: CommitmentLevel,
+        message: Message,
+    ) -> impl Future<Output = Result<Option<u64>, BanksClientError>> + '_ {
+        self.inner
+            .get_fee_for_message_with_commitment_and_context(ctx, commitment, message)
+            .map_err(Into::into)
+    }
 }
 
-pub async fn start_client<C>(transport: C) -> io::Result<BanksClient>
+pub async fn start_client<C>(transport: C) -> Result<BanksClient, BanksClientError>
 where
     C: Transport<ClientMessage<BanksRequest>, Response<BanksResponse>> + Send + 'static,
 {
     Ok(BanksClient {
-        inner: TarpcClient::new(client::Config::default(), transport).spawn()?,
+        inner: TarpcClient::new(client::Config::default(), transport).spawn(),
     })
 }
 
-pub async fn start_tcp_client<T: ToSocketAddrs>(addr: T) -> io::Result<BanksClient> {
+pub async fn start_tcp_client<T: ToSocketAddrs>(addr: T) -> Result<BanksClient, BanksClientError> {
     let transport = tcp::connect(addr, Bincode::default).await?;
     Ok(BanksClient {
-        inner: TarpcClient::new(client::Config::default(), transport).spawn()?,
+        inner: TarpcClient::new(client::Config::default(), transport).spawn(),
     })
 }
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use solana_banks_server::banks_server::start_local_server;
-    use solana_runtime::{
-        bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache,
-        genesis_utils::create_genesis_config,
+    use {
+        super::*,
+        solana_banks_server::banks_server::start_local_server,
+        solana_runtime::{
+            bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache,
+            genesis_utils::create_genesis_config,
+        },
+        solana_sdk::{message::Message, signature::Signer, system_instruction},
+        std::sync::{Arc, RwLock},
+        tarpc::transport,
+        tokio::{runtime::Runtime, time::sleep},
     };
-    use solana_sdk::{message::Message, signature::Signer, system_instruction};
-    use std::sync::{Arc, RwLock};
-    use tarpc::transport;
-    use tokio::{runtime::Runtime, time::sleep};
 
     #[test]
     fn test_banks_client_new() {
@@ -331,13 +487,13 @@ mod tests {
     }
 
     #[test]
-    fn test_banks_server_transfer_via_server() -> io::Result<()> {
+    fn test_banks_server_transfer_via_server() -> Result<(), BanksClientError> {
         // This test shows the preferred way to interact with BanksServer.
         // It creates a runtime explicitly (no globals via tokio macros) and calls
         // `runtime.block_on()` just once, to run all the async code.
 
         let genesis = create_genesis_config(10);
-        let bank = Bank::new(&genesis.genesis_config);
+        let bank = Bank::new_for_tests(&genesis.genesis_config);
         let slot = bank.slot();
         let block_commitment_cache = Arc::new(RwLock::new(
             BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
@@ -350,10 +506,12 @@ mod tests {
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
+            let client_transport =
+                start_local_server(bank_forks, block_commitment_cache, Duration::from_millis(1))
+                    .await;
             let mut banks_client = start_client(client_transport).await?;
 
-            let recent_blockhash = banks_client.get_recent_blockhash().await?;
+            let recent_blockhash = banks_client.get_latest_blockhash().await?;
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
             banks_client.process_transaction(transaction).await.unwrap();
             assert_eq!(banks_client.get_balance(bob_pubkey).await?, 1);
@@ -362,13 +520,13 @@ mod tests {
     }
 
     #[test]
-    fn test_banks_server_transfer_via_client() -> io::Result<()> {
+    fn test_banks_server_transfer_via_client() -> Result<(), BanksClientError> {
         // The caller may not want to hold the connection open until the transaction
         // is processed (or blockhash expires). In this test, we verify the
         // server-side functionality is available to the client.
 
         let genesis = create_genesis_config(10);
-        let bank = Bank::new(&genesis.genesis_config);
+        let bank = Bank::new_for_tests(&genesis.genesis_config);
         let slot = bank.slot();
         let block_commitment_cache = Arc::new(RwLock::new(
             BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
@@ -377,13 +535,18 @@ mod tests {
 
         let mint_pubkey = &genesis.mint_keypair.pubkey();
         let bob_pubkey = solana_sdk::pubkey::new_rand();
-        let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
-        let message = Message::new(&[instruction], Some(&mint_pubkey));
+        let instruction = system_instruction::transfer(mint_pubkey, &bob_pubkey, 1);
+        let message = Message::new(&[instruction], Some(mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
+            let client_transport =
+                start_local_server(bank_forks, block_commitment_cache, Duration::from_millis(1))
+                    .await;
             let mut banks_client = start_client(client_transport).await?;
-            let (_, recent_blockhash, last_valid_slot) = banks_client.get_fees().await?;
+            let (recent_blockhash, last_valid_block_height) = banks_client
+                .get_latest_blockhash_with_commitment(CommitmentLevel::default())
+                .await?
+                .unwrap();
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
             let signature = transaction.signatures[0];
             banks_client.send_transaction(transaction).await?;
@@ -391,8 +554,8 @@ mod tests {
             let mut status = banks_client.get_transaction_status(signature).await?;
 
             while status.is_none() {
-                let root_slot = banks_client.get_root_slot().await?;
-                if root_slot > last_valid_slot {
+                let root_block_height = banks_client.get_root_block_height().await?;
+                if root_block_height > last_valid_block_height {
                     break;
                 }
                 sleep(Duration::from_millis(100)).await;

banks-interface/Cargo.toml

@@ -1,22 +1,18 @@
 [package]
 name = "solana-banks-interface"
-version = "1.7.0"
+version = "1.10.0"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-banks-interface"
-edition = "2018"
+edition = "2021"
 
 [dependencies]
-mio = "0.7.6"
-serde = { version = "1.0.122", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-tarpc = { version = "0.24.1", features = ["full"] }
-
-[dev-dependencies]
-tokio = { version = "1", features = ["full"] }
+serde = { version = "1.0.133", features = ["derive"] }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+tarpc = { version = "0.27.2", features = ["full"] }
 
 [lib]
 crate-type = ["lib"]

banks-interface/src/lib.rs

@@ -1,13 +1,18 @@
-use serde::{Deserialize, Serialize};
-use solana_sdk::{
-    account::Account,
-    clock::Slot,
-    commitment_config::CommitmentLevel,
-    fee_calculator::FeeCalculator,
-    hash::Hash,
-    pubkey::Pubkey,
-    signature::Signature,
-    transaction::{self, Transaction, TransactionError},
+#![allow(deprecated)]
+
+use {
+    serde::{Deserialize, Serialize},
+    solana_sdk::{
+        account::Account,
+        clock::Slot,
+        commitment_config::CommitmentLevel,
+        fee_calculator::FeeCalculator,
+        hash::Hash,
+        message::Message,
+        pubkey::Pubkey,
+        signature::Signature,
+        transaction::{self, Transaction, TransactionError},
+    },
 };
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -25,15 +30,37 @@ pub struct TransactionStatus {
     pub confirmation_status: Option<TransactionConfirmationStatus>,
 }
 
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TransactionSimulationDetails {
+    pub logs: Vec<String>,
+    pub units_consumed: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct BanksTransactionResultWithSimulation {
+    pub result: Option<transaction::Result<()>>,
+    pub simulation_details: Option<TransactionSimulationDetails>,
+}
+
 #[tarpc::service]
 pub trait Banks {
     async fn send_transaction_with_context(transaction: Transaction);
+    #[deprecated(
+        since = "1.9.0",
+        note = "Please use `get_fee_for_message_with_commitment_and_context` instead"
+    )]
     async fn get_fees_with_commitment_and_context(
         commitment: CommitmentLevel,
     ) -> (FeeCalculator, Hash, Slot);
     async fn get_transaction_status_with_context(signature: Signature)
         -> Option<TransactionStatus>;
     async fn get_slot_with_context(commitment: CommitmentLevel) -> Slot;
+    async fn get_block_height_with_context(commitment: CommitmentLevel) -> u64;
+    async fn process_transaction_with_preflight_and_commitment_and_context(
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> BanksTransactionResultWithSimulation;
     async fn process_transaction_with_commitment_and_context(
         transaction: Transaction,
         commitment: CommitmentLevel,
@@ -42,12 +69,22 @@ pub trait Banks {
         address: Pubkey,
         commitment: CommitmentLevel,
     ) -> Option<Account>;
+    async fn get_latest_blockhash_with_context() -> Hash;
+    async fn get_latest_blockhash_with_commitment_and_context(
+        commitment: CommitmentLevel,
+    ) -> Option<(Hash, u64)>;
+    async fn get_fee_for_message_with_commitment_and_context(
+        commitment: CommitmentLevel,
+        message: Message,
+    ) -> Option<u64>;
 }
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use tarpc::{client, transport};
+    use {
+        super::*,
+        tarpc::{client, transport},
+    };
 
     #[test]
     fn test_banks_client_new() {

banks-server/Cargo.toml

@@ -1,24 +1,22 @@
 [package]
 name = "solana-banks-server"
-version = "1.7.0"
+version = "1.10.0"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-banks-server"
-edition = "2018"
+edition = "2021"
 
 [dependencies]
-bincode = "1.3.1"
+bincode = "1.3.3"
 futures = "0.3"
-log = "0.4.11"
-mio = "0.7.6"
-solana-banks-interface = { path = "../banks-interface", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-metrics = { path = "../metrics", version = "=1.7.0" }
-tarpc = { version = "0.24.1", features = ["full"] }
+solana-banks-interface = { path = "../banks-interface", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.10.0" }
+tarpc = { version = "0.27.2", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 tokio-stream = "0.1"

banks-server/src/banks_server.rs

@@ -1,48 +1,59 @@
-use crate::send_transaction_service::{SendTransactionService, TransactionInfo};
-use bincode::{deserialize, serialize};
-use futures::{
-    future,
-    prelude::stream::{self, StreamExt},
-};
-use solana_banks_interface::{
-    Banks, BanksRequest, BanksResponse, TransactionConfirmationStatus, TransactionStatus,
-};
-use solana_runtime::{bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache};
-use solana_sdk::{
-    account::Account,
-    clock::Slot,
-    commitment_config::CommitmentLevel,
-    fee_calculator::FeeCalculator,
-    hash::Hash,
-    pubkey::Pubkey,
-    signature::Signature,
-    transaction::{self, Transaction},
-};
-use std::{
-    io,
-    net::{Ipv4Addr, SocketAddr},
-    sync::{
-        mpsc::{channel, Receiver, Sender},
-        Arc, RwLock,
+use {
+    bincode::{deserialize, serialize},
+    futures::{future, prelude::stream::StreamExt},
+    solana_banks_interface::{
+        Banks, BanksRequest, BanksResponse, BanksTransactionResultWithSimulation,
+        TransactionConfirmationStatus, TransactionSimulationDetails, TransactionStatus,
     },
-    thread::Builder,
-    time::Duration,
+    solana_runtime::{
+        bank::{Bank, TransactionSimulationResult},
+        bank_forks::BankForks,
+        commitment::BlockCommitmentCache,
+    },
+    solana_sdk::{
+        account::Account,
+        clock::Slot,
+        commitment_config::CommitmentLevel,
+        feature_set::FeatureSet,
+        fee_calculator::FeeCalculator,
+        hash::Hash,
+        message::{Message, SanitizedMessage},
+        pubkey::Pubkey,
+        signature::Signature,
+        transaction::{self, SanitizedTransaction, Transaction},
+    },
+    solana_send_transaction_service::{
+        send_transaction_service::{SendTransactionService, TransactionInfo},
+        tpu_info::NullTpuInfo,
+    },
+    std::{
+        convert::TryFrom,
+        io,
+        net::{Ipv4Addr, SocketAddr},
+        sync::{
+            mpsc::{channel, Receiver, Sender},
+            Arc, RwLock,
+        },
+        thread::Builder,
+        time::Duration,
+    },
+    tarpc::{
+        context::Context,
+        serde_transport::tcp,
+        server::{self, incoming::Incoming, Channel},
+        transport::{self, channel::UnboundedChannel},
+        ClientMessage, Response,
+    },
+    tokio::time::sleep,
+    tokio_serde::formats::Bincode,
 };
-use tarpc::{
-    context::Context,
-    rpc::{transport::channel::UnboundedChannel, ClientMessage, Response},
-    serde_transport::tcp,
-    server::{self, Channel, Handler},
-    transport,
-};
-use tokio::time::sleep;
-use tokio_serde::formats::Bincode;
 
 #[derive(Clone)]
 struct BanksServer {
     bank_forks: Arc<RwLock<BankForks>>,
     block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
     transaction_sender: Sender<TransactionInfo>,
+    poll_signature_status_sleep_duration: Duration,
 }
 
 impl BanksServer {
@@ -54,11 +65,13 @@ impl BanksServer {
         bank_forks: Arc<RwLock<BankForks>>,
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
         transaction_sender: Sender<TransactionInfo>,
+        poll_signature_status_sleep_duration: Duration,
     ) -> Self {
         Self {
             bank_forks,
             block_commitment_cache,
             transaction_sender,
+            poll_signature_status_sleep_duration,
         }
     }
 
@@ -73,7 +86,7 @@ impl BanksServer {
                 .map(|info| deserialize(&info.wire_transaction).unwrap())
                 .collect();
             let bank = bank_forks.read().unwrap().working_bank();
-            let _ = bank.process_transactions(&transactions);
+            let _ = bank.try_process_transactions(transactions.iter());
         }
     }
 
@@ -81,6 +94,7 @@ impl BanksServer {
     fn new_loopback(
         bank_forks: Arc<RwLock<BankForks>>,
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+        poll_signature_status_sleep_duration: Duration,
     ) -> Self {
         let (transaction_sender, transaction_receiver) = channel();
         let bank = bank_forks.read().unwrap().working_bank();
@@ -95,7 +109,12 @@ impl BanksServer {
             .name("solana-bank-forks-client".to_string())
             .spawn(move || Self::run(server_bank_forks, transaction_receiver))
             .unwrap();
-        Self::new(bank_forks, block_commitment_cache, transaction_sender)
+        Self::new(
+            bank_forks,
+            block_commitment_cache,
+            transaction_sender,
+            poll_signature_status_sleep_duration,
+        )
     }
 
     fn slot(&self, commitment: CommitmentLevel) -> Slot {
@@ -113,16 +132,16 @@ impl BanksServer {
         self,
         signature: &Signature,
         blockhash: &Hash,
-        last_valid_slot: Slot,
+        last_valid_block_height: u64,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
         let mut status = self
             .bank(commitment)
             .get_signature_status_with_blockhash(signature, blockhash);
         while status.is_none() {
-            sleep(Duration::from_millis(200)).await;
+            sleep(self.poll_signature_status_sleep_duration).await;
             let bank = self.bank(commitment);
-            if bank.slot() > last_valid_slot {
+            if bank.block_height() > last_valid_block_height {
                 break;
             }
             status = bank.get_signature_status_with_blockhash(signature, blockhash);
@@ -131,10 +150,13 @@ impl BanksServer {
     }
 }
 
-fn verify_transaction(transaction: &Transaction) -> transaction::Result<()> {
+fn verify_transaction(
+    transaction: &Transaction,
+    feature_set: &Arc<FeatureSet>,
+) -> transaction::Result<()> {
     if let Err(err) = transaction.verify() {
         Err(err)
-    } else if let Err(err) = transaction.verify_precompiles() {
+    } else if let Err(err) = transaction.verify_precompiles(feature_set) {
         Err(err)
     } else {
         Ok(())
@@ -145,16 +167,21 @@ fn verify_transaction(transaction: &Transaction) -> transaction::Result<()> {
 impl Banks for BanksServer {
     async fn send_transaction_with_context(self, _: Context, transaction: Transaction) {
         let blockhash = &transaction.message.recent_blockhash;
-        let last_valid_slot = self
+        let last_valid_block_height = self
             .bank_forks
             .read()
             .unwrap()
             .root_bank()
-            .get_blockhash_last_valid_slot(&blockhash)
+            .get_blockhash_last_valid_block_height(blockhash)
             .unwrap();
         let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
-        let info =
-            TransactionInfo::new(signature, serialize(&transaction).unwrap(), last_valid_slot);
+        let info = TransactionInfo::new(
+            signature,
+            serialize(&transaction).unwrap(),
+            last_valid_block_height,
+            None,
+            None,
+        );
         self.transaction_sender.send(info).unwrap();
     }
 
@@ -162,11 +189,18 @@ impl Banks for BanksServer {
         self,
         _: Context,
         commitment: CommitmentLevel,
-    ) -> (FeeCalculator, Hash, Slot) {
+    ) -> (FeeCalculator, Hash, u64) {
         let bank = self.bank(commitment);
-        let (blockhash, fee_calculator) = bank.last_blockhash_with_fee_calculator();
-        let last_valid_slot = bank.get_blockhash_last_valid_slot(&blockhash).unwrap();
-        (fee_calculator, blockhash, last_valid_slot)
+        let blockhash = bank.last_blockhash();
+        let lamports_per_signature = bank.get_lamports_per_signature();
+        let last_valid_block_height = bank
+            .get_blockhash_last_valid_block_height(&blockhash)
+            .unwrap();
+        (
+            FeeCalculator::new(lamports_per_signature),
+            blockhash,
+            last_valid_block_height,
+        )
     }
 
     async fn get_transaction_status_with_context(
@@ -209,29 +243,76 @@ impl Banks for BanksServer {
         self.slot(commitment)
     }
 
+    async fn get_block_height_with_context(self, _: Context, commitment: CommitmentLevel) -> u64 {
+        self.bank(commitment).block_height()
+    }
+
+    async fn process_transaction_with_preflight_and_commitment_and_context(
+        self,
+        ctx: Context,
+        transaction: Transaction,
+        commitment: CommitmentLevel,
+    ) -> BanksTransactionResultWithSimulation {
+        let sanitized_transaction =
+            match SanitizedTransaction::try_from_legacy_transaction(transaction.clone()) {
+                Err(err) => {
+                    return BanksTransactionResultWithSimulation {
+                        result: Some(Err(err)),
+                        simulation_details: None,
+                    };
+                }
+                Ok(tx) => tx,
+            };
+        if let TransactionSimulationResult {
+            result: Err(err),
+            logs,
+            post_simulation_accounts: _,
+            units_consumed,
+        } = self
+            .bank(commitment)
+            .simulate_transaction_unchecked(sanitized_transaction)
+        {
+            return BanksTransactionResultWithSimulation {
+                result: Some(Err(err)),
+                simulation_details: Some(TransactionSimulationDetails {
+                    logs,
+                    units_consumed,
+                }),
+            };
+        }
+        BanksTransactionResultWithSimulation {
+            result: self
+                .process_transaction_with_commitment_and_context(ctx, transaction, commitment)
+                .await,
+            simulation_details: None,
+        }
+    }
+
     async fn process_transaction_with_commitment_and_context(
         self,
         _: Context,
         transaction: Transaction,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
-        if let Err(err) = verify_transaction(&transaction) {
+        if let Err(err) = verify_transaction(&transaction, &self.bank(commitment).feature_set) {
             return Some(Err(err));
         }
 
         let blockhash = &transaction.message.recent_blockhash;
-        let last_valid_slot = self
-            .bank_forks
-            .read()
-            .unwrap()
-            .root_bank()
-            .get_blockhash_last_valid_slot(blockhash)
+        let last_valid_block_height = self
+            .bank(commitment)
+            .get_blockhash_last_valid_block_height(blockhash)
             .unwrap();
         let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
-        let info =
-            TransactionInfo::new(signature, serialize(&transaction).unwrap(), last_valid_slot);
+        let info = TransactionInfo::new(
+            signature,
+            serialize(&transaction).unwrap(),
+            last_valid_block_height,
+            None,
+            None,
+        );
         self.transaction_sender.send(info).unwrap();
-        self.poll_signature_status(&signature, blockhash, last_valid_slot, commitment)
+        self.poll_signature_status(&signature, blockhash, last_valid_block_height, commitment)
             .await
     }
 
@@ -244,17 +325,47 @@ impl Banks for BanksServer {
         let bank = self.bank(commitment);
         bank.get_account(&address).map(Account::from)
     }
+
+    async fn get_latest_blockhash_with_context(self, _: Context) -> Hash {
+        let bank = self.bank(CommitmentLevel::default());
+        bank.last_blockhash()
+    }
+
+    async fn get_latest_blockhash_with_commitment_and_context(
+        self,
+        _: Context,
+        commitment: CommitmentLevel,
+    ) -> Option<(Hash, u64)> {
+        let bank = self.bank(commitment);
+        let blockhash = bank.last_blockhash();
+        let last_valid_block_height = bank.get_blockhash_last_valid_block_height(&blockhash)?;
+        Some((blockhash, last_valid_block_height))
+    }
+
+    async fn get_fee_for_message_with_commitment_and_context(
+        self,
+        _: Context,
+        commitment: CommitmentLevel,
+        message: Message,
+    ) -> Option<u64> {
+        let bank = self.bank(commitment);
+        let sanitized_message = SanitizedMessage::try_from(message).ok()?;
+        bank.get_fee_for_message(&sanitized_message)
+    }
 }
 
 pub async fn start_local_server(
     bank_forks: Arc<RwLock<BankForks>>,
     block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
+    poll_signature_status_sleep_duration: Duration,
 ) -> UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>> {
-    let banks_server = BanksServer::new_loopback(bank_forks, block_commitment_cache);
+    let banks_server = BanksServer::new_loopback(
+        bank_forks,
+        block_commitment_cache,
+        poll_signature_status_sleep_duration,
+    );
     let (client_transport, server_transport) = transport::channel::unbounded();
-    let server = server::new(server::Config::default())
-        .incoming(stream::once(future::ready(server_transport)))
-        .respond_with(banks_server.serve());
+    let server = server::BaseChannel::with_defaults(server_transport).execute(banks_server.serve());
     tokio::spawn(server);
     client_transport
 }
@@ -283,11 +394,22 @@ pub async fn start_tcp_server(
         .map(move |chan| {
             let (sender, receiver) = channel();
 
-            SendTransactionService::new(tpu_addr, &bank_forks, receiver);
+            SendTransactionService::new::<NullTpuInfo>(
+                tpu_addr,
+                &bank_forks,
+                None,
+                receiver,
+                5_000,
+                0,
+            );
 
-            let server =
-                BanksServer::new(bank_forks.clone(), block_commitment_cache.clone(), sender);
-            chan.respond_with(server.serve()).execute()
+            let server = BanksServer::new(
+                bank_forks.clone(),
+                block_commitment_cache.clone(),
+                sender,
+                Duration::from_millis(200),
+            );
+            chan.execute(server.serve())
         })
         // Max 10 channels.
         .buffer_unordered(10)

banks-server/src/lib.rs

@@ -1,7 +1,3 @@
 #![allow(clippy::integer_arithmetic)]
 pub mod banks_server;
 pub mod rpc_banks_service;
-pub mod send_transaction_service;
-
-#[macro_use]
-extern crate solana_metrics;

banks-server/src/rpc_banks_service.rs

@@ -1,21 +1,23 @@
 //! The `rpc_banks_service` module implements the Solana Banks RPC API.
 
-use crate::banks_server::start_tcp_server;
-use futures::{future::FutureExt, pin_mut, prelude::stream::StreamExt, select};
-use solana_runtime::{bank_forks::BankForks, commitment::BlockCommitmentCache};
-use std::{
-    net::SocketAddr,
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc, RwLock,
+use {
+    crate::banks_server::start_tcp_server,
+    futures::{future::FutureExt, pin_mut, prelude::stream::StreamExt, select},
+    solana_runtime::{bank_forks::BankForks, commitment::BlockCommitmentCache},
+    std::{
+        net::SocketAddr,
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc, RwLock,
+        },
+        thread::{self, Builder, JoinHandle},
     },
-    thread::{self, Builder, JoinHandle},
+    tokio::{
+        runtime::Runtime,
+        time::{self, Duration},
+    },
+    tokio_stream::wrappers::IntervalStream,
 };
-use tokio::{
-    runtime::Runtime,
-    time::{self, Duration},
-};
-use tokio_stream::wrappers::IntervalStream;
 
 pub struct RpcBanksService {
     thread_hdl: JoinHandle<()>,
@@ -101,12 +103,11 @@ impl RpcBanksService {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use solana_runtime::bank::Bank;
+    use {super::*, solana_runtime::bank::Bank};
 
     #[test]
     fn test_rpc_banks_server_exit() {
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::default())));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::default_for_tests())));
         let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
         let exit = Arc::new(AtomicBool::new(false));
         let addr = "127.0.0.1:0".parse().unwrap();

banks-server/src/send_transaction_service.rs

@@ -1,343 +0,0 @@
-// TODO: Merge this implementation with the one at `core/src/send_transaction_service.rs`
-use log::*;
-use solana_metrics::{datapoint_warn, inc_new_counter_info};
-use solana_runtime::{bank::Bank, bank_forks::BankForks};
-use solana_sdk::{clock::Slot, signature::Signature};
-use std::{
-    collections::HashMap,
-    net::{SocketAddr, UdpSocket},
-    sync::{
-        mpsc::{Receiver, RecvTimeoutError},
-        Arc, RwLock,
-    },
-    thread::{self, Builder, JoinHandle},
-    time::{Duration, Instant},
-};
-
-/// Maximum size of the transaction queue
-const MAX_TRANSACTION_QUEUE_SIZE: usize = 10_000; // This seems like a lot but maybe it needs to be bigger one day
-
-pub struct SendTransactionService {
-    thread: JoinHandle<()>,
-}
-
-pub struct TransactionInfo {
-    pub signature: Signature,
-    pub wire_transaction: Vec<u8>,
-    pub last_valid_slot: Slot,
-}
-
-impl TransactionInfo {
-    pub fn new(signature: Signature, wire_transaction: Vec<u8>, last_valid_slot: Slot) -> Self {
-        Self {
-            signature,
-            wire_transaction,
-            last_valid_slot,
-        }
-    }
-}
-
-#[derive(Default, Debug, PartialEq)]
-struct ProcessTransactionsResult {
-    rooted: u64,
-    expired: u64,
-    retried: u64,
-    failed: u64,
-    retained: u64,
-}
-
-impl SendTransactionService {
-    pub fn new(
-        tpu_address: SocketAddr,
-        bank_forks: &Arc<RwLock<BankForks>>,
-        receiver: Receiver<TransactionInfo>,
-    ) -> Self {
-        let thread = Self::retry_thread(receiver, bank_forks.clone(), tpu_address);
-        Self { thread }
-    }
-
-    fn retry_thread(
-        receiver: Receiver<TransactionInfo>,
-        bank_forks: Arc<RwLock<BankForks>>,
-        tpu_address: SocketAddr,
-    ) -> JoinHandle<()> {
-        let mut last_status_check = Instant::now();
-        let mut transactions = HashMap::new();
-        let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
-
-        Builder::new()
-            .name("send-tx-svc".to_string())
-            .spawn(move || loop {
-                match receiver.recv_timeout(Duration::from_secs(1)) {
-                    Err(RecvTimeoutError::Disconnected) => break,
-                    Err(RecvTimeoutError::Timeout) => {}
-                    Ok(transaction_info) => {
-                        Self::send_transaction(
-                            &send_socket,
-                            &tpu_address,
-                            &transaction_info.wire_transaction,
-                        );
-                        if transactions.len() < MAX_TRANSACTION_QUEUE_SIZE {
-                            transactions.insert(transaction_info.signature, transaction_info);
-                        } else {
-                            datapoint_warn!("send_transaction_service-queue-overflow");
-                        }
-                    }
-                }
-
-                if Instant::now().duration_since(last_status_check).as_secs() >= 5 {
-                    if !transactions.is_empty() {
-                        datapoint_info!(
-                            "send_transaction_service-queue-size",
-                            ("len", transactions.len(), i64)
-                        );
-                        let bank_forks = bank_forks.read().unwrap();
-                        let root_bank = bank_forks.root_bank();
-                        let working_bank = bank_forks.working_bank();
-
-                        let _result = Self::process_transactions(
-                            &working_bank,
-                            &root_bank,
-                            &send_socket,
-                            &tpu_address,
-                            &mut transactions,
-                        );
-                    }
-                    last_status_check = Instant::now();
-                }
-            })
-            .unwrap()
-    }
-
-    fn process_transactions(
-        working_bank: &Arc<Bank>,
-        root_bank: &Arc<Bank>,
-        send_socket: &UdpSocket,
-        tpu_address: &SocketAddr,
-        transactions: &mut HashMap<Signature, TransactionInfo>,
-    ) -> ProcessTransactionsResult {
-        let mut result = ProcessTransactionsResult::default();
-
-        transactions.retain(|signature, transaction_info| {
-            if root_bank.has_signature(signature) {
-                info!("Transaction is rooted: {}", signature);
-                result.rooted += 1;
-                inc_new_counter_info!("send_transaction_service-rooted", 1);
-                false
-            } else if transaction_info.last_valid_slot < root_bank.slot() {
-                info!("Dropping expired transaction: {}", signature);
-                result.expired += 1;
-                inc_new_counter_info!("send_transaction_service-expired", 1);
-                false
-            } else {
-                match working_bank.get_signature_status_slot(signature) {
-                    None => {
-                        // Transaction is unknown to the working bank, it might have been
-                        // dropped or landed in another fork.  Re-send it
-                        info!("Retrying transaction: {}", signature);
-                        result.retried += 1;
-                        inc_new_counter_info!("send_transaction_service-retry", 1);
-                        Self::send_transaction(
-                            &send_socket,
-                            &tpu_address,
-                            &transaction_info.wire_transaction,
-                        );
-                        true
-                    }
-                    Some((_slot, status)) => {
-                        if status.is_err() {
-                            info!("Dropping failed transaction: {}", signature);
-                            result.failed += 1;
-                            inc_new_counter_info!("send_transaction_service-failed", 1);
-                            false
-                        } else {
-                            result.retained += 1;
-                            true
-                        }
-                    }
-                }
-            }
-        });
-
-        result
-    }
-
-    fn send_transaction(
-        send_socket: &UdpSocket,
-        tpu_address: &SocketAddr,
-        wire_transaction: &[u8],
-    ) {
-        if let Err(err) = send_socket.send_to(wire_transaction, tpu_address) {
-            warn!("Failed to send transaction to {}: {:?}", tpu_address, err);
-        }
-    }
-
-    pub fn join(self) -> thread::Result<()> {
-        self.thread.join()
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use solana_sdk::{
-        genesis_config::create_genesis_config, pubkey::Pubkey, signature::Signer,
-        system_transaction,
-    };
-    use std::sync::mpsc::channel;
-
-    #[test]
-    fn service_exit() {
-        let tpu_address = "127.0.0.1:0".parse().unwrap();
-        let bank = Bank::default();
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
-        let (sender, receiver) = channel();
-
-        let send_tranaction_service =
-            SendTransactionService::new(tpu_address, &bank_forks, receiver);
-
-        drop(sender);
-        send_tranaction_service.join().unwrap();
-    }
-
-    #[test]
-    fn process_transactions() {
-        let (genesis_config, mint_keypair) = create_genesis_config(4);
-        let bank = Bank::new(&genesis_config);
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
-        let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
-        let tpu_address = "127.0.0.1:0".parse().unwrap();
-
-        let root_bank = Arc::new(Bank::new_from_parent(
-            &bank_forks.read().unwrap().working_bank(),
-            &Pubkey::default(),
-            1,
-        ));
-        let rooted_signature = root_bank
-            .transfer(1, &mint_keypair, &mint_keypair.pubkey())
-            .unwrap();
-
-        let working_bank = Arc::new(Bank::new_from_parent(&root_bank, &Pubkey::default(), 2));
-
-        let non_rooted_signature = working_bank
-            .transfer(2, &mint_keypair, &mint_keypair.pubkey())
-            .unwrap();
-
-        let failed_signature = {
-            let blockhash = working_bank.last_blockhash();
-            let transaction =
-                system_transaction::transfer(&mint_keypair, &Pubkey::default(), 1, blockhash);
-            let signature = transaction.signatures[0];
-            working_bank.process_transaction(&transaction).unwrap_err();
-            signature
-        };
-
-        let mut transactions = HashMap::new();
-
-        info!("Expired transactions are dropped..");
-        transactions.insert(
-            Signature::default(),
-            TransactionInfo::new(Signature::default(), vec![], root_bank.slot() - 1),
-        );
-        let result = SendTransactionService::process_transactions(
-            &working_bank,
-            &root_bank,
-            &send_socket,
-            &tpu_address,
-            &mut transactions,
-        );
-        assert!(transactions.is_empty());
-        assert_eq!(
-            result,
-            ProcessTransactionsResult {
-                expired: 1,
-                ..ProcessTransactionsResult::default()
-            }
-        );
-
-        info!("Rooted transactions are dropped...");
-        transactions.insert(
-            rooted_signature,
-            TransactionInfo::new(rooted_signature, vec![], working_bank.slot()),
-        );
-        let result = SendTransactionService::process_transactions(
-            &working_bank,
-            &root_bank,
-            &send_socket,
-            &tpu_address,
-            &mut transactions,
-        );
-        assert!(transactions.is_empty());
-        assert_eq!(
-            result,
-            ProcessTransactionsResult {
-                rooted: 1,
-                ..ProcessTransactionsResult::default()
-            }
-        );
-
-        info!("Failed transactions are dropped...");
-        transactions.insert(
-            failed_signature,
-            TransactionInfo::new(failed_signature, vec![], working_bank.slot()),
-        );
-        let result = SendTransactionService::process_transactions(
-            &working_bank,
-            &root_bank,
-            &send_socket,
-            &tpu_address,
-            &mut transactions,
-        );
-        assert!(transactions.is_empty());
-        assert_eq!(
-            result,
-            ProcessTransactionsResult {
-                failed: 1,
-                ..ProcessTransactionsResult::default()
-            }
-        );
-
-        info!("Non-rooted transactions are kept...");
-        transactions.insert(
-            non_rooted_signature,
-            TransactionInfo::new(non_rooted_signature, vec![], working_bank.slot()),
-        );
-        let result = SendTransactionService::process_transactions(
-            &working_bank,
-            &root_bank,
-            &send_socket,
-            &tpu_address,
-            &mut transactions,
-        );
-        assert_eq!(transactions.len(), 1);
-        assert_eq!(
-            result,
-            ProcessTransactionsResult {
-                retained: 1,
-                ..ProcessTransactionsResult::default()
-            }
-        );
-        transactions.clear();
-
-        info!("Unknown transactions are retried...");
-        transactions.insert(
-            Signature::default(),
-            TransactionInfo::new(Signature::default(), vec![], working_bank.slot()),
-        );
-        let result = SendTransactionService::process_transactions(
-            &working_bank,
-            &root_bank,
-            &send_socket,
-            &tpu_address,
-            &mut transactions,
-        );
-        assert_eq!(transactions.len(), 1);
-        assert_eq!(
-            result,
-            ProcessTransactionsResult {
-                retried: 1,
-                ..ProcessTransactionsResult::default()
-            }
-        );
-    }
-}

bench-exchange/.gitignore

@@ -1,4 +0,0 @@
-/target/
-/config/
-/config-local/
-/farf/

bench-exchange/Cargo.toml

@@ -1,39 +0,0 @@
-[package]
-authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2018"
-name = "solana-bench-exchange"
-version = "1.7.0"
-repository = "https://github.com/solana-labs/solana"
-license = "Apache-2.0"
-homepage = "https://solana.com/"
-publish = false
-
-[dependencies]
-clap = "2.33.1"
-itertools = "0.9.0"
-log = "0.4.11"
-num-derive = "0.3"
-num-traits = "0.2"
-rand = "0.7.0"
-rayon = "1.5.0"
-serde_json = "1.0.56"
-serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-core = { path = "../core", version = "=1.7.0" }
-solana-genesis = { path = "../genesis", version = "=1.7.0" }
-solana-client = { path = "../client", version = "=1.7.0" }
-solana-exchange-program = { path = "../programs/exchange", version = "=1.7.0" }
-solana-faucet = { path = "../faucet", version = "=1.7.0" }
-solana-gossip = { path = "../gossip", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-metrics = { path = "../metrics", version = "=1.7.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
-
-[dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "=1.7.0" }
-
-[package.metadata.docs.rs]
-targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/README.md

@@ -1,479 +0,0 @@
-# token-exchange
-Solana Token Exchange Bench
-
-If you can't wait; jump to [Running the exchange](#Running-the-exchange) to
-learn how to start and interact with the exchange.
-
-### Table of Contents
-[Overview](#Overview)<br>
-[Premise](#Premise)<br>
-[Exchange startup](#Exchange-startup)<br>
-[Order Requests](#Trade-requests)<br>
-[Order Cancellations](#Trade-cancellations)<br>
-[Trade swap](#Trade-swap)<br>
-[Exchange program operations](#Exchange-program-operations)<br>
-[Quotes and OHLCV](#Quotes-and-OHLCV)<br>
-[Investor strategies](#Investor-strategies)<br>
-[Running the exchange](#Running-the-exchange)<br>
-
-## Overview
-
-An exchange is a marketplace where one asset can be traded for another.  This
-demo demonstrates one way to host an exchange on the Solana blockchain by
-emulating a currency exchange.
-
-The assets are virtual tokens held by investors who may post order requests to
-the exchange.  A Matcher monitors the exchange and posts swap requests for
-matching orders.  All the transactions can execute concurrently.
-
-## Premise
-
-- Exchange
-  -  An exchange is a marketplace where one asset can be traded for another.
-     The exchange in this demo is the on-chain program that implements the
-     tokens and the policies for trading those tokens.
-- Token
-  - A virtual asset that can be owned, traded, and holds virtual intrinsic value
-    compared to other assets.  There are four types of tokens in this demo, A,
-    B, C, D.  Each one may be traded for another.
-- Token account
-  - An account owned by the exchange that holds a quantity of one type of token.
-- Account request
-  - A request to create a token account
-- Token request
-  - A request to deposit tokens of a particular type into a token account.
-- Asset pair
-  - A struct with fields Base and Quote, representing the two assets which make up a 
-    trading pair,  which themselves are Tokens. The Base or 'primary' asset is the
-    numerator and the Quote is the denominator for pricing purposes.
-- Order side
-  - Describes which side of the market an investor wants to place a trade on. Options
-    are "Bid" or "Ask", where a bid represents an offer to purchase the Base asset of
-    the AssetPair for a sum of the Quote Asset and an Ask is an offer to sell Base asset
-    for the Quote asset.
-- Price ratio
-  -  An expression of the relative prices of two tokens. Calculated with the Base
-     Asset as the numerator and the Quote Asset as the denominator. Ratios are 
-     represented as fixed point numbers.  The fixed point scaler is defined in
-     [exchange_state.rs](https://github.com/solana-labs/solana/blob/c2fdd1362a029dcf89c8907c562d2079d977df11/programs/exchange_api/src/exchange_state.rs#L7)
-- Order request
-  - A Solana transaction sent by a trader to the exchange to submit an order. 
-    Order requests are made up of the token pair, the order side (bid or ask),
-    quantity of the primary token, the price ratio, and the two token accounts
-    to be credited/deducted.  An example trade request looks like "T AB 5 2" 
-    which reads "Exchange 5 A tokens to B tokens at a price ratio of 1:2"  A fulfilled trade would result in 5 A tokens
-    deducted and 10 B tokens credited to the trade initiator's token accounts.
-    Successful order requests result in an order.
-- Order
-  - The result of a successful order request.  orders are stored in
-    accounts owned by the submitter of the order request.  They can only be
-    canceled by their owner but can be used by anyone in a trade swap.  They
-    contain the same information as the order request.
-- Price spread
-  - The difference between the two matching orders. The spread is the
-    profit of the Matcher initiating the swap request.
-- Match requirements
-  - Policies that result in a successful trade swap.
-- Match request
-  - A request to fill two complementary orders (bid/ask), resulting if successful,
-    in a trade being created.
-- Trade
-  - A successful trade is created from two matching orders that meet
-    swap requirements which are submitted in a Match Request by a Matcher and
-    executed by the exchange. A trade may not wholly satisfy one or both of the
-    orders in which case the orders are adjusted appropriately. Upon execution,
-    tokens are distributed to the traders' accounts and any overlap or 
-    "negative spread" between orders is deposited into the Matcher's profit
-    account.  All successful trades are recorded in the data of a new solana 
-    account for posterity.
-- Investor
-  - Individual investors who hold a number of tokens and wish to trade them on
-    the exchange.  Investors operate as Solana thin clients who own a set of
-    accounts containing tokens and/or order requests.  Investors post
-    transactions to the exchange in order to request tokens and post or cancel
-    order requests.
-- Matcher
-  - An agent who facilitates trading between investors.  Matchers operate as
-    Solana thin clients who monitor all the orders looking for a trade
-    match.  Once found, the Matcher issues a swap request to the exchange.
-    Matchers are the engine of the exchange and are rewarded for their efforts by
-    accumulating the price spreads of the swaps they initiate.  Matchers also
-    provide current bid/ask price and OHLCV (Open, High, Low, Close, Volume)
-    information on demand via a public network port.
-- Transaction fees
-  - Solana transaction fees are paid for by the transaction submitters who are
-    the Investors and Matchers.
-
-## Exchange startup
-
-The exchange is up and running when it reaches a state where it can take
-investors' trades and Matchers' match requests.  To achieve this state the
-following must occur in order:
-
-- Start the Solana blockchain
-- Start the  thin-client
-- The Matcher subscribes to change notifications for all the accounts owned by
-  the exchange program id.  The subscription is managed via Solana's JSON RPC
-  interface.
-- The Matcher starts responding to queries for bid/ask price and OHLCV
-
-The Matcher responding successfully to price and OHLCV requests is the signal to
-the investors that trades submitted after that point will be analyzed.  <!--This
-is not ideal, and instead investors should be able to submit trades at any time,
-and the Matcher could come and go without missing a trade.  One way to achieve
-this is for the Matcher to read the current state of all accounts looking for all
-open orders.-->
-
-Investors will initially query the exchange to discover their current balance
-for each type of token.  If the investor does not already have an account for
-each type of token, they will submit account requests.  Matcher as well will
-request accounts to hold the tokens they earn by initiating trade swaps.
-
-```rust
-/// Supported token types
-pub enum Token {
-    A,
-    B,
-    C,
-    D,
-}
-
-/// Supported token pairs
-pub enum TokenPair {
-    AB,
-    AC,
-    AD,
-    BC,
-    BD,
-    CD,
-}
-
-pub enum ExchangeInstruction {
-    /// New token account
-    /// key 0 - Signer
-    /// key 1 - New token account
-    AccountRequest,
-}
-
-/// Token accounts are populated with this structure
-pub struct TokenAccountInfo {
-    /// Investor who owns this account
-    pub owner: Pubkey,
-    /// Current number of tokens this account holds
-    pub tokens: Tokens,
-}
-```
-
-For this demo investors or Matcher can request more tokens from the exchange at
-any time by submitting token requests. In non-demos, an exchange of this type
-would provide another way to exchange a 3rd party asset into tokens.
-
-To request tokens, investors submit transfer requests:
-
-```rust
-pub enum ExchangeInstruction {
-    /// Transfer tokens between two accounts
-    /// key 0 - Account to transfer tokens to
-    /// key 1 - Account to transfer tokens from.  This can be the exchange program itself,
-    ///         the exchange has a limitless number of tokens it can transfer.
-    TransferRequest(Token, u64),
-}
-```
-
-## Order Requests
-
-When an investor decides to exchange a token of one type for another, they
-submit a transaction to the Solana Blockchain containing an order request, which,
-if successful, is turned into an order.  orders do not expire but are
-cancellable. <!-- orders should have a timestamp to enable trade
-expiration -->  When an order is created, tokens are deducted from a token
-account and the order acts as an escrow.  The tokens are held until the
-order is fulfilled or canceled. If the direction is `To`, then the number
-of `tokens` are deducted from the primary account, if `From` then `tokens`
-multiplied by `price` are deducted from the secondary account.  orders are
-no longer valid when the number of `tokens` goes to zero, at which point they
-can no longer be used. <!-- Could support refilling orders, so order
-accounts are refilled rather than accumulating -->
-
-```rust
-/// Direction of the exchange between two tokens in a pair
-pub enum Direction {
-    /// Trade first token type (primary) in the pair 'To' the second
-    To,
-    /// Trade first token type in the pair 'From' the second (secondary)
-    From,
-}
-
-pub struct OrderRequestInfo {
-    /// Direction of trade
-    pub direction: Direction,
-
-    /// Token pair to trade
-    pub pair: TokenPair,
-
-    /// Number of tokens to exchange; refers to the primary or the secondary depending on the direction
-    pub tokens: u64,
-
-    /// The price ratio the primary price over the secondary price.  The primary price is fixed
-    /// and equal to the variable `SCALER`.
-    pub price: u64,
-
-    /// Token account to deposit tokens on successful swap
-    pub dst_account: Pubkey,
-}
-
-pub enum ExchangeInstruction {
-    /// order request
-    /// key 0 - Signer
-    /// key 1 - Account in which to record the swap
-    /// key 2 - Token account associated with this trade
-    TradeRequest(TradeRequestInfo),
-}
-
-/// Trade accounts are populated with this structure
-pub struct TradeOrderInfo {
-    /// Owner of the order
-    pub owner: Pubkey,
-    /// Direction of the exchange
-    pub direction: Direction,
-    /// Token pair indicating two tokens to exchange, first is primary
-    pub pair: TokenPair,
-    /// Number of tokens to exchange; primary or secondary depending on direction
-    pub tokens: u64,
-    /// Scaled price of the secondary token given the primary is equal to the scale value
-    /// If scale is 1 and price is 2 then ratio is 1:2 or 1 primary token for 2 secondary tokens
-    pub price: u64,
-    /// account which the tokens were source from.  The trade account holds the tokens in escrow
-    /// until either one or more part of a swap or the trade is canceled.
-    pub src_account: Pubkey,
-    /// account which the tokens the tokens will be deposited into on a successful trade
-    pub dst_account: Pubkey,
-}
-```
-
-## Order cancellations
-
-An investor may cancel a trade at anytime, but only trades they own.  If the
-cancellation is successful, any tokens held in escrow are returned to the
-account from which they came.
-
-```rust
-pub enum ExchangeInstruction {
-    /// order cancellation
-    /// key 0 - Signer
-    /// key 1 -order to cancel
-    TradeCancellation,
-}
-```
-
-## Trade swaps
-
-The Matcher is monitoring the accounts assigned to the exchange program and
-building a trade-order table.  The order table is used to identify
-matching orders which could be fulfilled.  When a match is found the
-Matcher should issue a swap request.  Swap requests may not satisfy the entirety
-of either order, but the exchange will greedily fulfill it.  Any leftover tokens
-in either account will keep the order valid for further swap requests in
-the future.
-
-Matching orders are defined by the following swap requirements:
-
-- Opposite polarity (one `To` and one `From`)
-- Operate on the same token pair
-- The price ratio of the `From` order is greater than or equal to the `To` order
-- There are sufficient tokens to perform the trade
-
-Orders can be written in the following format:
-
-`investor direction pair quantity price-ratio`
-
-For example:
-
-- `1 T AB 2 1`
-  - Investor 1 wishes to exchange 2 A tokens to B tokens at a ratio of 1 A to 1
-    B
-- `2 F AC 6 1.2`
-  - Investor 2 wishes to exchange A tokens from 6 B tokens at a ratio of 1 A
-    from 1.2 B
-
-An order table could look something like the following. Notice how the columns
-are sorted low to high and high to low, respectively.  Prices are dramatic and
-whole for clarity.
-
-|Row| To          | From       |
-|---|-------------|------------|
-| 1 | 1 T AB 2 4  | 2 F AB 2 8 |
-| 2 | 1 T AB 1 4  | 2 F AB 2 8 |
-| 3 | 1 T AB 6 6  | 2 F AB 2 7 |
-| 4 | 1 T AB 2 8  | 2 F AB 3 6 |
-| 5 | 1 T AB 2 10 | 2 F AB 1 5 |
-
-As part of a successful swap request, the exchange will credit tokens to the
-Matcher's account equal to the difference in the price ratios or the two orders.
-These tokens are considered the Matcher's profit for initiating the trade.
-
-The Matcher would initiate the following swap on the order table above:
-
-  - Row 1, To:   Investor 1 trades 2 A tokens to 8 B tokens
-  - Row 1, From: Investor 2 trades 2 A tokens from 8 B tokens
-  - Matcher takes 8 B tokens as profit
-
-Both row 1 trades are fully realized, table becomes:
-
-|Row| To          | From       |
-|---|-------------|------------|
-| 1 | 1 T AB 1 4  | 2 F AB 2 8 |
-| 2 | 1 T AB 6 6  | 2 F AB 2 7 |
-| 3 | 1 T AB 2 8  | 2 F AB 3 6 |
-| 4 | 1 T AB 2 10 | 2 F AB 1 5 |
-
-The Matcher would initiate the following swap:
-
-  - Row 1, To:   Investor 1 trades 1 A token to 4 B tokens
-  - Row 1, From: Investor 2 trades 1 A token from 4 B tokens
-  - Matcher takes 4 B tokens as profit
-
-Row 1 From is not fully realized, table becomes:
-
-|Row| To          | From       |
-|---|-------------|------------|
-| 1 | 1 T AB 6 6  | 2 F AB 1 8 |
-| 2 | 1 T AB 2 8  | 2 F AB 2 7 |
-| 3 | 1 T AB 2 10 | 2 F AB 3 6 |
-| 4 |             | 2 F AB 1 5 |
-
-The Matcher would initiate the following swap:
-
-  - Row 1, To:   Investor 1 trades 1 A token to 6 B tokens
-  - Row 1, From: Investor 2 trades 1 A token from 6 B tokens
-  - Matcher takes 2 B tokens as profit
-
-Row 1 To is now fully realized, table becomes:
-
-|Row| To          | From       |
-|---|-------------|------------|
-| 1 | 1 T AB 5 6  | 2 F AB 2 7 |
-| 2 | 1 T AB 2 8  | 2 F AB 3 5 |
-| 3 | 1 T AB 2 10 | 2 F AB 1 5 |
-
-The Matcher would initiate the following last swap:
-
-  - Row 1, To:   Investor 1 trades 2 A token to 12 B tokens
-  - Row 1, From: Investor 2 trades 2 A token from 12 B tokens
-  - Matcher takes 2 B tokens as profit
-
-Table becomes:
-
-|Row| To          | From       |
-|---|-------------|------------|
-| 1 | 1 T AB 3 6  | 2 F AB 3 5 |
-| 2 | 1 T AB 2 8  | 2 F AB 1 5 |
-| 3 | 1 T AB 2 10 |            |
-
-At this point the lowest To's price is larger than the largest From's price so
-no more swaps would be initiated until new orders came in.
-
-```rust
-pub enum ExchangeInstruction {
-    /// Trade swap request
-    /// key 0 - Signer
-    /// key 1 - Account in which to record the swap
-    /// key 2 - 'To' order
-    /// key 3 - `From` order
-    /// key 4 - Token account associated with the To Trade
-    /// key 5 - Token account associated with From trade
-    /// key 6 - Token account in which to deposit the Matcher profit from the swap.
-    SwapRequest,
-}
-
-/// Swap accounts are populated with this structure
-pub struct TradeSwapInfo {
-    /// Pair swapped
-    pub pair: TokenPair,
-    /// `To` order
-    pub to_trade_order: Pubkey,
-    /// `From` order
-    pub from_trade_order: Pubkey,
-    /// Number of primary tokens exchanged
-    pub primary_tokens: u64,
-    /// Price the primary tokens were exchanged for
-    pub primary_price: u64,
-    /// Number of secondary tokens exchanged
-    pub secondary_tokens: u64,
-    /// Price the secondary tokens were exchanged for
-    pub secondary_price: u64,
-}
-```
-
-## Exchange program operations
-
-Putting all the commands together from above, the following operations will be
-supported by the on-chain exchange program:
-
-```rust
-pub enum ExchangeInstruction {
-    /// New token account
-    /// key 0 - Signer
-    /// key 1 - New token account
-    AccountRequest,
-
-    /// Transfer tokens between two accounts
-    /// key 0 - Account to transfer tokens to
-    /// key 1 - Account to transfer tokens from.  This can be the exchange program itself,
-    ///         the exchange has a limitless number of tokens it can transfer.
-    TransferRequest(Token, u64),
-
-    /// order request
-    /// key 0 - Signer
-    /// key 1 - Account in which to record the swap
-    /// key 2 - Token account associated with this trade
-    TradeRequest(TradeRequestInfo),
-
-    /// order cancellation
-    /// key 0 - Signer
-    /// key 1 -order to cancel
-    TradeCancellation,
-
-    /// Trade swap request
-    /// key 0 - Signer
-    /// key 1 - Account in which to record the swap
-    /// key 2 - 'To' order
-    /// key 3 - `From` order
-    /// key 4 - Token account associated with the To Trade
-    /// key 5 - Token account associated with From trade
-    /// key 6 - Token account in which to deposit the Matcher profit from the swap.
-    SwapRequest,
-}
-```
-
-## Quotes and OHLCV
-
-The Matcher will provide current bid/ask price quotes based on trade actively and
-also provide OHLCV based on some time window.  The details of how the bid/ask
-price quotes are calculated are yet to be decided.
-
-## Investor strategies
-
-To make a compelling demo, the investors needs to provide interesting trade
-behavior.  Something as simple as a randomly twiddled baseline would be a
-minimum starting point.
-
-## Running the exchange
-
-The exchange bench posts trades and swaps matches as fast as it can.  
-
-You might want to bump the duration up
-to 60 seconds and the batch size to 1000 for better numbers.  You can modify those
-in client_demo/src/demo.rs::test_exchange_local_cluster.
-
-The following command runs the bench:
-
-```bash
-$ RUST_LOG=solana_bench_exchange=info cargo test --release -- --nocapture test_exchange_local_cluster
-```
-
-To also see the cluster messages:
-
-```bash
-$ RUST_LOG=solana_bench_exchange=info,solana=info cargo test --release -- --nocapture test_exchange_local_cluster
-```

bench-exchange/src/cli.rs

@@ -1,221 +0,0 @@
-use clap::{crate_description, crate_name, value_t, App, Arg, ArgMatches};
-use solana_core::gen_keys::GenKeys;
-use solana_faucet::faucet::FAUCET_PORT;
-use solana_sdk::signature::{read_keypair_file, Keypair};
-use std::net::SocketAddr;
-use std::process::exit;
-use std::time::Duration;
-
-pub struct Config {
-    pub entrypoint_addr: SocketAddr,
-    pub faucet_addr: SocketAddr,
-    pub identity: Keypair,
-    pub threads: usize,
-    pub num_nodes: usize,
-    pub duration: Duration,
-    pub transfer_delay: u64,
-    pub fund_amount: u64,
-    pub batch_size: usize,
-    pub chunk_size: usize,
-    pub account_groups: usize,
-    pub client_ids_and_stake_file: String,
-    pub write_to_client_file: bool,
-    pub read_from_client_file: bool,
-}
-
-impl Default for Config {
-    fn default() -> Self {
-        Self {
-            entrypoint_addr: SocketAddr::from(([127, 0, 0, 1], 8001)),
-            faucet_addr: SocketAddr::from(([127, 0, 0, 1], FAUCET_PORT)),
-            identity: Keypair::new(),
-            num_nodes: 1,
-            threads: 4,
-            duration: Duration::new(u64::max_value(), 0),
-            transfer_delay: 0,
-            fund_amount: 100_000,
-            batch_size: 100,
-            chunk_size: 100,
-            account_groups: 100,
-            client_ids_and_stake_file: String::new(),
-            write_to_client_file: false,
-            read_from_client_file: false,
-        }
-    }
-}
-
-pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
-    App::new(crate_name!())
-        .about(crate_description!())
-        .version(version)
-        .arg(
-            Arg::with_name("entrypoint")
-                .short("n")
-                .long("entrypoint")
-                .value_name("HOST:PORT")
-                .takes_value(true)
-                .required(false)
-                .default_value("127.0.0.1:8001")
-                .help("Cluster entry point; defaults to 127.0.0.1:8001"),
-        )
-        .arg(
-            Arg::with_name("faucet")
-                .short("d")
-                .long("faucet")
-                .value_name("HOST:PORT")
-                .takes_value(true)
-                .required(false)
-                .default_value("127.0.0.1:9900")
-                .help("Location of the faucet; defaults to 127.0.0.1:9900"),
-        )
-        .arg(
-            Arg::with_name("identity")
-                .short("i")
-                .long("identity")
-                .value_name("PATH")
-                .takes_value(true)
-                .help("File containing a client identity (keypair)"),
-        )
-        .arg(
-            Arg::with_name("threads")
-                .long("threads")
-                .value_name("<threads>")
-                .takes_value(true)
-                .required(false)
-                .default_value("1")
-                .help("Number of threads submitting transactions"),
-        )
-        .arg(
-            Arg::with_name("num-nodes")
-                .long("num-nodes")
-                .value_name("NUM")
-                .takes_value(true)
-                .required(false)
-                .default_value("1")
-                .help("Wait for NUM nodes to converge"),
-        )
-        .arg(
-            Arg::with_name("duration")
-                .long("duration")
-                .value_name("SECS")
-                .takes_value(true)
-                .default_value("60")
-                .help("Seconds to run benchmark, then exit; default is forever"),
-        )
-        .arg(
-            Arg::with_name("transfer-delay")
-                .long("transfer-delay")
-                .value_name("<delay>")
-                .takes_value(true)
-                .required(false)
-                .default_value("0")
-                .help("Delay between each chunk"),
-        )
-        .arg(
-            Arg::with_name("fund-amount")
-                .long("fund-amount")
-                .value_name("<fund>")
-                .takes_value(true)
-                .required(false)
-                .default_value("100000")
-                .help("Number of lamports to fund to each signer"),
-        )
-        .arg(
-            Arg::with_name("batch-size")
-                .long("batch-size")
-                .value_name("<batch>")
-                .takes_value(true)
-                .required(false)
-                .default_value("1000")
-                .help("Number of transactions before the signer rolls over"),
-        )
-        .arg(
-            Arg::with_name("chunk-size")
-                .long("chunk-size")
-                .value_name("<cunk>")
-                .takes_value(true)
-                .required(false)
-                .default_value("500")
-                .help("Number of transactions to generate and send at a time"),
-        )
-        .arg(
-            Arg::with_name("account-groups")
-                .long("account-groups")
-                .value_name("<groups>")
-                .takes_value(true)
-                .required(false)
-                .default_value("10")
-                .help("Number of account groups to cycle for each batch"),
-        )
-        .arg(
-            Arg::with_name("write-client-keys")
-                .long("write-client-keys")
-                .value_name("FILENAME")
-                .takes_value(true)
-                .help("Generate client keys and stakes and write the list to YAML file"),
-        )
-        .arg(
-            Arg::with_name("read-client-keys")
-                .long("read-client-keys")
-                .value_name("FILENAME")
-                .takes_value(true)
-                .help("Read client keys and stakes from the YAML file"),
-        )
-}
-
-#[allow(clippy::field_reassign_with_default)]
-pub fn extract_args(matches: &ArgMatches) -> Config {
-    let mut args = Config::default();
-
-    args.entrypoint_addr = solana_net_utils::parse_host_port(
-        matches.value_of("entrypoint").unwrap(),
-    )
-    .unwrap_or_else(|e| {
-        eprintln!("failed to parse entrypoint address: {}", e);
-        exit(1)
-    });
-
-    args.faucet_addr = solana_net_utils::parse_host_port(matches.value_of("faucet").unwrap())
-        .unwrap_or_else(|e| {
-            eprintln!("failed to parse faucet address: {}", e);
-            exit(1)
-        });
-
-    if matches.is_present("identity") {
-        args.identity = read_keypair_file(matches.value_of("identity").unwrap())
-            .expect("can't read client identity");
-    } else {
-        args.identity = {
-            let seed = [42_u8; 32];
-            let mut rnd = GenKeys::new(seed);
-            rnd.gen_keypair()
-        };
-    }
-    args.threads = value_t!(matches.value_of("threads"), usize).expect("Failed to parse threads");
-    args.num_nodes =
-        value_t!(matches.value_of("num-nodes"), usize).expect("Failed to parse num-nodes");
-    let duration = value_t!(matches.value_of("duration"), u64).expect("Failed to parse duration");
-    args.duration = Duration::from_secs(duration);
-    args.transfer_delay =
-        value_t!(matches.value_of("transfer-delay"), u64).expect("Failed to parse transfer-delay");
-    args.fund_amount =
-        value_t!(matches.value_of("fund-amount"), u64).expect("Failed to parse fund-amount");
-    args.batch_size =
-        value_t!(matches.value_of("batch-size"), usize).expect("Failed to parse batch-size");
-    args.chunk_size =
-        value_t!(matches.value_of("chunk-size"), usize).expect("Failed to parse chunk-size");
-    args.account_groups = value_t!(matches.value_of("account-groups"), usize)
-        .expect("Failed to parse account-groups");
-
-    if let Some(s) = matches.value_of("write-client-keys") {
-        args.write_to_client_file = true;
-        args.client_ids_and_stake_file = s.to_string();
-    }
-
-    if let Some(s) = matches.value_of("read-client-keys") {
-        assert!(!args.write_to_client_file);
-        args.read_from_client_file = true;
-        args.client_ids_and_stake_file = s.to_string();
-    }
-    args
-}

bench-exchange/src/lib.rs

@@ -1,3 +0,0 @@
-pub mod bench;
-pub mod cli;
-mod order_book;

bench-exchange/src/main.rs

@@ -1,83 +0,0 @@
-#![allow(clippy::integer_arithmetic)]
-pub mod bench;
-mod cli;
-pub mod order_book;
-
-use crate::bench::{airdrop_lamports, create_client_accounts_file, do_bench_exchange, Config};
-use log::*;
-use solana_gossip::gossip_service::{discover_cluster, get_multi_client};
-use solana_sdk::signature::Signer;
-
-fn main() {
-    solana_logger::setup();
-    solana_metrics::set_panic_hook("bench-exchange");
-
-    let matches = cli::build_args(solana_version::version!()).get_matches();
-    let cli_config = cli::extract_args(&matches);
-
-    let cli::Config {
-        entrypoint_addr,
-        faucet_addr,
-        identity,
-        threads,
-        num_nodes,
-        duration,
-        transfer_delay,
-        fund_amount,
-        batch_size,
-        chunk_size,
-        account_groups,
-        client_ids_and_stake_file,
-        write_to_client_file,
-        read_from_client_file,
-        ..
-    } = cli_config;
-
-    let config = Config {
-        identity,
-        threads,
-        duration,
-        transfer_delay,
-        fund_amount,
-        batch_size,
-        chunk_size,
-        account_groups,
-        client_ids_and_stake_file,
-        read_from_client_file,
-    };
-
-    if write_to_client_file {
-        create_client_accounts_file(
-            &config.client_ids_and_stake_file,
-            config.batch_size,
-            config.account_groups,
-            config.fund_amount,
-        );
-    } else {
-        info!("Connecting to the cluster");
-        let nodes = discover_cluster(&entrypoint_addr, num_nodes).unwrap_or_else(|_| {
-            panic!("Failed to discover nodes");
-        });
-
-        let (client, num_clients) = get_multi_client(&nodes);
-
-        info!("{} nodes found", num_clients);
-        if num_clients < num_nodes {
-            panic!("Error: Insufficient nodes discovered");
-        }
-
-        if !read_from_client_file {
-            info!("Funding keypair: {}", config.identity.pubkey());
-
-            let accounts_in_groups = batch_size * account_groups;
-            const NUM_SIGNERS: u64 = 2;
-            airdrop_lamports(
-                &client,
-                &faucet_addr,
-                &config.identity,
-                fund_amount * (accounts_in_groups + 1) as u64 * NUM_SIGNERS,
-            );
-        }
-        do_bench_exchange(vec![client], config);
-    }
-}

bench-exchange/src/order_book.rs

@@ -1,134 +0,0 @@
-use itertools::EitherOrBoth::{Both, Left, Right};
-use itertools::Itertools;
-use log::*;
-use solana_exchange_program::exchange_state::*;
-use solana_sdk::pubkey::Pubkey;
-use std::cmp::Ordering;
-use std::collections::BinaryHeap;
-use std::{error, fmt};
-
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct ToOrder {
-    pub pubkey: Pubkey,
-    pub info: OrderInfo,
-}
-
-impl Ord for ToOrder {
-    fn cmp(&self, other: &Self) -> Ordering {
-        other.info.price.cmp(&self.info.price)
-    }
-}
-impl PartialOrd for ToOrder {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-#[derive(Clone, Debug, Eq, PartialEq)]
-pub struct FromOrder {
-    pub pubkey: Pubkey,
-    pub info: OrderInfo,
-}
-
-impl Ord for FromOrder {
-    fn cmp(&self, other: &Self) -> Ordering {
-        self.info.price.cmp(&other.info.price)
-    }
-}
-impl PartialOrd for FromOrder {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-#[derive(Default)]
-pub struct OrderBook {
-    // TODO scale to x token types
-    to_ab: BinaryHeap<ToOrder>,
-    from_ab: BinaryHeap<FromOrder>,
-}
-impl fmt::Display for OrderBook {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        writeln!(
-            f,
-            "+-Order Book--------------------------+-------------------------------------+"
-        )?;
-        for (i, it) in self
-            .to_ab
-            .iter()
-            .zip_longest(self.from_ab.iter())
-            .enumerate()
-        {
-            match it {
-                Both(to, from) => writeln!(
-                    f,
-                    "| T AB {:8} for {:8}/{:8} | F AB {:8} for {:8}/{:8} |{}",
-                    to.info.tokens,
-                    SCALER,
-                    to.info.price,
-                    from.info.tokens,
-                    SCALER,
-                    from.info.price,
-                    i
-                )?,
-                Left(to) => writeln!(
-                    f,
-                    "| T AB {:8} for {:8}/{:8} |                                     |{}",
-                    to.info.tokens, SCALER, to.info.price, i
-                )?,
-                Right(from) => writeln!(
-                    f,
-                    "|                                     | F AB {:8} for {:8}/{:8} |{}",
-                    from.info.tokens, SCALER, from.info.price, i
-                )?,
-            }
-        }
-        write!(
-            f,
-            "+-------------------------------------+-------------------------------------+"
-        )?;
-        Ok(())
-    }
-}
-
-impl OrderBook {
-    // TODO
-    // pub fn cancel(&mut self, pubkey: Pubkey) -> Result<(), Box<dyn error::Error>> {
-    //     Ok(())
-    // }
-    pub fn push(&mut self, pubkey: Pubkey, info: OrderInfo) -> Result<(), Box<dyn error::Error>> {
-        check_trade(info.side, info.tokens, info.price)?;
-        match info.side {
-            OrderSide::Ask => {
-                self.to_ab.push(ToOrder { pubkey, info });
-            }
-            OrderSide::Bid => {
-                self.from_ab.push(FromOrder { pubkey, info });
-            }
-        }
-        Ok(())
-    }
-    pub fn pop(&mut self) -> Option<(ToOrder, FromOrder)> {
-        if let Some(pair) = Self::pop_pair(&mut self.to_ab, &mut self.from_ab) {
-            return Some(pair);
-        }
-        None
-    }
-    pub fn get_num_outstanding(&self) -> (usize, usize) {
-        (self.to_ab.len(), self.from_ab.len())
-    }
-
-    fn pop_pair(
-        to_ab: &mut BinaryHeap<ToOrder>,
-        from_ab: &mut BinaryHeap<FromOrder>,
-    ) -> Option<(ToOrder, FromOrder)> {
-        let to = to_ab.peek()?;
-        let from = from_ab.peek()?;
-        if from.info.price < to.info.price {
-            debug!("Trade not viable");
-            return None;
-        }
-        let to = to_ab.pop()?;
-        let from = from_ab.pop()?;
-        Some((to, from))
-    }
-}

bench-exchange/tests/bench_exchange.rs

@@ -1,113 +0,0 @@
-use log::*;
-use solana_bench_exchange::bench::{airdrop_lamports, do_bench_exchange, Config};
-use solana_core::validator::ValidatorConfig;
-use solana_exchange_program::{
-    exchange_processor::process_instruction, id, solana_exchange_program,
-};
-use solana_faucet::faucet::run_local_faucet_with_port;
-use solana_gossip::gossip_service::{discover_cluster, get_multi_client};
-use solana_local_cluster::{
-    local_cluster::{ClusterConfig, LocalCluster},
-    validator_configs::make_identical_validator_configs,
-};
-use solana_runtime::{bank::Bank, bank_client::BankClient};
-use solana_sdk::{
-    genesis_config::create_genesis_config,
-    signature::{Keypair, Signer},
-};
-use std::{process::exit, sync::mpsc::channel, time::Duration};
-
-#[test]
-#[ignore]
-fn test_exchange_local_cluster() {
-    solana_logger::setup();
-
-    const NUM_NODES: usize = 1;
-
-    let config = Config {
-        identity: Keypair::new(),
-        duration: Duration::from_secs(1),
-        fund_amount: 100_000,
-        threads: 1,
-        transfer_delay: 20, // 15
-        batch_size: 100,    // 1000
-        chunk_size: 10,     // 200
-        account_groups: 1,  // 10
-        ..Config::default()
-    };
-    let Config {
-        fund_amount,
-        batch_size,
-        account_groups,
-        ..
-    } = config;
-    let accounts_in_groups = batch_size * account_groups;
-
-    let cluster = LocalCluster::new(&mut ClusterConfig {
-        node_stakes: vec![100_000; NUM_NODES],
-        cluster_lamports: 100_000_000_000_000,
-        validator_configs: make_identical_validator_configs(&ValidatorConfig::default(), NUM_NODES),
-        native_instruction_processors: [solana_exchange_program!()].to_vec(),
-        ..ClusterConfig::default()
-    });
-
-    let faucet_keypair = Keypair::new();
-    cluster.transfer(
-        &cluster.funding_keypair,
-        &faucet_keypair.pubkey(),
-        2_000_000_000_000,
-    );
-
-    let (addr_sender, addr_receiver) = channel();
-    run_local_faucet_with_port(faucet_keypair, addr_sender, Some(1_000_000_000_000), 0);
-    let faucet_addr = addr_receiver
-        .recv_timeout(Duration::from_secs(2))
-        .expect("run_local_faucet")
-        .expect("faucet_addr");
-
-    info!("Connecting to the cluster");
-    let nodes =
-        discover_cluster(&cluster.entry_point_info.gossip, NUM_NODES).unwrap_or_else(|err| {
-            error!("Failed to discover {} nodes: {:?}", NUM_NODES, err);
-            exit(1);
-        });
-
-    let (client, num_clients) = get_multi_client(&nodes);
-
-    info!("clients: {}", num_clients);
-    assert!(num_clients >= NUM_NODES);
-
-    const NUM_SIGNERS: u64 = 2;
-    airdrop_lamports(
-        &client,
-        &faucet_addr,
-        &config.identity,
-        fund_amount * (accounts_in_groups + 1) as u64 * NUM_SIGNERS,
-    );
-
-    do_bench_exchange(vec![client], config);
-}
-
-#[test]
-fn test_exchange_bank_client() {
-    solana_logger::setup();
-    let (genesis_config, identity) = create_genesis_config(100_000_000_000_000);
-    let mut bank = Bank::new(&genesis_config);
-    bank.add_builtin("exchange_program", id(), process_instruction);
-    let clients = vec![BankClient::new(bank)];
-
-    do_bench_exchange(
-        clients,
-        Config {
-            identity,
-            duration: Duration::from_secs(1),
-            fund_amount: 100_000,
-            threads: 1,
-            transfer_delay: 20, // 0;
-            batch_size: 100,    // 1500;
-            chunk_size: 10,     // 1500;
-            account_groups: 1,  // 50;
-            ..Config::default()
-        },
-    );
-}

bench-streamer/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2018"
+edition = "2021"
 name = "solana-bench-streamer"
-version = "1.7.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,9 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-streamer = { path = "../streamer", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+solana-streamer = { path = "../streamer", version = "=1.10.0" }
+solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/src/main.rs

@@ -1,32 +1,38 @@
 #![allow(clippy::integer_arithmetic)]
-use clap::{crate_description, crate_name, App, Arg};
-use solana_streamer::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
-use solana_streamer::streamer::{receiver, PacketReceiver};
-use std::cmp::max;
-use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
-use std::sync::mpsc::channel;
-use std::sync::Arc;
-use std::thread::sleep;
-use std::thread::{spawn, JoinHandle, Result};
-use std::time::Duration;
-use std::time::SystemTime;
+use {
+    clap::{crate_description, crate_name, App, Arg},
+    solana_streamer::{
+        packet::{Packet, PacketBatch, PacketBatchRecycler, PACKET_DATA_SIZE},
+        streamer::{receiver, PacketBatchReceiver},
+    },
+    std::{
+        cmp::max,
+        net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
+        sync::{
+            atomic::{AtomicBool, AtomicUsize, Ordering},
+            mpsc::channel,
+            Arc,
+        },
+        thread::{sleep, spawn, JoinHandle, Result},
+        time::{Duration, SystemTime},
+    },
+};
 
 fn producer(addr: &SocketAddr, exit: Arc<AtomicBool>) -> JoinHandle<()> {
     let send = UdpSocket::bind("0.0.0.0:0").unwrap();
-    let mut msgs = Packets::default();
-    msgs.packets.resize(10, Packet::default());
-    for w in msgs.packets.iter_mut() {
+    let mut packet_batch = PacketBatch::default();
+    packet_batch.packets.resize(10, Packet::default());
+    for w in packet_batch.packets.iter_mut() {
         w.meta.size = PACKET_DATA_SIZE;
-        w.meta.set_addr(&addr);
+        w.meta.set_addr(addr);
     }
-    let msgs = Arc::new(msgs);
+    let packet_batch = Arc::new(packet_batch);
     spawn(move || loop {
         if exit.load(Ordering::Relaxed) {
             return;
         }
         let mut num = 0;
-        for p in &msgs.packets {
+        for p in &packet_batch.packets {
             let a = p.meta.addr();
             assert!(p.meta.size <= PACKET_DATA_SIZE);
             send.send_to(&p.data[..p.meta.size], &a).unwrap();
@@ -36,14 +42,14 @@ fn producer(addr: &SocketAddr, exit: Arc<AtomicBool>) -> JoinHandle<()> {
     })
 }
 
-fn sink(exit: Arc<AtomicBool>, rvs: Arc<AtomicUsize>, r: PacketReceiver) -> JoinHandle<()> {
+fn sink(exit: Arc<AtomicBool>, rvs: Arc<AtomicUsize>, r: PacketBatchReceiver) -> JoinHandle<()> {
     spawn(move || loop {
         if exit.load(Ordering::Relaxed) {
             return;
         }
         let timer = Duration::new(1, 0);
-        if let Ok(msgs) = r.recv_timeout(timer) {
-            rvs.fetch_add(msgs.packets.len(), Ordering::Relaxed);
+        if let Ok(packet_batch) = r.recv_timeout(timer) {
+            rvs.fetch_add(packet_batch.packets.len(), Ordering::Relaxed);
         }
     })
 }
@@ -75,7 +81,7 @@ fn main() -> Result<()> {
 
     let mut read_channels = Vec::new();
     let mut read_threads = Vec::new();
-    let recycler = PacketsRecycler::default();
+    let recycler = PacketBatchRecycler::default();
     for _ in 0..num_sockets {
         let read = solana_net_utils::bind_to(ip_addr, port, false).unwrap();
         read.set_read_timeout(Some(Duration::new(1, 0))).unwrap();
@@ -92,6 +98,7 @@ fn main() -> Result<()> {
             recycler.clone(),
             "bench-streamer-test",
             1,
+            true,
         ));
     }
 

bench-tps/Cargo.toml

@@ -1,37 +1,36 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2018"
+edition = "2021"
 name = "solana-bench-tps"
-version = "1.7.0"
+version = "1.10.0"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 publish = false
 
 [dependencies]
-bincode = "1.3.1"
 clap = "2.33.1"
-log = "0.4.11"
-rayon = "1.5.0"
-serde_json = "1.0.56"
-serde_yaml = "0.8.13"
-solana-clap-utils = { path = "../clap-utils", version = "=1.7.0" }
-solana-core = { path = "../core", version = "=1.7.0" }
-solana-genesis = { path = "../genesis", version = "=1.7.0" }
-solana-client = { path = "../client", version = "=1.7.0" }
-solana-faucet = { path = "../faucet", version = "=1.7.0" }
-solana-gossip = { path = "../gossip", version = "=1.7.0" }
-solana-logger = { path = "../logger", version = "=1.7.0" }
-solana-metrics = { path = "../metrics", version = "=1.7.0" }
-solana-measure = { path = "../measure", version = "=1.7.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.7.0" }
-solana-runtime = { path = "../runtime", version = "=1.7.0" }
-solana-sdk = { path = "../sdk", version = "=1.7.0" }
-solana-version = { path = "../version", version = "=1.7.0" }
+log = "0.4.14"
+rayon = "1.5.1"
+serde_json = "1.0.74"
+serde_yaml = "0.8.23"
+solana-core = { path = "../core", version = "=1.10.0" }
+solana-genesis = { path = "../genesis", version = "=1.10.0" }
+solana-client = { path = "../client", version = "=1.10.0" }
+solana-faucet = { path = "../faucet", version = "=1.10.0" }
+solana-gossip = { path = "../gossip", version = "=1.10.0" }
+solana-logger = { path = "../logger", version = "=1.10.0" }
+solana-metrics = { path = "../metrics", version = "=1.10.0" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.10.0" }
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+solana-streamer = { path = "../streamer", version = "=1.10.0" }
+solana-version = { path = "../version", version = "=1.10.0" }
 
 [dev-dependencies]
-serial_test = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "=1.7.0" }
+serial_test = "0.5.1"
+solana-local-cluster = { path = "../local-cluster", version = "=1.10.0" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/src/bench.rs

@@ -1,34 +1,36 @@
-use crate::cli::Config;
-use log::*;
-use rayon::prelude::*;
-use solana_client::perf_utils::{sample_txs, SampleStats};
-use solana_core::gen_keys::GenKeys;
-use solana_faucet::faucet::request_airdrop_transaction;
-use solana_measure::measure::Measure;
-use solana_metrics::{self, datapoint_info};
-use solana_sdk::{
-    client::Client,
-    clock::{DEFAULT_S_PER_SLOT, MAX_PROCESSING_AGE},
-    commitment_config::CommitmentConfig,
-    fee_calculator::FeeCalculator,
-    hash::Hash,
-    message::Message,
-    pubkey::Pubkey,
-    signature::{Keypair, Signer},
-    system_instruction, system_transaction,
-    timing::{duration_as_ms, duration_as_s, duration_as_us, timestamp},
-    transaction::Transaction,
-};
-use std::{
-    collections::{HashSet, VecDeque},
-    net::SocketAddr,
-    process::exit,
-    sync::{
-        atomic::{AtomicBool, AtomicIsize, AtomicUsize, Ordering},
-        Arc, Mutex, RwLock,
+use {
+    crate::cli::Config,
+    log::*,
+    rayon::prelude::*,
+    solana_client::perf_utils::{sample_txs, SampleStats},
+    solana_core::gen_keys::GenKeys,
+    solana_faucet::faucet::request_airdrop_transaction,
+    solana_measure::measure::Measure,
+    solana_metrics::{self, datapoint_info},
+    solana_sdk::{
+        client::Client,
+        clock::{DEFAULT_MS_PER_SLOT, DEFAULT_S_PER_SLOT, MAX_PROCESSING_AGE},
+        commitment_config::CommitmentConfig,
+        hash::Hash,
+        instruction::{AccountMeta, Instruction},
+        message::Message,
+        pubkey::Pubkey,
+        signature::{Keypair, Signer},
+        system_instruction, system_transaction,
+        timing::{duration_as_ms, duration_as_s, duration_as_us, timestamp},
+        transaction::Transaction,
+    },
+    std::{
+        collections::{HashSet, VecDeque},
+        net::SocketAddr,
+        process::exit,
+        sync::{
+            atomic::{AtomicBool, AtomicIsize, AtomicUsize, Ordering},
+            Arc, Mutex, RwLock,
+        },
+        thread::{sleep, Builder, JoinHandle},
+        time::{Duration, Instant},
     },
-    thread::{sleep, Builder, JoinHandle},
-    time::{Duration, Instant},
 };
 
 // The point at which transactions become "too old", in seconds.
@@ -45,14 +47,12 @@ pub type Result<T> = std::result::Result<T, BenchTpsError>;
 
 pub type SharedTransactions = Arc<RwLock<VecDeque<Vec<(Transaction, u64)>>>>;
 
-fn get_recent_blockhash<T: Client>(client: &T) -> (Hash, FeeCalculator) {
+fn get_latest_blockhash<T: Client>(client: &T) -> Hash {
     loop {
-        match client.get_recent_blockhash_with_commitment(CommitmentConfig::processed()) {
-            Ok((blockhash, fee_calculator, _last_valid_slot)) => {
-                return (blockhash, fee_calculator)
-            }
+        match client.get_latest_blockhash_with_commitment(CommitmentConfig::processed()) {
+            Ok((blockhash, _)) => return blockhash,
             Err(err) => {
-                info!("Couldn't get recent blockhash: {:?}", err);
+                info!("Couldn't get last blockhash: {:?}", err);
                 sleep(Duration::from_secs(1));
             }
         };
@@ -239,19 +239,19 @@ where
 
     let shared_txs: SharedTransactions = Arc::new(RwLock::new(VecDeque::new()));
 
-    let recent_blockhash = Arc::new(RwLock::new(get_recent_blockhash(client.as_ref()).0));
+    let blockhash = Arc::new(RwLock::new(get_latest_blockhash(client.as_ref())));
     let shared_tx_active_thread_count = Arc::new(AtomicIsize::new(0));
     let total_tx_sent_count = Arc::new(AtomicUsize::new(0));
 
     let blockhash_thread = {
         let exit_signal = exit_signal.clone();
-        let recent_blockhash = recent_blockhash.clone();
+        let blockhash = blockhash.clone();
         let client = client.clone();
         let id = id.pubkey();
         Builder::new()
             .name("solana-blockhash-poller".to_string())
             .spawn(move || {
-                poll_blockhash(&exit_signal, &recent_blockhash, &client, &id);
+                poll_blockhash(&exit_signal, &blockhash, &client, &id);
             })
             .unwrap()
     };
@@ -271,7 +271,7 @@ where
     let start = Instant::now();
 
     generate_chunked_transfers(
-        recent_blockhash,
+        blockhash,
         &shared_txs,
         shared_tx_active_thread_count,
         source_keypair_chunks,
@@ -391,6 +391,22 @@ fn generate_txs(
     }
 }
 
+fn get_new_latest_blockhash<T: Client>(client: &Arc<T>, blockhash: &Hash) -> Option<Hash> {
+    let start = Instant::now();
+    while start.elapsed().as_secs() < 5 {
+        if let Ok(new_blockhash) = client.get_latest_blockhash() {
+            if new_blockhash != *blockhash {
+                return Some(new_blockhash);
+            }
+        }
+        debug!("Got same blockhash ({:?}), will retry...", blockhash);
+
+        // Retry ~twice during a slot
+        sleep(Duration::from_millis(DEFAULT_MS_PER_SLOT / 2));
+    }
+    None
+}
+
 fn poll_blockhash<T: Client>(
     exit_signal: &Arc<AtomicBool>,
     blockhash: &Arc<RwLock<Hash>>,
@@ -402,7 +418,7 @@ fn poll_blockhash<T: Client>(
     loop {
         let blockhash_updated = {
             let old_blockhash = *blockhash.read().unwrap();
-            if let Ok((new_blockhash, _fee)) = client.get_new_blockhash(&old_blockhash) {
+            if let Some(new_blockhash) = get_new_latest_blockhash(client, &old_blockhash) {
                 *blockhash.write().unwrap() = new_blockhash;
                 blockhash_last_updated = Instant::now();
                 true
@@ -540,16 +556,16 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
                 self.len(),
             );
 
-            let (blockhash, _fee_calculator) = get_recent_blockhash(client.as_ref());
+            let blockhash = get_latest_blockhash(client.as_ref());
 
             // re-sign retained to_fund_txes with updated blockhash
             self.sign(blockhash);
-            self.send(&client);
+            self.send(client);
 
             // Sleep a few slots to allow transactions to process
             sleep(Duration::from_secs(1));
 
-            self.verify(&client, to_lamports);
+            self.verify(client, to_lamports);
 
             // retry anything that seems to have dropped through cracks
             //  again since these txs are all or nothing, they're fine to
@@ -564,7 +580,7 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
         let to_fund_txs: Vec<(&Keypair, Transaction)> = to_fund
             .par_iter()
             .map(|(k, t)| {
-                let instructions = system_instruction::transfer_many(&k.pubkey(), &t);
+                let instructions = system_instruction::transfer_many(&k.pubkey(), t);
                 let message = Message::new(&instructions, Some(&k.pubkey()));
                 (*k, Transaction::new_unsigned(message))
             })
@@ -617,7 +633,7 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
                         return None;
                     }
 
-                    let verified = if verify_funding_transfer(&client, &tx, to_lamports) {
+                    let verified = if verify_funding_transfer(&client, tx, to_lamports) {
                         verified_txs.fetch_add(1, Ordering::Relaxed);
                         Some(k.pubkey())
                     } else {
@@ -732,8 +748,8 @@ pub fn airdrop_lamports<T: Client>(
             id.pubkey(),
         );
 
-        let (blockhash, _fee_calculator) = get_recent_blockhash(client);
-        match request_airdrop_transaction(&faucet_addr, &id.pubkey(), airdrop_amount, blockhash) {
+        let blockhash = get_latest_blockhash(client);
+        match request_airdrop_transaction(faucet_addr, &id.pubkey(), airdrop_amount, blockhash) {
             Ok(transaction) => {
                 let mut tries = 0;
                 loop {
@@ -890,8 +906,16 @@ pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
     //   pay for the transaction fees in a new run.
     let enough_lamports = 8 * lamports_per_account / 10;
     if first_keypair_balance < enough_lamports || last_keypair_balance < enough_lamports {
-        let fee_rate_governor = client.get_fee_rate_governor().unwrap();
-        let max_fee = fee_rate_governor.max_lamports_per_signature;
+        let single_sig_message = Message::new_with_blockhash(
+            &[Instruction::new_with_bytes(
+                Pubkey::new_unique(),
+                &[],
+                vec![AccountMeta::new(Pubkey::new_unique(), true)],
+            )],
+            None,
+            &client.get_latest_blockhash().unwrap(),
+        );
+        let max_fee = client.get_fee_for_message(&single_sig_message).unwrap();
         let extra_fees = extra * max_fee;
         let total_keypairs = keypairs.len() as u64 + 1; // Add one for funding keypair
         let total = lamports_per_account * total_keypairs + extra_fees;
@@ -924,17 +948,19 @@ pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use solana_runtime::bank::Bank;
-    use solana_runtime::bank_client::BankClient;
-    use solana_sdk::client::SyncClient;
-    use solana_sdk::fee_calculator::FeeRateGovernor;
-    use solana_sdk::genesis_config::create_genesis_config;
+    use {
+        super::*,
+        solana_runtime::{bank::Bank, bank_client::BankClient},
+        solana_sdk::{
+            client::SyncClient, fee_calculator::FeeRateGovernor,
+            genesis_config::create_genesis_config,
+        },
+    };
 
     #[test]
     fn test_bench_tps_bank_client() {
         let (genesis_config, id) = create_genesis_config(10_000);
-        let bank = Bank::new(&genesis_config);
+        let bank = Bank::new_for_tests(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
 
         let config = Config {
@@ -955,7 +981,7 @@ mod tests {
     #[test]
     fn test_bench_tps_fund_keys() {
         let (genesis_config, id) = create_genesis_config(10_000);
-        let bank = Bank::new(&genesis_config);
+        let bank = Bank::new_for_tests(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
         let keypair_count = 20;
         let lamports = 20;
@@ -978,7 +1004,7 @@ mod tests {
         let (mut genesis_config, id) = create_genesis_config(10_000);
         let fee_rate_governor = FeeRateGovernor::new(11, 0);
         genesis_config.fee_rate_governor = fee_rate_governor;
-        let bank = Bank::new(&genesis_config);
+        let bank = Bank::new_for_tests(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
         let keypair_count = 20;
         let lamports = 20;

bench-tps/src/cli.rs

@@ -1,11 +1,13 @@
-use clap::{crate_description, crate_name, App, Arg, ArgMatches};
-use solana_faucet::faucet::FAUCET_PORT;
-use solana_sdk::fee_calculator::FeeRateGovernor;
-use solana_sdk::{
-    pubkey::Pubkey,
-    signature::{read_keypair_file, Keypair},
+use {
+    clap::{crate_description, crate_name, App, Arg, ArgMatches},
+    solana_faucet::faucet::FAUCET_PORT,
+    solana_sdk::{
+        fee_calculator::FeeRateGovernor,
+        pubkey::Pubkey,
+        signature::{read_keypair_file, Keypair},
+    },
+    std::{net::SocketAddr, process::exit, time::Duration},
 };
-use std::{net::SocketAddr, process::exit, time::Duration};
 
 const NUM_LAMPORTS_PER_ACCOUNT_DEFAULT: u64 = solana_sdk::native_token::LAMPORTS_PER_SOL;
 

bench-tps/src/main.rs

@@ -1,13 +1,20 @@
 #![allow(clippy::integer_arithmetic)]
-use log::*;
-use solana_bench_tps::bench::{do_bench_tps, generate_and_fund_keypairs, generate_keypairs};
-use solana_bench_tps::cli;
-use solana_genesis::Base64Account;
-use solana_gossip::gossip_service::{discover_cluster, get_client, get_multi_client};
-use solana_sdk::fee_calculator::FeeRateGovernor;
-use solana_sdk::signature::{Keypair, Signer};
-use solana_sdk::system_program;
-use std::{collections::HashMap, fs::File, io::prelude::*, path::Path, process::exit, sync::Arc};
+use {
+    log::*,
+    solana_bench_tps::{
+        bench::{do_bench_tps, generate_and_fund_keypairs, generate_keypairs},
+        cli,
+    },
+    solana_genesis::Base64Account,
+    solana_gossip::gossip_service::{discover_cluster, get_client, get_multi_client},
+    solana_sdk::{
+        fee_calculator::FeeRateGovernor,
+        signature::{Keypair, Signer},
+        system_program,
+    },
+    solana_streamer::socket::SocketAddrSpace,
+    std::{collections::HashMap, fs::File, io::prelude::*, path::Path, process::exit, sync::Arc},
+};
 
 /// Number of signatures for all transactions in ~1 week at ~100K TPS
 pub const NUM_SIGNATURES_FOR_TXS: u64 = 100_000 * 60 * 60 * 24 * 7;
@@ -39,7 +46,7 @@ fn main() {
     let keypair_count = *tx_count * keypair_multiplier;
     if *write_to_client_file {
         info!("Generating {} keypairs", keypair_count);
-        let (keypairs, _) = generate_keypairs(&id, keypair_count as u64);
+        let (keypairs, _) = generate_keypairs(id, keypair_count as u64);
         let num_accounts = keypairs.len() as u64;
         let max_fee =
             FeeRateGovernor::new(*target_lamports_per_signature, 0).max_lamports_per_signature;
@@ -68,13 +75,14 @@ fn main() {
     }
 
     info!("Connecting to the cluster");
-    let nodes = discover_cluster(&entrypoint_addr, *num_nodes).unwrap_or_else(|err| {
-        eprintln!("Failed to discover {} nodes: {:?}", num_nodes, err);
-        exit(1);
-    });
+    let nodes = discover_cluster(entrypoint_addr, *num_nodes, SocketAddrSpace::Unspecified)
+        .unwrap_or_else(|err| {
+            eprintln!("Failed to discover {} nodes: {:?}", num_nodes, err);
+            exit(1);
+        });
 
     let client = if *multi_client {
-        let (client, num_clients) = get_multi_client(&nodes);
+        let (client, num_clients) = get_multi_client(&nodes, &SocketAddrSpace::Unspecified);
         if nodes.len() < num_clients {
             eprintln!(
                 "Error: Insufficient nodes discovered.  Expecting {} or more",
@@ -88,7 +96,7 @@ fn main() {
         let mut target_client = None;
         for node in nodes {
             if node.id == *target_node {
-                target_client = Some(Arc::new(get_client(&[node])));
+                target_client = Some(Arc::new(get_client(&[node], &SocketAddrSpace::Unspecified)));
                 break;
             }
         }
@@ -97,7 +105,7 @@ fn main() {
             exit(1);
         })
     } else {
-        Arc::new(get_client(&nodes))
+        Arc::new(get_client(&nodes, &SocketAddrSpace::Unspecified))
     };
 
     let keypairs = if *read_from_client_file {
@@ -135,7 +143,7 @@ fn main() {
         generate_and_fund_keypairs(
             client.clone(),
             Some(*faucet_addr),
-            &id,
+            id,
             keypair_count,
             *num_lamports_per_account,
         )

bench-tps/tests/bench_tps.rs

@@ -1,21 +1,24 @@
 #![allow(clippy::integer_arithmetic)]
-use serial_test::serial;
-use solana_bench_tps::{
-    bench::{do_bench_tps, generate_and_fund_keypairs},
-    cli::Config,
-};
-use solana_client::thin_client::create_client;
-use solana_core::validator::ValidatorConfig;
-use solana_faucet::faucet::run_local_faucet_with_port;
-use solana_gossip::cluster_info::VALIDATOR_PORT_RANGE;
-use solana_local_cluster::{
-    local_cluster::{ClusterConfig, LocalCluster},
-    validator_configs::make_identical_validator_configs,
-};
-use solana_sdk::signature::{Keypair, Signer};
-use std::{
-    sync::{mpsc::channel, Arc},
-    time::Duration,
+use {
+    serial_test::serial,
+    solana_bench_tps::{
+        bench::{do_bench_tps, generate_and_fund_keypairs},
+        cli::Config,
+    },
+    solana_client::thin_client::create_client,
+    solana_core::validator::ValidatorConfig,
+    solana_faucet::faucet::run_local_faucet_with_port,
+    solana_gossip::cluster_info::VALIDATOR_PORT_RANGE,
+    solana_local_cluster::{
+        local_cluster::{ClusterConfig, LocalCluster},
+        validator_configs::make_identical_validator_configs,
+    },
+    solana_sdk::signature::{Keypair, Signer},
+    solana_streamer::socket::SocketAddrSpace,
+    std::{
+        sync::{mpsc::channel, Arc},
+        time::Duration,
+    },
 };
 
 fn test_bench_tps_local_cluster(config: Config) {
@@ -23,13 +26,19 @@ fn test_bench_tps_local_cluster(config: Config) {
 
     solana_logger::setup();
     const NUM_NODES: usize = 1;
-    let cluster = LocalCluster::new(&mut ClusterConfig {
-        node_stakes: vec![999_990; NUM_NODES],
-        cluster_lamports: 200_000_000,
-        validator_configs: make_identical_validator_configs(&ValidatorConfig::default(), NUM_NODES),
-        native_instruction_processors,
-        ..ClusterConfig::default()
-    });
+    let cluster = LocalCluster::new(
+        &mut ClusterConfig {
+            node_stakes: vec![999_990; NUM_NODES],
+            cluster_lamports: 200_000_000,
+            validator_configs: make_identical_validator_configs(
+                &ValidatorConfig::default(),
+                NUM_NODES,
+            ),
+            native_instruction_processors,
+            ..ClusterConfig::default()
+        },
+        SocketAddrSpace::Unspecified,
+    );
 
     let faucet_keypair = Keypair::new();
     cluster.transfer(

book/src/proposals/handle-duplicate-block.md

@@ -0,0 +1,88 @@
+# Leader Duplicate Block Slashing
+
+This design describes how the cluster slashes leaders that produce duplicate
+blocks.
+
+Leaders that produce multiple blocks for the same slot increase the number of
+potential forks that the cluster has to resolve.
+
+## Primitives
+1. gossip_root: Nodes now gossip their current root
+2. gossip_duplicate_slots: Nodes can gossip up to `N` duplicate slot proofs.
+3. `DUPLICATE_THRESHOLD`: The minimum percentage of stake that needs to vote on a fork with version `X` of a duplicate slot, in order for that fork to become votable.
+
+## Protocol
+1. When WindowStage detects a duplicate slot proof `P`, it checks the new `gossip_root` to see if `<= 1/3` of the nodes have rooted a slot `S >= P`. If so, it pushes a proof to `gossip_duplicate_slots` to gossip. WindowStage then signals ReplayStage about this duplicate slot `S`. These proofs can be purged from gossip once the validator sees > 2/3 of people gossiping roots `R > S`.
+
+2. When ReplayStage receives the signal for a duplicate slot `S` from `1)` above, the validator monitors gossip and replay waiting for`>= DUPLICATE_THRESHOLD` votes for the same hash which implies the same version of the slot. If this conditon is met for some version with hash `H` of slot `S`, this is then known as the `duplicate_confirmed` version of the slot.
+
+Before a duplicate slot `S` is `duplicate_confirmed`, it's first excluded from the vote candidate set in the fork choice rules. In addition, ReplayStage also resets PoH to the *latest* ancestor of the *earliest* `non-duplicate/confirmed_duplicate_slot`, so that block generation can start happening on the earliest known *safe* block.
+
+Some notes about the `DUPLICATE_THRESHOLD`. In the cases below, assume `DUPLICATE_THRESHOLD = 52`:
+
+a) If less than `2 * DUPLICATE_THRESHOLD - 1` percentage of the network is malicious, then there can only be one such `duplicate_confirmed` version of the slot. With `DUPLICATE_THRESHOLD = 52`, this is
+a malcious tolerance of `4%`
+
+b) The liveness of the network is at most `1 - DUPLICATE_THRESHOLD - SWITCH_THRESHOLD`. This is because if you need at least `SWITCH_THRESHOLD` percentage of the stake voting on a different fork in order to switch off of a duplicate fork that has `< DUPLICATE_THRESHOLD` stake voting on it, and is *not* `duplicate_confirmed`. For `DUPLICATE_THRESHOLD = 52` and `DUPLICATE_THRESHOLD = 38`, this implies a liveness tolerance of `10%`.
+
+For example in the situation below, validators that voted on `2` can't vote any further on fork `2` because it's been removed from fork choice. Now slot 6 better have enough stake for a switching proof, or the network halts.
+
+```text
+    |-------- 2 (51% voted, then detected this slot was a duplicate and removed this slot from fork choice)
+0---|
+    |---------- 6 (39%)
+
+```
+
+3. Switching proofs need to be extended to allow including vote hashes from different versions of the same same slot (detected through 1). Right now this is not supported since switching proofs can
+only be built using votes from banks in BankForks, and two different versions of the same slot cannot
+simultaneously exist in BankForks. For instance:
+
+```text
+    |-------- 2
+    |
+0------------- 1 ------ 2'
+    |
+    |---------- 6
+
+```
+
+Imagine each version of slot 2 and 2' have `DUPLICATE_THRESHOLD / 2` of the votes on them, so neither duplicate can be confirmed. At most slot 6 has `1 - DUPLICATE_THRESHOLD / 2` of the votes
+on it, which is less than the switching threshold. Thus, in order for validators voting on `2` or `2'` to switch to slot 6, and make progress, they need to incorporate votes from the other version of the slot into their switching proofs.
+
+
+### The repair problem.
+Now what happens if one of the following occurs:
+
+1) Due to network blips/latencies, some validators fail to observe the gossip votes before they are overwritten by newer votes? Then some validators may conclude a slot `S` is `duplicate_confirmed` while others don't.
+
+2) Due to lockouts, no version of duplicate slot `S` reaches `duplicate_confirmed` status, but one of its descendants may reach `duplicate_confirmed` after those lockouts expire, which by definition, means `S` is also `duplicate_confirmed`.
+
+3) People who are catching up and don't see the votes in gossip encounter a dup block and can't make progress.
+
+We assume that given a network is eventually stable, if at least one correct validator observed `S` is `duplicate_confirmed`, then if `S` is part of the heaviest fork, then eventually all validators will observe some descendant of `S` is duplicate confirmed.
+
+This problem we need to solve is modeled simply by the below scenario:
+
+```text
+1 -> 2 (duplicate) -> 3 -> 4 (duplicate)
+```
+Assume the following:
+
+1. Due to gossiping duplciate proofs, we assume everyone will eventually see duplicate proofs for 2 and 4, so everyone agrees to remove them from fork choice until they are `duplicate_confirmed`.
+
+2. Due to lockouts, `> DUPLICATE_THRESHOLD` of the stake votes on 4, but not 2. This means at least `DUPLICATE_THRESHOLD` of people have the "correct" version of both slots 2 and 4.
+
+3. However, the remaining `1-DUPLICATE_THRESHOLD` of people have wrong version of 2. This means in replay, their slot 3 will be marked dead, *even though the faulty slot is 2*. The goal is to get these people on the right fork again.
+
+Possible solution:
+
+1. Change `EpochSlots` to signal when a bank is frozen, not when a slot is complete. If we see > `DUPLICATE_THRESHOLD` have frozen the dead slot 3, then we attempt recovery. Note this does not mean that all `DUPLICATE_THRESHOLD` have frozen the same version of the bank, it's just a signal to us that something may be wrong with our version of the bank.
+
+2. Recovery takes the form of a special repair request, `RepairDuplicateConfirmed(dead_slot, Vec<(Slot, Hash)>)`, which specifies a  dead slot, and then a vector of `(slot, hash)` of `N` of its latest parents.
+
+3. The repairer sees this request and responds with the correct hash only if any element of the `(slot, hash)` vector is both `duplicate_confirmed` and the hash doesn't match the requester's hash in the vector.
+
+4. Once the requester sees the "correct" hash is different than their frozen hash, they dump the block so that they can accept a new block, and ask the network for the block with the correct hash.
+
+Of course the repairer might lie to you, and you'll get the wrong version of the block, in which case you'll end up with another dead block and repeat the procedure.

bucket_map/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "solana-bucket-map"
+version = "1.10.0"
+description = "solana-bucket-map"
+homepage = "https://solana.com/"
+documentation = "https://docs.rs/solana-bucket-map"
+readme = "../README.md"
+repository = "https://github.com/solana-labs/solana"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+license = "Apache-2.0"
+edition = "2021"
+
+[dependencies]
+solana-sdk = { path = "../sdk", version = "=1.10.0" }
+memmap2 = "0.5.0"
+log = { version = "0.4.11" }
+solana-measure = { path = "../measure", version = "=1.10.0" }
+rand = "0.7.0"
+tempfile = "3.2.0"
+
+[dev-dependencies]
+fs_extra = "1.2.0"
+rayon = "1.5.0"
+solana-logger = { path = "../logger", version = "=1.10.0" }
+
+[lib]
+crate-type = ["lib"]
+name = "solana_bucket_map"
+
+[[bench]]
+name = "bucket_map"

bucket_map/benches/bucket_map.rs

@@ -0,0 +1,77 @@
+#![feature(test)]
+
+macro_rules! DEFINE_NxM_BENCH {
+    ($i:ident, $n:literal, $m:literal) => {
+        mod $i {
+            use super::*;
+
+            #[bench]
+            fn bench_insert_baseline_hashmap(bencher: &mut Bencher) {
+                do_bench_insert_baseline_hashmap(bencher, $n, $m);
+            }
+
+            #[bench]
+            fn bench_insert_bucket_map(bencher: &mut Bencher) {
+                do_bench_insert_bucket_map(bencher, $n, $m);
+            }
+        }
+    };
+}
+
+extern crate test;
+use {
+    rayon::prelude::*,
+    solana_bucket_map::bucket_map::{BucketMap, BucketMapConfig},
+    solana_sdk::pubkey::Pubkey,
+    std::{collections::hash_map::HashMap, sync::RwLock},
+    test::Bencher,
+};
+
+type IndexValue = u64;
+
+DEFINE_NxM_BENCH!(dim_01x02, 1, 2);
+DEFINE_NxM_BENCH!(dim_02x04, 2, 4);
+DEFINE_NxM_BENCH!(dim_04x08, 4, 8);
+DEFINE_NxM_BENCH!(dim_08x16, 8, 16);
+DEFINE_NxM_BENCH!(dim_16x32, 16, 32);
+DEFINE_NxM_BENCH!(dim_32x64, 32, 64);
+
+/// Benchmark insert with Hashmap as baseline for N threads inserting M keys each
+fn do_bench_insert_baseline_hashmap(bencher: &mut Bencher, n: usize, m: usize) {
+    let index = RwLock::new(HashMap::new());
+    (0..n).into_iter().into_par_iter().for_each(|i| {
+        let key = Pubkey::new_unique();
+        index
+            .write()
+            .unwrap()
+            .insert(key, vec![(i, IndexValue::default())]);
+    });
+    bencher.iter(|| {
+        (0..n).into_iter().into_par_iter().for_each(|_| {
+            for j in 0..m {
+                let key = Pubkey::new_unique();
+                index
+                    .write()
+                    .unwrap()
+                    .insert(key, vec![(j, IndexValue::default())]);
+            }
+        })
+    });
+}
+
+/// Benchmark insert with BucketMap with N buckets for N threads inserting M keys each
+fn do_bench_insert_bucket_map(bencher: &mut Bencher, n: usize, m: usize) {
+    let index = BucketMap::new(BucketMapConfig::new(n));
+    (0..n).into_iter().into_par_iter().for_each(|i| {
+        let key = Pubkey::new_unique();
+        index.update(&key, |_| Some((vec![(i, IndexValue::default())], 0)));
+    });
+    bencher.iter(|| {
+        (0..n).into_iter().into_par_iter().for_each(|_| {
+            for j in 0..m {
+                let key = Pubkey::new_unique();
+                index.update(&key, |_| Some((vec![(j, IndexValue::default())], 0)));
+            }
+        })
+    });
+}

bucket_map/src/bucket.rs

@@ -0,0 +1,501 @@
+use {
+    crate::{
+        bucket_item::BucketItem,
+        bucket_map::BucketMapError,
+        bucket_stats::BucketMapStats,
+        bucket_storage::{BucketStorage, Uid, DEFAULT_CAPACITY_POW2},
+        index_entry::IndexEntry,
+        MaxSearch, RefCount,
+    },
+    rand::{thread_rng, Rng},
+    solana_measure::measure::Measure,
+    solana_sdk::pubkey::Pubkey,
+    std::{
+        collections::hash_map::DefaultHasher,
+        hash::{Hash, Hasher},
+        marker::PhantomData,
+        ops::RangeBounds,
+        path::PathBuf,
+        sync::{
+            atomic::{AtomicU64, AtomicUsize, Ordering},
+            Arc, Mutex,
+        },
+    },
+};
+
+#[derive(Default)]
+pub struct ReallocatedItems {
+    // Some if the index was reallocated
+    // u64 is random associated with the new index
+    pub index: Option<(u64, BucketStorage)>,
+    // Some for a data bucket reallocation
+    // u64 is data bucket index
+    pub data: Option<(u64, BucketStorage)>,
+}
+
+#[derive(Default)]
+pub struct Reallocated {
+    /// > 0 if reallocations are encoded
+    pub active_reallocations: AtomicUsize,
+
+    /// actual reallocated bucket
+    /// mutex because bucket grow code runs with a read lock
+    pub items: Mutex<ReallocatedItems>,
+}
+
+impl Reallocated {
+    /// specify that a reallocation has occurred
+    pub fn add_reallocation(&self) {
+        assert_eq!(
+            0,
+            self.active_reallocations.fetch_add(1, Ordering::Relaxed),
+            "Only 1 reallocation can occur at a time"
+        );
+    }
+    /// Return true IFF a reallocation has occurred.
+    /// Calling this takes conceptual ownership of the reallocation encoded in the struct.
+    pub fn get_reallocated(&self) -> bool {
+        self.active_reallocations
+            .compare_exchange(1, 0, Ordering::Acquire, Ordering::Relaxed)
+            .is_ok()
+    }
+}
+
+// >= 2 instances of BucketStorage per 'bucket' in the bucket map. 1 for index, >= 1 for data
+pub struct Bucket<T> {
+    drives: Arc<Vec<PathBuf>>,
+    //index
+    pub index: BucketStorage,
+    //random offset for the index
+    random: u64,
+    //storage buckets to store SlotSlice up to a power of 2 in len
+    pub data: Vec<BucketStorage>,
+    _phantom: PhantomData<T>,
+    stats: Arc<BucketMapStats>,
+
+    pub reallocated: Reallocated,
+}
+
+impl<T: Clone + Copy> Bucket<T> {
+    pub fn new(
+        drives: Arc<Vec<PathBuf>>,
+        max_search: MaxSearch,
+        stats: Arc<BucketMapStats>,
+        count: Arc<AtomicU64>,
+    ) -> Self {
+        let index = BucketStorage::new(
+            Arc::clone(&drives),
+            1,
+            std::mem::size_of::<IndexEntry>() as u64,
+            max_search,
+            Arc::clone(&stats.index),
+            count,
+        );
+        Self {
+            random: thread_rng().gen(),
+            drives,
+            index,
+            data: vec![],
+            _phantom: PhantomData::default(),
+            stats,
+            reallocated: Reallocated::default(),
+        }
+    }
+
+    pub fn keys(&self) -> Vec<Pubkey> {
+        let mut rv = vec![];
+        for i in 0..self.index.capacity() {
+            if self.index.is_free(i) {
+                continue;
+            }
+            let ix: &IndexEntry = self.index.get(i);
+            rv.push(ix.key);
+        }
+        rv
+    }
+
+    pub fn items_in_range<R>(&self, range: &Option<&R>) -> Vec<BucketItem<T>>
+    where
+        R: RangeBounds<Pubkey>,
+    {
+        let mut result = Vec::with_capacity(self.index.count.load(Ordering::Relaxed) as usize);
+        for i in 0..self.index.capacity() {
+            let ii = i % self.index.capacity();
+            if self.index.is_free(ii) {
+                continue;
+            }
+            let ix: &IndexEntry = self.index.get(ii);
+            let key = ix.key;
+            if range.map(|r| r.contains(&key)).unwrap_or(true) {
+                let val = ix.read_value(self);
+                result.push(BucketItem {
+                    pubkey: key,
+                    ref_count: ix.ref_count(),
+                    slot_list: val.map(|(v, _ref_count)| v.to_vec()).unwrap_or_default(),
+                });
+            }
+        }
+        result
+    }
+
+    pub fn find_entry(&self, key: &Pubkey) -> Option<(&IndexEntry, u64)> {
+        Self::bucket_find_entry(&self.index, key, self.random)
+    }
+
+    fn find_entry_mut(&self, key: &Pubkey) -> Option<(&mut IndexEntry, u64)> {
+        Self::bucket_find_entry_mut(&self.index, key, self.random)
+    }
+
+    fn bucket_find_entry_mut<'a>(
+        index: &'a BucketStorage,
+        key: &Pubkey,
+        random: u64,
+    ) -> Option<(&'a mut IndexEntry, u64)> {
+        let ix = Self::bucket_index_ix(index, key, random);
+        for i in ix..ix + index.max_search() {
+            let ii = i % index.capacity();
+            if index.is_free(ii) {
+                continue;
+            }
+            let elem: &mut IndexEntry = index.get_mut(ii);
+            if elem.key == *key {
+                return Some((elem, ii));
+            }
+        }
+        None
+    }
+
+    fn bucket_find_entry<'a>(
+        index: &'a BucketStorage,
+        key: &Pubkey,
+        random: u64,
+    ) -> Option<(&'a IndexEntry, u64)> {
+        let ix = Self::bucket_index_ix(index, key, random);
+        for i in ix..ix + index.max_search() {
+            let ii = i % index.capacity();
+            if index.is_free(ii) {
+                continue;
+            }
+            let elem: &IndexEntry = index.get(ii);
+            if elem.key == *key {
+                return Some((elem, ii));
+            }
+        }
+        None
+    }
+
+    fn bucket_create_key(
+        index: &mut BucketStorage,
+        key: &Pubkey,
+        elem_uid: Uid,
+        random: u64,
+        is_resizing: bool,
+    ) -> Result<u64, BucketMapError> {
+        let ix = Self::bucket_index_ix(index, key, random);
+        for i in ix..ix + index.max_search() {
+            let ii = i as u64 % index.capacity();
+            if !index.is_free(ii) {
+                continue;
+            }
+            index.allocate(ii, elem_uid, is_resizing).unwrap();
+            let elem: &mut IndexEntry = index.get_mut(ii);
+            // These fields will be overwritten after allocation by callers.
+            // Since this part of the mmapped file could have previously been used by someone else, there can be garbage here.
+            elem.init(key);
+            //debug!(                "INDEX ALLOC {:?} {} {} {}",                key, ii, index.capacity, elem_uid            );
+            return Ok(ii);
+        }
+        Err(BucketMapError::IndexNoSpace(index.capacity_pow2))
+    }
+
+    pub fn addref(&mut self, key: &Pubkey) -> Option<RefCount> {
+        let (elem, _) = self.find_entry_mut(key)?;
+        elem.ref_count += 1;
+        Some(elem.ref_count)
+    }
+
+    pub fn unref(&mut self, key: &Pubkey) -> Option<RefCount> {
+        let (elem, _) = self.find_entry_mut(key)?;
+        elem.ref_count -= 1;
+        Some(elem.ref_count)
+    }
+
+    fn create_key(&mut self, key: &Pubkey) -> Result<u64, BucketMapError> {
+        Self::bucket_create_key(
+            &mut self.index,
+            key,
+            IndexEntry::key_uid(key),
+            self.random,
+            false,
+        )
+    }
+
+    pub fn read_value(&self, key: &Pubkey) -> Option<(&[T], RefCount)> {
+        //debug!("READ_VALUE: {:?}", key);
+        let (elem, _) = self.find_entry(key)?;
+        elem.read_value(self)
+    }
+
+    pub fn try_write(
+        &mut self,
+        key: &Pubkey,
+        data: &[T],
+        ref_count: u64,
+    ) -> Result<(), BucketMapError> {
+        let best_fit_bucket = IndexEntry::data_bucket_from_num_slots(data.len() as u64);
+        if self.data.get(best_fit_bucket as usize).is_none() {
+            // fail early if the data bucket we need doesn't exist - we don't want the index entry partially allocated
+            return Err(BucketMapError::DataNoSpace((best_fit_bucket, 0)));
+        }
+        let index_entry = self.find_entry_mut(key);
+        let (elem, elem_ix) = match index_entry {
+            None => {
+                let ii = self.create_key(key)?;
+                let elem: &mut IndexEntry = self.index.get_mut(ii);
+                (elem, ii)
+            }
+            Some(res) => res,
+        };
+        elem.ref_count = ref_count;
+        let elem_uid = self.index.uid_unchecked(elem_ix);
+        let bucket_ix = elem.data_bucket_ix();
+        let current_bucket = &self.data[bucket_ix as usize];
+        let num_slots = data.len() as u64;
+        if best_fit_bucket == bucket_ix && elem.num_slots > 0 {
+            // in place update
+            let elem_loc = elem.data_loc(current_bucket);
+            let slice: &mut [T] = current_bucket.get_mut_cell_slice(elem_loc, data.len() as u64);
+            assert_eq!(current_bucket.uid(elem_loc), Some(elem_uid));
+            elem.num_slots = num_slots;
+            slice.copy_from_slice(data);
+            Ok(())
+        } else {
+            // need to move the allocation to a best fit spot
+            let best_bucket = &self.data[best_fit_bucket as usize];
+            let cap_power = best_bucket.capacity_pow2;
+            let cap = best_bucket.capacity();
+            let pos = thread_rng().gen_range(0, cap);
+            for i in pos..pos + self.index.max_search() {
+                let ix = i % cap;
+                if best_bucket.is_free(ix) {
+                    let elem_loc = elem.data_loc(current_bucket);
+                    let old_slots = elem.num_slots;
+                    elem.set_storage_offset(ix);
+                    elem.set_storage_capacity_when_created_pow2(best_bucket.capacity_pow2);
+                    elem.num_slots = num_slots;
+                    if old_slots > 0 {
+                        let current_bucket = &mut self.data[bucket_ix as usize];
+                        current_bucket.free(elem_loc, elem_uid);
+                    }
+                    //debug!(                        "DATA ALLOC {:?} {} {} {}",                        key, elem.data_location, best_bucket.capacity, elem_uid                    );
+                    if num_slots > 0 {
+                        let best_bucket = &mut self.data[best_fit_bucket as usize];
+                        best_bucket.allocate(ix, elem_uid, false).unwrap();
+                        let slice = best_bucket.get_mut_cell_slice(ix, num_slots);
+                        slice.copy_from_slice(data);
+                    }
+                    return Ok(());
+                }
+            }
+            Err(BucketMapError::DataNoSpace((best_fit_bucket, cap_power)))
+        }
+    }
+
+    pub fn delete_key(&mut self, key: &Pubkey) {
+        if let Some((elem, elem_ix)) = self.find_entry(key) {
+            let elem_uid = self.index.uid_unchecked(elem_ix);
+            if elem.num_slots > 0 {
+                let ix = elem.data_bucket_ix() as usize;
+                let data_bucket = &self.data[ix];
+                let loc = elem.data_loc(data_bucket);
+                let data_bucket = &mut self.data[ix];
+                //debug!(                    "DATA FREE {:?} {} {} {}",                    key, elem.data_location, data_bucket.capacity, elem_uid                );
+                data_bucket.free(loc, elem_uid);
+            }
+            //debug!("INDEX FREE {:?} {}", key, elem_uid);
+            self.index.free(elem_ix, elem_uid);
+        }
+    }
+
+    pub fn grow_index(&self, current_capacity_pow2: u8) {
+        if self.index.capacity_pow2 == current_capacity_pow2 {
+            let mut m = Measure::start("grow_index");
+            //debug!("GROW_INDEX: {}", current_capacity_pow2);
+            let increment = 1;
+            for i in increment.. {
+                //increasing the capacity by ^4 reduces the
+                //likelyhood of a re-index collision of 2^(max_search)^2
+                //1 in 2^32
+                let mut index = BucketStorage::new_with_capacity(
+                    Arc::clone(&self.drives),
+                    1,
+                    std::mem::size_of::<IndexEntry>() as u64,
+                    // *2 causes rapid growth of index buckets
+                    self.index.capacity_pow2 + i, // * 2,
+                    self.index.max_search,
+                    Arc::clone(&self.stats.index),
+                    Arc::clone(&self.index.count),
+                );
+                let random = thread_rng().gen();
+                let mut valid = true;
+                for ix in 0..self.index.capacity() {
+                    let uid = self.index.uid(ix);
+                    if let Some(uid) = uid {
+                        let elem: &IndexEntry = self.index.get(ix);
+                        let new_ix =
+                            Self::bucket_create_key(&mut index, &elem.key, uid, random, true);
+                        if new_ix.is_err() {
+                            valid = false;
+                            break;
+                        }
+                        let new_ix = new_ix.unwrap();
+                        let new_elem: &mut IndexEntry = index.get_mut(new_ix);
+                        *new_elem = *elem;
+                        /*
+                        let dbg_elem: IndexEntry = *new_elem;
+                        assert_eq!(
+                            Self::bucket_find_entry(&index, &elem.key, random).unwrap(),
+                            (&dbg_elem, new_ix)
+                        );
+                        */
+                    }
+                }
+                if valid {
+                    let sz = index.capacity();
+                    {
+                        let mut max = self.stats.index.max_size.lock().unwrap();
+                        *max = std::cmp::max(*max, sz);
+                    }
+                    let mut items = self.reallocated.items.lock().unwrap();
+                    items.index = Some((random, index));
+                    self.reallocated.add_reallocation();
+                    break;
+                }
+            }
+            m.stop();
+            self.stats.index.resizes.fetch_add(1, Ordering::Relaxed);
+            self.stats
+                .index
+                .resize_us
+                .fetch_add(m.as_us(), Ordering::Relaxed);
+        }
+    }
+
+    pub fn apply_grow_index(&mut self, random: u64, index: BucketStorage) {
+        self.random = random;
+        self.index = index;
+    }
+
+    fn elem_size() -> u64 {
+        std::mem::size_of::<T>() as u64
+    }
+
+    pub fn apply_grow_data(&mut self, ix: usize, bucket: BucketStorage) {
+        if self.data.get(ix).is_none() {
+            for i in self.data.len()..ix {
+                // insert empty data buckets
+                self.data.push(BucketStorage::new(
+                    Arc::clone(&self.drives),
+                    1 << i,
+                    Self::elem_size(),
+                    self.index.max_search,
+                    Arc::clone(&self.stats.data),
+                    Arc::default(),
+                ))
+            }
+            self.data.push(bucket);
+        } else {
+            self.data[ix] = bucket;
+        }
+    }
+
+    /// grow a data bucket
+    /// The application of the new bucket is deferred until the next write lock.
+    pub fn grow_data(&self, data_index: u64, current_capacity_pow2: u8) {
+        let new_bucket = BucketStorage::new_resized(
+            &self.drives,
+            self.index.max_search,
+            self.data.get(data_index as usize),
+            std::cmp::max(current_capacity_pow2 + 1, DEFAULT_CAPACITY_POW2),
+            1 << data_index,
+            Self::elem_size(),
+            &self.stats.data,
+        );
+        self.reallocated.add_reallocation();
+        let mut items = self.reallocated.items.lock().unwrap();
+        items.data = Some((data_index, new_bucket));
+    }
+
+    fn bucket_index_ix(index: &BucketStorage, key: &Pubkey, random: u64) -> u64 {
+        let uid = IndexEntry::key_uid(key);
+        let mut s = DefaultHasher::new();
+        uid.hash(&mut s);
+        //the locally generated random will make it hard for an attacker
+        //to deterministically cause all the pubkeys to land in the same
+        //location in any bucket on all validators
+        random.hash(&mut s);
+        let ix = s.finish();
+        ix % index.capacity()
+        //debug!(            "INDEX_IX: {:?} uid:{} loc: {} cap:{}",            key,            uid,            location,            index.capacity()        );
+    }
+
+    /// grow the appropriate piece. Note this takes an immutable ref.
+    /// The actual grow is set into self.reallocated and applied later on a write lock
+    pub fn grow(&self, err: BucketMapError) {
+        match err {
+            BucketMapError::DataNoSpace((data_index, current_capacity_pow2)) => {
+                //debug!("GROWING SPACE {:?}", (data_index, current_capacity_pow2));
+                self.grow_data(data_index, current_capacity_pow2);
+            }
+            BucketMapError::IndexNoSpace(current_capacity_pow2) => {
+                //debug!("GROWING INDEX {}", sz);
+                self.grow_index(current_capacity_pow2);
+            }
+        }
+    }
+
+    /// if a bucket was resized previously with a read lock, then apply that resize now
+    pub fn handle_delayed_grows(&mut self) {
+        if self.reallocated.get_reallocated() {
+            // swap out the bucket that was resized previously with a read lock
+            let mut items = ReallocatedItems::default();
+            std::mem::swap(&mut items, &mut self.reallocated.items.lock().unwrap());
+
+            if let Some((random, bucket)) = items.index.take() {
+                self.apply_grow_index(random, bucket);
+            } else {
+                // data bucket
+                let (i, new_bucket) = items.data.take().unwrap();
+                self.apply_grow_data(i as usize, new_bucket);
+            }
+        }
+    }
+
+    pub fn insert(&mut self, key: &Pubkey, value: (&[T], RefCount)) {
+        let (new, refct) = value;
+        loop {
+            let rv = self.try_write(key, new, refct);
+            match rv {
+                Ok(_) => return,
+                Err(err) => {
+                    self.grow(err);
+                    self.handle_delayed_grows();
+                }
+            }
+        }
+    }
+
+    pub fn update<F>(&mut self, key: &Pubkey, mut updatefn: F)
+    where
+        F: FnMut(Option<(&[T], RefCount)>) -> Option<(Vec<T>, RefCount)>,
+    {
+        let current = self.read_value(key);
+        let new = updatefn(current);
+        if new.is_none() {
+            self.delete_key(key);
+            return;
+        }
+        let (new, refct) = new.unwrap();
+        self.insert(key, (&new, refct));
+    }
+}

bucket_map/src/bucket_api.rs

@@ -0,0 +1,142 @@
+use {
+    crate::{
+        bucket::Bucket, bucket_item::BucketItem, bucket_map::BucketMapError,
+        bucket_stats::BucketMapStats, MaxSearch, RefCount,
+    },
+    solana_sdk::pubkey::Pubkey,
+    std::{
+        ops::RangeBounds,
+        path::PathBuf,
+        sync::{
+            atomic::{AtomicU64, Ordering},
+            Arc, RwLock, RwLockWriteGuard,
+        },
+    },
+};
+
+type LockedBucket<T> = RwLock<Option<Bucket<T>>>;
+
+pub struct BucketApi<T: Clone + Copy> {
+    drives: Arc<Vec<PathBuf>>,
+    max_search: MaxSearch,
+    pub stats: Arc<BucketMapStats>,
+
+    bucket: LockedBucket<T>,
+    count: Arc<AtomicU64>,
+}
+
+impl<T: Clone + Copy> BucketApi<T> {
+    pub fn new(
+        drives: Arc<Vec<PathBuf>>,
+        max_search: MaxSearch,
+        stats: Arc<BucketMapStats>,
+    ) -> Self {
+        Self {
+            drives,
+            max_search,
+            stats,
+            bucket: RwLock::default(),
+            count: Arc::default(),
+        }
+    }
+
+    /// Get the items for bucket
+    pub fn items_in_range<R>(&self, range: &Option<&R>) -> Vec<BucketItem<T>>
+    where
+        R: RangeBounds<Pubkey>,
+    {
+        self.bucket
+            .read()
+            .unwrap()
+            .as_ref()
+            .map(|bucket| bucket.items_in_range(range))
+            .unwrap_or_default()
+    }
+
+    /// Get the Pubkeys
+    pub fn keys(&self) -> Vec<Pubkey> {
+        self.bucket
+            .read()
+            .unwrap()
+            .as_ref()
+            .map_or_else(Vec::default, |bucket| bucket.keys())
+    }
+
+    /// Get the values for Pubkey `key`
+    pub fn read_value(&self, key: &Pubkey) -> Option<(Vec<T>, RefCount)> {
+        self.bucket.read().unwrap().as_ref().and_then(|bucket| {
+            bucket
+                .read_value(key)
+                .map(|(value, ref_count)| (value.to_vec(), ref_count))
+        })
+    }
+
+    pub fn bucket_len(&self) -> u64 {
+        self.count.load(Ordering::Relaxed)
+    }
+
+    pub fn delete_key(&self, key: &Pubkey) {
+        let mut bucket = self.get_write_bucket();
+        if let Some(bucket) = bucket.as_mut() {
+            bucket.delete_key(key)
+        }
+    }
+
+    fn get_write_bucket(&self) -> RwLockWriteGuard<Option<Bucket<T>>> {
+        let mut bucket = self.bucket.write().unwrap();
+        if bucket.is_none() {
+            *bucket = Some(Bucket::new(
+                Arc::clone(&self.drives),
+                self.max_search,
+                Arc::clone(&self.stats),
+                Arc::clone(&self.count),
+            ));
+        } else {
+            let write = bucket.as_mut().unwrap();
+            write.handle_delayed_grows();
+        }
+        bucket
+    }
+
+    pub fn addref(&self, key: &Pubkey) -> Option<RefCount> {
+        self.get_write_bucket()
+            .as_mut()
+            .and_then(|bucket| bucket.addref(key))
+    }
+
+    pub fn unref(&self, key: &Pubkey) -> Option<RefCount> {
+        self.get_write_bucket()
+            .as_mut()
+            .and_then(|bucket| bucket.unref(key))
+    }
+
+    pub fn insert(&self, pubkey: &Pubkey, value: (&[T], RefCount)) {
+        let mut bucket = self.get_write_bucket();
+        bucket.as_mut().unwrap().insert(pubkey, value)
+    }
+
+    pub fn grow(&self, err: BucketMapError) {
+        // grows are special - they get a read lock and modify 'reallocated'
+        // the grown changes are applied the next time there is a write lock taken
+        if let Some(bucket) = self.bucket.read().unwrap().as_ref() {
+            bucket.grow(err)
+        }
+    }
+
+    pub fn update<F>(&self, key: &Pubkey, updatefn: F)
+    where
+        F: FnMut(Option<(&[T], RefCount)>) -> Option<(Vec<T>, RefCount)>,
+    {
+        let mut bucket = self.get_write_bucket();
+        bucket.as_mut().unwrap().update(key, updatefn)
+    }
+
+    pub fn try_write(
+        &self,
+        pubkey: &Pubkey,
+        value: (&[T], RefCount),
+    ) -> Result<(), BucketMapError> {
+        let mut bucket = self.get_write_bucket();
+        bucket.as_mut().unwrap().try_write(pubkey, value.0, value.1)
+    }
+}

bucket_map/src/bucket_item.rs

@@ -0,0 +1,8 @@
+use {crate::RefCount, solana_sdk::pubkey::Pubkey};
+
+#[derive(Debug, Default, Clone)]
+pub struct BucketItem<T> {
+    pub pubkey: Pubkey,
+    pub ref_count: RefCount,
+    pub slot_list: Vec<T>,
+}

bucket_map/src/bucket_map.rs

@@ -0,0 +1,522 @@
+//! BucketMap is a mostly contention free concurrent map backed by MmapMut
+
+use {
+    crate::{bucket_api::BucketApi, bucket_stats::BucketMapStats, MaxSearch, RefCount},
+    solana_sdk::pubkey::Pubkey,
+    std::{convert::TryInto, fmt::Debug, fs, path::PathBuf, sync::Arc},
+    tempfile::TempDir,
+};
+
+#[derive(Debug, Default, Clone)]
+pub struct BucketMapConfig {
+    pub max_buckets: usize,
+    pub drives: Option<Vec<PathBuf>>,
+    pub max_search: Option<MaxSearch>,
+}
+
+impl BucketMapConfig {
+    /// Create a new BucketMapConfig
+    /// NOTE: BucketMap requires that max_buckets is a power of two
+    pub fn new(max_buckets: usize) -> BucketMapConfig {
+        BucketMapConfig {
+            max_buckets,
+            ..BucketMapConfig::default()
+        }
+    }
+}
+
+pub struct BucketMap<T: Clone + Copy + Debug> {
+    buckets: Vec<Arc<BucketApi<T>>>,
+    drives: Arc<Vec<PathBuf>>,
+    max_buckets_pow2: u8,
+    pub stats: Arc<BucketMapStats>,
+    pub temp_dir: Option<TempDir>,
+}
+
+impl<T: Clone + Copy + Debug> Drop for BucketMap<T> {
+    fn drop(&mut self) {
+        if self.temp_dir.is_none() {
+            BucketMap::<T>::erase_previous_drives(&self.drives);
+        }
+    }
+}
+
+impl<T: Clone + Copy + Debug> std::fmt::Debug for BucketMap<T> {
+    fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+pub enum BucketMapError {
+    /// (bucket_index, current_capacity_pow2)
+    DataNoSpace((u64, u8)),
+    /// current_capacity_pow2
+    IndexNoSpace(u8),
+}
+
+impl<T: Clone + Copy + Debug> BucketMap<T> {
+    pub fn new(config: BucketMapConfig) -> Self {
+        assert_ne!(
+            config.max_buckets, 0,
+            "Max number of buckets must be non-zero"
+        );
+        assert!(
+            config.max_buckets.is_power_of_two(),
+            "Max number of buckets must be a power of two"
+        );
+        // this should be <= 1 << DEFAULT_CAPACITY or we end up searching the same items over and over - probably not a big deal since it is so small anyway
+        const MAX_SEARCH: MaxSearch = 32;
+        let max_search = config.max_search.unwrap_or(MAX_SEARCH);
+
+        if let Some(drives) = config.drives.as_ref() {
+            Self::erase_previous_drives(drives);
+        }
+        let mut temp_dir = None;
+        let drives = config.drives.unwrap_or_else(|| {
+            temp_dir = Some(TempDir::new().unwrap());
+            vec![temp_dir.as_ref().unwrap().path().to_path_buf()]
+        });
+        let drives = Arc::new(drives);
+
+        let stats = Arc::default();
+        let buckets = (0..config.max_buckets)
+            .into_iter()
+            .map(|_| {
+                Arc::new(BucketApi::new(
+                    Arc::clone(&drives),
+                    max_search,
+                    Arc::clone(&stats),
+                ))
+            })
+            .collect();
+
+        // A simple log2 function that is correct if x is a power of two
+        let log2 = |x: usize| usize::BITS - x.leading_zeros() - 1;
+
+        Self {
+            buckets,
+            drives,
+            max_buckets_pow2: log2(config.max_buckets) as u8,
+            stats,
+            temp_dir,
+        }
+    }
+
+    fn erase_previous_drives(drives: &[PathBuf]) {
+        drives.iter().for_each(|folder| {
+            let _ = fs::remove_dir_all(&folder);
+            let _ = fs::create_dir_all(&folder);
+        })
+    }
+
+    pub fn num_buckets(&self) -> usize {
+        self.buckets.len()
+    }
+
+    /// Get the values for Pubkey `key`
+    pub fn read_value(&self, key: &Pubkey) -> Option<(Vec<T>, RefCount)> {
+        self.get_bucket(key).read_value(key)
+    }
+
+    /// Delete the Pubkey `key`
+    pub fn delete_key(&self, key: &Pubkey) {
+        self.get_bucket(key).delete_key(key);
+    }
+
+    /// Update Pubkey `key`'s value with 'value'
+    pub fn insert(&self, key: &Pubkey, value: (&[T], RefCount)) {
+        self.get_bucket(key).insert(key, value)
+    }
+
+    /// Update Pubkey `key`'s value with 'value'
+    pub fn try_insert(&self, key: &Pubkey, value: (&[T], RefCount)) -> Result<(), BucketMapError> {
+        self.get_bucket(key).try_write(key, value)
+    }
+
+    /// Update Pubkey `key`'s value with function `updatefn`
+    pub fn update<F>(&self, key: &Pubkey, updatefn: F)
+    where
+        F: FnMut(Option<(&[T], RefCount)>) -> Option<(Vec<T>, RefCount)>,
+    {
+        self.get_bucket(key).update(key, updatefn)
+    }
+
+    pub fn get_bucket(&self, key: &Pubkey) -> &Arc<BucketApi<T>> {
+        self.get_bucket_from_index(self.bucket_ix(key))
+    }
+
+    pub fn get_bucket_from_index(&self, ix: usize) -> &Arc<BucketApi<T>> {
+        &self.buckets[ix]
+    }
+
+    /// Get the bucket index for Pubkey `key`
+    pub fn bucket_ix(&self, key: &Pubkey) -> usize {
+        if self.max_buckets_pow2 > 0 {
+            let location = read_be_u64(key.as_ref());
+            (location >> (u64::BITS - self.max_buckets_pow2 as u32)) as usize
+        } else {
+            0
+        }
+    }
+
+    /// Increment the refcount for Pubkey `key`
+    pub fn addref(&self, key: &Pubkey) -> Option<RefCount> {
+        let ix = self.bucket_ix(key);
+        let bucket = &self.buckets[ix];
+        bucket.addref(key)
+    }
+
+    /// Decrement the refcount for Pubkey `key`
+    pub fn unref(&self, key: &Pubkey) -> Option<RefCount> {
+        let ix = self.bucket_ix(key);
+        let bucket = &self.buckets[ix];
+        bucket.unref(key)
+    }
+}
+
+/// Look at the first 8 bytes of the input and reinterpret them as a u64
+fn read_be_u64(input: &[u8]) -> u64 {
+    assert!(input.len() >= std::mem::size_of::<u64>());
+    u64::from_be_bytes(input[0..std::mem::size_of::<u64>()].try_into().unwrap())
+}
+
+#[cfg(test)]
+mod tests {
+    use {
+        super::*,
+        rand::{thread_rng, Rng},
+        std::{collections::HashMap, sync::RwLock},
+    };
+
+    #[test]
+    fn bucket_map_test_insert() {
+        let key = Pubkey::new_unique();
+        let config = BucketMapConfig::new(1 << 1);
+        let index = BucketMap::new(config);
+        index.update(&key, |_| Some((vec![0], 0)));
+        assert_eq!(index.read_value(&key), Some((vec![0], 0)));
+    }
+
+    #[test]
+    fn bucket_map_test_insert2() {
+        for pass in 0..3 {
+            let key = Pubkey::new_unique();
+            let config = BucketMapConfig::new(1 << 1);
+            let index = BucketMap::new(config);
+            let bucket = index.get_bucket(&key);
+            if pass == 0 {
+                index.insert(&key, (&[0], 0));
+            } else {
+                let result = index.try_insert(&key, (&[0], 0));
+                assert!(result.is_err());
+                assert_eq!(index.read_value(&key), None);
+                if pass == 2 {
+                    // another call to try insert again - should still return an error
+                    let result = index.try_insert(&key, (&[0], 0));
+                    assert!(result.is_err());
+                    assert_eq!(index.read_value(&key), None);
+                }
+                bucket.grow(result.unwrap_err());
+                let result = index.try_insert(&key, (&[0], 0));
+                assert!(result.is_ok());
+            }
+            assert_eq!(index.read_value(&key), Some((vec![0], 0)));
+        }
+    }
+
+    #[test]
+    fn bucket_map_test_update2() {
+        let key = Pubkey::new_unique();
+        let config = BucketMapConfig::new(1 << 1);
+        let index = BucketMap::new(config);
+        index.insert(&key, (&[0], 0));
+        assert_eq!(index.read_value(&key), Some((vec![0], 0)));
+        index.insert(&key, (&[1], 0));
+        assert_eq!(index.read_value(&key), Some((vec![1], 0)));
+    }
+
+    #[test]
+    fn bucket_map_test_update() {
+        let key = Pubkey::new_unique();
+        let config = BucketMapConfig::new(1 << 1);
+        let index = BucketMap::new(config);
+        index.update(&key, |_| Some((vec![0], 0)));
+        assert_eq!(index.read_value(&key), Some((vec![0], 0)));
+        index.update(&key, |_| Some((vec![1], 0)));
+        assert_eq!(index.read_value(&key), Some((vec![1], 0)));
+    }
+
+    #[test]
+    fn bucket_map_test_update_to_0_len() {
+        solana_logger::setup();
+        let key = Pubkey::new_unique();
+        let config = BucketMapConfig::new(1 << 1);
+        let index = BucketMap::new(config);
+        index.update(&key, |_| Some((vec![0], 1)));
+        assert_eq!(index.read_value(&key), Some((vec![0], 1)));
+        // sets len to 0, updates in place
+        index.update(&key, |_| Some((vec![], 1)));
+        assert_eq!(index.read_value(&key), Some((vec![], 1)));
+        // sets len to 0, doesn't update in place - finds a new place, which causes us to no longer have an allocation in data
+        index.update(&key, |_| Some((vec![], 2)));
+        assert_eq!(index.read_value(&key), Some((vec![], 2)));
+        // sets len to 1, doesn't update in place - finds a new place
+        index.update(&key, |_| Some((vec![1], 2)));
+        assert_eq!(index.read_value(&key), Some((vec![1], 2)));
+    }
+
+    #[test]
+    fn bucket_map_test_delete() {
+        let config = BucketMapConfig::new(1 << 1);
+        let index = BucketMap::new(config);
+        for i in 0..10 {
+            let key = Pubkey::new_unique();
+            assert_eq!(index.read_value(&key), None);
+
+            index.update(&key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
+
+            index.delete_key(&key);
+            assert_eq!(index.read_value(&key), None);
+
+            index.update(&key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
+            index.delete_key(&key);
+        }
+    }
+
+    #[test]
+    fn bucket_map_test_delete_2() {
+        let config = BucketMapConfig::new(1 << 2);
+        let index = BucketMap::new(config);
+        for i in 0..100 {
+            let key = Pubkey::new_unique();
+            assert_eq!(index.read_value(&key), None);
+
+            index.update(&key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
+
+            index.delete_key(&key);
+            assert_eq!(index.read_value(&key), None);
+
+            index.update(&key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
+            index.delete_key(&key);
+        }
+    }
+
+    #[test]
+    fn bucket_map_test_n_drives() {
+        let config = BucketMapConfig::new(1 << 2);
+        let index = BucketMap::new(config);
+        for i in 0..100 {
+            let key = Pubkey::new_unique();
+            index.update(&key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
+        }
+    }
+    #[test]
+    fn bucket_map_test_grow_read() {
+        let config = BucketMapConfig::new(1 << 2);
+        let index = BucketMap::new(config);
+        let keys: Vec<Pubkey> = (0..100).into_iter().map(|_| Pubkey::new_unique()).collect();
+        for k in 0..keys.len() {
+            let key = &keys[k];
+            let i = read_be_u64(key.as_ref());
+            index.update(key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(key), Some((vec![i], 0)));
+            for (ix, key) in keys.iter().enumerate() {
+                let i = read_be_u64(key.as_ref());
+                //debug!("READ: {:?} {}", key, i);
+                let expected = if ix <= k { Some((vec![i], 0)) } else { None };
+                assert_eq!(index.read_value(key), expected);
+            }
+        }
+    }
+
+    #[test]
+    fn bucket_map_test_n_delete() {
+        let config = BucketMapConfig::new(1 << 2);
+        let index = BucketMap::new(config);
+        let keys: Vec<Pubkey> = (0..20).into_iter().map(|_| Pubkey::new_unique()).collect();
+        for key in keys.iter() {
+            let i = read_be_u64(key.as_ref());
+            index.update(key, |_| Some((vec![i], 0)));
+            assert_eq!(index.read_value(key), Some((vec![i], 0)));
+        }
+        for key in keys.iter() {
+            let i = read_be_u64(key.as_ref());
+            //debug!("READ: {:?} {}", key, i);
+            assert_eq!(index.read_value(key), Some((vec![i], 0)));
+        }
+        for k in 0..keys.len() {
+            let key = &keys[k];
+            index.delete_key(key);
+            assert_eq!(index.read_value(key), None);
+            for key in keys.iter().skip(k + 1) {
+                let i = read_be_u64(key.as_ref());
+                assert_eq!(index.read_value(key), Some((vec![i], 0)));
+            }
+        }
+    }
+
+    #[test]
+    fn hashmap_compare() {
+        use std::sync::Mutex;
+        solana_logger::setup();
+        let maps = (0..2)
+            .into_iter()
+            .map(|max_buckets_pow2| {
+                let config = BucketMapConfig::new(1 << max_buckets_pow2);
+                BucketMap::new(config)
+            })
+            .collect::<Vec<_>>();
+        let hash_map = RwLock::new(HashMap::<Pubkey, (Vec<(usize, usize)>, RefCount)>::new());
+        let max_slot_list_len = 3;
+        let all_keys = Mutex::new(vec![]);
+
+        let gen_rand_value = || {
+            let count = thread_rng().gen_range(0, max_slot_list_len);
+            let v = (0..count)
+                .into_iter()
+                .map(|x| (x as usize, x as usize /*thread_rng().gen::<usize>()*/))
+                .collect::<Vec<_>>();
+            let rc = thread_rng().gen::<RefCount>();
+            (v, rc)
+        };
+
+        let get_key = || {
+            let mut keys = all_keys.lock().unwrap();
+            if keys.is_empty() {
+                return None;
+            }
+            let len = keys.len();
+            Some(keys.remove(thread_rng().gen_range(0, len)))
+        };
+        let return_key = |key| {
+            let mut keys = all_keys.lock().unwrap();
+            keys.push(key);
+        };
+
+        let verify = || {
+            let mut maps = maps
+                .iter()
+                .map(|map| {
+                    let mut r = vec![];
+                    for bin in 0..map.num_buckets() {
+                        r.append(
+                            &mut map.buckets[bin]
+                                .items_in_range(&None::<&std::ops::RangeInclusive<Pubkey>>),
+                        );
+                    }
+                    r
+                })
+                .collect::<Vec<_>>();
+            let hm = hash_map.read().unwrap();
+            for (k, v) in hm.iter() {
+                for map in maps.iter_mut() {
+                    for i in 0..map.len() {
+                        if k == &map[i].pubkey {
+                            assert_eq!(map[i].slot_list, v.0);
+                            assert_eq!(map[i].ref_count, v.1);
+                            map.remove(i);
+                            break;
+                        }
+                    }
+                }
+            }
+            for map in maps.iter() {
+                assert!(map.is_empty());
+            }
+        };
+        let mut initial = 100; // put this many items in to start
+
+        // do random operations: insert, update, delete, add/unref in random order
+        // verify consistency between hashmap and all bucket maps
+        for i in 0..10000 {
+            if initial > 0 {
+                initial -= 1;
+            }
+            if initial > 0 || thread_rng().gen_range(0, 5) == 0 {
+                // insert
+                let k = solana_sdk::pubkey::new_rand();
+                let v = gen_rand_value();
+                hash_map.write().unwrap().insert(k, v.clone());
+                let insert = thread_rng().gen_range(0, 2) == 0;
+                maps.iter().for_each(|map| {
+                    if insert {
+                        map.insert(&k, (&v.0, v.1))
+                    } else {
+                        map.update(&k, |current| {
+                            assert!(current.is_none());
+                            Some(v.clone())
+                        })
+                    }
+                });
+                return_key(k);
+            }
+            if thread_rng().gen_range(0, 10) == 0 {
+                // update
+                if let Some(k) = get_key() {
+                    let hm = hash_map.read().unwrap();
+                    let (v, rc) = gen_rand_value();
+                    let v_old = hm.get(&k);
+                    let insert = thread_rng().gen_range(0, 2) == 0;
+                    maps.iter().for_each(|map| {
+                        if insert {
+                            map.insert(&k, (&v, rc))
+                        } else {
+                            map.update(&k, |current| {
+                                assert_eq!(current, v_old.map(|(v, rc)| (&v[..], *rc)), "{}", k);
+                                Some((v.clone(), rc))
+                            })
+                        }
+                    });
+                    drop(hm);
+                    hash_map.write().unwrap().insert(k, (v, rc));
+                    return_key(k);
+                }
+            }
+            if thread_rng().gen_range(0, 20) == 0 {
+                // delete
+                if let Some(k) = get_key() {
+                    let mut hm = hash_map.write().unwrap();
+                    hm.remove(&k);
+                    maps.iter().for_each(|map| {
+                        map.delete_key(&k);
+                    });
+                }
+            }
+            if thread_rng().gen_range(0, 10) == 0 {
+                // add/unref
+                if let Some(k) = get_key() {
+                    let mut inc = thread_rng().gen_range(0, 2) == 0;
+                    let mut hm = hash_map.write().unwrap();
+                    let (v, mut rc) = hm.get(&k).map(|(v, rc)| (v.to_vec(), *rc)).unwrap();
+                    if !inc && rc == 0 {
+                        // can't decrement rc=0
+                        inc = true;
+                    }
+                    rc = if inc { rc + 1 } else { rc - 1 };
+                    hm.insert(k, (v.to_vec(), rc));
+                    maps.iter().for_each(|map| {
+                        if thread_rng().gen_range(0, 2) == 0 {
+                            map.update(&k, |current| Some((current.unwrap().0.to_vec(), rc)))
+                        } else if inc {
+                            map.addref(&k);
+                        } else {
+                            map.unref(&k);
+                        }
+                    });
+
+                    return_key(k);
+                }
+            }
+            if i % 1000 == 0 {
+                verify();
+            }
+        }
+        verify();
+    }
+}

bucket_map/src/bucket_stats.rs

@@ -0,0 +1,17 @@
+use std::sync::{atomic::AtomicU64, Arc, Mutex};
+
+#[derive(Debug, Default)]
+pub struct BucketStats {
+    pub resizes: AtomicU64,
+    pub max_size: Mutex<u64>,
+    pub resize_us: AtomicU64,
+    pub new_file_us: AtomicU64,
+    pub flush_file_us: AtomicU64,
+    pub mmap_us: AtomicU64,
+}
+
+#[derive(Debug, Default)]
+pub struct BucketMapStats {
+    pub index: Arc<BucketStats>,
+    pub data: Arc<BucketStats>,
+}

bucket_map/src/bucket_storage.rs

@@ -0,0 +1,424 @@
+use {
+    crate::{bucket_stats::BucketStats, MaxSearch},
+    memmap2::MmapMut,
+    rand::{thread_rng, Rng},
+    solana_measure::measure::Measure,
+    std::{
+        fs::{remove_file, OpenOptions},
+        io::{Seek, SeekFrom, Write},
+        path::PathBuf,
+        sync::{
+            atomic::{AtomicU64, Ordering},
+            Arc,
+        },
+    },
+};
+
+/*
+1	2
+2	4
+3	8
+4	16
+5	32
+6	64
+7	128
+8	256
+9	512
+10	1,024
+11	2,048
+12	4,096
+13	8,192
+14	16,384
+23  8,388,608
+24  16,777,216
+*/
+pub const DEFAULT_CAPACITY_POW2: u8 = 5;
+
+/// A Header UID of 0 indicates that the header is unlocked
+const UID_UNLOCKED: Uid = 0;
+
+pub(crate) type Uid = u64;
+
+#[repr(C)]
+struct Header {
+    lock: u64,
+}
+
+impl Header {
+    /// try to lock this entry with 'uid'
+    /// return true if it could be locked
+    fn try_lock(&mut self, uid: Uid) -> bool {
+        if self.lock == UID_UNLOCKED {
+            self.lock = uid;
+            true
+        } else {
+            false
+        }
+    }
+    /// mark this entry as unlocked
+    fn unlock(&mut self, expected: Uid) {
+        assert_eq!(expected, self.lock);
+        self.lock = UID_UNLOCKED;
+    }
+    /// uid that has locked this entry or None if unlocked
+    fn uid(&self) -> Option<Uid> {
+        if self.lock == UID_UNLOCKED {
+            None
+        } else {
+            Some(self.lock)
+        }
+    }
+    /// true if this entry is unlocked
+    fn is_unlocked(&self) -> bool {
+        self.lock == UID_UNLOCKED
+    }
+}
+
+pub struct BucketStorage {
+    path: PathBuf,
+    mmap: MmapMut,
+    pub cell_size: u64,
+    pub capacity_pow2: u8,
+    pub count: Arc<AtomicU64>,
+    pub stats: Arc<BucketStats>,
+    pub max_search: MaxSearch,
+}
+
+#[derive(Debug)]
+pub enum BucketStorageError {
+    AlreadyAllocated,
+}
+
+impl Drop for BucketStorage {
+    fn drop(&mut self) {
+        let _ = remove_file(&self.path);
+    }
+}
+
+impl BucketStorage {
+    pub fn new_with_capacity(
+        drives: Arc<Vec<PathBuf>>,
+        num_elems: u64,
+        elem_size: u64,
+        capacity_pow2: u8,
+        max_search: MaxSearch,
+        stats: Arc<BucketStats>,
+        count: Arc<AtomicU64>,
+    ) -> Self {
+        let cell_size = elem_size * num_elems + std::mem::size_of::<Header>() as u64;
+        let (mmap, path) = Self::new_map(&drives, cell_size as usize, capacity_pow2, &stats);
+        Self {
+            path,
+            mmap,
+            cell_size,
+            count,
+            capacity_pow2,
+            stats,
+            max_search,
+        }
+    }
+
+    pub fn max_search(&self) -> u64 {
+        self.max_search as u64
+    }
+
+    pub fn new(
+        drives: Arc<Vec<PathBuf>>,
+        num_elems: u64,
+        elem_size: u64,
+        max_search: MaxSearch,
+        stats: Arc<BucketStats>,
+        count: Arc<AtomicU64>,
+    ) -> Self {
+        Self::new_with_capacity(
+            drives,
+            num_elems,
+            elem_size,
+            DEFAULT_CAPACITY_POW2,
+            max_search,
+            stats,
+            count,
+        )
+    }
+
+    /// return ref to header of item 'ix' in mmapped file
+    fn header_ptr(&self, ix: u64) -> &Header {
+        let ix = (ix * self.cell_size) as usize;
+        let hdr_slice: &[u8] = &self.mmap[ix..ix + std::mem::size_of::<Header>()];
+        unsafe {
+            let hdr = hdr_slice.as_ptr() as *const Header;
+            hdr.as_ref().unwrap()
+        }
+    }
+
+    /// return ref to header of item 'ix' in mmapped file
+    fn header_mut_ptr(&mut self, ix: u64) -> &mut Header {
+        let ix = (ix * self.cell_size) as usize;
+        let hdr_slice: &mut [u8] = &mut self.mmap[ix..ix + std::mem::size_of::<Header>()];
+        unsafe {
+            let hdr = hdr_slice.as_mut_ptr() as *mut Header;
+            hdr.as_mut().unwrap()
+        }
+    }
+
+    /// return uid allocated at index 'ix' or None if vacant
+    pub fn uid(&self, ix: u64) -> Option<Uid> {
+        assert!(ix < self.capacity(), "bad index size");
+        self.header_ptr(ix).uid()
+    }
+
+    /// true if the entry at index 'ix' is free (as opposed to being allocated)
+    pub fn is_free(&self, ix: u64) -> bool {
+        // note that the terminology in the implementation is locked or unlocked.
+        // but our api is allocate/free
+        self.header_ptr(ix).is_unlocked()
+    }
+
+    /// caller knows id is not empty
+    pub fn uid_unchecked(&self, ix: u64) -> Uid {
+        self.uid(ix).unwrap()
+    }
+
+    /// 'is_resizing' true if caller is resizing the index (so don't increment count)
+    /// 'is_resizing' false if caller is adding an item to the index (so increment count)
+    pub fn allocate(
+        &mut self,
+        ix: u64,
+        uid: Uid,
+        is_resizing: bool,
+    ) -> Result<(), BucketStorageError> {
+        assert!(ix < self.capacity(), "allocate: bad index size");
+        assert!(UID_UNLOCKED != uid, "allocate: bad uid");
+        let mut e = Err(BucketStorageError::AlreadyAllocated);
+        //debug!("ALLOC {} {}", ix, uid);
+        if self.header_mut_ptr(ix).try_lock(uid) {
+            e = Ok(());
+            if !is_resizing {
+                self.count.fetch_add(1, Ordering::Relaxed);
+            }
+        }
+        e
+    }
+
+    pub fn free(&mut self, ix: u64, uid: Uid) {
+        assert!(ix < self.capacity(), "bad index size");
+        assert!(UID_UNLOCKED != uid, "free: bad uid");
+        self.header_mut_ptr(ix).unlock(uid);
+        self.count.fetch_sub(1, Ordering::Relaxed);
+    }
+
+    pub fn get<T: Sized>(&self, ix: u64) -> &T {
+        assert!(ix < self.capacity(), "bad index size");
+        let start = (ix * self.cell_size) as usize + std::mem::size_of::<Header>();
+        let end = start + std::mem::size_of::<T>();
+        let item_slice: &[u8] = &self.mmap[start..end];
+        unsafe {
+            let item = item_slice.as_ptr() as *const T;
+            &*item
+        }
+    }
+
+    pub fn get_empty_cell_slice<T: Sized>(&self) -> &[T] {
+        let len = 0;
+        let item_slice: &[u8] = &self.mmap[0..0];
+        unsafe {
+            let item = item_slice.as_ptr() as *const T;
+            std::slice::from_raw_parts(item, len as usize)
+        }
+    }
+
+    pub fn get_cell_slice<T: Sized>(&self, ix: u64, len: u64) -> &[T] {
+        assert!(ix < self.capacity(), "bad index size");
+        let ix = self.cell_size * ix;
+        let start = ix as usize + std::mem::size_of::<Header>();
+        let end = start + std::mem::size_of::<T>() * len as usize;
+        //debug!("GET slice {} {}", start, end);
+        let item_slice: &[u8] = &self.mmap[start..end];
+        unsafe {
+            let item = item_slice.as_ptr() as *const T;
+            std::slice::from_raw_parts(item, len as usize)
+        }
+    }
+
+    #[allow(clippy::mut_from_ref)]
+    pub fn get_mut<T: Sized>(&self, ix: u64) -> &mut T {
+        assert!(ix < self.capacity(), "bad index size");
+        let start = (ix * self.cell_size) as usize + std::mem::size_of::<Header>();
+        let end = start + std::mem::size_of::<T>();
+        let item_slice: &[u8] = &self.mmap[start..end];
+        unsafe {
+            let item = item_slice.as_ptr() as *mut T;
+            &mut *item
+        }
+    }
+
+    #[allow(clippy::mut_from_ref)]
+    pub fn get_mut_cell_slice<T: Sized>(&self, ix: u64, len: u64) -> &mut [T] {
+        assert!(ix < self.capacity(), "bad index size");
+        let ix = self.cell_size * ix;
+        let start = ix as usize + std::mem::size_of::<Header>();
+        let end = start + std::mem::size_of::<T>() * len as usize;
+        //debug!("GET mut slice {} {}", start, end);
+        let item_slice: &[u8] = &self.mmap[start..end];
+        unsafe {
+            let item = item_slice.as_ptr() as *mut T;
+            std::slice::from_raw_parts_mut(item, len as usize)
+        }
+    }
+
+    fn new_map(
+        drives: &[PathBuf],
+        cell_size: usize,
+        capacity_pow2: u8,
+        stats: &BucketStats,
+    ) -> (MmapMut, PathBuf) {
+        let mut measure_new_file = Measure::start("measure_new_file");
+        let capacity = 1u64 << capacity_pow2;
+        let r = thread_rng().gen_range(0, drives.len());
+        let drive = &drives[r];
+        let pos = format!("{}", thread_rng().gen_range(0, u128::MAX),);
+        let file = drive.join(pos);
+        let mut data = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .open(file.clone())
+            .map_err(|e| {
+                panic!(
+                    "Unable to create data file {} in current dir({:?}): {:?}",
+                    file.display(),
+                    std::env::current_dir(),
+                    e
+                );
+            })
+            .unwrap();
+
+        // Theoretical performance optimization: write a zero to the end of
+        // the file so that we won't have to resize it later, which may be
+        // expensive.
+        //debug!("GROWING file {}", capacity * cell_size as u64);
+        data.seek(SeekFrom::Start(capacity * cell_size as u64 - 1))
+            .unwrap();
+        data.write_all(&[0]).unwrap();
+        data.seek(SeekFrom::Start(0)).unwrap();
+        measure_new_file.stop();
+        let mut measure_flush = Measure::start("measure_flush");
+        data.flush().unwrap(); // can we skip this?
+        measure_flush.stop();
+        let mut measure_mmap = Measure::start("measure_mmap");
+        let res = (unsafe { MmapMut::map_mut(&data).unwrap() }, file);
+        measure_mmap.stop();
+        stats
+            .new_file_us
+            .fetch_add(measure_new_file.as_us(), Ordering::Relaxed);
+        stats
+            .flush_file_us
+            .fetch_add(measure_flush.as_us(), Ordering::Relaxed);
+        stats
+            .mmap_us
+            .fetch_add(measure_mmap.as_us(), Ordering::Relaxed);
+        res
+    }
+
+    /// copy contents from 'old_bucket' to 'self'
+    fn copy_contents(&mut self, old_bucket: &Self) {
+        let mut m = Measure::start("grow");
+        let old_cap = old_bucket.capacity();
+        let old_map = &old_bucket.mmap;
+
+        let increment = self.capacity_pow2 - old_bucket.capacity_pow2;
+        let index_grow = 1 << increment;
+        (0..old_cap as usize).into_iter().for_each(|i| {
+            let old_ix = i * old_bucket.cell_size as usize;
+            let new_ix = old_ix * index_grow;
+            let dst_slice: &[u8] = &self.mmap[new_ix..new_ix + old_bucket.cell_size as usize];
+            let src_slice: &[u8] = &old_map[old_ix..old_ix + old_bucket.cell_size as usize];
+
+            unsafe {
+                let dst = dst_slice.as_ptr() as *mut u8;
+                let src = src_slice.as_ptr() as *const u8;
+                std::ptr::copy_nonoverlapping(src, dst, old_bucket.cell_size as usize);
+            };
+        });
+        m.stop();
+        self.stats.resizes.fetch_add(1, Ordering::Relaxed);
+        self.stats.resize_us.fetch_add(m.as_us(), Ordering::Relaxed);
+    }
+
+    /// allocate a new bucket, copying data from 'bucket'
+    pub fn new_resized(
+        drives: &Arc<Vec<PathBuf>>,
+        max_search: MaxSearch,
+        bucket: Option<&Self>,
+        capacity_pow_2: u8,
+        num_elems: u64,
+        elem_size: u64,
+        stats: &Arc<BucketStats>,
+    ) -> Self {
+        let mut new_bucket = Self::new_with_capacity(
+            Arc::clone(drives),
+            num_elems,
+            elem_size,
+            capacity_pow_2,
+            max_search,
+            Arc::clone(stats),
+            bucket
+                .map(|bucket| Arc::clone(&bucket.count))
+                .unwrap_or_default(),
+        );
+        if let Some(bucket) = bucket {
+            new_bucket.copy_contents(bucket);
+        }
+        let sz = new_bucket.capacity();
+        {
+            let mut max = new_bucket.stats.max_size.lock().unwrap();
+            *max = std::cmp::max(*max, sz);
+        }
+        new_bucket
+    }
+
+    /// Return the number of cells currently allocated
+    pub fn capacity(&self) -> u64 {
+        1 << self.capacity_pow2
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_bucket_storage() {
+        let tmpdir1 = std::env::temp_dir().join("bucket_map_test_mt");
+        let paths: Vec<PathBuf> = [tmpdir1]
+            .iter()
+            .filter(|x| std::fs::create_dir_all(x).is_ok())
+            .cloned()
+            .collect();
+        assert!(!paths.is_empty());
+
+        let mut storage =
+            BucketStorage::new(Arc::new(paths), 1, 1, 1, Arc::default(), Arc::default());
+        let ix = 0;
+        let uid = Uid::MAX;
+        assert!(storage.is_free(ix));
+        assert!(storage.allocate(ix, uid, false).is_ok());
+        assert!(storage.allocate(ix, uid, false).is_err());
+        assert!(!storage.is_free(ix));
+        assert_eq!(storage.uid(ix), Some(uid));
+        assert_eq!(storage.uid_unchecked(ix), uid);
+        storage.free(ix, uid);
+        assert!(storage.is_free(ix));
+        assert_eq!(storage.uid(ix), None);
+        let uid = 1;
+        assert!(storage.is_free(ix));
+        assert!(storage.allocate(ix, uid, false).is_ok());
+        assert!(storage.allocate(ix, uid, false).is_err());
+        assert!(!storage.is_free(ix));
+        assert_eq!(storage.uid(ix), Some(uid));
+        assert_eq!(storage.uid_unchecked(ix), uid);
+        storage.free(ix, uid);
+        assert!(storage.is_free(ix));
+        assert_eq!(storage.uid(ix), None);
+    }
+}

bucket_map/src/index_entry.rs

@@ -0,0 +1,86 @@
+use {
+    crate::{
+        bucket::Bucket,
+        bucket_storage::{BucketStorage, Uid},
+        RefCount,
+    },
+    solana_sdk::{clock::Slot, pubkey::Pubkey},
+    std::{
+        collections::hash_map::DefaultHasher,
+        fmt::Debug,
+        hash::{Hash, Hasher},
+    },
+};
+
+#[repr(C)]
+#[derive(Debug, Copy, Clone, PartialEq)]
+// one instance of this per item in the index
+// stored in the index bucket
+pub struct IndexEntry {
+    pub key: Pubkey, // can this be smaller if we have reduced the keys into buckets already?
+    pub ref_count: RefCount, // can this be smaller? Do we ever need more than 4B refcounts?
+    storage_offset: u64, // smaller? since these are variably sized, this could get tricky. well, actually accountinfo is not variable sized...
+    // if the bucket doubled, the index can be recomputed using create_bucket_capacity_pow2
+    storage_capacity_when_created_pow2: u8, // see data_location
+    pub num_slots: Slot, // can this be smaller? epoch size should ~ be the max len. this is the num elements in the slot list
+}
+
+impl IndexEntry {
+    pub fn init(&mut self, pubkey: &Pubkey) {
+        self.key = *pubkey;
+        self.ref_count = 0;
+        self.storage_offset = 0;
+        self.storage_capacity_when_created_pow2 = 0;
+        self.num_slots = 0;
+    }
+
+    pub fn set_storage_capacity_when_created_pow2(
+        &mut self,
+        storage_capacity_when_created_pow2: u8,
+    ) {
+        self.storage_capacity_when_created_pow2 = storage_capacity_when_created_pow2;
+    }
+
+    pub fn set_storage_offset(&mut self, storage_offset: u64) {
+        self.storage_offset = storage_offset;
+    }
+
+    pub fn data_bucket_from_num_slots(num_slots: Slot) -> u64 {
+        (num_slots as f64).log2().ceil() as u64 // use int log here?
+    }
+
+    pub fn data_bucket_ix(&self) -> u64 {
+        Self::data_bucket_from_num_slots(self.num_slots)
+    }
+
+    pub fn ref_count(&self) -> RefCount {
+        self.ref_count
+    }
+
+    // This function maps the original data location into an index in the current bucket storage.
+    // This is coupled with how we resize bucket storages.
+    pub fn data_loc(&self, storage: &BucketStorage) -> u64 {
+        self.storage_offset << (storage.capacity_pow2 - self.storage_capacity_when_created_pow2)
+    }
+
+    pub fn read_value<'a, T>(&self, bucket: &'a Bucket<T>) -> Option<(&'a [T], RefCount)> {
+        let data_bucket_ix = self.data_bucket_ix();
+        let data_bucket = &bucket.data[data_bucket_ix as usize];
+        let slice = if self.num_slots > 0 {
+            let loc = self.data_loc(data_bucket);
+            let uid = Self::key_uid(&self.key);
+            assert_eq!(Some(uid), bucket.data[data_bucket_ix as usize].uid(loc));
+            bucket.data[data_bucket_ix as usize].get_cell_slice(loc, self.num_slots)
+        } else {
+            // num_slots is 0. This means we don't have an actual allocation.
+            // can we trust that the data_bucket is even safe?
+            bucket.data[data_bucket_ix as usize].get_empty_cell_slice()
+        };
+        Some((slice, self.ref_count))
+    }
+    pub fn key_uid(key: &Pubkey) -> Uid {
+        let mut s = DefaultHasher::new();
+        key.hash(&mut s);
+        s.finish().max(1u64)
+    }
+}

bucket_map/src/lib.rs

@@ -0,0 +1,11 @@
+#![allow(clippy::integer_arithmetic)]
+mod bucket;
+pub mod bucket_api;
+mod bucket_item;
+pub mod bucket_map;
+mod bucket_stats;
+mod bucket_storage;
+mod index_entry;
+
+pub type MaxSearch = u8;
+pub type RefCount = u64;

bucket_map/tests/bucket_map.rs

@@ -0,0 +1,48 @@
+use {
+    rayon::prelude::*,
+    solana_bucket_map::bucket_map::{BucketMap, BucketMapConfig},
+    solana_measure::measure::Measure,
+    solana_sdk::pubkey::Pubkey,
+    std::path::PathBuf,
+};
+#[test]
+#[ignore]
+fn bucket_map_test_mt() {
+    let threads = 4096;
+    let items = 4096;
+    let tmpdir1 = std::env::temp_dir().join("bucket_map_test_mt");
+    let tmpdir2 = PathBuf::from("/mnt/data/0").join("bucket_map_test_mt");
+    let paths: Vec<PathBuf> = [tmpdir1, tmpdir2]
+        .iter()
+        .filter(|x| std::fs::create_dir_all(x).is_ok())
+        .cloned()
+        .collect();
+    assert!(!paths.is_empty());
+    let index = BucketMap::new(BucketMapConfig {
+        max_buckets: 1 << 12,
+        drives: Some(paths.clone()),
+        ..BucketMapConfig::default()
+    });
+    (0..threads).into_iter().into_par_iter().for_each(|_| {
+        let key = Pubkey::new_unique();
+        index.update(&key, |_| Some((vec![0u64], 0)));
+    });
+    let mut timer = Measure::start("bucket_map_test_mt");
+    (0..threads).into_iter().into_par_iter().for_each(|_| {
+        for _ in 0..items {
+            let key = Pubkey::new_unique();
+            let ix: u64 = index.bucket_ix(&key) as u64;
+            index.update(&key, |_| Some((vec![ix], 0)));
+            assert_eq!(index.read_value(&key), Some((vec![ix], 0)));
+        }
+    });
+    timer.stop();
+    println!("time: {}ns per item", timer.as_ns() / (threads * items));
+    let mut total = 0;
+    for tmpdir in paths.iter() {
+        let folder_size = fs_extra::dir::get_size(tmpdir).unwrap();
+        total += folder_size;
+        std::fs::remove_dir_all(tmpdir).unwrap();
+    }
+    println!("overhead: {}bytes per item", total / (threads * items));
+}

cargo-build-bpf

@@ -9,5 +9,8 @@ for a in "$@"; do
   fi
 done
 
-set -x
+set -ex
+if [[ ! -f sdk/bpf/syscalls.txt ]]; then
+  "$here"/cargo build --manifest-path "$here"/programs/bpf_loader/gen-syscall-list/Cargo.toml
+fi
 exec "$here"/cargo run --manifest-path "$here"/sdk/cargo-build-bpf/Cargo.toml -- $maybe_bpf_sdk "$@"

ci/buildkite-pipeline.sh

@@ -137,7 +137,7 @@ all_test_steps() {
              ^ci/test-coverage.sh \
              ^scripts/coverage.sh \
       ; then
-    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 30
+    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 40
     wait_step
   else
     annotate --style info --context test-coverage \
@@ -148,6 +148,33 @@ all_test_steps() {
   command_step stable ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-stable.sh" 60
   wait_step
 
+  # BPF test suite
+  if affects \
+             .rs$ \
+             Cargo.lock$ \
+             Cargo.toml$ \
+             ^ci/rust-version.sh \
+             ^ci/test-stable-bpf.sh \
+             ^ci/test-stable.sh \
+             ^ci/test-local-cluster.sh \
+             ^core/build.rs \
+             ^fetch-perf-libs.sh \
+             ^programs/ \
+             ^sdk/ \
+      ; then
+    cat >> "$output_file" <<"EOF"
+  - command: "ci/test-stable-bpf.sh"
+    name: "stable-bpf"
+    timeout_in_minutes: 20
+    artifact_paths: "bpf-dumps.tar.bz2"
+    agents:
+      - "queue=default"
+EOF
+  else
+    annotate --style info \
+      "Stable-BPF skipped as no relevant files were modified"
+  fi
+
   # Perf test suite
   if affects \
              .rs$ \
@@ -165,7 +192,7 @@ all_test_steps() {
     cat >> "$output_file" <<"EOF"
   - command: "ci/test-stable-perf.sh"
     name: "stable-perf"
-    timeout_in_minutes: 40
+    timeout_in_minutes: 20
     artifact_paths: "log-*.txt"
     agents:
       - "queue=cuda"
@@ -199,6 +226,19 @@ EOF
     annotate --style info \
       "downstream-projects skipped as no relevant files were modified"
   fi
+
+  # Wasm support
+  if affects \
+             ^ci/test-wasm.sh \
+             ^ci/test-stable.sh \
+             ^sdk/ \
+      ; then
+    command_step wasm ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-wasm.sh" 20
+  else
+    annotate --style info \
+      "wasm skipped as no relevant files were modified"
+  fi
+
   # Benches...
   if affects \
              .rs$ \
@@ -216,7 +256,15 @@ EOF
 
   command_step "local-cluster" \
     ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster.sh" \
-    45
+    40
+
+  command_step "local-cluster-flakey" \
+    ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster-flakey.sh" \
+    10
+
+  command_step "local-cluster-slow" \
+    ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster-slow.sh" \
+    25
 }
 
 pull_or_push_steps() {
@@ -235,7 +283,7 @@ pull_or_push_steps() {
     all_test_steps
   fi
 
-  # web3.js, explorer and docs changes run on Travis...
+  # web3.js, explorer and docs changes run on Travis or Github actions...
 }
 
 

ci/buildkite-secondary.yml

@@ -3,13 +3,19 @@
 # Pull requests to not run these steps.
 steps:
   - command: "ci/publish-tarball.sh"
+    agents:
+      - "queue=release-build"
     timeout_in_minutes: 60
     name: "publish tarball"
   - wait
   - command: "sdk/docker-solana/build.sh"
+    agents:
+      - "queue=release-build"
     timeout_in_minutes: 60
     name: "publish docker"
   - command: "ci/publish-crate.sh"
+    agents:
+      - "queue=release-build"
     timeout_in_minutes: 240
     name: "publish crate"
     branches: "!master"

ci/dependabot-pr.sh

@@ -23,7 +23,7 @@ echo --- "(FAILING) Backpropagating dependabot-triggered Cargo.lock updates"
 name="dependabot-buildkite"
 api_base="https://api.github.com/repos/solana-labs/solana/pulls"
 pr_num=$(echo "$BUILDKITE_BRANCH" | grep -Eo '[0-9]+')
-branch=$(curl -s "$api_base/$pr_num" | python -c 'import json,sys;print json.load(sys.stdin)["head"]["ref"]')
+branch=$(curl -s "$api_base/$pr_num" | python3 -c 'import json,sys;print(json.load(sys.stdin)["head"]["ref"])')
 
 git add :**/Cargo.lock
 EMAIL="dependabot-buildkite@noreply.solana.com" \

ci/do-audit.sh

@@ -28,16 +28,29 @@ cargo_audit_ignores=(
   # Blocked on multiple crates updating `time` to >= 0.2.23
   --ignore RUSTSEC-2020-0071
 
-  # difference is unmaintained
-  #
-  # Blocked on predicates v1.0.6 removing its dependency on `difference`
-  --ignore RUSTSEC-2020-0095
-
   # generic-array: arr! macro erases lifetimes
   #
   # Blocked on libsecp256k1 releasing with upgraded dependencies
   # https://github.com/paritytech/libsecp256k1/issues/66
   --ignore RUSTSEC-2020-0146
 
+  # hyper: Lenient `hyper` header parsing of `Content-Length` could allow request smuggling
+  #
+  # Blocked on jsonrpc removing dependency on unmaintained `websocket`
+  # https://github.com/paritytech/jsonrpc/issues/605
+  --ignore RUSTSEC-2021-0078
+
+  # hyper: Integer overflow in `hyper`'s parsing of the `Transfer-Encoding` header leads to data loss
+  #
+  # Blocked on jsonrpc removing dependency on unmaintained `websocket`
+  # https://github.com/paritytech/jsonrpc/issues/605
+  --ignore RUSTSEC-2021-0079
+
+  # chrono: Potential segfault in `localtime_r` invocations
+  #
+  # Blocked due to no safe upgrade
+  # https://github.com/chronotope/chrono/issues/499
+  --ignore RUSTSEC-2020-0159
+
 )
 scripts/cargo-for-all-lock-files.sh stable audit "${cargo_audit_ignores[@]}"

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.52.1
+FROM solanalabs/rust:1.57.0
 ARG date
 
 RUN set -x \