Add RecentItems metrics (#20484) (#20486)

(cherry picked from commit e13ed8a627)

Co-authored-by: Tyera Eulberg <teulberg@gmail.com>

.github/workflows/explorer.yml

@@ -1,28 +0,0 @@
-name: Explorer_build&test_on_PR
-on:
-  pull_request:
-    branches:
-     - master
-    paths:
-      - 'explorer/**'
-jobs:
-  check-explorer:
-    runs-on: ubuntu-latest
-
-    defaults:
-      run:
-        working-directory: explorer
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-          cache: 'npm'
-          cache-dependency-path: explorer/package-lock.json
-      - run: npm i -g npm@7
-      - run: npm ci
-      - run: npm run format
-      - run: npm run build
-      - run: npm run test

.github/workflows/explorer_preview.yml

@@ -1,22 +0,0 @@
-name : explorer_preview
-on:
-  workflow_run:
-    workflows: ["Explorer_build&test_on_PR"]
-#     types: 
-#       - completed
-jobs:
-   explorer_preview:
-    runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-      - uses: amondnet/vercel-action@v20
-        with:
-          vercel-token: ${{ secrets.VERCEL_TOKEN }} # Required
-          github-token: ${{ secrets.GITHUB_TOKEN }} #Optional 
-          vercel-org-id: ${{ secrets.ORG_ID}}  #Required
-          vercel-project-id: ${{ secrets.PROJECT_ID}} #Required 
-          working-directory: ./explorer
-          scope: ${{ secrets.TEAM_ID }}

.github/workflows/explorer_production.yml

@@ -1,46 +0,0 @@
-name: Explorer_production_build&test
-on:
-  push:
-   branches: [master]
-   paths:
-      - 'explorer/**'
-jobs:
-    Explorer_production_build_test:
-      runs-on: ubuntu-latest
-      defaults:
-        run:
-          working-directory: explorer
-      steps:
-       - uses: actions/checkout@v2
-         with:
-          ref: ${{ github.event.pull_request.head.sha }}
-       - uses: actions/setup-node@v2
-         with:
-           node-version: '14'
-           cache: 'npm'
-           cache-dependency-path: explorer/package-lock.json
-       - run: npm i -g npm@7
-       - run: npm ci
-       - run: npm run format
-       - run: npm run build
-       - run: npm run test
-       
-    Explorer_production_deploy:
-      needs: Explorer_production_build_test
-      runs-on: ubuntu-latest
-      defaults:
-        run:
-          working-directory: explorer
-      steps:
-       - uses: actions/checkout@v2
-         with:
-           ref: ${{ github.event.pull_request.head.sha }}
-       - uses: amondnet/vercel-action@v20
-         with:
-           vercel-token: ${{ secrets.VERCEL_TOKEN }} # Required
-           github-token: ${{ secrets.GITHUB_TOKEN }} #Optional 
-           vercel-args: '--prod' #for production
-           vercel-org-id: ${{ secrets.ORG_ID}}  #Required
-           vercel-project-id: ${{ secrets.PROJECT_ID}} #Required 
-           working-directory: ./explorer
-           scope: ${{ secrets.TEAM_ID }}

.github/workflows/solana-action.yml.txt

@@ -1,208 +0,0 @@
-name : minimal 
-
-on:
-  push:
-    branches:  [master]
-  pull_request:
-    branches:  [master]
-
-jobs:
-  Export_Github_Repositories:
-    runs-on: ubuntu-latest
-    env: 
-       VERCEL_TOKEN: ${{secrets.VERCEL_TOKEN}}
-       GITHUB_TOKEN: ${{secrets.PAT_ANM}} 
-       COMMIT_RANGE: ${{ github.event.before}}...${{ github.event.after}}
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 2
-
-      - run: echo "COMMIT_DIFF_RANGE=$(echo $COMMIT_RANGE)" >> $GITHUB_ENV
-#       - run: echo "$COMMIT_DIFF_RANGE"
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          GITHUB_TOKEN: ${{secrets.PAT_ANM}}
-        if: ${{ github.event_name == 'push' && 'cron'&& github.ref == 'refs/heads/master'}}
-        
-      - name: cmd
-        run : |
-            .travis/export-github-repo.sh web3.js/ solana-web3.js
-            
-  macos-artifacts:
-    needs: [Export_Github_Repositories]
-    strategy:
-      fail-fast: false
-    runs-on: macos-latest
-    if : ${{ github.event_name == 'api' && 'cron' || 'push' || startsWith(github.ref, 'refs/tags/v')}}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-      - name: Setup | Rust
-        uses: ATiltedTree/setup-rust@v1
-        with:
-          rust-version: stable
-      - name:  release artifact
-        run: |
-           source ci/rust-version.sh
-           brew install coreutils
-           export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
-           greadlink -f .
-           source ci/env.sh
-           rustup set profile default
-           ci/publish-tarball.sh
-        shell: bash
-        
-      - name: Cache modules
-        uses: actions/cache@master
-        id: yarn-cache
-        with:
-          path: node_modules
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: ${{ runner.os }}-yarn-
-          
-
-# - To stop from uploading on the production
-#      - uses: ochanje210/simple-s3-upload-action@master
-#        with:
-#          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_KEY_ID }}
-#          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
-#          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
-#          SOURCE_DIR: 'travis-s3-upload1'
-#          DEST_DIR: 'giitsol'
-
-#      - uses: ochanje210/simple-s3-upload-action@master
-#        with:
-#          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_KEY_ID }}
-#          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
-#          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
-#          SOURCE_DIR: './docs/'
-#          DEST_DIR: 'giitsol'
-
-  
-  windows-artifact:
-    needs: [Export_Github_Repositories]
-    strategy:
-      fail-fast: false 
-    runs-on: windows-latest
-    if : ${{ github.event_name == 'api' && 'cron' || 'push' || startsWith(github.ref, 'refs/tags/v')}}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-      - name: Setup | Rust
-        uses: ATiltedTree/setup-rust@v1
-        with:
-          rust-version: stable
-  release-artifact:
-    needs: windows-artifact
-    runs-on: windows-latest
-    if : ${{ github.event_name == 'api' && 'cron' || github.ref == 'refs/heads/master'}}
-    steps:
-      - name: release artifact 
-        run: |
-             git clone git://git.openssl.org/openssl.git
-             cd openssl
-             make
-             make test
-             make install
-             openssl version
-#            choco install openssl 
-#            vcpkg integrate install
-#            refreshenv
-
-      - name: Checkout repository
-        uses: actions/checkout@v2 
-      - uses: actions/checkout@v2
-      - run: choco install msys2
-      - uses: actions/checkout@v2
-      - run: |
-           openssl version
-           bash ci/rust-version.sh
-           readlink -f .
-           bash ci/env.sh
-           rustup set profile default
-           bash ci/publish-tarball.sh
-        shell: bash
-        
-      - name: Cache modules
-        uses: actions/cache@v1
-        id: yarn-cache
-        with:
-          path: node_modules
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: ${{ runner.os }}-yarn-
-
-# - To stop from uploading on the production      
-#      - name: Config. aws cred
-#        uses: aws-actions/configure-aws-credentials@v1
-#        with:
-#          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-#          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-#          aws-region: us-east-2
-#      - name: Deploy
-#        uses: shallwefootball/s3-upload-action@master
-#        with:
-#            folder: build
-#            aws_bucket: ${{ secrets.AWS_S3_BUCKET }}
-#            aws_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-#            aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-#            destination_dir: /
-#            bucket-region: us-east-2
-#            delete-removed: true
-#            no-cache: true
-#            private: true
-  
-#  Docs:
-#    needs: [windows-artifact,release-artifact]
-#    runs-on: ubuntu-latest
-#    env: 
-#       GITHUB_TOKEN: ${{secrets.PAT_NEW}} 
-#       GITHUB_EVENT_BEFORE: ${{ github.event.before }}
-#       GITHUB_EVENT_AFTER: ${{ github.event.after }}
-#       COMMIT_RANGE: ${{ github.event.before}}...${{ github.event.after}}
-#    steps:
-#      - name: Checkout repo
-#        uses: actions/checkout@v2
-#        with:
-#          fetch-depth: 2
-#      - name: docs
-#        if: ${{github.event_name == 'pull_request' || startsWith(github.ref, 'refs/tags/v')}}
-#        run: |
-#             touch .env
-#             echo "COMMIT_RANGE=($COMMIT_RANGE)" > .env
-#             source ci/env.sh
-#             .travis/channel_restriction.sh edge beta || exit 0
-#             .travis/affects.sh docs/ .travis || exit 0
-#             cd docs/
-#             source .travis/before_install.sh
-#             source .travis/script.sh
-#      - name: setup-node
-#        uses: actions/checkout@v2
-#      - name: setup-node
-#        uses: actions/setup-node@v2
-#        with:
-#          node-version: 'lts/*'
-#      - name: Cache
-#        uses: actions/cache@v1
-#        with:
-#          path: ~/.npm
-#          key: ${{ runner.OS }}-npm-cache-${{ hashFiles('**/package-lock.json') }}
-#          restore-keys: |
-#            ${{ runner.OS }}-npm-cache-2  
-
-#  auto_bump:
-#     needs: [windows-artifact,release-artifact,Docs]
-#     runs-on: ubuntu-latest
-#     steps:
-#       - name : checkout repo
-#         uses: actions/checkout@v2
-#         with:
-#          fetch-depth: '0'
-#       - name: Bump version and push tag
-#         uses: anothrNick/github-tag-action@1.26.0
-#         env:
-#           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-#           WITH_V: true
-#           DEFAULT_BUMP: patch

.github/workflows/web3.yml

@@ -1,84 +0,0 @@
-name: Web3
-
-on:
-  push:
-    branches: [ master ]
-    paths:
-      - "web3.js/**"
-  pull_request:
-    branches: [ master ]
-    paths:
-      - "web3.js/**"
-
-jobs:
-  # needed for grouping check-web3 strategies into one check for mergify
-  all-web3-checks:
-    runs-on: ubuntu-latest
-    needs:
-      - check-web3
-    steps:
-      - run: echo "Done"
-
-  web3-commit-lint:
-    runs-on: ubuntu-latest
-    # Set to true in order to avoid cancelling other workflow jobs.
-    # Mergify will still require web3-commit-lint for automerge
-    continue-on-error: true
-
-    defaults:
-      run:
-        working-directory: web3.js
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          # maybe needed for base sha below
-          fetch-depth: 0
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '16'
-          cache: 'npm'
-          cache-dependency-path: web3.js/package-lock.json
-      - run: npm ci
-      - name: commit-lint
-        if: ${{ github.event_name == 'pull_request' }}
-        run: bash commitlint.sh
-        env:
-          COMMIT_RANGE: ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}
-
-  check-web3:
-    runs-on: ubuntu-latest
-
-    defaults:
-      run:
-        working-directory: web3.js
-
-    strategy:
-      matrix:
-        node: [ '12', '14', '16' ]
-
-    name: Node ${{ matrix.node }}
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
-        with:
-          node-version: ${{ matrix.node }}
-          cache: 'npm'
-          cache-dependency-path: web3.js/package-lock.json
-      - run: npm i -g npm@7
-      - run: npm ci
-      - run: npm run lint
-      - run: |
-          npm run build
-          ls -l lib
-          test -r lib/index.iife.js
-          test -r lib/index.cjs.js
-          test -r lib/index.esm.js
-      - run: npm run doc
-      - run: npm run codecov
-      - run: |
-          sh -c "$(curl -sSfL https://release.solana.com/edge/install)"
-          echo "$HOME/.local/share/solana/install/active_release/bin" >> $GITHUB_PATH
-          PATH="$HOME/.local/share/solana/install/active_release/bin:$PATH"
-          solana --version
-      - run: npm run test:live-with-test-validator

.gitignore

@@ -27,5 +27,3 @@ log-*/
 /spl_*.so
 
 .DS_Store
-# scripts that may be generated by cargo *-bpf commands
-**/cargo-*-bpf-child-script-*.sh

.mergify.yml

@@ -4,71 +4,24 @@
 #
 # https://doc.mergify.io/
 pull_request_rules:
-  - name: label changes from community
-    conditions:
-      - author≠@core-contributors
-      - author≠mergify[bot]
-      - author≠dependabot[bot]
-    actions:
-      label:
-        add:
-          - community
-  - name: request review for community changes
-    conditions:
-      - author≠@core-contributors
-      - author≠mergify[bot]
-      - author≠dependabot[bot]
-      # Only request reviews from the pr subscribers group if no one
-      # has reviewed the community PR yet. These checks only match
-      # reviewers with admin, write or maintain permission on the repository.
-      - "#approved-reviews-by=0"
-      - "#commented-reviews-by=0"
-      - "#changes-requested-reviews-by=0"
-    actions:
-      request_reviews:
-        teams:
-          - "@solana-labs/community-pr-subscribers"
   - name: automatic merge (squash) on CI success
     conditions:
-      - and:
       - status-success=buildkite/solana
+      - status-success=Travis CI - Pull Request
       - status-success=ci-gate
       - label=automerge
       - author≠@dont-squash-my-commits
-        - or:
-          # only require travis success if docs files changed
-          - status-success=Travis CI - Pull Request
-          - -files~=^docs/
-        - or:
-          # only require explorer checks if explorer files changed
-          - status-success=check-explorer
-          - -files~=^explorer/
-        - or:
-          - and:
-            - status-success=all-web3-checks
-            - status-success=web3-commit-lint
-          # only require web3 checks if web3.js files changed
-          - -files~=^web3.js/
     actions:
       merge:
         method: squash
   # Join the dont-squash-my-commits group if you won't like your commits squashed
   - name: automatic merge (rebase) on CI success
     conditions:
-      - and:
       - status-success=buildkite/solana
       - status-success=Travis CI - Pull Request
       - status-success=ci-gate
       - label=automerge
       - author=@dont-squash-my-commits
-        - or:
-          # only require explorer checks if explorer files changed
-          - status-success=check-explorer
-          - -files~=^explorer/
-        - or:
-          # only require web3 checks if web3.js files changed
-          - status-success=all-web3-checks
-          - -files~=^web3.js/
     actions:
       merge:
         method: rebase
@@ -97,19 +50,35 @@ pull_request_rules:
       label:
         add:
           - automerge
-  - name: v1.8 backport
+  - name: v1.4 backport
     conditions:
-      - label=v1.8
+      - label=v1.4
     actions:
       backport:
         ignore_conflicts: true
         branches:
-          - v1.8
-  - name: v1.9 backport
+          - v1.4
+  - name: v1.5 backport
     conditions:
-      - label=v1.9
+      - label=v1.5
     actions:
       backport:
         ignore_conflicts: true
         branches:
-          - v1.9
+          - v1.5
+  - name: v1.6 backport
+    conditions:
+      - label=v1.6
+    actions:
+      backport:
+        ignore_conflicts: true
+        branches:
+          - v1.6
+  - name: v1.7 backport
+    conditions:
+      - label=v1.7
+    actions:
+      backport:
+        ignore_conflicts: true
+        branches:
+          - v1.7

.travis.yml

@@ -23,12 +23,12 @@ jobs:
         depth: false
       script:
         - .travis/export-github-repo.sh web3.js/ solana-web3.js
+        - .travis/export-github-repo.sh explorer/ explorer
 
     - &release-artifacts
       if: type IN (api, cron) OR tag IS present
       name: "macOS release artifacts"
       os: osx
-      osx_image: xcode12
       language: rust
       rust:
         - stable
@@ -36,9 +36,6 @@ jobs:
         - source ci/rust-version.sh
         - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
         - readlink -f .
-        - brew install gnu-tar
-        - PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH"
-        - tar --version
       script:
         - source ci/env.sh
         - rustup set profile default
@@ -64,12 +61,6 @@ jobs:
     - <<: *release-artifacts
       name: "Windows release artifacts"
       os: windows
-      install:
-        - choco install openssl
-        - export OPENSSL_DIR="C:\Program Files\OpenSSL-Win64"
-        - source ci/rust-version.sh
-        - PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
-        - readlink -f .
     #  Linux release artifacts are still built by ci/buildkite-secondary.yml
     #- <<: *release-artifacts
     #  name: "Linux release artifacts"
@@ -77,12 +68,56 @@ jobs:
     #  before_install:
     #    - sudo apt-get install libssl-dev libudev-dev
 
+    # explorer pull request
+    - name: "explorer"
+      if: type = pull_request AND branch = master
+
+      language: node_js
+      node_js:
+        - "node"
+
+      cache:
+        directories:
+          - ~/.npm
+
+      before_install:
+        - .travis/affects.sh explorer/ .travis || travis_terminate 0
+        - cd explorer
+
+      script:
+        - npm run build
+        - npm run format
+
+    # web3.js pull request
+    - name: "web3.js"
+      if: type = pull_request AND branch = master
+
+      language: node_js
+      node_js:
+        - "lts/*"
+
+      services:
+        - docker
+
+      cache:
+        directories:
+          - ~/.npm
+
+      before_install:
+        - .travis/affects.sh web3.js/ .travis || travis_terminate 0
+        - cd web3.js/
+        - source .travis/before_install.sh
+
+      script:
+        - ../.travis/commitlint.sh
+        - source .travis/script.sh
+
     # docs pull request
     - name: "docs"
       if: type IN (push, pull_request) OR tag IS present
       language: node_js
       node_js:
-        - "lts/*"
+        - "node"
 
       services:
         - docker

.travis/commitlint.sh

@@ -20,13 +20,13 @@ if [[ ! -f "$basedir"/commitlint.config.js ]]; then
   exit 1
 fi
 
-if [[ -z $COMMIT_RANGE ]]; then
-  echo "Error: COMMIT_RANGE not defined"
+if [[ -z $TRAVIS_COMMIT_RANGE ]]; then
+  echo "Error: TRAVIS_COMMIT_RANGE not defined"
   exit 1
 fi
 
 cd "$basedir"
-echo "Checking commits in COMMIT_RANGE: $COMMIT_RANGE"
+echo "Checking commits in TRAVIS_COMMIT_RANGE: $TRAVIS_COMMIT_RANGE"
 while IFS= read -r line; do
   echo "$line" | npx commitlint
-done < <(git log "$COMMIT_RANGE" --format=%s -- .)
+done < <(git log "$TRAVIS_COMMIT_RANGE" --format=%s -- .)

Cargo.toml

@@ -1,9 +1,7 @@
 [workspace]
 members = [
-    "accountsdb-plugin-interface",
-    "accountsdb-plugin-manager",
-    "accountsdb-plugin-postgres",
     "accounts-cluster-bench",
+    "bench-exchange",
     "bench-streamer",
     "bench-tps",
     "accounts-bench",
@@ -11,7 +9,6 @@ members = [
     "banks-client",
     "banks-interface",
     "banks-server",
-    "bucket_map",
     "clap-utils",
     "cli-config",
     "cli-output",
@@ -19,13 +16,11 @@ members = [
     "core",
     "dos",
     "download-utils",
-    "entry",
     "faucet",
     "frozen-abi",
     "perf",
     "validator",
     "genesis",
-    "genesis-utils",
     "gossip",
     "install",
     "keygen",
@@ -36,6 +31,7 @@ members = [
     "log-analyzer",
     "merkle-root-bench",
     "merkle-tree",
+    "stake-o-matic",
     "storage-bigtable",
     "storage-proto",
     "streamer",
@@ -43,29 +39,30 @@ members = [
     "metrics",
     "net-shaper",
     "notifier",
-    "poh",
     "poh-bench",
     "program-test",
-    "programs/address-lookup-table",
-    "programs/address-lookup-table-tests",
+    "programs/secp256k1",
     "programs/bpf_loader",
-    "programs/compute-budget",
+    "programs/budget",
     "programs/config",
+    "programs/exchange",
+    "programs/failure",
+    "programs/noop",
+    "programs/ownable",
     "programs/stake",
+    "programs/vest",
     "programs/vote",
-    "rbpf-cli",
     "remote-wallet",
-    "rpc",
+    "ramp-tps",
     "runtime",
     "runtime/store-tool",
     "sdk",
     "sdk/cargo-build-bpf",
     "sdk/cargo-test-bpf",
-    "send-transaction-service",
+    "scripts",
     "stake-accounts",
     "sys-tuner",
     "tokens",
-    "transaction-dos",
     "transaction-status",
     "account-decoder",
     "upload-perf",
@@ -74,18 +71,11 @@ members = [
     "cli",
     "rayon-threadlimit",
     "watchtower",
-    "replica-node",
-    "replica-lib",
-    "test-validator",
-    "rpc-test",
-    "client-test",
 ]
 
 exclude = [
     "programs/bpf",
 ]
 
-# TODO: Remove once the "simd-accel" feature from the reed-solomon-erasure
-# dependency is supported on Apple M1. v2 of the feature resolver is needed to
-# specify arch-specific features.
-resolver = "2"
+[profile.dev]
+split-debuginfo = "unpacked"

README.md

@@ -1,6 +1,6 @@
 <p align="center">
   <a href="https://solana.com">
-    <img alt="Solana" src="https://i.imgur.com/IKyzQ6T.png" width="250" />
+    <img alt="Solana" src="https://i.imgur.com/OMnvVEz.png" width="250" />
   </a>
 </p>
 
@@ -19,18 +19,12 @@ $ source $HOME/.cargo/env
 $ rustup component add rustfmt
 ```
 
-When building the master branch, please make sure you are using the latest stable rust version by running:
+Please make sure you are always using the latest stable rust version by running:
 
 ```bash
 $ rustup update
 ```
 
-When building a specific release branch, you should check the rust version in `ci/rust-version.sh` and if necessary, install that version by running:
-```bash
-$ rustup install VERSION
-```
-Note that if this is not the latest rust version on your machine, cargo commands may require an [override](https://rust-lang.github.io/rustup/overrides.html) in order to use the correct version.
-
 On Linux systems you may need to install libssl-dev, pkg-config, zlib1g-dev, etc.  On Ubuntu:
 
 ```bash
@@ -38,12 +32,6 @@ $ sudo apt-get update
 $ sudo apt-get install libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang make
 ```
 
-On Mac M1s, make sure you set up your terminal & homebrew [to use](https://5balloons.info/correct-way-to-install-and-use-homebrew-on-m1-macs/) Rosetta. You can install it with:
-
-```bash
-$ softwareupdate --install-rosetta
-```
-
 ## **2. Download the source code.**
 
 ```bash
@@ -57,6 +45,11 @@ $ cd solana
 $ cargo build
 ```
 
+## **4. Run a minimal local cluster.**
+```bash
+$ ./run.sh
+```
+
 # Testing
 
 **Run the test suite:**
@@ -116,13 +109,13 @@ send us that patch!
 
 All claims, content, designs, algorithms, estimates, roadmaps,
 specifications, and performance measurements described in this project
-are done with the Solana Foundation's ("SF") good faith efforts. It is up to
+are done with the Solana Foundation's ("SF") best efforts. It is up to
 the reader to check and validate their accuracy and truthfulness.
-Furthermore, nothing in this project constitutes a solicitation for
+Furthermore nothing in this project constitutes a solicitation for
 investment.
 
 Any content produced by SF or developer resources that SF provides, are
-for educational and inspirational purposes only. SF does not encourage,
+for educational and inspiration purposes only. SF does not encourage,
 induce or sanction the deployment, integration or use of any such
 applications (including the code comprising the Solana blockchain
 protocol) in violation of applicable laws or regulations and hereby

SECURITY.md

@@ -18,24 +18,24 @@ Expect a response as fast as possible, within one business day at the latest.
 We offer bounties for critical security issues. Please see below for more details.
 
 Loss of Funds:
-$2,000,000 USD in locked SOL tokens (locked for 12 months)
+$500,000 USD in locked SOL tokens (locked for 12 months)
 * Theft of funds without users signature from any account
 * Theft of funds without users interaction in system, token, stake, vote programs
 * Theft of funds that requires users signature - creating a vote program that drains the delegated stakes.
 
 Consensus/Safety Violations:
-$1,000,000 USD in locked SOL tokens (locked for 12 months)
+$250,000 USD in locked SOL tokens (locked for 12 months)
 * Consensus safety violation
 * Tricking a validator to accept an optimistic confirmation or rooted slot without a double vote, etc..
 
 Other Attacks:
- $400,000 USD in locked SOL tokens (locked for 12 months)
+ $100,000 USD in locked SOL tokens (locked for 12 months)
 * Protocol liveness attacks,
 * Eclipse attacks,
 * Remote attacks that partition the network,
 
 DoS Attacks:
-$100,000 USD in locked SOL tokens (locked for 12 months)
+$25,000 USD in locked SOL tokens (locked for 12 months)
 * Remote resource exaustion via Non-RPC protocols
 
 RPC DoS/Crashes:
@@ -51,27 +51,13 @@ The following components are out of scope for the bounty program
 * Attacks that require social engineering
 
 Eligibility:
-* The participant submitting the bug report shall follow the process outlined within this document
+* The participant submitting the bug bounty shall follow the process outlined within this document
 * Valid exploits can be eligible even if they are not successfully executed on the cluster
 * Multiple submissions for the same class of exploit are still eligible for compensation, though may be compensated at a lower rate, however these will be assessed on a case-by-case basis
 * Participants must complete KYC and sign the participation agreement here when the registrations are open https://solana.com/validator-registration. Security exploits will still be assessed and open for submission at all times. This needs only be done prior to distribution of tokens.
 
-Payment of Bug Bounties:
-* Payments for eligible bug reports are distributed monthly.
-* Bounties for all bug reports submitted in a given month are paid out in the middle of the
-following month.
-* The SOL/USD conversion rate used for payments is the market price at the end of
-  the last day of the month for the month in which the bug was submitted.
-* The reference for this price is the Closing Price given by Coingecko.com on
-  that date given here:
-  https://www.coingecko.com/en/coins/solana/historical_data/usd#panel
-* For example, for all bugs submitted in March 2021, the SOL/USD price for bug
-  payouts is the Close price on 2021-03-31 of $19.49.  This applies to all bugs
-  submitted in March 2021, to be paid in mid-April 2021.
-* Bug bounties are paid out in
-[stake accounts](https://solana.com/staking) with a
-[lockup](https://docs.solana.com/staking/stake-accounts#lockups)
-expiring 12 months from the last day of the month in which the bug was submitted.
+Notes:
+* All locked tokens can be staked during the lockup period
 
 <a name="process"></a>
 ## Incident Response Process

account-decoder/Cargo.toml

@@ -1,30 +1,31 @@
 [package]
 name = "solana-account-decoder"
-version = "1.10.0"
+version = "1.6.28"
 description = "Solana account decoder"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-account-decoder"
 license = "Apache-2.0"
-edition = "2021"
+edition = "2018"
 
 [dependencies]
 base64 = "0.12.3"
-bincode = "1.3.3"
-bs58 = "0.4.0"
+bincode = "1.3.1"
+bs58 = "0.3.1"
 bv = "0.11.1"
 Inflector = "0.11.4"
 lazy_static = "1.4.0"
-serde = "1.0.131"
+serde = "1.0.122"
 serde_derive = "1.0.103"
-serde_json = "1.0.72"
-solana-config-program = { path = "../programs/config", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-vote-program = { path = "../programs/vote", version = "=1.10.0" }
-spl-token = { version = "=3.2.0", features = ["no-entrypoint"] }
+serde_json = "1.0.56"
+solana-config-program = { path = "../programs/config", version = "=1.6.28" }
+solana-sdk = { path = "../sdk", version = "=1.6.28" }
+solana-stake-program = { path = "../programs/stake", version = "=1.6.28" }
+solana-vote-program = { path = "../programs/vote", version = "=1.6.28" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.1.1", features = ["no-entrypoint"] }
 thiserror = "1.0"
-zstd = "0.9.0"
+zstd = "0.5.1"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

account-decoder/src/lib.rs

@@ -17,10 +17,8 @@ pub mod validator_info;
 use {
     crate::parse_account_data::{parse_account_data, AccountAdditionalData, ParsedAccount},
     solana_sdk::{
-        account::{ReadableAccount, WritableAccount},
-        clock::Epoch,
-        fee_calculator::FeeCalculator,
-        pubkey::Pubkey,
+        account::ReadableAccount, account::WritableAccount, clock::Epoch,
+        fee_calculator::FeeCalculator, pubkey::Pubkey,
     },
     std::{
         io::{Read, Write},
@@ -30,7 +28,6 @@ use {
 
 pub type StringAmount = String;
 pub type StringDecimals = String;
-pub const MAX_BASE58_BYTES: usize = 128;
 
 /// A duplicate representation of an Account for pretty JSON serialization
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -63,17 +60,6 @@ pub enum UiAccountEncoding {
 }
 
 impl UiAccount {
-    fn encode_bs58<T: ReadableAccount>(
-        account: &T,
-        data_slice_config: Option<UiDataSliceConfig>,
-    ) -> String {
-        if account.data().len() <= MAX_BASE58_BYTES {
-            bs58::encode(slice_data(account.data(), data_slice_config)).into_string()
-        } else {
-            "error: data too large for bs58 encoding".to_string()
-        }
-    }
-
     pub fn encode<T: ReadableAccount>(
         pubkey: &Pubkey,
         account: &T,
@@ -82,34 +68,33 @@ impl UiAccount {
         data_slice_config: Option<UiDataSliceConfig>,
     ) -> Self {
         let data = match encoding {
-            UiAccountEncoding::Binary => {
-                let data = Self::encode_bs58(account, data_slice_config);
-                UiAccountData::LegacyBinary(data)
-            }
-            UiAccountEncoding::Base58 => {
-                let data = Self::encode_bs58(account, data_slice_config);
-                UiAccountData::Binary(data, encoding)
-            }
+            UiAccountEncoding::Binary => UiAccountData::LegacyBinary(
+                bs58::encode(slice_data(&account.data(), data_slice_config)).into_string(),
+            ),
+            UiAccountEncoding::Base58 => UiAccountData::Binary(
+                bs58::encode(slice_data(&account.data(), data_slice_config)).into_string(),
+                encoding,
+            ),
             UiAccountEncoding::Base64 => UiAccountData::Binary(
-                base64::encode(slice_data(account.data(), data_slice_config)),
+                base64::encode(slice_data(&account.data(), data_slice_config)),
                 encoding,
             ),
             UiAccountEncoding::Base64Zstd => {
                 let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 0).unwrap();
                 match encoder
-                    .write_all(slice_data(account.data(), data_slice_config))
+                    .write_all(slice_data(&account.data(), data_slice_config))
                     .and_then(|()| encoder.finish())
                 {
                     Ok(zstd_data) => UiAccountData::Binary(base64::encode(zstd_data), encoding),
                     Err(_) => UiAccountData::Binary(
-                        base64::encode(slice_data(account.data(), data_slice_config)),
+                        base64::encode(slice_data(&account.data(), data_slice_config)),
                         UiAccountEncoding::Base64,
                     ),
                 }
             }
             UiAccountEncoding::JsonParsed => {
                 if let Ok(parsed_data) =
-                    parse_account_data(pubkey, account.owner(), account.data(), additional_data)
+                    parse_account_data(pubkey, &account.owner(), &account.data(), additional_data)
                 {
                     UiAccountData::Json(parsed_data)
                 } else {
@@ -204,10 +189,8 @@ fn slice_data(data: &[u8], data_slice_config: Option<UiDataSliceConfig>) -> &[u8
 
 #[cfg(test)]
 mod test {
-    use {
-        super::*,
-        solana_sdk::account::{Account, AccountSharedData},
-    };
+    use super::*;
+    use solana_sdk::account::{Account, AccountSharedData};
 
     #[test]
     fn test_slice_data() {

account-decoder/src/parse_account_data.rs

@@ -1,27 +1,25 @@
-use {
-    crate::{
+use crate::{
     parse_bpf_loader::parse_bpf_upgradeable_loader,
     parse_config::parse_config,
     parse_nonce::parse_nonce,
     parse_stake::parse_stake,
     parse_sysvar::parse_sysvar,
-        parse_token::{parse_token, spl_token_id},
+    parse_token::{parse_token, spl_token_id_v2_0},
     parse_vote::parse_vote,
-    },
-    inflector::Inflector,
-    serde_json::Value,
-    solana_sdk::{instruction::InstructionError, pubkey::Pubkey, stake, system_program, sysvar},
-    std::collections::HashMap,
-    thiserror::Error,
 };
+use inflector::Inflector;
+use serde_json::Value;
+use solana_sdk::{instruction::InstructionError, pubkey::Pubkey, system_program, sysvar};
+use std::collections::HashMap;
+use thiserror::Error;
 
 lazy_static! {
     static ref BPF_UPGRADEABLE_LOADER_PROGRAM_ID: Pubkey = solana_sdk::bpf_loader_upgradeable::id();
     static ref CONFIG_PROGRAM_ID: Pubkey = solana_config_program::id();
-    static ref STAKE_PROGRAM_ID: Pubkey = stake::program::id();
+    static ref STAKE_PROGRAM_ID: Pubkey = solana_stake_program::id();
     static ref SYSTEM_PROGRAM_ID: Pubkey = system_program::id();
     static ref SYSVAR_PROGRAM_ID: Pubkey = sysvar::id();
-    static ref TOKEN_PROGRAM_ID: Pubkey = spl_token_id();
+    static ref TOKEN_PROGRAM_ID: Pubkey = spl_token_id_v2_0();
     static ref VOTE_PROGRAM_ID: Pubkey = solana_vote_program::id();
     pub static ref PARSABLE_PROGRAM_IDS: HashMap<Pubkey, ParsableAccount> = {
         let mut m = HashMap::new();
@@ -114,14 +112,12 @@ pub fn parse_account_data(
 
 #[cfg(test)]
 mod test {
-    use {
-        super::*,
-        solana_sdk::nonce::{
+    use super::*;
+    use solana_sdk::nonce::{
         state::{Data, Versions},
         State,
-        },
-        solana_vote_program::vote_state::{VoteState, VoteStateVersions},
     };
+    use solana_vote_program::vote_state::{VoteState, VoteStateVersions};
 
     #[test]
     fn test_parse_account_data() {

account-decoder/src/parse_bpf_loader.rs

@@ -1,11 +1,9 @@
-use {
-    crate::{
+use crate::{
     parse_account_data::{ParsableAccount, ParseAccountError},
     UiAccountData, UiAccountEncoding,
-    },
-    bincode::{deserialize, serialized_size},
-    solana_sdk::{bpf_loader_upgradeable::UpgradeableLoaderState, pubkey::Pubkey},
 };
+use bincode::{deserialize, serialized_size};
+use solana_sdk::{bpf_loader_upgradeable::UpgradeableLoaderState, pubkey::Pubkey};
 
 pub fn parse_bpf_upgradeable_loader(
     data: &[u8],
@@ -92,7 +90,9 @@ pub struct UiProgramData {
 
 #[cfg(test)]
 mod test {
-    use {super::*, bincode::serialize, solana_sdk::pubkey::Pubkey};
+    use super::*;
+    use bincode::serialize;
+    use solana_sdk::pubkey::Pubkey;
 
     #[test]
     fn test_parse_bpf_upgradeable_loader_accounts() {

account-decoder/src/parse_config.rs

@@ -1,19 +1,15 @@
-use {
-    crate::{
+use crate::{
     parse_account_data::{ParsableAccount, ParseAccountError},
     validator_info,
-    },
-    bincode::deserialize,
-    serde_json::Value,
-    solana_config_program::{get_config_data, ConfigKeys},
-    solana_sdk::{
-        pubkey::Pubkey,
-        stake::config::{self as stake_config, Config as StakeConfig},
-    },
 };
+use bincode::deserialize;
+use serde_json::Value;
+use solana_config_program::{get_config_data, ConfigKeys};
+use solana_sdk::pubkey::Pubkey;
+use solana_stake_program::config::Config as StakeConfig;
 
 pub fn parse_config(data: &[u8], pubkey: &Pubkey) -> Result<ConfigAccountType, ParseAccountError> {
-    let parsed_account = if pubkey == &stake_config::id() {
+    let parsed_account = if pubkey == &solana_stake_program::config::id() {
         get_config_data(data)
             .ok()
             .and_then(|data| deserialize::<StakeConfig>(data).ok())
@@ -41,7 +37,7 @@ fn parse_config_data<T>(data: &[u8], keys: Vec<(Pubkey, bool)>) -> Option<UiConf
 where
     T: serde::de::DeserializeOwned,
 {
-    let config_data: T = deserialize(get_config_data(data).ok()?).ok()?;
+    let config_data: T = deserialize(&get_config_data(data).ok()?).ok()?;
     let keys = keys
         .iter()
         .map(|key| UiConfigKey {
@@ -91,10 +87,11 @@ pub struct UiConfig<T> {
 
 #[cfg(test)]
 mod test {
-    use {
-        super::*, crate::validator_info::ValidatorInfo, serde_json::json,
-        solana_config_program::create_config_account, solana_sdk::account::ReadableAccount,
-    };
+    use super::*;
+    use crate::validator_info::ValidatorInfo;
+    use serde_json::json;
+    use solana_config_program::create_config_account;
+    use solana_sdk::account::ReadableAccount;
 
     #[test]
     fn test_parse_config() {
@@ -104,7 +101,11 @@ mod test {
         };
         let stake_config_account = create_config_account(vec![], &stake_config, 10);
         assert_eq!(
-            parse_config(stake_config_account.data(), &stake_config::id()).unwrap(),
+            parse_config(
+                &stake_config_account.data(),
+                &solana_stake_program::config::id()
+            )
+            .unwrap(),
             ConfigAccountType::StakeConfig(UiStakeConfig {
                 warmup_cooldown_rate: 0.25,
                 slash_penalty: 50,
@@ -124,7 +125,7 @@ mod test {
             10,
         );
         assert_eq!(
-            parse_config(validator_info_config_account.data(), &info_pubkey).unwrap(),
+            parse_config(&validator_info_config_account.data(), &info_pubkey).unwrap(),
             ConfigAccountType::ValidatorInfo(UiConfig {
                 keys: vec![
                     UiConfigKey {

account-decoder/src/parse_nonce.rs

@@ -1,9 +1,7 @@
-use {
-    crate::{parse_account_data::ParseAccountError, UiFeeCalculator},
-    solana_sdk::{
+use crate::{parse_account_data::ParseAccountError, UiFeeCalculator};
+use solana_sdk::{
     instruction::InstructionError,
     nonce::{state::Versions, State},
-    },
 };
 
 pub fn parse_nonce(data: &[u8]) -> Result<UiNonceState, ParseAccountError> {
@@ -44,16 +42,14 @@ pub struct UiNonceData {
 
 #[cfg(test)]
 mod test {
-    use {
-        super::*,
-        solana_sdk::{
+    use super::*;
+    use solana_sdk::{
         hash::Hash,
         nonce::{
             state::{Data, Versions},
             State,
         },
         pubkey::Pubkey,
-        },
     };
 
     #[test]

account-decoder/src/parse_stake.rs

@@ -1,14 +1,10 @@
-use {
-    crate::{
+use crate::{
     parse_account_data::{ParsableAccount, ParseAccountError},
     StringAmount,
-    },
-    bincode::deserialize,
-    solana_sdk::{
-        clock::{Epoch, UnixTimestamp},
-        stake::state::{Authorized, Delegation, Lockup, Meta, Stake, StakeState},
-    },
 };
+use bincode::deserialize;
+use solana_sdk::clock::{Epoch, UnixTimestamp};
+use solana_stake_program::stake_state::{Authorized, Delegation, Lockup, Meta, Stake, StakeState};
 
 pub fn parse_stake(data: &[u8]) -> Result<StakeAccountType, ParseAccountError> {
     let stake_state: StakeState = deserialize(data)
@@ -136,7 +132,8 @@ impl From<Delegation> for UiDelegation {
 
 #[cfg(test)]
 mod test {
-    use {super::*, bincode::serialize};
+    use super::*;
+    use bincode::serialize;
 
     #[test]
     fn test_parse_stake() {

account-decoder/src/parse_sysvar.rs

@@ -1,13 +1,10 @@
-#[allow(deprecated)]
-use solana_sdk::sysvar::{fees::Fees, recent_blockhashes::RecentBlockhashes};
-use {
-    crate::{
+use crate::{
     parse_account_data::{ParsableAccount, ParseAccountError},
     StringAmount, UiFeeCalculator,
-    },
-    bincode::deserialize,
-    bv::BitVec,
-    solana_sdk::{
+};
+use bincode::deserialize;
+use bv::BitVec;
+use solana_sdk::{
     clock::{Clock, Epoch, Slot, UnixTimestamp},
     epoch_schedule::EpochSchedule,
     pubkey::Pubkey,
@@ -15,12 +12,10 @@ use {
     slot_hashes::SlotHashes,
     slot_history::{self, SlotHistory},
     stake_history::{StakeHistory, StakeHistoryEntry},
-        sysvar::{self, rewards::Rewards},
-    },
+    sysvar::{self, fees::Fees, recent_blockhashes::RecentBlockhashes, rewards::Rewards},
 };
 
 pub fn parse_sysvar(data: &[u8], pubkey: &Pubkey) -> Result<SysvarAccountType, ParseAccountError> {
-    #[allow(deprecated)]
     let parsed_account = {
         if pubkey == &sysvar::clock::id() {
             deserialize::<Clock>(data)
@@ -96,9 +91,7 @@ pub fn parse_sysvar(data: &[u8], pubkey: &Pubkey) -> Result<SysvarAccountType, P
 pub enum SysvarAccountType {
     Clock(UiClock),
     EpochSchedule(EpochSchedule),
-    #[allow(deprecated)]
     Fees(UiFees),
-    #[allow(deprecated)]
     RecentBlockhashes(Vec<UiRecentBlockhashesEntry>),
     Rent(UiRent),
     Rewards(UiRewards),
@@ -134,7 +127,6 @@ impl From<Clock> for UiClock {
 pub struct UiFees {
     pub fee_calculator: UiFeeCalculator,
 }
-#[allow(deprecated)]
 impl From<Fees> for UiFees {
     fn from(fees: Fees) -> Self {
         Self {
@@ -220,17 +212,14 @@ pub struct UiStakeHistoryEntry {
 
 #[cfg(test)]
 mod test {
-    #[allow(deprecated)]
-    use solana_sdk::sysvar::recent_blockhashes::IterItem;
-    use {
-        super::*,
-        solana_sdk::{account::create_account_for_test, fee_calculator::FeeCalculator, hash::Hash},
+    use super::*;
+    use solana_sdk::{
+        account::create_account_for_test, fee_calculator::FeeCalculator, hash::Hash,
+        sysvar::recent_blockhashes::IterItem,
     };
 
     #[test]
     fn test_parse_sysvars() {
-        let hash = Hash::new(&[1; 32]);
-
         let clock_sysvar = create_account_for_test(&Clock::default());
         assert_eq!(
             parse_sysvar(&clock_sysvar.data, &sysvar::clock::id()).unwrap(),
@@ -250,16 +239,19 @@ mod test {
             SysvarAccountType::EpochSchedule(epoch_schedule),
         );
 
-        #[allow(deprecated)]
-        {
         let fees_sysvar = create_account_for_test(&Fees::default());
         assert_eq!(
             parse_sysvar(&fees_sysvar.data, &sysvar::fees::id()).unwrap(),
             SysvarAccountType::Fees(UiFees::default()),
         );
 
-            let recent_blockhashes: RecentBlockhashes =
-                vec![IterItem(0, &hash, 10)].into_iter().collect();
+        let hash = Hash::new(&[1; 32]);
+        let fee_calculator = FeeCalculator {
+            lamports_per_signature: 10,
+        };
+        let recent_blockhashes: RecentBlockhashes = vec![IterItem(0, &hash, &fee_calculator)]
+            .into_iter()
+            .collect();
         let recent_blockhashes_sysvar = create_account_for_test(&recent_blockhashes);
         assert_eq!(
             parse_sysvar(
@@ -269,10 +261,9 @@ mod test {
             .unwrap(),
             SysvarAccountType::RecentBlockhashes(vec![UiRecentBlockhashesEntry {
                 blockhash: hash.to_string(),
-                    fee_calculator: FeeCalculator::new(10).into(),
+                fee_calculator: fee_calculator.into(),
             }]),
         );
-        }
 
         let rent = Rent {
             lamports_per_byte_year: 10,

account-decoder/src/parse_token.rs

@@ -1,37 +1,35 @@
-use {
-    crate::{
+use crate::{
     parse_account_data::{ParsableAccount, ParseAccountError},
     StringAmount, StringDecimals,
-    },
-    solana_sdk::pubkey::Pubkey,
-    spl_token::{
+};
+use solana_sdk::pubkey::Pubkey;
+use spl_token_v2_0::{
     solana_program::{
         program_option::COption, program_pack::Pack, pubkey::Pubkey as SplTokenPubkey,
     },
     state::{Account, AccountState, Mint, Multisig},
-    },
-    std::str::FromStr,
 };
+use std::str::FromStr;
 
-// A helper function to convert spl_token::id() as spl_sdk::pubkey::Pubkey to
+// A helper function to convert spl_token_v2_0::id() as spl_sdk::pubkey::Pubkey to
 // solana_sdk::pubkey::Pubkey
-pub fn spl_token_id() -> Pubkey {
-    Pubkey::new_from_array(spl_token::id().to_bytes())
+pub fn spl_token_id_v2_0() -> Pubkey {
+    Pubkey::new_from_array(spl_token_v2_0::id().to_bytes())
 }
 
-// A helper function to convert spl_token::native_mint::id() as spl_sdk::pubkey::Pubkey to
+// A helper function to convert spl_token_v2_0::native_mint::id() as spl_sdk::pubkey::Pubkey to
 // solana_sdk::pubkey::Pubkey
-pub fn spl_token_native_mint() -> Pubkey {
-    Pubkey::new_from_array(spl_token::native_mint::id().to_bytes())
+pub fn spl_token_v2_0_native_mint() -> Pubkey {
+    Pubkey::new_from_array(spl_token_v2_0::native_mint::id().to_bytes())
 }
 
 // A helper function to convert a solana_sdk::pubkey::Pubkey to spl_sdk::pubkey::Pubkey
-pub fn spl_token_pubkey(pubkey: &Pubkey) -> SplTokenPubkey {
+pub fn spl_token_v2_0_pubkey(pubkey: &Pubkey) -> SplTokenPubkey {
     SplTokenPubkey::new_from_array(pubkey.to_bytes())
 }
 
 // A helper function to convert a spl_sdk::pubkey::Pubkey to solana_sdk::pubkey::Pubkey
-pub fn pubkey_from_spl_token(pubkey: &SplTokenPubkey) -> Pubkey {
+pub fn pubkey_from_spl_token_v2_0(pubkey: &SplTokenPubkey) -> Pubkey {
     Pubkey::new_from_array(pubkey.to_bytes())
 }
 
@@ -118,7 +116,6 @@ pub fn parse_token(
 
 #[derive(Debug, Serialize, Deserialize, PartialEq)]
 #[serde(rename_all = "camelCase", tag = "type", content = "info")]
-#[allow(clippy::large_enum_variant)]
 pub enum TokenAccountType {
     Account(UiTokenAccount),
     Mint(UiMint),

account-decoder/src/parse_vote.rs

@@ -1,11 +1,9 @@
-use {
-    crate::{parse_account_data::ParseAccountError, StringAmount},
-    solana_sdk::{
+use crate::{parse_account_data::ParseAccountError, StringAmount};
+use solana_sdk::{
     clock::{Epoch, Slot},
     pubkey::Pubkey,
-    },
-    solana_vote_program::vote_state::{BlockTimestamp, Lockout, VoteState},
 };
+use solana_vote_program::vote_state::{BlockTimestamp, Lockout, VoteState};
 
 pub fn parse_vote(data: &[u8]) -> Result<VoteAccountType, ParseAccountError> {
     let mut vote_state = VoteState::deserialize(data).map_err(ParseAccountError::from)?;
@@ -123,7 +121,8 @@ struct UiEpochCredits {
 
 #[cfg(test)]
 mod test {
-    use {super::*, solana_vote_program::vote_state::VoteStateVersions};
+    use super::*;
+    use solana_vote_program::vote_state::VoteStateVersions;
 
     #[test]
     fn test_parse_vote() {

accounts-bench/Cargo.toml

@@ -1,22 +1,24 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2021"
+edition = "2018"
 name = "solana-accounts-bench"
-version = "1.10.0"
+version = "1.6.28"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 publish = false
 
 [dependencies]
-log = "0.4.14"
-rayon = "1.5.1"
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
+log = "0.4.11"
+rayon = "1.5.0"
+solana-logger = { path = "../logger", version = "=1.6.28" }
+solana-runtime = { path = "../runtime", version = "=1.6.28" }
+solana-measure = { path = "../measure", version = "=1.6.28" }
+solana-sdk = { path = "../sdk", version = "=1.6.28" }
+solana-version = { path = "../version", version = "=1.6.28" }
+rand = "0.7.0"
 clap = "2.33.1"
+crossbeam-channel = "0.4"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accounts-bench/src/main.rs

@@ -1,19 +1,15 @@
 #![allow(clippy::integer_arithmetic)]
 #[macro_use]
 extern crate log;
-use {
-    clap::{crate_description, crate_name, value_t, App, Arg},
-    rayon::prelude::*,
-    solana_measure::measure::Measure,
-    solana_runtime::{
+use clap::{crate_description, crate_name, value_t, App, Arg};
+use rayon::prelude::*;
+use solana_measure::measure::Measure;
+use solana_runtime::{
     accounts::{create_test_accounts, update_accounts_bench, Accounts},
-        accounts_db::AccountShrinkThreshold,
-        accounts_index::AccountSecondaryIndexes,
-        ancestors::Ancestors,
-    },
-    solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey},
-    std::{env, fs, path::PathBuf},
+    accounts_index::{AccountSecondaryIndexes, Ancestors},
 };
+use solana_sdk::{genesis_config::ClusterType, pubkey::Pubkey};
+use std::{env, fs, path::PathBuf};
 
 fn main() {
     solana_logger::setup();
@@ -62,12 +58,11 @@ fn main() {
     if fs::remove_dir_all(path.clone()).is_err() {
         println!("Warning: Couldn't remove {:?}", path);
     }
-    let accounts = Accounts::new_with_config_for_benches(
+    let accounts = Accounts::new_with_config(
         vec![path],
         &ClusterType::Testnet,
         AccountSecondaryIndexes::default(),
         false,
-        AccountShrinkThreshold::default(),
     );
     println!("Creating {} accounts", num_accounts);
     let mut create_time = Measure::start("create accounts");
@@ -92,19 +87,17 @@ fn main() {
         num_slots,
         create_time
     );
-    let mut ancestors = Vec::with_capacity(num_slots);
-    ancestors.push(0);
+    let mut ancestors: Ancestors = vec![(0, 0)].into_iter().collect();
     for i in 1..num_slots {
-        ancestors.push(i as u64);
+        ancestors.insert(i as u64, i - 1);
         accounts.add_root(i as u64);
     }
-    let ancestors = Ancestors::from(ancestors);
     let mut elapsed = vec![0; iterations];
     let mut elapsed_store = vec![0; iterations];
     for x in 0..iterations {
         if clean {
             let mut time = Measure::start("clean");
-            accounts.accounts_db.clean_accounts(None, false, None);
+            accounts.accounts_db.clean_accounts(None);
             time.stop();
             println!("{}", time);
             for slot in 0..num_slots {
@@ -123,9 +116,6 @@ fn main() {
                 solana_sdk::clock::Slot::default(),
                 &ancestors,
                 None,
-                false,
-                None,
-                false,
             );
             time_store.stop();
             if results != results_store {

accounts-cluster-bench/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2021"
+edition = "2018"
 name = "solana-accounts-cluster-bench"
-version = "1.10.0"
+version = "1.6.28"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,28 +10,25 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-log = "0.4.14"
+log = "0.4.11"
 rand = "0.7.0"
-rayon = "1.5.1"
-solana-account-decoder = { path = "../account-decoder", version = "=1.10.0" }
-solana-clap-utils = { path = "../clap-utils", version = "=1.10.0" }
-solana-client = { path = "../client", version = "=1.10.0" }
-solana-core = { path = "../core", version = "=1.10.0" }
-solana-faucet = { path = "../faucet", version = "=1.10.0" }
-solana-gossip = { path = "../gossip", version = "=1.10.0" }
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-streamer = { path = "../streamer", version = "=1.10.0" }
-solana-test-validator = { path = "../test-validator", version = "=1.10.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
-spl-token = { version = "=3.2.0", features = ["no-entrypoint"] }
+rayon = "1.4.1"
+solana-account-decoder = { path = "../account-decoder", version = "=1.6.28" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.28" }
+solana-client = { path = "../client", version = "=1.6.28" }
+solana-core = { path = "../core", version = "=1.6.28" }
+solana-measure = { path = "../measure", version = "=1.6.28" }
+solana-logger = { path = "../logger", version = "=1.6.28" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.28" }
+solana-faucet = { path = "../faucet", version = "=1.6.28" }
+solana-runtime = { path = "../runtime", version = "=1.6.28" }
+solana-sdk = { path = "../sdk", version = "=1.6.28" }
+solana-transaction-status = { path = "../transaction-status", version = "=1.6.28" }
+solana-version = { path = "../version", version = "=1.6.28" }
+spl-token-v2-0 = { package = "spl-token", version = "=3.1.1", features = ["no-entrypoint"] }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "=1.10.0" }
+solana-local-cluster = { path = "../local-cluster", version = "=1.6.28" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

accounts-cluster-bench/src/main.rs

@@ -1,40 +1,41 @@
 #![allow(clippy::integer_arithmetic)]
-use {
-    clap::{crate_description, crate_name, value_t, values_t_or_exit, App, Arg},
-    log::*,
-    rand::{thread_rng, Rng},
-    rayon::prelude::*,
-    solana_account_decoder::parse_token::spl_token_pubkey,
-    solana_clap_utils::input_parsers::pubkey_of,
-    solana_client::{rpc_client::RpcClient, transaction_executor::TransactionExecutor},
-    solana_faucet::faucet::{request_airdrop_transaction, FAUCET_PORT},
-    solana_gossip::gossip_service::discover,
-    solana_runtime::inline_spl_token,
-    solana_sdk::{
+use clap::{crate_description, crate_name, value_t, values_t_or_exit, App, Arg};
+use log::*;
+use rand::{thread_rng, Rng};
+use rayon::prelude::*;
+use solana_account_decoder::parse_token::spl_token_v2_0_pubkey;
+use solana_clap_utils::input_parsers::pubkey_of;
+use solana_client::rpc_client::RpcClient;
+use solana_core::gossip_service::discover;
+use solana_faucet::faucet::{request_airdrop_transaction, FAUCET_PORT};
+use solana_measure::measure::Measure;
+use solana_runtime::inline_spl_token_v2_0;
+use solana_sdk::{
     commitment_config::CommitmentConfig,
-        instruction::{AccountMeta, Instruction},
     message::Message,
     pubkey::Pubkey,
     rpc_port::DEFAULT_RPC_PORT,
-        signature::{read_keypair_file, Keypair, Signer},
+    signature::{read_keypair_file, Keypair, Signature, Signer},
     system_instruction, system_program,
+    timing::timestamp,
     transaction::Transaction,
-    },
-    solana_streamer::socket::SocketAddrSpace,
-    solana_transaction_status::parse_token::spl_token_instruction,
-    std::{
-        cmp::min,
+};
+use solana_transaction_status::parse_token::spl_token_v2_0_instruction;
+use std::{
     net::SocketAddr,
     process::exit,
     sync::{
-            atomic::{AtomicU64, Ordering},
-            Arc,
+        atomic::{AtomicBool, AtomicU64, Ordering},
+        Arc, RwLock,
     },
-        thread::sleep,
+    thread::{sleep, Builder, JoinHandle},
     time::{Duration, Instant},
-    },
 };
 
+// Create and close messages both require 2 signatures; if transaction construction changes, update
+// this magic number
+const NUM_SIGNATURES: u64 = 2;
+
 pub fn airdrop_lamports(
     client: &RpcClient,
     faucet_addr: &SocketAddr,
@@ -53,8 +54,8 @@ pub fn airdrop_lamports(
             id.pubkey(),
         );
 
-        let blockhash = client.get_latest_blockhash().unwrap();
-        match request_airdrop_transaction(faucet_addr, &id.pubkey(), airdrop_amount, blockhash) {
+        let (blockhash, _fee_calculator) = client.get_recent_blockhash().unwrap();
+        match request_airdrop_transaction(&faucet_addr, &id.pubkey(), airdrop_amount, blockhash) {
             Ok(transaction) => {
                 let mut tries = 0;
                 loop {
@@ -98,6 +99,160 @@ pub fn airdrop_lamports(
     true
 }
 
+// signature, timestamp, id
+type PendingQueue = Vec<(Signature, u64, u64)>;
+
+struct TransactionExecutor {
+    sig_clear_t: JoinHandle<()>,
+    sigs: Arc<RwLock<PendingQueue>>,
+    cleared: Arc<RwLock<Vec<u64>>>,
+    exit: Arc<AtomicBool>,
+    counter: AtomicU64,
+    client: RpcClient,
+}
+
+impl TransactionExecutor {
+    fn new(entrypoint_addr: SocketAddr) -> Self {
+        let sigs = Arc::new(RwLock::new(Vec::new()));
+        let cleared = Arc::new(RwLock::new(Vec::new()));
+        let exit = Arc::new(AtomicBool::new(false));
+        let sig_clear_t = Self::start_sig_clear_thread(&exit, &sigs, &cleared, entrypoint_addr);
+        let client =
+            RpcClient::new_socket_with_commitment(entrypoint_addr, CommitmentConfig::confirmed());
+        Self {
+            sigs,
+            cleared,
+            sig_clear_t,
+            exit,
+            counter: AtomicU64::new(0),
+            client,
+        }
+    }
+
+    fn num_outstanding(&self) -> usize {
+        self.sigs.read().unwrap().len()
+    }
+
+    fn push_transactions(&self, txs: Vec<Transaction>) -> Vec<u64> {
+        let mut ids = vec![];
+        let new_sigs = txs.into_iter().filter_map(|tx| {
+            let id = self.counter.fetch_add(1, Ordering::Relaxed);
+            ids.push(id);
+            match self.client.send_transaction(&tx) {
+                Ok(sig) => {
+                    return Some((sig, timestamp(), id));
+                }
+                Err(e) => {
+                    info!("error: {:#?}", e);
+                }
+            }
+            None
+        });
+        let mut sigs_w = self.sigs.write().unwrap();
+        sigs_w.extend(new_sigs);
+        ids
+    }
+
+    fn drain_cleared(&self) -> Vec<u64> {
+        std::mem::take(&mut *self.cleared.write().unwrap())
+    }
+
+    fn close(self) {
+        self.exit.store(true, Ordering::Relaxed);
+        self.sig_clear_t.join().unwrap();
+    }
+
+    fn start_sig_clear_thread(
+        exit: &Arc<AtomicBool>,
+        sigs: &Arc<RwLock<PendingQueue>>,
+        cleared: &Arc<RwLock<Vec<u64>>>,
+        entrypoint_addr: SocketAddr,
+    ) -> JoinHandle<()> {
+        let sigs = sigs.clone();
+        let exit = exit.clone();
+        let cleared = cleared.clone();
+        Builder::new()
+            .name("sig_clear".to_string())
+            .spawn(move || {
+                let client = RpcClient::new_socket_with_commitment(
+                    entrypoint_addr,
+                    CommitmentConfig::confirmed(),
+                );
+                let mut success = 0;
+                let mut error_count = 0;
+                let mut timed_out = 0;
+                let mut last_log = Instant::now();
+                while !exit.load(Ordering::Relaxed) {
+                    let sigs_len = sigs.read().unwrap().len();
+                    if sigs_len > 0 {
+                        let mut sigs_w = sigs.write().unwrap();
+                        let mut start = Measure::start("sig_status");
+                        let statuses: Vec<_> = sigs_w
+                            .chunks(200)
+                            .flat_map(|sig_chunk| {
+                                let only_sigs: Vec<_> = sig_chunk.iter().map(|s| s.0).collect();
+                                client
+                                    .get_signature_statuses(&only_sigs)
+                                    .expect("status fail")
+                                    .value
+                            })
+                            .collect();
+                        let mut num_cleared = 0;
+                        let start_len = sigs_w.len();
+                        let now = timestamp();
+                        let mut new_ids = vec![];
+                        let mut i = 0;
+                        let mut j = 0;
+                        while i != sigs_w.len() {
+                            let mut retain = true;
+                            let sent_ts = sigs_w[i].1;
+                            if let Some(e) = &statuses[j] {
+                                debug!("error: {:?}", e);
+                                if e.status.is_ok() {
+                                    success += 1;
+                                } else {
+                                    error_count += 1;
+                                }
+                                num_cleared += 1;
+                                retain = false;
+                            } else if now - sent_ts > 30_000 {
+                                retain = false;
+                                timed_out += 1;
+                            }
+                            if !retain {
+                                new_ids.push(sigs_w.remove(i).2);
+                            } else {
+                                i += 1;
+                            }
+                            j += 1;
+                        }
+                        let final_sigs_len = sigs_w.len();
+                        drop(sigs_w);
+                        cleared.write().unwrap().extend(new_ids);
+                        start.stop();
+                        debug!(
+                            "sigs len: {:?} success: {} took: {}ms cleared: {}/{}",
+                            final_sigs_len,
+                            success,
+                            start.as_ms(),
+                            num_cleared,
+                            start_len,
+                        );
+                        if last_log.elapsed().as_millis() > 5000 {
+                            info!(
+                                "success: {} error: {} timed_out: {}",
+                                success, error_count, timed_out,
+                            );
+                            last_log = Instant::now();
+                        }
+                    }
+                    sleep(Duration::from_millis(200));
+                }
+            })
+            .unwrap()
+    }
+}
+
 struct SeedTracker {
     max_created: Arc<AtomicU64>,
     max_closed: Arc<AtomicU64>,
@@ -118,7 +273,7 @@ fn make_create_message(
         .into_iter()
         .map(|_| {
             let program_id = if mint.is_some() {
-                inline_spl_token::id()
+                inline_spl_token_v2_0::id()
             } else {
                 system_program::id()
             };
@@ -135,12 +290,12 @@ fn make_create_message(
                 &program_id,
             )];
             if let Some(mint_address) = mint {
-                instructions.push(spl_token_instruction(
-                    spl_token::instruction::initialize_account(
-                        &spl_token::id(),
-                        &spl_token_pubkey(&to_pubkey),
-                        &spl_token_pubkey(&mint_address),
-                        &spl_token_pubkey(&base_keypair.pubkey()),
+                instructions.push(spl_token_v2_0_instruction(
+                    spl_token_v2_0::instruction::initialize_account(
+                        &spl_token_v2_0::id(),
+                        &spl_token_v2_0_pubkey(&to_pubkey),
+                        &spl_token_v2_0_pubkey(&mint_address),
+                        &spl_token_v2_0_pubkey(&base_keypair.pubkey()),
                     )
                     .unwrap(),
                 ));
@@ -148,8 +303,8 @@ fn make_create_message(
 
             instructions
         })
-        .flatten()
         .collect();
+    let instructions: Vec<_> = instructions.into_iter().flatten().collect();
 
     Message::new(&instructions, Some(&keypair.pubkey()))
 }
@@ -157,48 +312,42 @@ fn make_create_message(
 fn make_close_message(
     keypair: &Keypair,
     base_keypair: &Keypair,
-    max_created: Arc<AtomicU64>,
-    max_closed: Arc<AtomicU64>,
+    max_closed_seed: Arc<AtomicU64>,
     num_instructions: usize,
     balance: u64,
     spl_token: bool,
 ) -> Message {
     let instructions: Vec<_> = (0..num_instructions)
         .into_iter()
-        .filter_map(|_| {
+        .map(|_| {
             let program_id = if spl_token {
-                inline_spl_token::id()
+                inline_spl_token_v2_0::id()
             } else {
                 system_program::id()
             };
-            let max_created_seed = max_created.load(Ordering::Relaxed);
-            let max_closed_seed = max_closed.load(Ordering::Relaxed);
-            if max_closed_seed >= max_created_seed {
-                return None;
-            }
-            let seed = max_closed.fetch_add(1, Ordering::Relaxed).to_string();
+            let seed = max_closed_seed.fetch_add(1, Ordering::Relaxed).to_string();
             let address =
                 Pubkey::create_with_seed(&base_keypair.pubkey(), &seed, &program_id).unwrap();
             if spl_token {
-                Some(spl_token_instruction(
-                    spl_token::instruction::close_account(
-                        &spl_token::id(),
-                        &spl_token_pubkey(&address),
-                        &spl_token_pubkey(&keypair.pubkey()),
-                        &spl_token_pubkey(&base_keypair.pubkey()),
+                spl_token_v2_0_instruction(
+                    spl_token_v2_0::instruction::close_account(
+                        &spl_token_v2_0::id(),
+                        &spl_token_v2_0_pubkey(&address),
+                        &spl_token_v2_0_pubkey(&keypair.pubkey()),
+                        &spl_token_v2_0_pubkey(&base_keypair.pubkey()),
                         &[],
                     )
                     .unwrap(),
-                ))
+                )
             } else {
-                Some(system_instruction::transfer_with_seed(
+                system_instruction::transfer_with_seed(
                     &address,
                     &base_keypair.pubkey(),
                     seed,
                     &program_id,
                     &keypair.pubkey(),
                     balance,
-                ))
+                )
             }
         })
         .collect();
@@ -214,11 +363,10 @@ fn run_accounts_bench(
     iterations: usize,
     maybe_space: Option<u64>,
     batch_size: usize,
-    close_nth_batch: u64,
+    close_nth: u64,
     maybe_lamports: Option<u64>,
     num_instructions: usize,
     mint: Option<Pubkey>,
-    reclaim_accounts: bool,
 ) {
     assert!(num_instructions > 0);
     let client =
@@ -226,10 +374,10 @@ fn run_accounts_bench(
 
     info!("Targeting {}", entrypoint_addr);
 
-    let mut latest_blockhash = Instant::now();
+    let mut last_blockhash = Instant::now();
     let mut last_log = Instant::now();
     let mut count = 0;
-    let mut blockhash = client.get_latest_blockhash().expect("blockhash");
+    let mut recent_blockhash = client.get_recent_blockhash().expect("blockhash");
     let mut tx_sent_count = 0;
     let mut total_accounts_created = 0;
     let mut total_accounts_closed = 0;
@@ -257,33 +405,16 @@ fn run_accounts_bench(
 
     let executor = TransactionExecutor::new(entrypoint_addr);
 
-    // Create and close messages both require 2 signatures, fake a 2 signature message to calculate fees
-    let mut message = Message::new(
-        &[
-            Instruction::new_with_bytes(
-                Pubkey::new_unique(),
-                &[],
-                vec![AccountMeta::new(Pubkey::new_unique(), true)],
-            ),
-            Instruction::new_with_bytes(
-                Pubkey::new_unique(),
-                &[],
-                vec![AccountMeta::new(Pubkey::new_unique(), true)],
-            ),
-        ],
-        None,
-    );
-
     loop {
-        if latest_blockhash.elapsed().as_millis() > 10_000 {
-            blockhash = client.get_latest_blockhash().expect("blockhash");
-            latest_blockhash = Instant::now();
+        if last_blockhash.elapsed().as_millis() > 10_000 {
+            recent_blockhash = client.get_recent_blockhash().expect("blockhash");
+            last_blockhash = Instant::now();
         }
 
-        message.recent_blockhash = blockhash;
-        let fee = client
-            .get_fee_for_message(&message)
-            .expect("get_fee_for_message");
+        let fee = recent_blockhash
+            .1
+            .lamports_per_signature
+            .saturating_mul(NUM_SIGNATURES);
         let lamports = min_balance + fee;
 
         for (i, balance) in balances.iter_mut().enumerate() {
@@ -300,7 +431,7 @@ fn run_accounts_bench(
                     if !airdrop_lamports(
                         &client,
                         &faucet_addr,
-                        payer_keypairs[i],
+                        &payer_keypairs[i],
                         lamports * 100_000,
                     ) {
                         warn!("failed airdrop, exiting");
@@ -310,7 +441,6 @@ fn run_accounts_bench(
             }
         }
 
-        // Create accounts
         let sigs_len = executor.num_outstanding();
         if sigs_len < batch_size {
             let num_to_create = batch_size - sigs_len;
@@ -332,7 +462,7 @@ fn run_accounts_bench(
                                     mint,
                                 );
                                 let signers: Vec<&Keypair> = vec![keypair, &base_keypair];
-                                Transaction::new(&signers, message, blockhash)
+                                Transaction::new(&signers, message, recent_blockhash.0)
                             })
                             .collect();
                         balances[i] = balances[i].saturating_sub(lamports * txs.len() as u64);
@@ -345,27 +475,22 @@ fn run_accounts_bench(
                 }
             }
 
-            if close_nth_batch > 0 {
-                let num_batches_to_close =
-                    total_accounts_created as u64 / (close_nth_batch * batch_size as u64);
-                let expected_closed = num_batches_to_close * batch_size as u64;
-                let max_closed_seed = seed_tracker.max_closed.load(Ordering::Relaxed);
-                // Close every account we've created with seed between max_closed_seed..expected_closed
-                if max_closed_seed < expected_closed {
-                    let txs: Vec<_> = (0..expected_closed - max_closed_seed)
+            if close_nth > 0 {
+                let expected_closed = total_accounts_created as u64 / close_nth;
+                if expected_closed > total_accounts_closed {
+                    let txs: Vec<_> = (0..expected_closed - total_accounts_closed)
                         .into_par_iter()
                         .map(|_| {
                             let message = make_close_message(
-                                payer_keypairs[0],
+                                &payer_keypairs[0],
                                 &base_keypair,
-                                seed_tracker.max_created.clone(),
                                 seed_tracker.max_closed.clone(),
                                 1,
                                 min_balance,
                                 mint.is_some(),
                             );
-                            let signers: Vec<&Keypair> = vec![payer_keypairs[0], &base_keypair];
-                            Transaction::new(&signers, message, blockhash)
+                            let signers: Vec<&Keypair> = vec![&payer_keypairs[0], &base_keypair];
+                            Transaction::new(&signers, message, recent_blockhash.0)
                         })
                         .collect();
                     balances[0] = balances[0].saturating_sub(fee * txs.len() as u64);
@@ -381,7 +506,7 @@ fn run_accounts_bench(
         }
 
         count += 1;
-        if last_log.elapsed().as_millis() > 3000 || count >= iterations {
+        if last_log.elapsed().as_millis() > 3000 {
             info!(
                 "total_accounts_created: {} total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance(s): {:?}",
                 total_accounts_created, total_accounts_closed, tx_sent_count, count, balances
@@ -396,83 +521,6 @@ fn run_accounts_bench(
         }
     }
     executor.close();
-
-    if reclaim_accounts {
-        let executor = TransactionExecutor::new(entrypoint_addr);
-        loop {
-            let max_closed_seed = seed_tracker.max_closed.load(Ordering::Relaxed);
-            let max_created_seed = seed_tracker.max_created.load(Ordering::Relaxed);
-
-            if latest_blockhash.elapsed().as_millis() > 10_000 {
-                blockhash = client.get_latest_blockhash().expect("blockhash");
-                latest_blockhash = Instant::now();
-            }
-            message.recent_blockhash = blockhash;
-            let fee = client
-                .get_fee_for_message(&message)
-                .expect("get_fee_for_message");
-
-            let sigs_len = executor.num_outstanding();
-            if sigs_len < batch_size && max_closed_seed < max_created_seed {
-                let num_to_close = min(
-                    batch_size - sigs_len,
-                    (max_created_seed - max_closed_seed) as usize,
-                );
-                if num_to_close >= payer_keypairs.len() {
-                    info!("closing {} accounts", num_to_close);
-                    let chunk_size = num_to_close / payer_keypairs.len();
-                    info!("{:?} chunk_size", chunk_size);
-                    if chunk_size > 0 {
-                        for (i, keypair) in payer_keypairs.iter().enumerate() {
-                            let txs: Vec<_> = (0..chunk_size)
-                                .into_par_iter()
-                                .filter_map(|_| {
-                                    let message = make_close_message(
-                                        keypair,
-                                        &base_keypair,
-                                        seed_tracker.max_created.clone(),
-                                        seed_tracker.max_closed.clone(),
-                                        num_instructions,
-                                        min_balance,
-                                        mint.is_some(),
-                                    );
-                                    if message.instructions.is_empty() {
-                                        return None;
-                                    }
-                                    let signers: Vec<&Keypair> = vec![keypair, &base_keypair];
-                                    Some(Transaction::new(&signers, message, blockhash))
-                                })
-                                .collect();
-                            balances[i] = balances[i].saturating_sub(fee * txs.len() as u64);
-                            info!("close txs: {}", txs.len());
-                            let new_ids = executor.push_transactions(txs);
-                            info!("close ids: {}", new_ids.len());
-                            tx_sent_count += new_ids.len();
-                            total_accounts_closed += (num_instructions * new_ids.len()) as u64;
-                        }
-                    }
-                }
-            } else {
-                let _ = executor.drain_cleared();
-            }
-            count += 1;
-            if last_log.elapsed().as_millis() > 3000 || max_closed_seed >= max_created_seed {
-                info!(
-                    "total_accounts_closed: {} tx_sent_count: {} loop_count: {} balance(s): {:?}",
-                    total_accounts_closed, tx_sent_count, count, balances
-                );
-                last_log = Instant::now();
-            }
-
-            if max_closed_seed >= max_created_seed {
-                break;
-            }
-            if executor.num_outstanding() >= batch_size {
-                sleep(Duration::from_millis(500));
-            }
-        }
-        executor.close();
-    }
 }
 
 fn main() {
@@ -524,14 +572,14 @@ fn main() {
                 .help("Number of transactions to send per batch"),
         )
         .arg(
-            Arg::with_name("close_nth_batch")
+            Arg::with_name("close_nth")
                 .long("close-frequency")
                 .takes_value(true)
                 .value_name("BYTES")
                 .help(
-                    "Every `n` batches, create a batch of close transactions for
-                    the earliest remaining batch of accounts created.
-                    Note: Should be > 1 to avoid situations where the close \
+                    "Send close transactions after this many accounts created. \
+                    Note: a `close-frequency` value near or below `batch-size` \
+                    may result in transaction-simulation errors, as the close \
                     transactions will be submitted before the corresponding \
                     create transactions have been confirmed",
                 ),
@@ -548,7 +596,7 @@ fn main() {
                 .long("iterations")
                 .takes_value(true)
                 .value_name("NUM")
-                .help("Number of iterations to make. 0 = unlimited iterations."),
+                .help("Number of iterations to make"),
         )
         .arg(
             Arg::with_name("check_gossip")
@@ -561,12 +609,6 @@ fn main() {
                 .takes_value(true)
                 .help("Mint address to initialize account"),
         )
-        .arg(
-            Arg::with_name("reclaim_accounts")
-                .long("reclaim-accounts")
-                .takes_value(false)
-                .help("Reclaim accounts after session ends; incompatible with --iterations 0"),
-        )
         .get_matches();
 
     let skip_gossip = !matches.is_present("check_gossip");
@@ -590,7 +632,7 @@ fn main() {
     let space = value_t!(matches, "space", u64).ok();
     let lamports = value_t!(matches, "lamports", u64).ok();
     let batch_size = value_t!(matches, "batch_size", usize).unwrap_or(4);
-    let close_nth_batch = value_t!(matches, "close_nth_batch", u64).unwrap_or(0);
+    let close_nth = value_t!(matches, "close_nth", u64).unwrap_or(0);
     let iterations = value_t!(matches, "iterations", usize).unwrap_or(10);
     let num_instructions = value_t!(matches, "num_instructions", usize).unwrap_or(1);
     if num_instructions == 0 || num_instructions > 500 {
@@ -623,7 +665,6 @@ fn main() {
             Some(&entrypoint_addr),  // find_node_by_gossip_addr
             None,                    // my_gossip_addr
             0,                       // my_shred_version
-            SocketAddrSpace::Unspecified,
         )
         .unwrap_or_else(|err| {
             eprintln!("Failed to discover {} node: {:?}", entrypoint_addr, err);
@@ -644,32 +685,22 @@ fn main() {
         iterations,
         space,
         batch_size,
-        close_nth_batch,
+        close_nth,
         lamports,
         num_instructions,
         mint,
-        matches.is_present("reclaim_accounts"),
     );
 }
 
 #[cfg(test)]
 pub mod test {
-    use {
-        super::*,
-        solana_core::validator::ValidatorConfig,
-        solana_faucet::faucet::run_local_faucet,
-        solana_local_cluster::{
+    use super::*;
+    use solana_core::validator::ValidatorConfig;
+    use solana_local_cluster::{
         local_cluster::{ClusterConfig, LocalCluster},
         validator_configs::make_identical_validator_configs,
-        },
-        solana_measure::measure::Measure,
-        solana_sdk::{native_token::sol_to_lamports, poh_config::PohConfig},
-        solana_test_validator::TestValidator,
-        spl_token::{
-            solana_program::program_pack::Pack,
-            state::{Account, Mint},
-        },
     };
+    use solana_sdk::poh_config::PohConfig;
 
     #[test]
     fn test_accounts_cluster_bench() {
@@ -685,11 +716,11 @@ pub mod test {
         };
 
         let faucet_addr = SocketAddr::from(([127, 0, 0, 1], 9900));
-        let cluster = LocalCluster::new(&mut config, SocketAddrSpace::Unspecified);
+        let cluster = LocalCluster::new(&mut config);
         let iterations = 10;
         let maybe_space = None;
         let batch_size = 100;
-        let close_nth_batch = 100;
+        let close_nth = 100;
         let maybe_lamports = None;
         let num_instructions = 2;
         let mut start = Measure::start("total accounts run");
@@ -700,112 +731,10 @@ pub mod test {
             iterations,
             maybe_space,
             batch_size,
-            close_nth_batch,
+            close_nth,
             maybe_lamports,
             num_instructions,
             None,
-            false,
-        );
-        start.stop();
-        info!("{}", start);
-    }
-
-    #[test]
-    fn test_create_then_reclaim_spl_token_accounts() {
-        solana_logger::setup();
-        let mint_keypair = Keypair::new();
-        let mint_pubkey = mint_keypair.pubkey();
-        let faucet_addr = run_local_faucet(mint_keypair, None);
-        let test_validator = TestValidator::with_custom_fees(
-            mint_pubkey,
-            1,
-            Some(faucet_addr),
-            SocketAddrSpace::Unspecified,
-        );
-        let rpc_client =
-            RpcClient::new_with_commitment(test_validator.rpc_url(), CommitmentConfig::processed());
-
-        // Created funder
-        let funder = Keypair::new();
-        let latest_blockhash = rpc_client.get_latest_blockhash().unwrap();
-        let signature = rpc_client
-            .request_airdrop_with_blockhash(
-                &funder.pubkey(),
-                sol_to_lamports(1.0),
-                &latest_blockhash,
-            )
-            .unwrap();
-        rpc_client
-            .confirm_transaction_with_spinner(
-                &signature,
-                &latest_blockhash,
-                CommitmentConfig::confirmed(),
-            )
-            .unwrap();
-
-        // Create Mint
-        let spl_mint_keypair = Keypair::new();
-        let spl_mint_len = Mint::get_packed_len();
-        let spl_mint_rent = rpc_client
-            .get_minimum_balance_for_rent_exemption(spl_mint_len)
-            .unwrap();
-        let transaction = Transaction::new_signed_with_payer(
-            &[
-                system_instruction::create_account(
-                    &funder.pubkey(),
-                    &spl_mint_keypair.pubkey(),
-                    spl_mint_rent,
-                    spl_mint_len as u64,
-                    &inline_spl_token::id(),
-                ),
-                spl_token_instruction(
-                    spl_token::instruction::initialize_mint(
-                        &spl_token::id(),
-                        &spl_token_pubkey(&spl_mint_keypair.pubkey()),
-                        &spl_token_pubkey(&spl_mint_keypair.pubkey()),
-                        None,
-                        2,
-                    )
-                    .unwrap(),
-                ),
-            ],
-            Some(&funder.pubkey()),
-            &[&funder, &spl_mint_keypair],
-            latest_blockhash,
-        );
-        let _sig = rpc_client
-            .send_and_confirm_transaction(&transaction)
-            .unwrap();
-
-        let account_len = Account::get_packed_len();
-        let minimum_balance = rpc_client
-            .get_minimum_balance_for_rent_exemption(account_len)
-            .unwrap();
-
-        let iterations = 5;
-        let batch_size = 100;
-        let close_nth_batch = 0;
-        let num_instructions = 4;
-        let mut start = Measure::start("total accounts run");
-        let keypair0 = Keypair::new();
-        let keypair1 = Keypair::new();
-        let keypair2 = Keypair::new();
-        run_accounts_bench(
-            test_validator
-                .rpc_url()
-                .replace("http://", "")
-                .parse()
-                .unwrap(),
-            faucet_addr,
-            &[&keypair0, &keypair1, &keypair2],
-            iterations,
-            Some(account_len as u64),
-            batch_size,
-            close_nth_batch,
-            Some(minimum_balance),
-            num_instructions,
-            Some(spl_mint_keypair.pubkey()),
-            true,
         );
         start.stop();
         info!("{}", start);

accountsdb-plugin-interface/Cargo.toml

@@ -1,19 +0,0 @@
-[package]
-authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2021"
-name = "solana-accountsdb-plugin-interface"
-description = "The Solana AccountsDb plugin interface."
-version = "1.10.0"
-repository = "https://github.com/solana-labs/solana"
-license = "Apache-2.0"
-homepage = "https://solana.com/"
-documentation = "https://docs.rs/solana-accountsdb-plugin-interface"
-
-[dependencies]
-log = "0.4.11"
-thiserror = "1.0.30"
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
-
-[package.metadata.docs.rs]
-targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-interface/README.md

@@ -1,20 +0,0 @@
-<p align="center">
-  <a href="https://solana.com">
-    <img alt="Solana" src="https://i.imgur.com/IKyzQ6T.png" width="250" />
-  </a>
-</p>
-
-# Solana AccountsDb Plugin Interface
-
-This crate enables an AccountsDb plugin to be plugged into the Solana Validator runtime to take actions
-at the time of each account update; for example, saving the account state to an external database. The plugin must implement the `AccountsDbPlugin` trait. Please see the detail of the `accountsdb_plugin_interface.rs` for the interface definition.
-
-The plugin should produce a `cdylib` dynamic library, which must expose a `C` function `_create_plugin()` that
-instantiates the implementation of the interface.
-
-The `solana-accountsdb-plugin-postgres` crate provides an example of how to create a plugin which saves the accounts data into an
-external PostgreSQL databases.
-
-More information about Solana is available in the [Solana documentation](https://docs.solana.com/).
-
-Still have questions?  Ask us on [Discord](https://discordapp.com/invite/pquxPsq)

accountsdb-plugin-interface/src/accountsdb_plugin_interface.rs

@@ -1,189 +0,0 @@
-/// The interface for AccountsDb plugins. A plugin must implement
-/// the AccountsDbPlugin trait to work with the runtime.
-/// In addition, the dynamic library must export a "C" function _create_plugin which
-/// creates the implementation of the plugin.
-use {
-    solana_sdk::{signature::Signature, transaction::SanitizedTransaction},
-    solana_transaction_status::TransactionStatusMeta,
-    std::{any::Any, error, io},
-    thiserror::Error,
-};
-
-impl Eq for ReplicaAccountInfo<'_> {}
-
-#[derive(Clone, PartialEq, Debug)]
-/// Information about an account being updated
-pub struct ReplicaAccountInfo<'a> {
-    /// The Pubkey for the account
-    pub pubkey: &'a [u8],
-
-    /// The lamports for the account
-    pub lamports: u64,
-
-    /// The Pubkey of the owner program account
-    pub owner: &'a [u8],
-
-    /// This account's data contains a loaded program (and is now read-only)
-    pub executable: bool,
-
-    /// The epoch at which this account will next owe rent
-    pub rent_epoch: u64,
-
-    /// The data held in this account.
-    pub data: &'a [u8],
-
-    /// A global monotonically increasing atomic number, which can be used
-    /// to tell the order of the account update. For example, when an
-    /// account is updated in the same slot multiple times, the update
-    /// with higher write_version should supersede the one with lower
-    /// write_version.
-    pub write_version: u64,
-}
-
-/// A wrapper to future-proof ReplicaAccountInfo handling.
-/// If there were a change to the structure of ReplicaAccountInfo,
-/// there would be new enum entry for the newer version, forcing
-/// plugin implementations to handle the change.
-pub enum ReplicaAccountInfoVersions<'a> {
-    V0_0_1(&'a ReplicaAccountInfo<'a>),
-}
-
-#[derive(Clone, Debug)]
-pub struct ReplicaTransactionInfo<'a> {
-    pub signature: &'a Signature,
-    pub is_vote: bool,
-    pub transaction: &'a SanitizedTransaction,
-    pub transaction_status_meta: &'a TransactionStatusMeta,
-}
-
-pub enum ReplicaTransactionInfoVersions<'a> {
-    V0_0_1(&'a ReplicaTransactionInfo<'a>),
-}
-
-/// Errors returned by plugin calls
-#[derive(Error, Debug)]
-pub enum AccountsDbPluginError {
-    /// Error opening the configuration file; for example, when the file
-    /// is not found or when the validator process has no permission to read it.
-    #[error("Error opening config file. Error detail: ({0}).")]
-    ConfigFileOpenError(#[from] io::Error),
-
-    /// Error in reading the content of the config file or the content
-    /// is not in the expected format.
-    #[error("Error reading config file. Error message: ({msg})")]
-    ConfigFileReadError { msg: String },
-
-    /// Error when updating the account.
-    #[error("Error updating account. Error message: ({msg})")]
-    AccountsUpdateError { msg: String },
-
-    /// Error when updating the slot status
-    #[error("Error updating slot status. Error message: ({msg})")]
-    SlotStatusUpdateError { msg: String },
-
-    /// Any custom error defined by the plugin.
-    #[error("Plugin-defined custom error. Error message: ({0})")]
-    Custom(Box<dyn error::Error + Send + Sync>),
-}
-
-/// The current status of a slot
-#[derive(Debug, Clone)]
-pub enum SlotStatus {
-    /// The highest slot of the heaviest fork processed by the node. Ledger state at this slot is
-    /// not derived from a confirmed or finalized block, but if multiple forks are present, is from
-    /// the fork the validator believes is most likely to finalize.
-    Processed,
-
-    /// The highest slot having reached max vote lockout.
-    Rooted,
-
-    /// The highest slot that has been voted on by supermajority of the cluster, ie. is confirmed.
-    Confirmed,
-}
-
-impl SlotStatus {
-    pub fn as_str(&self) -> &'static str {
-        match self {
-            SlotStatus::Confirmed => "confirmed",
-            SlotStatus::Processed => "processed",
-            SlotStatus::Rooted => "rooted",
-        }
-    }
-}
-
-pub type Result<T> = std::result::Result<T, AccountsDbPluginError>;
-
-/// Defines an AccountsDb plugin, to stream data from the runtime.
-/// AccountsDb plugins must describe desired behavior for load and unload,
-/// as well as how they will handle streamed data.
-pub trait AccountsDbPlugin: Any + Send + Sync + std::fmt::Debug {
-    fn name(&self) -> &'static str;
-
-    /// The callback called when a plugin is loaded by the system,
-    /// used for doing whatever initialization is required by the plugin.
-    /// The _config_file contains the name of the
-    /// of the config file. The config must be in JSON format and
-    /// include a field "libpath" indicating the full path
-    /// name of the shared library implementing this interface.
-    fn on_load(&mut self, _config_file: &str) -> Result<()> {
-        Ok(())
-    }
-
-    /// The callback called right before a plugin is unloaded by the system
-    /// Used for doing cleanup before unload.
-    fn on_unload(&mut self) {}
-
-    /// Called when an account is updated at a slot.
-    /// When `is_startup` is true, it indicates the account is loaded from
-    /// snapshots when the validator starts up. When `is_startup` is false,
-    /// the account is updated during transaction processing.
-    #[allow(unused_variables)]
-    fn update_account(
-        &mut self,
-        account: ReplicaAccountInfoVersions,
-        slot: u64,
-        is_startup: bool,
-    ) -> Result<()> {
-        Ok(())
-    }
-
-    /// Called when all accounts are notified of during startup.
-    fn notify_end_of_startup(&mut self) -> Result<()> {
-        Ok(())
-    }
-
-    /// Called when a slot status is updated
-    #[allow(unused_variables)]
-    fn update_slot_status(
-        &mut self,
-        slot: u64,
-        parent: Option<u64>,
-        status: SlotStatus,
-    ) -> Result<()> {
-        Ok(())
-    }
-
-    /// Called when a transaction is updated at a slot.
-    #[allow(unused_variables)]
-    fn notify_transaction(
-        &mut self,
-        transaction: ReplicaTransactionInfoVersions,
-        slot: u64,
-    ) -> Result<()> {
-        Ok(())
-    }
-
-    /// Check if the plugin is interested in account data
-    /// Default is true -- if the plugin is not interested in
-    /// account data, please return false.
-    fn account_data_notifications_enabled(&self) -> bool {
-        true
-    }
-
-    /// Check if the plugin is interested in transaction data
-    /// Default is false -- if the plugin is not interested in
-    /// transaction data, please return false.
-    fn transaction_notifications_enabled(&self) -> bool {
-        false
-    }
-}

accountsdb-plugin-interface/src/lib.rs

@@ -1 +0,0 @@
-pub mod accountsdb_plugin_interface;

accountsdb-plugin-manager/Cargo.toml

@@ -1,31 +0,0 @@
-[package]
-authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2021"
-name = "solana-accountsdb-plugin-manager"
-description = "The Solana AccountsDb plugin manager."
-version = "1.10.0"
-repository = "https://github.com/solana-labs/solana"
-license = "Apache-2.0"
-homepage = "https://solana.com/"
-documentation = "https://docs.rs/solana-validator"
-
-[dependencies]
-bs58 = "0.4.0"
-crossbeam-channel = "0.5"
-libloading = "0.7.2"
-log = "0.4.11"
-serde = "1.0.131"
-serde_derive = "1.0.103"
-serde_json = "1.0.72"
-solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.10.0" }
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-metrics = { path = "../metrics", version = "=1.10.0" }
-solana-rpc = { path = "../rpc", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
-thiserror = "1.0.30"
-
-[package.metadata.docs.rs]
-targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-manager/src/accounts_update_notifier.rs

@@ -1,180 +0,0 @@
-/// Module responsible for notifying plugins of account updates
-use {
-    crate::accountsdb_plugin_manager::AccountsDbPluginManager,
-    log::*,
-    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
-        ReplicaAccountInfo, ReplicaAccountInfoVersions,
-    },
-    solana_measure::measure::Measure,
-    solana_metrics::*,
-    solana_runtime::{
-        accounts_update_notifier_interface::AccountsUpdateNotifierInterface,
-        append_vec::{StoredAccountMeta, StoredMeta},
-    },
-    solana_sdk::{
-        account::{AccountSharedData, ReadableAccount},
-        clock::Slot,
-    },
-    std::sync::{Arc, RwLock},
-};
-#[derive(Debug)]
-pub(crate) struct AccountsUpdateNotifierImpl {
-    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
-}
-
-impl AccountsUpdateNotifierInterface for AccountsUpdateNotifierImpl {
-    fn notify_account_update(&self, slot: Slot, meta: &StoredMeta, account: &AccountSharedData) {
-        if let Some(account_info) = self.accountinfo_from_shared_account_data(meta, account) {
-            self.notify_plugins_of_account_update(account_info, slot, false);
-        }
-    }
-
-    fn notify_account_restore_from_snapshot(&self, slot: Slot, account: &StoredAccountMeta) {
-        let mut measure_all = Measure::start("accountsdb-plugin-notify-account-restore-all");
-        let mut measure_copy = Measure::start("accountsdb-plugin-copy-stored-account-info");
-
-        let account = self.accountinfo_from_stored_account_meta(account);
-        measure_copy.stop();
-
-        inc_new_counter_debug!(
-            "accountsdb-plugin-copy-stored-account-info-us",
-            measure_copy.as_us() as usize,
-            100000,
-            100000
-        );
-
-        if let Some(account_info) = account {
-            self.notify_plugins_of_account_update(account_info, slot, true);
-        }
-        measure_all.stop();
-
-        inc_new_counter_debug!(
-            "accountsdb-plugin-notify-account-restore-all-us",
-            measure_all.as_us() as usize,
-            100000,
-            100000
-        );
-    }
-
-    fn notify_end_of_restore_from_snapshot(&self) {
-        let mut plugin_manager = self.plugin_manager.write().unwrap();
-        if plugin_manager.plugins.is_empty() {
-            return;
-        }
-
-        for plugin in plugin_manager.plugins.iter_mut() {
-            let mut measure = Measure::start("accountsdb-plugin-end-of-restore-from-snapshot");
-            match plugin.notify_end_of_startup() {
-                Err(err) => {
-                    error!(
-                        "Failed to notify the end of restore from snapshot, error: {} to plugin {}",
-                        err,
-                        plugin.name()
-                    )
-                }
-                Ok(_) => {
-                    trace!(
-                        "Successfully notified the end of restore from snapshot to plugin {}",
-                        plugin.name()
-                    );
-                }
-            }
-            measure.stop();
-            inc_new_counter_debug!(
-                "accountsdb-plugin-end-of-restore-from-snapshot",
-                measure.as_us() as usize
-            );
-        }
-    }
-}
-
-impl AccountsUpdateNotifierImpl {
-    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
-        AccountsUpdateNotifierImpl { plugin_manager }
-    }
-
-    fn accountinfo_from_shared_account_data<'a>(
-        &self,
-        meta: &'a StoredMeta,
-        account: &'a AccountSharedData,
-    ) -> Option<ReplicaAccountInfo<'a>> {
-        Some(ReplicaAccountInfo {
-            pubkey: meta.pubkey.as_ref(),
-            lamports: account.lamports(),
-            owner: account.owner().as_ref(),
-            executable: account.executable(),
-            rent_epoch: account.rent_epoch(),
-            data: account.data(),
-            write_version: meta.write_version,
-        })
-    }
-
-    fn accountinfo_from_stored_account_meta<'a>(
-        &self,
-        stored_account_meta: &'a StoredAccountMeta,
-    ) -> Option<ReplicaAccountInfo<'a>> {
-        Some(ReplicaAccountInfo {
-            pubkey: stored_account_meta.meta.pubkey.as_ref(),
-            lamports: stored_account_meta.account_meta.lamports,
-            owner: stored_account_meta.account_meta.owner.as_ref(),
-            executable: stored_account_meta.account_meta.executable,
-            rent_epoch: stored_account_meta.account_meta.rent_epoch,
-            data: stored_account_meta.data,
-            write_version: stored_account_meta.meta.write_version,
-        })
-    }
-
-    fn notify_plugins_of_account_update(
-        &self,
-        account: ReplicaAccountInfo,
-        slot: Slot,
-        is_startup: bool,
-    ) {
-        let mut measure2 = Measure::start("accountsdb-plugin-notify_plugins_of_account_update");
-        let mut plugin_manager = self.plugin_manager.write().unwrap();
-
-        if plugin_manager.plugins.is_empty() {
-            return;
-        }
-        for plugin in plugin_manager.plugins.iter_mut() {
-            let mut measure = Measure::start("accountsdb-plugin-update-account");
-            match plugin.update_account(
-                ReplicaAccountInfoVersions::V0_0_1(&account),
-                slot,
-                is_startup,
-            ) {
-                Err(err) => {
-                    error!(
-                        "Failed to update account {} at slot {}, error: {} to plugin {}",
-                        bs58::encode(account.pubkey).into_string(),
-                        slot,
-                        err,
-                        plugin.name()
-                    )
-                }
-                Ok(_) => {
-                    trace!(
-                        "Successfully updated account {} at slot {} to plugin {}",
-                        bs58::encode(account.pubkey).into_string(),
-                        slot,
-                        plugin.name()
-                    );
-                }
-            }
-            measure.stop();
-            inc_new_counter_debug!(
-                "accountsdb-plugin-update-account-us",
-                measure.as_us() as usize,
-                100000,
-                100000
-            );
-        }
-        measure2.stop();
-        inc_new_counter_debug!(
-            "accountsdb-plugin-notify_plugins_of_account_update-us",
-            measure2.as_us() as usize,
-            100000,
-            100000
-        );
-    }
-}

accountsdb-plugin-manager/src/accountsdb_plugin_manager.rs

@@ -1,75 +0,0 @@
-/// Managing the AccountsDb plugins
-use {
-    libloading::{Library, Symbol},
-    log::*,
-    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::AccountsDbPlugin,
-    std::error::Error,
-};
-
-#[derive(Default, Debug)]
-pub struct AccountsDbPluginManager {
-    pub plugins: Vec<Box<dyn AccountsDbPlugin>>,
-    libs: Vec<Library>,
-}
-
-impl AccountsDbPluginManager {
-    pub fn new() -> Self {
-        AccountsDbPluginManager {
-            plugins: Vec::default(),
-            libs: Vec::default(),
-        }
-    }
-
-    /// # Safety
-    ///
-    /// This function loads the dynamically linked library specified in the path. The library
-    /// must do necessary initializations.
-    pub unsafe fn load_plugin(
-        &mut self,
-        libpath: &str,
-        config_file: &str,
-    ) -> Result<(), Box<dyn Error>> {
-        type PluginConstructor = unsafe fn() -> *mut dyn AccountsDbPlugin;
-        let lib = Library::new(libpath)?;
-        let constructor: Symbol<PluginConstructor> = lib.get(b"_create_plugin")?;
-        let plugin_raw = constructor();
-        let mut plugin = Box::from_raw(plugin_raw);
-        plugin.on_load(config_file)?;
-        self.plugins.push(plugin);
-        self.libs.push(lib);
-        Ok(())
-    }
-
-    /// Unload all plugins and loaded plugin libraries, making sure to fire
-    /// their `on_plugin_unload()` methods so they can do any necessary cleanup.
-    pub fn unload(&mut self) {
-        for mut plugin in self.plugins.drain(..) {
-            info!("Unloading plugin for {:?}", plugin.name());
-            plugin.on_unload();
-        }
-
-        for lib in self.libs.drain(..) {
-            drop(lib);
-        }
-    }
-
-    /// Check if there is any plugin interested in account data
-    pub fn account_data_notifications_enabled(&self) -> bool {
-        for plugin in &self.plugins {
-            if plugin.account_data_notifications_enabled() {
-                return true;
-            }
-        }
-        false
-    }
-
-    /// Check if there is any plugin interested in transaction data
-    pub fn transaction_notifications_enabled(&self) -> bool {
-        for plugin in &self.plugins {
-            if plugin.transaction_notifications_enabled() {
-                return true;
-            }
-        }
-        false
-    }
-}

accountsdb-plugin-manager/src/accountsdb_plugin_service.rs

@@ -1,196 +0,0 @@
-use {
-    crate::{
-        accounts_update_notifier::AccountsUpdateNotifierImpl,
-        accountsdb_plugin_manager::AccountsDbPluginManager,
-        slot_status_notifier::SlotStatusNotifierImpl, slot_status_observer::SlotStatusObserver,
-        transaction_notifier::TransactionNotifierImpl,
-    },
-    crossbeam_channel::Receiver,
-    log::*,
-    serde_json,
-    solana_rpc::{
-        optimistically_confirmed_bank_tracker::BankNotification,
-        transaction_notifier_interface::TransactionNotifierLock,
-    },
-    solana_runtime::accounts_update_notifier_interface::AccountsUpdateNotifier,
-    std::{
-        fs::File,
-        io::Read,
-        path::{Path, PathBuf},
-        sync::{Arc, RwLock},
-        thread,
-    },
-    thiserror::Error,
-};
-
-#[derive(Error, Debug)]
-pub enum AccountsdbPluginServiceError {
-    #[error("Cannot open the the plugin config file")]
-    CannotOpenConfigFile(String),
-
-    #[error("Cannot read the the plugin config file")]
-    CannotReadConfigFile(String),
-
-    #[error("The config file is not in a valid Json format")]
-    InvalidConfigFileFormat(String),
-
-    #[error("Plugin library path is not specified in the config file")]
-    LibPathNotSet,
-
-    #[error("Invalid plugin path")]
-    InvalidPluginPath,
-
-    #[error("Cannot load plugin shared library")]
-    PluginLoadError(String),
-}
-
-/// The service managing the AccountsDb plugin workflow.
-pub struct AccountsDbPluginService {
-    slot_status_observer: Option<SlotStatusObserver>,
-    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
-    accounts_update_notifier: Option<AccountsUpdateNotifier>,
-    transaction_notifier: Option<TransactionNotifierLock>,
-}
-
-impl AccountsDbPluginService {
-    /// Creates and returns the AccountsDbPluginService.
-    /// # Arguments
-    /// * `confirmed_bank_receiver` - The receiver for confirmed bank notification
-    /// * `accountsdb_plugin_config_file` - The config file path for the plugin. The
-    ///    config file controls the plugin responsible
-    ///    for transporting the data to external data stores. It is defined in JSON format.
-    ///    The `libpath` field should be pointed to the full path of the dynamic shared library
-    ///    (.so file) to be loaded. The shared library must implement the `AccountsDbPlugin`
-    ///    trait. And the shared library shall export a `C` function `_create_plugin` which
-    ///    shall create the implementation of `AccountsDbPlugin` and returns to the caller.
-    ///    The rest of the JSON fields' definition is up to to the concrete plugin implementation
-    ///    It is usually used to configure the connection information for the external data store.
-
-    pub fn new(
-        confirmed_bank_receiver: Receiver<BankNotification>,
-        accountsdb_plugin_config_files: &[PathBuf],
-    ) -> Result<Self, AccountsdbPluginServiceError> {
-        info!(
-            "Starting AccountsDbPluginService from config files: {:?}",
-            accountsdb_plugin_config_files
-        );
-        let mut plugin_manager = AccountsDbPluginManager::new();
-
-        for accountsdb_plugin_config_file in accountsdb_plugin_config_files {
-            Self::load_plugin(&mut plugin_manager, accountsdb_plugin_config_file)?;
-        }
-        let account_data_notifications_enabled =
-            plugin_manager.account_data_notifications_enabled();
-        let transaction_notifications_enabled = plugin_manager.transaction_notifications_enabled();
-
-        let plugin_manager = Arc::new(RwLock::new(plugin_manager));
-
-        let accounts_update_notifier: Option<AccountsUpdateNotifier> =
-            if account_data_notifications_enabled {
-                let accounts_update_notifier =
-                    AccountsUpdateNotifierImpl::new(plugin_manager.clone());
-                Some(Arc::new(RwLock::new(accounts_update_notifier)))
-            } else {
-                None
-            };
-
-        let transaction_notifier: Option<TransactionNotifierLock> =
-            if transaction_notifications_enabled {
-                let transaction_notifier = TransactionNotifierImpl::new(plugin_manager.clone());
-                Some(Arc::new(RwLock::new(transaction_notifier)))
-            } else {
-                None
-            };
-
-        let slot_status_observer =
-            if account_data_notifications_enabled || transaction_notifications_enabled {
-                let slot_status_notifier = SlotStatusNotifierImpl::new(plugin_manager.clone());
-                let slot_status_notifier = Arc::new(RwLock::new(slot_status_notifier));
-                Some(SlotStatusObserver::new(
-                    confirmed_bank_receiver,
-                    slot_status_notifier,
-                ))
-            } else {
-                None
-            };
-
-        info!("Started AccountsDbPluginService");
-        Ok(AccountsDbPluginService {
-            slot_status_observer,
-            plugin_manager,
-            accounts_update_notifier,
-            transaction_notifier,
-        })
-    }
-
-    fn load_plugin(
-        plugin_manager: &mut AccountsDbPluginManager,
-        accountsdb_plugin_config_file: &Path,
-    ) -> Result<(), AccountsdbPluginServiceError> {
-        let mut file = match File::open(accountsdb_plugin_config_file) {
-            Ok(file) => file,
-            Err(err) => {
-                return Err(AccountsdbPluginServiceError::CannotOpenConfigFile(format!(
-                    "Failed to open the plugin config file {:?}, error: {:?}",
-                    accountsdb_plugin_config_file, err
-                )));
-            }
-        };
-
-        let mut contents = String::new();
-        if let Err(err) = file.read_to_string(&mut contents) {
-            return Err(AccountsdbPluginServiceError::CannotReadConfigFile(format!(
-                "Failed to read the plugin config file {:?}, error: {:?}",
-                accountsdb_plugin_config_file, err
-            )));
-        }
-
-        let result: serde_json::Value = match serde_json::from_str(&contents) {
-            Ok(value) => value,
-            Err(err) => {
-                return Err(AccountsdbPluginServiceError::InvalidConfigFileFormat(
-                    format!(
-                        "The config file {:?} is not in a valid Json format, error: {:?}",
-                        accountsdb_plugin_config_file, err
-                    ),
-                ));
-            }
-        };
-
-        let libpath = result["libpath"]
-            .as_str()
-            .ok_or(AccountsdbPluginServiceError::LibPathNotSet)?;
-        let config_file = accountsdb_plugin_config_file
-            .as_os_str()
-            .to_str()
-            .ok_or(AccountsdbPluginServiceError::InvalidPluginPath)?;
-
-        unsafe {
-            let result = plugin_manager.load_plugin(libpath, config_file);
-            if let Err(err) = result {
-                let msg = format!(
-                    "Failed to load the plugin library: {:?}, error: {:?}",
-                    libpath, err
-                );
-                return Err(AccountsdbPluginServiceError::PluginLoadError(msg));
-            }
-        }
-        Ok(())
-    }
-
-    pub fn get_accounts_update_notifier(&self) -> Option<AccountsUpdateNotifier> {
-        self.accounts_update_notifier.clone()
-    }
-
-    pub fn get_transaction_notifier(&self) -> Option<TransactionNotifierLock> {
-        self.transaction_notifier.clone()
-    }
-
-    pub fn join(self) -> thread::Result<()> {
-        if let Some(mut slot_status_observer) = self.slot_status_observer {
-            slot_status_observer.join()?;
-        }
-        self.plugin_manager.write().unwrap().unload();
-        Ok(())
-    }
-}

accountsdb-plugin-manager/src/lib.rs

@@ -1,6 +0,0 @@
-pub mod accounts_update_notifier;
-pub mod accountsdb_plugin_manager;
-pub mod accountsdb_plugin_service;
-pub mod slot_status_notifier;
-pub mod slot_status_observer;
-pub mod transaction_notifier;

accountsdb-plugin-manager/src/slot_status_notifier.rs

@@ -1,81 +0,0 @@
-use {
-    crate::accountsdb_plugin_manager::AccountsDbPluginManager,
-    log::*,
-    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::SlotStatus,
-    solana_measure::measure::Measure,
-    solana_metrics::*,
-    solana_sdk::clock::Slot,
-    std::sync::{Arc, RwLock},
-};
-
-pub trait SlotStatusNotifierInterface {
-    /// Notified when a slot is optimistically confirmed
-    fn notify_slot_confirmed(&self, slot: Slot, parent: Option<Slot>);
-
-    /// Notified when a slot is marked frozen.
-    fn notify_slot_processed(&self, slot: Slot, parent: Option<Slot>);
-
-    /// Notified when a slot is rooted.
-    fn notify_slot_rooted(&self, slot: Slot, parent: Option<Slot>);
-}
-
-pub type SlotStatusNotifier = Arc<RwLock<dyn SlotStatusNotifierInterface + Sync + Send>>;
-
-pub struct SlotStatusNotifierImpl {
-    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
-}
-
-impl SlotStatusNotifierInterface for SlotStatusNotifierImpl {
-    fn notify_slot_confirmed(&self, slot: Slot, parent: Option<Slot>) {
-        self.notify_slot_status(slot, parent, SlotStatus::Confirmed);
-    }
-
-    fn notify_slot_processed(&self, slot: Slot, parent: Option<Slot>) {
-        self.notify_slot_status(slot, parent, SlotStatus::Processed);
-    }
-
-    fn notify_slot_rooted(&self, slot: Slot, parent: Option<Slot>) {
-        self.notify_slot_status(slot, parent, SlotStatus::Rooted);
-    }
-}
-
-impl SlotStatusNotifierImpl {
-    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
-        Self { plugin_manager }
-    }
-
-    pub fn notify_slot_status(&self, slot: Slot, parent: Option<Slot>, slot_status: SlotStatus) {
-        let mut plugin_manager = self.plugin_manager.write().unwrap();
-        if plugin_manager.plugins.is_empty() {
-            return;
-        }
-
-        for plugin in plugin_manager.plugins.iter_mut() {
-            let mut measure = Measure::start("accountsdb-plugin-update-slot");
-            match plugin.update_slot_status(slot, parent, slot_status.clone()) {
-                Err(err) => {
-                    error!(
-                        "Failed to update slot status at slot {}, error: {} to plugin {}",
-                        slot,
-                        err,
-                        plugin.name()
-                    )
-                }
-                Ok(_) => {
-                    trace!(
-                        "Successfully updated slot status at slot {} to plugin {}",
-                        slot,
-                        plugin.name()
-                    );
-                }
-            }
-            measure.stop();
-            inc_new_counter_debug!(
-                "accountsdb-plugin-update-slot-us",
-                measure.as_us() as usize,
-                1000,
-                1000
-            );
-        }
-    }
-}

accountsdb-plugin-manager/src/slot_status_observer.rs

@@ -1,80 +0,0 @@
-use {
-    crate::slot_status_notifier::SlotStatusNotifier,
-    crossbeam_channel::Receiver,
-    solana_rpc::optimistically_confirmed_bank_tracker::BankNotification,
-    std::{
-        sync::{
-            atomic::{AtomicBool, Ordering},
-            Arc,
-        },
-        thread::{self, Builder, JoinHandle},
-    },
-};
-
-#[derive(Debug)]
-pub(crate) struct SlotStatusObserver {
-    bank_notification_receiver_service: Option<JoinHandle<()>>,
-    exit_updated_slot_server: Arc<AtomicBool>,
-}
-
-impl SlotStatusObserver {
-    pub fn new(
-        bank_notification_receiver: Receiver<BankNotification>,
-        slot_status_notifier: SlotStatusNotifier,
-    ) -> Self {
-        let exit_updated_slot_server = Arc::new(AtomicBool::new(false));
-
-        Self {
-            bank_notification_receiver_service: Some(Self::run_bank_notification_receiver(
-                bank_notification_receiver,
-                exit_updated_slot_server.clone(),
-                slot_status_notifier,
-            )),
-            exit_updated_slot_server,
-        }
-    }
-
-    pub fn join(&mut self) -> thread::Result<()> {
-        self.exit_updated_slot_server.store(true, Ordering::Relaxed);
-        self.bank_notification_receiver_service
-            .take()
-            .map(JoinHandle::join)
-            .unwrap()
-    }
-
-    fn run_bank_notification_receiver(
-        bank_notification_receiver: Receiver<BankNotification>,
-        exit: Arc<AtomicBool>,
-        slot_status_notifier: SlotStatusNotifier,
-    ) -> JoinHandle<()> {
-        Builder::new()
-            .name("bank_notification_receiver".to_string())
-            .spawn(move || {
-                while !exit.load(Ordering::Relaxed) {
-                    if let Ok(slot) = bank_notification_receiver.recv() {
-                        match slot {
-                            BankNotification::OptimisticallyConfirmed(slot) => {
-                                slot_status_notifier
-                                    .read()
-                                    .unwrap()
-                                    .notify_slot_confirmed(slot, None);
-                            }
-                            BankNotification::Frozen(bank) => {
-                                slot_status_notifier
-                                    .read()
-                                    .unwrap()
-                                    .notify_slot_processed(bank.slot(), Some(bank.parent_slot()));
-                            }
-                            BankNotification::Root(bank) => {
-                                slot_status_notifier
-                                    .read()
-                                    .unwrap()
-                                    .notify_slot_rooted(bank.slot(), Some(bank.parent_slot()));
-                            }
-                        }
-                    }
-                }
-            })
-            .unwrap()
-    }
-}

accountsdb-plugin-manager/src/transaction_notifier.rs

@@ -1,93 +0,0 @@
-/// Module responsible for notifying plugins of transactions
-use {
-    crate::accountsdb_plugin_manager::AccountsDbPluginManager,
-    log::*,
-    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
-        ReplicaTransactionInfo, ReplicaTransactionInfoVersions,
-    },
-    solana_measure::measure::Measure,
-    solana_metrics::*,
-    solana_rpc::transaction_notifier_interface::TransactionNotifier,
-    solana_runtime::bank,
-    solana_sdk::{clock::Slot, signature::Signature, transaction::SanitizedTransaction},
-    solana_transaction_status::TransactionStatusMeta,
-    std::sync::{Arc, RwLock},
-};
-
-/// This implementation of TransactionNotifier is passed to the rpc's TransactionStatusService
-/// at the validator startup. TransactionStatusService invokes the notify_transaction method
-/// for new transactions. The implementation in turn invokes the notify_transaction of each
-/// plugin enabled with transaction notification managed by the AccountsDbPluginManager.
-pub(crate) struct TransactionNotifierImpl {
-    plugin_manager: Arc<RwLock<AccountsDbPluginManager>>,
-}
-
-impl TransactionNotifier for TransactionNotifierImpl {
-    fn notify_transaction(
-        &self,
-        slot: Slot,
-        signature: &Signature,
-        transaction_status_meta: &TransactionStatusMeta,
-        transaction: &SanitizedTransaction,
-    ) {
-        let mut measure = Measure::start("accountsdb-plugin-notify_plugins_of_transaction_info");
-        let transaction_log_info =
-            Self::build_replica_transaction_info(signature, transaction_status_meta, transaction);
-
-        let mut plugin_manager = self.plugin_manager.write().unwrap();
-
-        if plugin_manager.plugins.is_empty() {
-            return;
-        }
-
-        for plugin in plugin_manager.plugins.iter_mut() {
-            if !plugin.transaction_notifications_enabled() {
-                continue;
-            }
-            match plugin.notify_transaction(
-                ReplicaTransactionInfoVersions::V0_0_1(&transaction_log_info),
-                slot,
-            ) {
-                Err(err) => {
-                    error!(
-                        "Failed to notify transaction, error: ({}) to plugin {}",
-                        err,
-                        plugin.name()
-                    )
-                }
-                Ok(_) => {
-                    trace!(
-                        "Successfully notified transaction to plugin {}",
-                        plugin.name()
-                    );
-                }
-            }
-        }
-        measure.stop();
-        inc_new_counter_debug!(
-            "accountsdb-plugin-notify_plugins_of_transaction_info-us",
-            measure.as_us() as usize,
-            10000,
-            10000
-        );
-    }
-}
-
-impl TransactionNotifierImpl {
-    pub fn new(plugin_manager: Arc<RwLock<AccountsDbPluginManager>>) -> Self {
-        Self { plugin_manager }
-    }
-
-    fn build_replica_transaction_info<'a>(
-        signature: &'a Signature,
-        transaction_status_meta: &'a TransactionStatusMeta,
-        transaction: &'a SanitizedTransaction,
-    ) -> ReplicaTransactionInfo<'a> {
-        ReplicaTransactionInfo {
-            signature,
-            is_vote: bank::is_simple_vote_transaction(transaction),
-            transaction,
-            transaction_status_meta,
-        }
-    }
-}

accountsdb-plugin-postgres/Cargo.toml

@@ -1,39 +0,0 @@
-[package]
-authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2021"
-name = "solana-accountsdb-plugin-postgres"
-description = "The Solana AccountsDb plugin for PostgreSQL database."
-version = "1.10.0"
-repository = "https://github.com/solana-labs/solana"
-license = "Apache-2.0"
-homepage = "https://solana.com/"
-documentation = "https://docs.rs/solana-validator"
-
-[lib]
-crate-type = ["cdylib", "rlib"]
-
-[dependencies]
-bs58 = "0.4.0"
-chrono = { version = "0.4.11", features = ["serde"] }
-crossbeam-channel = "0.5"
-log = "0.4.14"
-postgres = { version = "0.19.2", features = ["with-chrono-0_4"] }
-postgres-types = { version = "0.2.2", features = ["derive"] }
-serde = "1.0.131"
-serde_derive = "1.0.103"
-serde_json = "1.0.72"
-solana-accountsdb-plugin-interface = { path = "../accountsdb-plugin-interface", version = "=1.10.0" }
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-metrics = { path = "../metrics", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-transaction-status = { path = "../transaction-status", version = "=1.10.0" }
-thiserror = "1.0.30"
-tokio-postgres = "0.7.4"
-
-[dev-dependencies]
-solana-account-decoder = { path = "../account-decoder", version = "=1.10.0" }
-
-[package.metadata.docs.rs]
-targets = ["x86_64-unknown-linux-gnu"]

accountsdb-plugin-postgres/README.md

@@ -1,5 +0,0 @@
-This is an example implementing the AccountsDb plugin for PostgreSQL database.
-Please see the `src/accountsdb_plugin_postgres.rs` for the format of the plugin's configuration file.
-
-To create the schema objects for the database, please use `scripts/create_schema.sql`.
-`scripts/drop_schema.sql` can be used to tear down the schema objects.

accountsdb-plugin-postgres/scripts/create_schema.sql

@@ -1,184 +0,0 @@
-/**
- * This plugin implementation for PostgreSQL requires the following tables
- */
--- The table storing accounts
-
-
-CREATE TABLE account (
-    pubkey BYTEA PRIMARY KEY,
-    owner BYTEA,
-    lamports BIGINT NOT NULL,
-    slot BIGINT NOT NULL,
-    executable BOOL NOT NULL,
-    rent_epoch BIGINT NOT NULL,
-    data BYTEA,
-    write_version BIGINT NOT NULL,
-    updated_on TIMESTAMP NOT NULL
-);
-
--- The table storing slot information
-CREATE TABLE slot (
-    slot BIGINT PRIMARY KEY,
-    parent BIGINT,
-    status VARCHAR(16) NOT NULL,
-    updated_on TIMESTAMP NOT NULL
-);
-
--- Types for Transactions
-
-Create TYPE "TransactionErrorCode" AS ENUM (
-    'AccountInUse',
-    'AccountLoadedTwice',
-    'AccountNotFound',
-    'ProgramAccountNotFound',
-    'InsufficientFundsForFee',
-    'InvalidAccountForFee',
-    'AlreadyProcessed',
-    'BlockhashNotFound',
-    'InstructionError',
-    'CallChainTooDeep',
-    'MissingSignatureForFee',
-    'InvalidAccountIndex',
-    'SignatureFailure',
-    'InvalidProgramForExecution',
-    'SanitizeFailure',
-    'ClusterMaintenance',
-    'AccountBorrowOutstanding',
-    'WouldExceedMaxAccountCostLimit',
-    'WouldExceedMaxBlockCostLimit',
-    'UnsupportedVersion',
-    'InvalidWritableAccount'
-);
-
-CREATE TYPE "TransactionError" AS (
-    error_code "TransactionErrorCode",
-    error_detail VARCHAR(256)
-);
-
-CREATE TYPE "CompiledInstruction" AS (
-    program_id_index SMALLINT,
-    accounts SMALLINT[],
-    data BYTEA
-);
-
-CREATE TYPE "InnerInstructions" AS (
-    index SMALLINT,
-    instructions "CompiledInstruction"[]
-);
-
-CREATE TYPE "TransactionTokenBalance" AS (
-    account_index SMALLINT,
-    mint VARCHAR(44),
-    ui_token_amount DOUBLE PRECISION,
-    owner VARCHAR(44)
-);
-
-Create TYPE "RewardType" AS ENUM (
-    'Fee',
-    'Rent',
-    'Staking',
-    'Voting'
-);
-
-CREATE TYPE "Reward" AS (
-    pubkey VARCHAR(44),
-    lamports BIGINT,
-    post_balance BIGINT,
-    reward_type "RewardType",
-    commission SMALLINT
-);
-
-CREATE TYPE "TransactionStatusMeta" AS (
-    error "TransactionError",
-    fee BIGINT,
-    pre_balances BIGINT[],
-    post_balances BIGINT[],
-    inner_instructions "InnerInstructions"[],
-    log_messages TEXT[],
-    pre_token_balances "TransactionTokenBalance"[],
-    post_token_balances "TransactionTokenBalance"[],
-    rewards "Reward"[]
-);
-
-CREATE TYPE "TransactionMessageHeader" AS (
-    num_required_signatures SMALLINT,
-    num_readonly_signed_accounts SMALLINT,
-    num_readonly_unsigned_accounts SMALLINT
-);
-
-CREATE TYPE "TransactionMessage" AS (
-    header "TransactionMessageHeader",
-    account_keys BYTEA[],
-    recent_blockhash BYTEA,
-    instructions "CompiledInstruction"[]
-);
-
-CREATE TYPE "TransactionMessageAddressTableLookup" AS (
-    account_key: BYTEA[],
-    writable_indexes SMALLINT[],
-    readonly_indexes SMALLINT[]
-);
-
-CREATE TYPE "TransactionMessageV0" AS (
-    header "TransactionMessageHeader",
-    account_keys BYTEA[],
-    recent_blockhash BYTEA,
-    instructions "CompiledInstruction"[],
-    address_table_lookups "TransactionMessageAddressTableLookup"[]
-);
-
-CREATE TYPE "LoadedAddresses" AS (
-    writable BYTEA[],
-    readonly BYTEA[]
-);
-
-CREATE TYPE "LoadedMessageV0" AS (
-    message "TransactionMessageV0",
-    loaded_addresses "LoadedAddresses"
-);
-
--- The table storing transactions
-CREATE TABLE transaction (
-    slot BIGINT NOT NULL,
-    signature BYTEA NOT NULL,
-    is_vote BOOL NOT NULL,
-    message_type SMALLINT, -- 0: legacy, 1: v0 message
-    legacy_message "TransactionMessage",
-    v0_loaded_message "LoadedMessageV0",
-    signatures BYTEA[],
-    message_hash BYTEA,
-    meta "TransactionStatusMeta",
-    updated_on TIMESTAMP NOT NULL,
-    CONSTRAINT transaction_pk PRIMARY KEY (slot, signature)
-);
-
-/**
- * The following is for keeping historical data for accounts and is not required for plugin to work.
- */
--- The table storing historical data for accounts
-CREATE TABLE account_audit (
-    pubkey BYTEA,
-    owner BYTEA,
-    lamports BIGINT NOT NULL,
-    slot BIGINT NOT NULL,
-    executable BOOL NOT NULL,
-    rent_epoch BIGINT NOT NULL,
-    data BYTEA,
-    write_version BIGINT NOT NULL,
-    updated_on TIMESTAMP NOT NULL
-);
-
-CREATE INDEX account_audit_account_key ON  account_audit (pubkey, write_version);
-
-CREATE FUNCTION audit_account_update() RETURNS trigger AS $audit_account_update$
-    BEGIN
-		INSERT INTO account_audit (pubkey, owner, lamports, slot, executable, rent_epoch, data, write_version, updated_on)
-            VALUES (OLD.pubkey, OLD.owner, OLD.lamports, OLD.slot,
-                    OLD.executable, OLD.rent_epoch, OLD.data, OLD.write_version, OLD.updated_on);
-        RETURN NEW;
-    END;
-
-$audit_account_update$ LANGUAGE plpgsql;
-
-CREATE TRIGGER account_update_trigger AFTER UPDATE OR DELETE ON account
-    FOR EACH ROW EXECUTE PROCEDURE audit_account_update();

accountsdb-plugin-postgres/scripts/drop_schema.sql

@@ -1,25 +0,0 @@
-/**
- * Script for cleaning up the schema for PostgreSQL used for the AccountsDb plugin.
- */
-
-DROP TRIGGER account_update_trigger ON account;
-DROP FUNCTION audit_account_update;
-DROP TABLE account_audit;
-DROP TABLE account;
-DROP TABLE slot;
-DROP TABLE transaction;
-
-DROP TYPE "TransactionError" CASCADE;
-DROP TYPE "TransactionErrorCode" CASCADE;
-DROP TYPE "LoadedMessageV0" CASCADE;
-DROP TYPE "LoadedAddresses" CASCADE;
-DROP TYPE "TransactionMessageV0" CASCADE;
-DROP TYPE "TransactionMessage" CASCADE;
-DROP TYPE "TransactionMessageHeader" CASCADE;
-DROP TYPE "TransactionMessageAddressTableLookup" CASCADE;
-DROP TYPE "TransactionStatusMeta" CASCADE;
-DROP TYPE "RewardType" CASCADE;
-DROP TYPE "Reward" CASCADE;
-DROP TYPE "TransactionTokenBalance" CASCADE;
-DROP TYPE "InnerInstructions" CASCADE;
-DROP TYPE "CompiledInstruction" CASCADE;

accountsdb-plugin-postgres/scripts/postgresql.conf

@@ -1,802 +0,0 @@
-# This a reference configuration file for the PostgreSQL database version 14.
-
-# -----------------------------
-# PostgreSQL configuration file
-# -----------------------------
-#
-# This file consists of lines of the form:
-#
-#   name = value
-#
-# (The "=" is optional.)  Whitespace may be used.  Comments are introduced with
-# "#" anywhere on a line.  The complete list of parameter names and allowed
-# values can be found in the PostgreSQL documentation.
-#
-# The commented-out settings shown in this file represent the default values.
-# Re-commenting a setting is NOT sufficient to revert it to the default value;
-# you need to reload the server.
-#
-# This file is read on server startup and when the server receives a SIGHUP
-# signal.  If you edit the file on a running system, you have to SIGHUP the
-# server for the changes to take effect, run "pg_ctl reload", or execute
-# "SELECT pg_reload_conf()".  Some parameters, which are marked below,
-# require a server shutdown and restart to take effect.
-#
-# Any parameter can also be given as a command-line option to the server, e.g.,
-# "postgres -c log_connections=on".  Some parameters can be changed at run time
-# with the "SET" SQL command.
-#
-# Memory units:  B  = bytes            Time units:  us  = microseconds
-#                kB = kilobytes                     ms  = milliseconds
-#                MB = megabytes                     s   = seconds
-#                GB = gigabytes                     min = minutes
-#                TB = terabytes                     h   = hours
-#                                                   d   = days
-
-
-#------------------------------------------------------------------------------
-# FILE LOCATIONS
-#------------------------------------------------------------------------------
-
-# The default values of these variables are driven from the -D command-line
-# option or PGDATA environment variable, represented here as ConfigDir.
-
-data_directory = '/var/lib/postgresql/14/main'		# use data in another directory
-					# (change requires restart)
-
-hba_file = '/etc/postgresql/14/main/pg_hba.conf'	# host-based authentication file
-					# (change requires restart)
-ident_file = '/etc/postgresql/14/main/pg_ident.conf'	# ident configuration file
-					# (change requires restart)
-
-# If external_pid_file is not explicitly set, no extra PID file is written.
-external_pid_file = '/var/run/postgresql/14-main.pid'			# write an extra PID file
-					# (change requires restart)
-
-
-#------------------------------------------------------------------------------
-# CONNECTIONS AND AUTHENTICATION
-#------------------------------------------------------------------------------
-
-# - Connection Settings -
-
-#listen_addresses = 'localhost'		# what IP address(es) to listen on;
-					# comma-separated list of addresses;
-					# defaults to 'localhost'; use '*' for all
-					# (change requires restart)
-listen_addresses = '*'
-port = 5433				# (change requires restart)
-max_connections = 200			# (change requires restart)
-#superuser_reserved_connections = 3	# (change requires restart)
-unix_socket_directories = '/var/run/postgresql'	# comma-separated list of directories
-					# (change requires restart)
-#unix_socket_group = ''			# (change requires restart)
-#unix_socket_permissions = 0777		# begin with 0 to use octal notation
-					# (change requires restart)
-#bonjour = off				# advertise server via Bonjour
-					# (change requires restart)
-#bonjour_name = ''			# defaults to the computer name
-					# (change requires restart)
-
-# - TCP settings -
-# see "man tcp" for details
-
-#tcp_keepalives_idle = 0		# TCP_KEEPIDLE, in seconds;
-					# 0 selects the system default
-#tcp_keepalives_interval = 0		# TCP_KEEPINTVL, in seconds;
-					# 0 selects the system default
-#tcp_keepalives_count = 0		# TCP_KEEPCNT;
-					# 0 selects the system default
-#tcp_user_timeout = 0			# TCP_USER_TIMEOUT, in milliseconds;
-					# 0 selects the system default
-
-#client_connection_check_interval = 0	# time between checks for client
-					# disconnection while running queries;
-					# 0 for never
-
-# - Authentication -
-
-#authentication_timeout = 1min		# 1s-600s
-#password_encryption = scram-sha-256	# scram-sha-256 or md5
-#db_user_namespace = off
-
-# GSSAPI using Kerberos
-#krb_server_keyfile = 'FILE:${sysconfdir}/krb5.keytab'
-#krb_caseins_users = off
-
-# - SSL -
-
-ssl = on
-#ssl_ca_file = ''
-ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem'
-#ssl_crl_file = ''
-#ssl_crl_dir = ''
-ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key'
-#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers
-#ssl_prefer_server_ciphers = on
-#ssl_ecdh_curve = 'prime256v1'
-#ssl_min_protocol_version = 'TLSv1.2'
-#ssl_max_protocol_version = ''
-#ssl_dh_params_file = ''
-#ssl_passphrase_command = ''
-#ssl_passphrase_command_supports_reload = off
-
-
-#------------------------------------------------------------------------------
-# RESOURCE USAGE (except WAL)
-#------------------------------------------------------------------------------
-
-# - Memory -
-
-shared_buffers = 1GB			# min 128kB
-					# (change requires restart)
-#huge_pages = try			# on, off, or try
-					# (change requires restart)
-#huge_page_size = 0			# zero for system default
-					# (change requires restart)
-#temp_buffers = 8MB			# min 800kB
-#max_prepared_transactions = 0		# zero disables the feature
-					# (change requires restart)
-# Caution: it is not advisable to set max_prepared_transactions nonzero unless
-# you actively intend to use prepared transactions.
-#work_mem = 4MB				# min 64kB
-#hash_mem_multiplier = 1.0		# 1-1000.0 multiplier on hash table work_mem
-#maintenance_work_mem = 64MB		# min 1MB
-#autovacuum_work_mem = -1		# min 1MB, or -1 to use maintenance_work_mem
-#logical_decoding_work_mem = 64MB	# min 64kB
-#max_stack_depth = 2MB			# min 100kB
-#shared_memory_type = mmap		# the default is the first option
-					# supported by the operating system:
-					#   mmap
-					#   sysv
-					#   windows
-					# (change requires restart)
-dynamic_shared_memory_type = posix	# the default is the first option
-					# supported by the operating system:
-					#   posix
-					#   sysv
-					#   windows
-					#   mmap
-					# (change requires restart)
-#min_dynamic_shared_memory = 0MB	# (change requires restart)
-
-# - Disk -
-
-#temp_file_limit = -1			# limits per-process temp file space
-					# in kilobytes, or -1 for no limit
-
-# - Kernel Resources -
-
-#max_files_per_process = 1000		# min 64
-					# (change requires restart)
-
-# - Cost-Based Vacuum Delay -
-
-#vacuum_cost_delay = 0			# 0-100 milliseconds (0 disables)
-#vacuum_cost_page_hit = 1		# 0-10000 credits
-#vacuum_cost_page_miss = 2		# 0-10000 credits
-#vacuum_cost_page_dirty = 20		# 0-10000 credits
-#vacuum_cost_limit = 200		# 1-10000 credits
-
-# - Background Writer -
-
-#bgwriter_delay = 200ms			# 10-10000ms between rounds
-#bgwriter_lru_maxpages = 100		# max buffers written/round, 0 disables
-#bgwriter_lru_multiplier = 2.0		# 0-10.0 multiplier on buffers scanned/round
-#bgwriter_flush_after = 512kB		# measured in pages, 0 disables
-
-# - Asynchronous Behavior -
-
-#backend_flush_after = 0		# measured in pages, 0 disables
-effective_io_concurrency = 1000		# 1-1000; 0 disables prefetching
-#maintenance_io_concurrency = 10	# 1-1000; 0 disables prefetching
-#max_worker_processes = 8		# (change requires restart)
-#max_parallel_workers_per_gather = 2	# taken from max_parallel_workers
-#max_parallel_maintenance_workers = 2	# taken from max_parallel_workers
-#max_parallel_workers = 8		# maximum number of max_worker_processes that
-					# can be used in parallel operations
-#parallel_leader_participation = on
-#old_snapshot_threshold = -1		# 1min-60d; -1 disables; 0 is immediate
-					# (change requires restart)
-
-
-#------------------------------------------------------------------------------
-# WRITE-AHEAD LOG
-#------------------------------------------------------------------------------
-
-# - Settings -
-
-wal_level = minimal 			# minimal, replica, or logical
-					# (change requires restart)
-fsync = off				# flush data to disk for crash safety
-					# (turning this off can cause
-					# unrecoverable data corruption)
-synchronous_commit = off		# synchronization level;
-					# off, local, remote_write, remote_apply, or on
-#wal_sync_method = fsync		# the default is the first option
-					# supported by the operating system:
-					#   open_datasync
-					#   fdatasync (default on Linux and FreeBSD)
-					#   fsync
-					#   fsync_writethrough
-					#   open_sync
-full_page_writes = off			# recover from partial page writes
-#wal_log_hints = off			# also do full page writes of non-critical updates
-					# (change requires restart)
-#wal_compression = off			# enable compression of full-page writes
-#wal_init_zero = on			# zero-fill new WAL files
-#wal_recycle = on			# recycle WAL files
-#wal_buffers = -1			# min 32kB, -1 sets based on shared_buffers
-					# (change requires restart)
-#wal_writer_delay = 200ms		# 1-10000 milliseconds
-#wal_writer_flush_after = 1MB		# measured in pages, 0 disables
-#wal_skip_threshold = 2MB
-
-#commit_delay = 0			# range 0-100000, in microseconds
-#commit_siblings = 5			# range 1-1000
-
-# - Checkpoints -
-
-#checkpoint_timeout = 5min		# range 30s-1d
-#checkpoint_completion_target = 0.9	# checkpoint target duration, 0.0 - 1.0
-#checkpoint_flush_after = 256kB		# measured in pages, 0 disables
-#checkpoint_warning = 30s		# 0 disables
-max_wal_size = 1GB
-min_wal_size = 80MB
-
-# - Archiving -
-
-#archive_mode = off		# enables archiving; off, on, or always
-				# (change requires restart)
-#archive_command = ''		# command to use to archive a logfile segment
-				# placeholders: %p = path of file to archive
-				#               %f = file name only
-				# e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f'
-#archive_timeout = 0		# force a logfile segment switch after this
-				# number of seconds; 0 disables
-
-# - Archive Recovery -
-
-# These are only used in recovery mode.
-
-#restore_command = ''		# command to use to restore an archived logfile segment
-				# placeholders: %p = path of file to restore
-				#               %f = file name only
-				# e.g. 'cp /mnt/server/archivedir/%f %p'
-#archive_cleanup_command = ''	# command to execute at every restartpoint
-#recovery_end_command = ''	# command to execute at completion of recovery
-
-# - Recovery Target -
-
-# Set these only when performing a targeted recovery.
-
-#recovery_target = ''		# 'immediate' to end recovery as soon as a
-                                # consistent state is reached
-				# (change requires restart)
-#recovery_target_name = ''	# the named restore point to which recovery will proceed
-				# (change requires restart)
-#recovery_target_time = ''	# the time stamp up to which recovery will proceed
-				# (change requires restart)
-#recovery_target_xid = ''	# the transaction ID up to which recovery will proceed
-				# (change requires restart)
-#recovery_target_lsn = ''	# the WAL LSN up to which recovery will proceed
-				# (change requires restart)
-#recovery_target_inclusive = on # Specifies whether to stop:
-				# just after the specified recovery target (on)
-				# just before the recovery target (off)
-				# (change requires restart)
-#recovery_target_timeline = 'latest'	# 'current', 'latest', or timeline ID
-				# (change requires restart)
-#recovery_target_action = 'pause'	# 'pause', 'promote', 'shutdown'
-				# (change requires restart)
-
-
-#------------------------------------------------------------------------------
-# REPLICATION
-#------------------------------------------------------------------------------
-
-# - Sending Servers -
-
-# Set these on the primary and on any standby that will send replication data.
-
-max_wal_senders = 0		# max number of walsender processes
-				# (change requires restart)
-#max_replication_slots = 10	# max number of replication slots
-				# (change requires restart)
-#wal_keep_size = 0		# in megabytes; 0 disables
-#max_slot_wal_keep_size = -1	# in megabytes; -1 disables
-#wal_sender_timeout = 60s	# in milliseconds; 0 disables
-#track_commit_timestamp = off	# collect timestamp of transaction commit
-				# (change requires restart)
-
-# - Primary Server -
-
-# These settings are ignored on a standby server.
-
-#synchronous_standby_names = ''	# standby servers that provide sync rep
-				# method to choose sync standbys, number of sync standbys,
-				# and comma-separated list of application_name
-				# from standby(s); '*' = all
-#vacuum_defer_cleanup_age = 0	# number of xacts by which cleanup is delayed
-
-# - Standby Servers -
-
-# These settings are ignored on a primary server.
-
-#primary_conninfo = ''			# connection string to sending server
-#primary_slot_name = ''			# replication slot on sending server
-#promote_trigger_file = ''		# file name whose presence ends recovery
-#hot_standby = on			# "off" disallows queries during recovery
-					# (change requires restart)
-#max_standby_archive_delay = 30s	# max delay before canceling queries
-					# when reading WAL from archive;
-					# -1 allows indefinite delay
-#max_standby_streaming_delay = 30s	# max delay before canceling queries
-					# when reading streaming WAL;
-					# -1 allows indefinite delay
-#wal_receiver_create_temp_slot = off	# create temp slot if primary_slot_name
-					# is not set
-#wal_receiver_status_interval = 10s	# send replies at least this often
-					# 0 disables
-#hot_standby_feedback = off		# send info from standby to prevent
-					# query conflicts
-#wal_receiver_timeout = 60s		# time that receiver waits for
-					# communication from primary
-					# in milliseconds; 0 disables
-#wal_retrieve_retry_interval = 5s	# time to wait before retrying to
-					# retrieve WAL after a failed attempt
-#recovery_min_apply_delay = 0		# minimum delay for applying changes during recovery
-
-# - Subscribers -
-
-# These settings are ignored on a publisher.
-
-#max_logical_replication_workers = 4	# taken from max_worker_processes
-					# (change requires restart)
-#max_sync_workers_per_subscription = 2	# taken from max_logical_replication_workers
-
-
-#------------------------------------------------------------------------------
-# QUERY TUNING
-#------------------------------------------------------------------------------
-
-# - Planner Method Configuration -
-
-#enable_async_append = on
-#enable_bitmapscan = on
-#enable_gathermerge = on
-#enable_hashagg = on
-#enable_hashjoin = on
-#enable_incremental_sort = on
-#enable_indexscan = on
-#enable_indexonlyscan = on
-#enable_material = on
-#enable_memoize = on
-#enable_mergejoin = on
-#enable_nestloop = on
-#enable_parallel_append = on
-#enable_parallel_hash = on
-#enable_partition_pruning = on
-#enable_partitionwise_join = off
-#enable_partitionwise_aggregate = off
-#enable_seqscan = on
-#enable_sort = on
-#enable_tidscan = on
-
-# - Planner Cost Constants -
-
-#seq_page_cost = 1.0			# measured on an arbitrary scale
-#random_page_cost = 4.0			# same scale as above
-#cpu_tuple_cost = 0.01			# same scale as above
-#cpu_index_tuple_cost = 0.005		# same scale as above
-#cpu_operator_cost = 0.0025		# same scale as above
-#parallel_setup_cost = 1000.0	# same scale as above
-#parallel_tuple_cost = 0.1		# same scale as above
-#min_parallel_table_scan_size = 8MB
-#min_parallel_index_scan_size = 512kB
-#effective_cache_size = 4GB
-
-#jit_above_cost = 100000		# perform JIT compilation if available
-					# and query more expensive than this;
-					# -1 disables
-#jit_inline_above_cost = 500000		# inline small functions if query is
-					# more expensive than this; -1 disables
-#jit_optimize_above_cost = 500000	# use expensive JIT optimizations if
-					# query is more expensive than this;
-					# -1 disables
-
-# - Genetic Query Optimizer -
-
-#geqo = on
-#geqo_threshold = 12
-#geqo_effort = 5			# range 1-10
-#geqo_pool_size = 0			# selects default based on effort
-#geqo_generations = 0			# selects default based on effort
-#geqo_selection_bias = 2.0		# range 1.5-2.0
-#geqo_seed = 0.0			# range 0.0-1.0
-
-# - Other Planner Options -
-
-#default_statistics_target = 100	# range 1-10000
-#constraint_exclusion = partition	# on, off, or partition
-#cursor_tuple_fraction = 0.1		# range 0.0-1.0
-#from_collapse_limit = 8
-#jit = on				# allow JIT compilation
-#join_collapse_limit = 8		# 1 disables collapsing of explicit
-					# JOIN clauses
-#plan_cache_mode = auto			# auto, force_generic_plan or
-					# force_custom_plan
-
-
-#------------------------------------------------------------------------------
-# REPORTING AND LOGGING
-#------------------------------------------------------------------------------
-
-# - Where to Log -
-
-#log_destination = 'stderr'		# Valid values are combinations of
-					# stderr, csvlog, syslog, and eventlog,
-					# depending on platform.  csvlog
-					# requires logging_collector to be on.
-
-# This is used when logging to stderr:
-#logging_collector = off		# Enable capturing of stderr and csvlog
-					# into log files. Required to be on for
-					# csvlogs.
-					# (change requires restart)
-
-# These are only used if logging_collector is on:
-#log_directory = 'log'			# directory where log files are written,
-					# can be absolute or relative to PGDATA
-#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'	# log file name pattern,
-					# can include strftime() escapes
-#log_file_mode = 0600			# creation mode for log files,
-					# begin with 0 to use octal notation
-#log_rotation_age = 1d			# Automatic rotation of logfiles will
-					# happen after that time.  0 disables.
-#log_rotation_size = 10MB		# Automatic rotation of logfiles will
-					# happen after that much log output.
-					# 0 disables.
-#log_truncate_on_rotation = off		# If on, an existing log file with the
-					# same name as the new log file will be
-					# truncated rather than appended to.
-					# But such truncation only occurs on
-					# time-driven rotation, not on restarts
-					# or size-driven rotation.  Default is
-					# off, meaning append to existing files
-					# in all cases.
-
-# These are relevant when logging to syslog:
-#syslog_facility = 'LOCAL0'
-#syslog_ident = 'postgres'
-#syslog_sequence_numbers = on
-#syslog_split_messages = on
-
-# This is only relevant when logging to eventlog (Windows):
-# (change requires restart)
-#event_source = 'PostgreSQL'
-
-# - When to Log -
-
-#log_min_messages = warning		# values in order of decreasing detail:
-					#   debug5
-					#   debug4
-					#   debug3
-					#   debug2
-					#   debug1
-					#   info
-					#   notice
-					#   warning
-					#   error
-					#   log
-					#   fatal
-					#   panic
-
-#log_min_error_statement = error	# values in order of decreasing detail:
-					#   debug5
-					#   debug4
-					#   debug3
-					#   debug2
-					#   debug1
-					#   info
-					#   notice
-					#   warning
-					#   error
-					#   log
-					#   fatal
-					#   panic (effectively off)
-
-#log_min_duration_statement = -1	# -1 is disabled, 0 logs all statements
-					# and their durations, > 0 logs only
-					# statements running at least this number
-					# of milliseconds
-
-#log_min_duration_sample = -1		# -1 is disabled, 0 logs a sample of statements
-					# and their durations, > 0 logs only a sample of
-					# statements running at least this number
-					# of milliseconds;
-					# sample fraction is determined by log_statement_sample_rate
-
-#log_statement_sample_rate = 1.0	# fraction of logged statements exceeding
-					# log_min_duration_sample to be logged;
-					# 1.0 logs all such statements, 0.0 never logs
-
-
-#log_transaction_sample_rate = 0.0	# fraction of transactions whose statements
-					# are logged regardless of their duration; 1.0 logs all
-					# statements from all transactions, 0.0 never logs
-
-# - What to Log -
-
-#debug_print_parse = off
-#debug_print_rewritten = off
-#debug_print_plan = off
-#debug_pretty_print = on
-#log_autovacuum_min_duration = -1	# log autovacuum activity;
-					# -1 disables, 0 logs all actions and
-					# their durations, > 0 logs only
-					# actions running at least this number
-					# of milliseconds.
-#log_checkpoints = off
-#log_connections = off
-#log_disconnections = off
-#log_duration = off
-#log_error_verbosity = default		# terse, default, or verbose messages
-#log_hostname = off
-log_line_prefix = '%m [%p] %q%u@%d '		# special values:
-					#   %a = application name
-					#   %u = user name
-					#   %d = database name
-					#   %r = remote host and port
-					#   %h = remote host
-					#   %b = backend type
-					#   %p = process ID
-					#   %P = process ID of parallel group leader
-					#   %t = timestamp without milliseconds
-					#   %m = timestamp with milliseconds
-					#   %n = timestamp with milliseconds (as a Unix epoch)
-					#   %Q = query ID (0 if none or not computed)
-					#   %i = command tag
-					#   %e = SQL state
-					#   %c = session ID
-					#   %l = session line number
-					#   %s = session start timestamp
-					#   %v = virtual transaction ID
-					#   %x = transaction ID (0 if none)
-					#   %q = stop here in non-session
-					#        processes
-					#   %% = '%'
-					# e.g. '<%u%%%d> '
-#log_lock_waits = off			# log lock waits >= deadlock_timeout
-#log_recovery_conflict_waits = off	# log standby recovery conflict waits
-					# >= deadlock_timeout
-#log_parameter_max_length = -1		# when logging statements, limit logged
-					# bind-parameter values to N bytes;
-					# -1 means print in full, 0 disables
-#log_parameter_max_length_on_error = 0	# when logging an error, limit logged
-					# bind-parameter values to N bytes;
-					# -1 means print in full, 0 disables
-#log_statement = 'none'			# none, ddl, mod, all
-#log_replication_commands = off
-#log_temp_files = -1			# log temporary files equal or larger
-					# than the specified size in kilobytes;
-					# -1 disables, 0 logs all temp files
-log_timezone = 'Etc/UTC'
-
-
-#------------------------------------------------------------------------------
-# PROCESS TITLE
-#------------------------------------------------------------------------------
-
-cluster_name = '14/main'			# added to process titles if nonempty
-					# (change requires restart)
-#update_process_title = on
-
-
-#------------------------------------------------------------------------------
-# STATISTICS
-#------------------------------------------------------------------------------
-
-# - Query and Index Statistics Collector -
-
-#track_activities = on
-#track_activity_query_size = 1024	# (change requires restart)
-#track_counts = on
-#track_io_timing = off
-#track_wal_io_timing = off
-#track_functions = none			# none, pl, all
-stats_temp_directory = '/var/run/postgresql/14-main.pg_stat_tmp'
-
-
-# - Monitoring -
-
-#compute_query_id = auto
-#log_statement_stats = off
-#log_parser_stats = off
-#log_planner_stats = off
-#log_executor_stats = off
-
-
-#------------------------------------------------------------------------------
-# AUTOVACUUM
-#------------------------------------------------------------------------------
-
-#autovacuum = on			# Enable autovacuum subprocess?  'on'
-					# requires track_counts to also be on.
-#autovacuum_max_workers = 3		# max number of autovacuum subprocesses
-					# (change requires restart)
-#autovacuum_naptime = 1min		# time between autovacuum runs
-#autovacuum_vacuum_threshold = 50	# min number of row updates before
-					# vacuum
-#autovacuum_vacuum_insert_threshold = 1000	# min number of row inserts
-					# before vacuum; -1 disables insert
-					# vacuums
-#autovacuum_analyze_threshold = 50	# min number of row updates before
-					# analyze
-#autovacuum_vacuum_scale_factor = 0.2	# fraction of table size before vacuum
-#autovacuum_vacuum_insert_scale_factor = 0.2	# fraction of inserts over table
-					# size before insert vacuum
-#autovacuum_analyze_scale_factor = 0.1	# fraction of table size before analyze
-#autovacuum_freeze_max_age = 200000000	# maximum XID age before forced vacuum
-					# (change requires restart)
-#autovacuum_multixact_freeze_max_age = 400000000	# maximum multixact age
-					# before forced vacuum
-					# (change requires restart)
-#autovacuum_vacuum_cost_delay = 2ms	# default vacuum cost delay for
-					# autovacuum, in milliseconds;
-					# -1 means use vacuum_cost_delay
-#autovacuum_vacuum_cost_limit = -1	# default vacuum cost limit for
-					# autovacuum, -1 means use
-					# vacuum_cost_limit
-
-
-#------------------------------------------------------------------------------
-# CLIENT CONNECTION DEFAULTS
-#------------------------------------------------------------------------------
-
-# - Statement Behavior -
-
-#client_min_messages = notice		# values in order of decreasing detail:
-					#   debug5
-					#   debug4
-					#   debug3
-					#   debug2
-					#   debug1
-					#   log
-					#   notice
-					#   warning
-					#   error
-#search_path = '"$user", public'	# schema names
-#row_security = on
-#default_table_access_method = 'heap'
-#default_tablespace = ''		# a tablespace name, '' uses the default
-#default_toast_compression = 'pglz'	# 'pglz' or 'lz4'
-#temp_tablespaces = ''			# a list of tablespace names, '' uses
-					# only default tablespace
-#check_function_bodies = on
-#default_transaction_isolation = 'read committed'
-#default_transaction_read_only = off
-#default_transaction_deferrable = off
-#session_replication_role = 'origin'
-#statement_timeout = 0			# in milliseconds, 0 is disabled
-#lock_timeout = 0			# in milliseconds, 0 is disabled
-#idle_in_transaction_session_timeout = 0	# in milliseconds, 0 is disabled
-#idle_session_timeout = 0		# in milliseconds, 0 is disabled
-#vacuum_freeze_table_age = 150000000
-#vacuum_freeze_min_age = 50000000
-#vacuum_failsafe_age = 1600000000
-#vacuum_multixact_freeze_table_age = 150000000
-#vacuum_multixact_freeze_min_age = 5000000
-#vacuum_multixact_failsafe_age = 1600000000
-#bytea_output = 'hex'			# hex, escape
-#xmlbinary = 'base64'
-#xmloption = 'content'
-#gin_pending_list_limit = 4MB
-
-# - Locale and Formatting -
-
-datestyle = 'iso, mdy'
-#intervalstyle = 'postgres'
-timezone = 'Etc/UTC'
-#timezone_abbreviations = 'Default'     # Select the set of available time zone
-					# abbreviations.  Currently, there are
-					#   Default
-					#   Australia (historical usage)
-					#   India
-					# You can create your own file in
-					# share/timezonesets/.
-#extra_float_digits = 1			# min -15, max 3; any value >0 actually
-					# selects precise output mode
-#client_encoding = sql_ascii		# actually, defaults to database
-					# encoding
-
-# These settings are initialized by initdb, but they can be changed.
-lc_messages = 'C.UTF-8'			# locale for system error message
-					# strings
-lc_monetary = 'C.UTF-8'			# locale for monetary formatting
-lc_numeric = 'C.UTF-8'			# locale for number formatting
-lc_time = 'C.UTF-8'				# locale for time formatting
-
-# default configuration for text search
-default_text_search_config = 'pg_catalog.english'
-
-# - Shared Library Preloading -
-
-#local_preload_libraries = ''
-#session_preload_libraries = ''
-#shared_preload_libraries = ''	# (change requires restart)
-#jit_provider = 'llvmjit'		# JIT library to use
-
-# - Other Defaults -
-
-#dynamic_library_path = '$libdir'
-#extension_destdir = ''			# prepend path when loading extensions
-					# and shared objects (added by Debian)
-#gin_fuzzy_search_limit = 0
-
-
-#------------------------------------------------------------------------------
-# LOCK MANAGEMENT
-#------------------------------------------------------------------------------
-
-#deadlock_timeout = 1s
-#max_locks_per_transaction = 64		# min 10
-					# (change requires restart)
-#max_pred_locks_per_transaction = 64	# min 10
-					# (change requires restart)
-#max_pred_locks_per_relation = -2	# negative values mean
-					# (max_pred_locks_per_transaction
-					#  / -max_pred_locks_per_relation) - 1
-#max_pred_locks_per_page = 2            # min 0
-
-
-#------------------------------------------------------------------------------
-# VERSION AND PLATFORM COMPATIBILITY
-#------------------------------------------------------------------------------
-
-# - Previous PostgreSQL Versions -
-
-#array_nulls = on
-#backslash_quote = safe_encoding	# on, off, or safe_encoding
-#escape_string_warning = on
-#lo_compat_privileges = off
-#quote_all_identifiers = off
-#standard_conforming_strings = on
-#synchronize_seqscans = on
-
-# - Other Platforms and Clients -
-
-#transform_null_equals = off
-
-
-#------------------------------------------------------------------------------
-# ERROR HANDLING
-#------------------------------------------------------------------------------
-
-#exit_on_error = off			# terminate session on any error?
-#restart_after_crash = on		# reinitialize after backend crash?
-#data_sync_retry = off			# retry or panic on failure to fsync
-					# data?
-					# (change requires restart)
-#recovery_init_sync_method = fsync	# fsync, syncfs (Linux 5.8+)
-
-
-#------------------------------------------------------------------------------
-# CONFIG FILE INCLUDES
-#------------------------------------------------------------------------------
-
-# These options allow settings to be loaded from files other than the
-# default postgresql.conf.  Note that these are directives, not variable
-# assignments, so they can usefully be given more than once.
-
-include_dir = 'conf.d'			# include files ending in '.conf' from
-					# a directory, e.g., 'conf.d'
-#include_if_exists = '...'		# include file only if it exists
-#include = '...'			# include file
-
-
-#------------------------------------------------------------------------------
-# CUSTOMIZED OPTIONS
-#------------------------------------------------------------------------------
-
-# Add settings for extensions here

accountsdb-plugin-postgres/src/accounts_selector.rs

@@ -1,74 +0,0 @@
-use {log::*, std::collections::HashSet};
-
-#[derive(Debug)]
-pub(crate) struct AccountsSelector {
-    pub accounts: HashSet<Vec<u8>>,
-    pub owners: HashSet<Vec<u8>>,
-    pub select_all_accounts: bool,
-}
-
-impl AccountsSelector {
-    pub fn default() -> Self {
-        AccountsSelector {
-            accounts: HashSet::default(),
-            owners: HashSet::default(),
-            select_all_accounts: true,
-        }
-    }
-
-    pub fn new(accounts: &[String], owners: &[String]) -> Self {
-        info!(
-            "Creating AccountsSelector from accounts: {:?}, owners: {:?}",
-            accounts, owners
-        );
-
-        let select_all_accounts = accounts.iter().any(|key| key == "*");
-        if select_all_accounts {
-            return AccountsSelector {
-                accounts: HashSet::default(),
-                owners: HashSet::default(),
-                select_all_accounts,
-            };
-        }
-        let accounts = accounts
-            .iter()
-            .map(|key| bs58::decode(key).into_vec().unwrap())
-            .collect();
-        let owners = owners
-            .iter()
-            .map(|key| bs58::decode(key).into_vec().unwrap())
-            .collect();
-        AccountsSelector {
-            accounts,
-            owners,
-            select_all_accounts,
-        }
-    }
-
-    pub fn is_account_selected(&self, account: &[u8], owner: &[u8]) -> bool {
-        self.select_all_accounts || self.accounts.contains(account) || self.owners.contains(owner)
-    }
-
-    /// Check if any account is of interested at all
-    pub fn is_enabled(&self) -> bool {
-        self.select_all_accounts || !self.accounts.is_empty() || !self.owners.is_empty()
-    }
-}
-
-#[cfg(test)]
-pub(crate) mod tests {
-    use super::*;
-
-    #[test]
-    fn test_create_accounts_selector() {
-        AccountsSelector::new(
-            &["9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin".to_string()],
-            &[],
-        );
-
-        AccountsSelector::new(
-            &[],
-            &["9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin".to_string()],
-        );
-    }
-}

accountsdb-plugin-postgres/src/accountsdb_plugin_postgres.rs

@@ -1,437 +0,0 @@
-use solana_measure::measure::Measure;
-/// Main entry for the PostgreSQL plugin
-use {
-    crate::{
-        accounts_selector::AccountsSelector,
-        postgres_client::{ParallelPostgresClient, PostgresClientBuilder},
-        transaction_selector::TransactionSelector,
-    },
-    bs58,
-    log::*,
-    serde_derive::{Deserialize, Serialize},
-    serde_json,
-    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
-        AccountsDbPlugin, AccountsDbPluginError, ReplicaAccountInfoVersions,
-        ReplicaTransactionInfoVersions, Result, SlotStatus,
-    },
-    solana_metrics::*,
-    std::{fs::File, io::Read},
-    thiserror::Error,
-};
-
-#[derive(Default)]
-pub struct AccountsDbPluginPostgres {
-    client: Option<ParallelPostgresClient>,
-    accounts_selector: Option<AccountsSelector>,
-    transaction_selector: Option<TransactionSelector>,
-}
-
-impl std::fmt::Debug for AccountsDbPluginPostgres {
-    fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        Ok(())
-    }
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct AccountsDbPluginPostgresConfig {
-    pub host: Option<String>,
-    pub user: Option<String>,
-    pub port: Option<u16>,
-    pub connection_str: Option<String>,
-    pub threads: Option<usize>,
-    pub batch_size: Option<usize>,
-    pub panic_on_db_errors: Option<bool>,
-}
-
-#[derive(Error, Debug)]
-pub enum AccountsDbPluginPostgresError {
-    #[error("Error connecting to the backend data store. Error message: ({msg})")]
-    DataStoreConnectionError { msg: String },
-
-    #[error("Error preparing data store schema. Error message: ({msg})")]
-    DataSchemaError { msg: String },
-
-    #[error("Error preparing data store schema. Error message: ({msg})")]
-    ConfigurationError { msg: String },
-}
-
-impl AccountsDbPlugin for AccountsDbPluginPostgres {
-    fn name(&self) -> &'static str {
-        "AccountsDbPluginPostgres"
-    }
-
-    /// Do initialization for the PostgreSQL plugin.
-    ///
-    /// # Format of the config file:
-    /// * The `accounts_selector` section allows the user to controls accounts selections.
-    /// "accounts_selector" : {
-    ///     "accounts" : \["pubkey-1", "pubkey-2", ..., "pubkey-n"\],
-    /// }
-    /// or:
-    /// "accounts_selector" = {
-    ///     "owners" : \["pubkey-1", "pubkey-2", ..., "pubkey-m"\]
-    /// }
-    /// Accounts either satisyfing the accounts condition or owners condition will be selected.
-    /// When only owners is specified,
-    /// all accounts belonging to the owners will be streamed.
-    /// The accounts field support wildcard to select all accounts:
-    /// "accounts_selector" : {
-    ///     "accounts" : \["*"\],
-    /// }
-    /// * "host", optional, specifies the PostgreSQL server.
-    /// * "user", optional, specifies the PostgreSQL user.
-    /// * "port", optional, specifies the PostgreSQL server's port.
-    /// * "connection_str", optional, the custom PostgreSQL connection string.
-    /// Please refer to https://docs.rs/postgres/0.19.2/postgres/config/struct.Config.html for the connection configuration.
-    /// When `connection_str` is set, the values in "host", "user" and "port" are ignored. If `connection_str` is not given,
-    /// `host` and `user` must be given.
-    /// * "threads" optional, specifies the number of worker threads for the plugin. A thread
-    /// maintains a PostgreSQL connection to the server. The default is '10'.
-    /// * "batch_size" optional, specifies the batch size of bulk insert when the AccountsDb is created
-    /// from restoring a snapshot. The default is '10'.
-    /// * "panic_on_db_errors", optional, contols if to panic when there are errors replicating data to the
-    /// PostgreSQL database. The default is 'false'.
-    /// * "transaction_selector", optional, controls if and what transaction to store. If this field is missing
-    /// None of the transction is stored.
-    /// "transaction_selector" : {
-    ///     "mentions" : \["pubkey-1", "pubkey-2", ..., "pubkey-n"\],
-    /// }
-    /// The `mentions` field support wildcard to select all transaction or all 'vote' transactions:
-    /// For example, to select all transactions:
-    /// "transaction_selector" : {
-    ///     "mentions" : \["*"\],
-    /// }
-    /// To select all vote transactions:
-    /// "transaction_selector" : {
-    ///     "mentions" : \["all_votes"\],
-    /// }
-    /// # Examples
-    ///
-    /// {
-    ///    "libpath": "/home/solana/target/release/libsolana_accountsdb_plugin_postgres.so",
-    ///    "host": "host_foo",
-    ///    "user": "solana",
-    ///    "threads": 10,
-    ///    "accounts_selector" : {
-    ///       "owners" : ["9oT9R5ZyRovSVnt37QvVoBttGpNqR3J7unkb567NP8k3"]
-    ///    }
-    /// }
-
-    fn on_load(&mut self, config_file: &str) -> Result<()> {
-        solana_logger::setup_with_default("info");
-        info!(
-            "Loading plugin {:?} from config_file {:?}",
-            self.name(),
-            config_file
-        );
-        let mut file = File::open(config_file)?;
-        let mut contents = String::new();
-        file.read_to_string(&mut contents)?;
-
-        let result: serde_json::Value = serde_json::from_str(&contents).unwrap();
-        self.accounts_selector = Some(Self::create_accounts_selector_from_config(&result));
-        self.transaction_selector = Some(Self::create_transaction_selector_from_config(&result));
-
-        let result: serde_json::Result<AccountsDbPluginPostgresConfig> =
-            serde_json::from_str(&contents);
-        match result {
-            Err(err) => {
-                return Err(AccountsDbPluginError::ConfigFileReadError {
-                    msg: format!(
-                        "The config file is not in the JSON format expected: {:?}",
-                        err
-                    ),
-                })
-            }
-            Ok(config) => {
-                let client = PostgresClientBuilder::build_pararallel_postgres_client(&config)?;
-                self.client = Some(client);
-            }
-        }
-
-        Ok(())
-    }
-
-    fn on_unload(&mut self) {
-        info!("Unloading plugin: {:?}", self.name());
-
-        match &mut self.client {
-            None => {}
-            Some(client) => {
-                client.join().unwrap();
-            }
-        }
-    }
-
-    fn update_account(
-        &mut self,
-        account: ReplicaAccountInfoVersions,
-        slot: u64,
-        is_startup: bool,
-    ) -> Result<()> {
-        let mut measure_all = Measure::start("accountsdb-plugin-postgres-update-account-main");
-        match account {
-            ReplicaAccountInfoVersions::V0_0_1(account) => {
-                let mut measure_select =
-                    Measure::start("accountsdb-plugin-postgres-update-account-select");
-                if let Some(accounts_selector) = &self.accounts_selector {
-                    if !accounts_selector.is_account_selected(account.pubkey, account.owner) {
-                        return Ok(());
-                    }
-                } else {
-                    return Ok(());
-                }
-                measure_select.stop();
-                inc_new_counter_debug!(
-                    "accountsdb-plugin-postgres-update-account-select-us",
-                    measure_select.as_us() as usize,
-                    100000,
-                    100000
-                );
-
-                debug!(
-                    "Updating account {:?} with owner {:?} at slot {:?} using account selector {:?}",
-                    bs58::encode(account.pubkey).into_string(),
-                    bs58::encode(account.owner).into_string(),
-                    slot,
-                    self.accounts_selector.as_ref().unwrap()
-                );
-
-                match &mut self.client {
-                    None => {
-                        return Err(AccountsDbPluginError::Custom(Box::new(
-                            AccountsDbPluginPostgresError::DataStoreConnectionError {
-                                msg: "There is no connection to the PostgreSQL database."
-                                    .to_string(),
-                            },
-                        )));
-                    }
-                    Some(client) => {
-                        let mut measure_update =
-                            Measure::start("accountsdb-plugin-postgres-update-account-client");
-                        let result = { client.update_account(account, slot, is_startup) };
-                        measure_update.stop();
-
-                        inc_new_counter_debug!(
-                            "accountsdb-plugin-postgres-update-account-client-us",
-                            measure_update.as_us() as usize,
-                            100000,
-                            100000
-                        );
-
-                        if let Err(err) = result {
-                            return Err(AccountsDbPluginError::AccountsUpdateError {
-                                msg: format!("Failed to persist the update of account to the PostgreSQL database. Error: {:?}", err)
-                            });
-                        }
-                    }
-                }
-            }
-        }
-
-        measure_all.stop();
-
-        inc_new_counter_debug!(
-            "accountsdb-plugin-postgres-update-account-main-us",
-            measure_all.as_us() as usize,
-            100000,
-            100000
-        );
-
-        Ok(())
-    }
-
-    fn update_slot_status(
-        &mut self,
-        slot: u64,
-        parent: Option<u64>,
-        status: SlotStatus,
-    ) -> Result<()> {
-        info!("Updating slot {:?} at with status {:?}", slot, status);
-
-        match &mut self.client {
-            None => {
-                return Err(AccountsDbPluginError::Custom(Box::new(
-                    AccountsDbPluginPostgresError::DataStoreConnectionError {
-                        msg: "There is no connection to the PostgreSQL database.".to_string(),
-                    },
-                )));
-            }
-            Some(client) => {
-                let result = client.update_slot_status(slot, parent, status);
-
-                if let Err(err) = result {
-                    return Err(AccountsDbPluginError::SlotStatusUpdateError{
-                        msg: format!("Failed to persist the update of slot to the PostgreSQL database. Error: {:?}", err)
-                    });
-                }
-            }
-        }
-
-        Ok(())
-    }
-
-    fn notify_end_of_startup(&mut self) -> Result<()> {
-        info!("Notifying the end of startup for accounts notifications");
-        match &mut self.client {
-            None => {
-                return Err(AccountsDbPluginError::Custom(Box::new(
-                    AccountsDbPluginPostgresError::DataStoreConnectionError {
-                        msg: "There is no connection to the PostgreSQL database.".to_string(),
-                    },
-                )));
-            }
-            Some(client) => {
-                let result = client.notify_end_of_startup();
-
-                if let Err(err) = result {
-                    return Err(AccountsDbPluginError::SlotStatusUpdateError{
-                        msg: format!("Failed to notify the end of startup for accounts notifications. Error: {:?}", err)
-                    });
-                }
-            }
-        }
-        Ok(())
-    }
-
-    fn notify_transaction(
-        &mut self,
-        transaction_info: ReplicaTransactionInfoVersions,
-        slot: u64,
-    ) -> Result<()> {
-        match &mut self.client {
-            None => {
-                return Err(AccountsDbPluginError::Custom(Box::new(
-                    AccountsDbPluginPostgresError::DataStoreConnectionError {
-                        msg: "There is no connection to the PostgreSQL database.".to_string(),
-                    },
-                )));
-            }
-            Some(client) => match transaction_info {
-                ReplicaTransactionInfoVersions::V0_0_1(transaction_info) => {
-                    if let Some(transaction_selector) = &self.transaction_selector {
-                        if !transaction_selector.is_transaction_selected(
-                            transaction_info.is_vote,
-                            transaction_info.transaction.message().account_keys_iter(),
-                        ) {
-                            return Ok(());
-                        }
-                    } else {
-                        return Ok(());
-                    }
-
-                    let result = client.log_transaction_info(transaction_info, slot);
-
-                    if let Err(err) = result {
-                        return Err(AccountsDbPluginError::SlotStatusUpdateError{
-                                msg: format!("Failed to persist the transaction info to the PostgreSQL database. Error: {:?}", err)
-                            });
-                    }
-                }
-            },
-        }
-
-        Ok(())
-    }
-
-    /// Check if the plugin is interested in account data
-    /// Default is true -- if the plugin is not interested in
-    /// account data, please return false.
-    fn account_data_notifications_enabled(&self) -> bool {
-        self.accounts_selector
-            .as_ref()
-            .map_or_else(|| false, |selector| selector.is_enabled())
-    }
-
-    /// Check if the plugin is interested in transaction data
-    fn transaction_notifications_enabled(&self) -> bool {
-        self.transaction_selector
-            .as_ref()
-            .map_or_else(|| false, |selector| selector.is_enabled())
-    }
-}
-
-impl AccountsDbPluginPostgres {
-    fn create_accounts_selector_from_config(config: &serde_json::Value) -> AccountsSelector {
-        let accounts_selector = &config["accounts_selector"];
-
-        if accounts_selector.is_null() {
-            AccountsSelector::default()
-        } else {
-            let accounts = &accounts_selector["accounts"];
-            let accounts: Vec<String> = if accounts.is_array() {
-                accounts
-                    .as_array()
-                    .unwrap()
-                    .iter()
-                    .map(|val| val.as_str().unwrap().to_string())
-                    .collect()
-            } else {
-                Vec::default()
-            };
-            let owners = &accounts_selector["owners"];
-            let owners: Vec<String> = if owners.is_array() {
-                owners
-                    .as_array()
-                    .unwrap()
-                    .iter()
-                    .map(|val| val.as_str().unwrap().to_string())
-                    .collect()
-            } else {
-                Vec::default()
-            };
-            AccountsSelector::new(&accounts, &owners)
-        }
-    }
-
-    fn create_transaction_selector_from_config(config: &serde_json::Value) -> TransactionSelector {
-        let transaction_selector = &config["transaction_selector"];
-
-        if transaction_selector.is_null() {
-            TransactionSelector::default()
-        } else {
-            let accounts = &transaction_selector["mentions"];
-            let accounts: Vec<String> = if accounts.is_array() {
-                accounts
-                    .as_array()
-                    .unwrap()
-                    .iter()
-                    .map(|val| val.as_str().unwrap().to_string())
-                    .collect()
-            } else {
-                Vec::default()
-            };
-            TransactionSelector::new(&accounts)
-        }
-    }
-
-    pub fn new() -> Self {
-        Self::default()
-    }
-}
-
-#[no_mangle]
-#[allow(improper_ctypes_definitions)]
-/// # Safety
-///
-/// This function returns the AccountsDbPluginPostgres pointer as trait AccountsDbPlugin.
-pub unsafe extern "C" fn _create_plugin() -> *mut dyn AccountsDbPlugin {
-    let plugin = AccountsDbPluginPostgres::new();
-    let plugin: Box<dyn AccountsDbPlugin> = Box::new(plugin);
-    Box::into_raw(plugin)
-}
-
-#[cfg(test)]
-pub(crate) mod tests {
-    use {super::*, serde_json};
-
-    #[test]
-    fn test_accounts_selector_from_config() {
-        let config = "{\"accounts_selector\" : { \
-           \"owners\" : [\"9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin\"] \
-        }}";
-
-        let config: serde_json::Value = serde_json::from_str(config).unwrap();
-        AccountsDbPluginPostgres::create_accounts_selector_from_config(&config);
-    }
-}

accountsdb-plugin-postgres/src/lib.rs

@@ -1,4 +0,0 @@
-pub mod accounts_selector;
-pub mod accountsdb_plugin_postgres;
-pub mod postgres_client;
-pub mod transaction_selector;

accountsdb-plugin-postgres/src/postgres_client.rs

@@ -1,910 +0,0 @@
-#![allow(clippy::integer_arithmetic)]
-mod postgres_client_transaction;
-
-/// A concurrent implementation for writing accounts into the PostgreSQL in parallel.
-use {
-    crate::accountsdb_plugin_postgres::{
-        AccountsDbPluginPostgresConfig, AccountsDbPluginPostgresError,
-    },
-    chrono::Utc,
-    crossbeam_channel::{bounded, Receiver, RecvTimeoutError, Sender},
-    log::*,
-    postgres::{Client, NoTls, Statement},
-    postgres_client_transaction::LogTransactionRequest,
-    solana_accountsdb_plugin_interface::accountsdb_plugin_interface::{
-        AccountsDbPluginError, ReplicaAccountInfo, SlotStatus,
-    },
-    solana_measure::measure::Measure,
-    solana_metrics::*,
-    solana_sdk::timing::AtomicInterval,
-    std::{
-        sync::{
-            atomic::{AtomicBool, AtomicUsize, Ordering},
-            Arc, Mutex,
-        },
-        thread::{self, sleep, Builder, JoinHandle},
-        time::Duration,
-    },
-    tokio_postgres::types,
-};
-
-/// The maximum asynchronous requests allowed in the channel to avoid excessive
-/// memory usage. The downside -- calls after this threshold is reached can get blocked.
-const MAX_ASYNC_REQUESTS: usize = 40960;
-const DEFAULT_POSTGRES_PORT: u16 = 5432;
-const DEFAULT_THREADS_COUNT: usize = 100;
-const DEFAULT_ACCOUNTS_INSERT_BATCH_SIZE: usize = 10;
-const ACCOUNT_COLUMN_COUNT: usize = 9;
-const DEFAULT_PANIC_ON_DB_ERROR: bool = false;
-
-struct PostgresSqlClientWrapper {
-    client: Client,
-    update_account_stmt: Statement,
-    bulk_account_insert_stmt: Statement,
-    update_slot_with_parent_stmt: Statement,
-    update_slot_without_parent_stmt: Statement,
-    update_transaction_log_stmt: Statement,
-}
-
-pub struct SimplePostgresClient {
-    batch_size: usize,
-    pending_account_updates: Vec<DbAccountInfo>,
-    client: Mutex<PostgresSqlClientWrapper>,
-}
-
-struct PostgresClientWorker {
-    client: SimplePostgresClient,
-    /// Indicating if accounts notification during startup is done.
-    is_startup_done: bool,
-}
-
-impl Eq for DbAccountInfo {}
-
-#[derive(Clone, PartialEq, Debug)]
-pub struct DbAccountInfo {
-    pub pubkey: Vec<u8>,
-    pub lamports: i64,
-    pub owner: Vec<u8>,
-    pub executable: bool,
-    pub rent_epoch: i64,
-    pub data: Vec<u8>,
-    pub slot: i64,
-    pub write_version: i64,
-}
-
-pub(crate) fn abort() -> ! {
-    #[cfg(not(test))]
-    {
-        // standard error is usually redirected to a log file, cry for help on standard output as
-        // well
-        eprintln!("Validator process aborted. The validator log may contain further details");
-        std::process::exit(1);
-    }
-
-    #[cfg(test)]
-    panic!("process::exit(1) is intercepted for friendly test failure...");
-}
-
-impl DbAccountInfo {
-    fn new<T: ReadableAccountInfo>(account: &T, slot: u64) -> DbAccountInfo {
-        let data = account.data().to_vec();
-        Self {
-            pubkey: account.pubkey().to_vec(),
-            lamports: account.lamports() as i64,
-            owner: account.owner().to_vec(),
-            executable: account.executable(),
-            rent_epoch: account.rent_epoch() as i64,
-            data,
-            slot: slot as i64,
-            write_version: account.write_version(),
-        }
-    }
-}
-
-pub trait ReadableAccountInfo: Sized {
-    fn pubkey(&self) -> &[u8];
-    fn owner(&self) -> &[u8];
-    fn lamports(&self) -> i64;
-    fn executable(&self) -> bool;
-    fn rent_epoch(&self) -> i64;
-    fn data(&self) -> &[u8];
-    fn write_version(&self) -> i64;
-}
-
-impl ReadableAccountInfo for DbAccountInfo {
-    fn pubkey(&self) -> &[u8] {
-        &self.pubkey
-    }
-
-    fn owner(&self) -> &[u8] {
-        &self.owner
-    }
-
-    fn lamports(&self) -> i64 {
-        self.lamports
-    }
-
-    fn executable(&self) -> bool {
-        self.executable
-    }
-
-    fn rent_epoch(&self) -> i64 {
-        self.rent_epoch
-    }
-
-    fn data(&self) -> &[u8] {
-        &self.data
-    }
-
-    fn write_version(&self) -> i64 {
-        self.write_version
-    }
-}
-
-impl<'a> ReadableAccountInfo for ReplicaAccountInfo<'a> {
-    fn pubkey(&self) -> &[u8] {
-        self.pubkey
-    }
-
-    fn owner(&self) -> &[u8] {
-        self.owner
-    }
-
-    fn lamports(&self) -> i64 {
-        self.lamports as i64
-    }
-
-    fn executable(&self) -> bool {
-        self.executable
-    }
-
-    fn rent_epoch(&self) -> i64 {
-        self.rent_epoch as i64
-    }
-
-    fn data(&self) -> &[u8] {
-        self.data
-    }
-
-    fn write_version(&self) -> i64 {
-        self.write_version as i64
-    }
-}
-
-pub trait PostgresClient {
-    fn join(&mut self) -> thread::Result<()> {
-        Ok(())
-    }
-
-    fn update_account(
-        &mut self,
-        account: DbAccountInfo,
-        is_startup: bool,
-    ) -> Result<(), AccountsDbPluginError>;
-
-    fn update_slot_status(
-        &mut self,
-        slot: u64,
-        parent: Option<u64>,
-        status: SlotStatus,
-    ) -> Result<(), AccountsDbPluginError>;
-
-    fn notify_end_of_startup(&mut self) -> Result<(), AccountsDbPluginError>;
-
-    fn log_transaction(
-        &mut self,
-        transaction_log_info: LogTransactionRequest,
-    ) -> Result<(), AccountsDbPluginError>;
-}
-
-impl SimplePostgresClient {
-    fn connect_to_db(
-        config: &AccountsDbPluginPostgresConfig,
-    ) -> Result<Client, AccountsDbPluginError> {
-        let port = config.port.unwrap_or(DEFAULT_POSTGRES_PORT);
-
-        let connection_str = if let Some(connection_str) = &config.connection_str {
-            connection_str.clone()
-        } else {
-            if config.host.is_none() || config.user.is_none() {
-                let msg = format!(
-                    "\"connection_str\": {:?}, or \"host\": {:?} \"user\": {:?} must be specified",
-                    config.connection_str, config.host, config.user
-                );
-                return Err(AccountsDbPluginError::Custom(Box::new(
-                    AccountsDbPluginPostgresError::ConfigurationError { msg },
-                )));
-            }
-            format!(
-                "host={} user={} port={}",
-                config.host.as_ref().unwrap(),
-                config.user.as_ref().unwrap(),
-                port
-            )
-        };
-
-        match Client::connect(&connection_str, NoTls) {
-            Err(err) => {
-                let msg = format!(
-                    "Error in connecting to the PostgreSQL database: {:?} connection_str: {:?}",
-                    err, connection_str
-                );
-                error!("{}", msg);
-                Err(AccountsDbPluginError::Custom(Box::new(
-                    AccountsDbPluginPostgresError::DataStoreConnectionError { msg },
-                )))
-            }
-            Ok(client) => Ok(client),
-        }
-    }
-
-    fn build_bulk_account_insert_statement(
-        client: &mut Client,
-        config: &AccountsDbPluginPostgresConfig,
-    ) -> Result<Statement, AccountsDbPluginError> {
-        let batch_size = config
-            .batch_size
-            .unwrap_or(DEFAULT_ACCOUNTS_INSERT_BATCH_SIZE);
-        let mut stmt = String::from("INSERT INTO account AS acct (pubkey, slot, owner, lamports, executable, rent_epoch, data, write_version, updated_on) VALUES");
-        for j in 0..batch_size {
-            let row = j * ACCOUNT_COLUMN_COUNT;
-            let val_str = format!(
-                "(${}, ${}, ${}, ${}, ${}, ${}, ${}, ${}, ${})",
-                row + 1,
-                row + 2,
-                row + 3,
-                row + 4,
-                row + 5,
-                row + 6,
-                row + 7,
-                row + 8,
-                row + 9,
-            );
-
-            if j == 0 {
-                stmt = format!("{} {}", &stmt, val_str);
-            } else {
-                stmt = format!("{}, {}", &stmt, val_str);
-            }
-        }
-
-        let handle_conflict = "ON CONFLICT (pubkey) DO UPDATE SET slot=excluded.slot, owner=excluded.owner, lamports=excluded.lamports, executable=excluded.executable, rent_epoch=excluded.rent_epoch, \
-            data=excluded.data, write_version=excluded.write_version, updated_on=excluded.updated_on WHERE acct.slot < excluded.slot OR (\
-            acct.slot = excluded.slot AND acct.write_version < excluded.write_version)";
-
-        stmt = format!("{} {}", stmt, handle_conflict);
-
-        info!("{}", stmt);
-        let bulk_stmt = client.prepare(&stmt);
-
-        match bulk_stmt {
-            Err(err) => {
-                return Err(AccountsDbPluginError::Custom(Box::new(AccountsDbPluginPostgresError::DataSchemaError {
-                    msg: format!(
-                        "Error in preparing for the accounts update PostgreSQL database: {} host: {:?} user: {:?} config: {:?}",
-                        err, config.host, config.user, config
-                    ),
-                })));
-            }
-            Ok(update_account_stmt) => Ok(update_account_stmt),
-        }
-    }
-
-    fn build_single_account_upsert_statement(
-        client: &mut Client,
-        config: &AccountsDbPluginPostgresConfig,
-    ) -> Result<Statement, AccountsDbPluginError> {
-        let stmt = "INSERT INTO account AS acct (pubkey, slot, owner, lamports, executable, rent_epoch, data, write_version, updated_on) \
-        VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) \
-        ON CONFLICT (pubkey) DO UPDATE SET slot=excluded.slot, owner=excluded.owner, lamports=excluded.lamports, executable=excluded.executable, rent_epoch=excluded.rent_epoch, \
-        data=excluded.data, write_version=excluded.write_version, updated_on=excluded.updated_on  WHERE acct.slot < excluded.slot OR (\
-        acct.slot = excluded.slot AND acct.write_version < excluded.write_version)";
-
-        let stmt = client.prepare(stmt);
-
-        match stmt {
-            Err(err) => {
-                return Err(AccountsDbPluginError::Custom(Box::new(AccountsDbPluginPostgresError::DataSchemaError {
-                    msg: format!(
-                        "Error in preparing for the accounts update PostgreSQL database: {} host: {:?} user: {:?} config: {:?}",
-                        err, config.host, config.user, config
-                    ),
-                })));
-            }
-            Ok(update_account_stmt) => Ok(update_account_stmt),
-        }
-    }
-
-    fn build_slot_upsert_statement_with_parent(
-        client: &mut Client,
-        config: &AccountsDbPluginPostgresConfig,
-    ) -> Result<Statement, AccountsDbPluginError> {
-        let stmt = "INSERT INTO slot (slot, parent, status, updated_on) \
-        VALUES ($1, $2, $3, $4) \
-        ON CONFLICT (slot) DO UPDATE SET parent=excluded.parent, status=excluded.status, updated_on=excluded.updated_on";
-
-        let stmt = client.prepare(stmt);
-
-        match stmt {
-            Err(err) => {
-                return Err(AccountsDbPluginError::Custom(Box::new(AccountsDbPluginPostgresError::DataSchemaError {
-                    msg: format!(
-                        "Error in preparing for the slot update PostgreSQL database: {} host: {:?} user: {:?} config: {:?}",
-                        err, config.host, config.user, config
-                    ),
-                })));
-            }
-            Ok(stmt) => Ok(stmt),
-        }
-    }
-
-    fn build_slot_upsert_statement_without_parent(
-        client: &mut Client,
-        config: &AccountsDbPluginPostgresConfig,
-    ) -> Result<Statement, AccountsDbPluginError> {
-        let stmt = "INSERT INTO slot (slot, status, updated_on) \
-        VALUES ($1, $2, $3) \
-        ON CONFLICT (slot) DO UPDATE SET status=excluded.status, updated_on=excluded.updated_on";
-
-        let stmt = client.prepare(stmt);
-
-        match stmt {
-            Err(err) => {
-                return Err(AccountsDbPluginError::Custom(Box::new(AccountsDbPluginPostgresError::DataSchemaError {
-                    msg: format!(
-                        "Error in preparing for the slot update PostgreSQL database: {} host: {:?} user: {:?} config: {:?}",
-                        err, config.host, config.user, config
-                    ),
-                })));
-            }
-            Ok(stmt) => Ok(stmt),
-        }
-    }
-
-    /// Internal function for updating or inserting a single account
-    fn upsert_account_internal(
-        account: &DbAccountInfo,
-        statement: &Statement,
-        client: &mut Client,
-    ) -> Result<(), AccountsDbPluginError> {
-        let lamports = account.lamports() as i64;
-        let rent_epoch = account.rent_epoch() as i64;
-        let updated_on = Utc::now().naive_utc();
-        let result = client.query(
-            statement,
-            &[
-                &account.pubkey(),
-                &account.slot,
-                &account.owner(),
-                &lamports,
-                &account.executable(),
-                &rent_epoch,
-                &account.data(),
-                &account.write_version(),
-                &updated_on,
-            ],
-        );
-
-        if let Err(err) = result {
-            let msg = format!(
-                "Failed to persist the update of account to the PostgreSQL database. Error: {:?}",
-                err
-            );
-            error!("{}", msg);
-            return Err(AccountsDbPluginError::AccountsUpdateError { msg });
-        }
-
-        Ok(())
-    }
-
-    /// Update or insert a single account
-    fn upsert_account(&mut self, account: &DbAccountInfo) -> Result<(), AccountsDbPluginError> {
-        let client = self.client.get_mut().unwrap();
-        let statement = &client.update_account_stmt;
-        let client = &mut client.client;
-        Self::upsert_account_internal(account, statement, client)
-    }
-
-    /// Insert accounts in batch to reduce network overhead
-    fn insert_accounts_in_batch(
-        &mut self,
-        account: DbAccountInfo,
-    ) -> Result<(), AccountsDbPluginError> {
-        self.pending_account_updates.push(account);
-
-        if self.pending_account_updates.len() == self.batch_size {
-            let mut measure = Measure::start("accountsdb-plugin-postgres-prepare-values");
-
-            let mut values: Vec<&(dyn types::ToSql + Sync)> =
-                Vec::with_capacity(self.batch_size * ACCOUNT_COLUMN_COUNT);
-            let updated_on = Utc::now().naive_utc();
-            for j in 0..self.batch_size {
-                let account = &self.pending_account_updates[j];
-
-                values.push(&account.pubkey);
-                values.push(&account.slot);
-                values.push(&account.owner);
-                values.push(&account.lamports);
-                values.push(&account.executable);
-                values.push(&account.rent_epoch);
-                values.push(&account.data);
-                values.push(&account.write_version);
-                values.push(&updated_on);
-            }
-            measure.stop();
-            inc_new_counter_debug!(
-                "accountsdb-plugin-postgres-prepare-values-us",
-                measure.as_us() as usize,
-                10000,
-                10000
-            );
-
-            let mut measure = Measure::start("accountsdb-plugin-postgres-update-account");
-            let client = self.client.get_mut().unwrap();
-            let result = client
-                .client
-                .query(&client.bulk_account_insert_stmt, &values);
-
-            self.pending_account_updates.clear();
-            if let Err(err) = result {
-                let msg = format!(
-                    "Failed to persist the update of account to the PostgreSQL database. Error: {:?}",
-                    err
-                );
-                error!("{}", msg);
-                return Err(AccountsDbPluginError::AccountsUpdateError { msg });
-            }
-            measure.stop();
-            inc_new_counter_debug!(
-                "accountsdb-plugin-postgres-update-account-us",
-                measure.as_us() as usize,
-                10000,
-                10000
-            );
-            inc_new_counter_debug!(
-                "accountsdb-plugin-postgres-update-account-count",
-                self.batch_size,
-                10000,
-                10000
-            );
-        }
-        Ok(())
-    }
-
-    /// Flush any left over accounts in batch which are not processed in the last batch
-    fn flush_buffered_writes(&mut self) -> Result<(), AccountsDbPluginError> {
-        if self.pending_account_updates.is_empty() {
-            return Ok(());
-        }
-
-        let client = self.client.get_mut().unwrap();
-        let statement = &client.update_account_stmt;
-        let client = &mut client.client;
-
-        for account in self.pending_account_updates.drain(..) {
-            Self::upsert_account_internal(&account, statement, client)?;
-        }
-
-        Ok(())
-    }
-
-    pub fn new(config: &AccountsDbPluginPostgresConfig) -> Result<Self, AccountsDbPluginError> {
-        info!("Creating SimplePostgresClient...");
-        let mut client = Self::connect_to_db(config)?;
-        let bulk_account_insert_stmt =
-            Self::build_bulk_account_insert_statement(&mut client, config)?;
-        let update_account_stmt = Self::build_single_account_upsert_statement(&mut client, config)?;
-
-        let update_slot_with_parent_stmt =
-            Self::build_slot_upsert_statement_with_parent(&mut client, config)?;
-        let update_slot_without_parent_stmt =
-            Self::build_slot_upsert_statement_without_parent(&mut client, config)?;
-        let update_transaction_log_stmt =
-            Self::build_transaction_info_upsert_statement(&mut client, config)?;
-
-        let batch_size = config
-            .batch_size
-            .unwrap_or(DEFAULT_ACCOUNTS_INSERT_BATCH_SIZE);
-        info!("Created SimplePostgresClient.");
-        Ok(Self {
-            batch_size,
-            pending_account_updates: Vec::with_capacity(batch_size),
-            client: Mutex::new(PostgresSqlClientWrapper {
-                client,
-                update_account_stmt,
-                bulk_account_insert_stmt,
-                update_slot_with_parent_stmt,
-                update_slot_without_parent_stmt,
-                update_transaction_log_stmt,
-            }),
-        })
-    }
-}
-
-impl PostgresClient for SimplePostgresClient {
-    fn update_account(
-        &mut self,
-        account: DbAccountInfo,
-        is_startup: bool,
-    ) -> Result<(), AccountsDbPluginError> {
-        trace!(
-            "Updating account {} with owner {} at slot {}",
-            bs58::encode(account.pubkey()).into_string(),
-            bs58::encode(account.owner()).into_string(),
-            account.slot,
-        );
-        if !is_startup {
-            return self.upsert_account(&account);
-        }
-        self.insert_accounts_in_batch(account)
-    }
-
-    fn update_slot_status(
-        &mut self,
-        slot: u64,
-        parent: Option<u64>,
-        status: SlotStatus,
-    ) -> Result<(), AccountsDbPluginError> {
-        info!("Updating slot {:?} at with status {:?}", slot, status);
-
-        let slot = slot as i64; // postgres only supports i64
-        let parent = parent.map(|parent| parent as i64);
-        let updated_on = Utc::now().naive_utc();
-        let status_str = status.as_str();
-        let client = self.client.get_mut().unwrap();
-
-        let result = match parent {
-            Some(parent) => client.client.execute(
-                &client.update_slot_with_parent_stmt,
-                &[&slot, &parent, &status_str, &updated_on],
-            ),
-            None => client.client.execute(
-                &client.update_slot_without_parent_stmt,
-                &[&slot, &status_str, &updated_on],
-            ),
-        };
-
-        match result {
-            Err(err) => {
-                let msg = format!(
-                    "Failed to persist the update of slot to the PostgreSQL database. Error: {:?}",
-                    err
-                );
-                error!("{:?}", msg);
-                return Err(AccountsDbPluginError::SlotStatusUpdateError { msg });
-            }
-            Ok(rows) => {
-                assert_eq!(1, rows, "Expected one rows to be updated a time");
-            }
-        }
-
-        Ok(())
-    }
-
-    fn notify_end_of_startup(&mut self) -> Result<(), AccountsDbPluginError> {
-        self.flush_buffered_writes()
-    }
-
-    fn log_transaction(
-        &mut self,
-        transaction_log_info: LogTransactionRequest,
-    ) -> Result<(), AccountsDbPluginError> {
-        self.log_transaction_impl(transaction_log_info)
-    }
-}
-
-struct UpdateAccountRequest {
-    account: DbAccountInfo,
-    is_startup: bool,
-}
-
-struct UpdateSlotRequest {
-    slot: u64,
-    parent: Option<u64>,
-    slot_status: SlotStatus,
-}
-
-#[warn(clippy::large_enum_variant)]
-enum DbWorkItem {
-    UpdateAccount(Box<UpdateAccountRequest>),
-    UpdateSlot(Box<UpdateSlotRequest>),
-    LogTransaction(Box<LogTransactionRequest>),
-}
-
-impl PostgresClientWorker {
-    fn new(config: AccountsDbPluginPostgresConfig) -> Result<Self, AccountsDbPluginError> {
-        let result = SimplePostgresClient::new(&config);
-        match result {
-            Ok(client) => Ok(PostgresClientWorker {
-                client,
-                is_startup_done: false,
-            }),
-            Err(err) => {
-                error!("Error in creating SimplePostgresClient: {}", err);
-                Err(err)
-            }
-        }
-    }
-
-    fn do_work(
-        &mut self,
-        receiver: Receiver<DbWorkItem>,
-        exit_worker: Arc<AtomicBool>,
-        is_startup_done: Arc<AtomicBool>,
-        startup_done_count: Arc<AtomicUsize>,
-        panic_on_db_errors: bool,
-    ) -> Result<(), AccountsDbPluginError> {
-        while !exit_worker.load(Ordering::Relaxed) {
-            let mut measure = Measure::start("accountsdb-plugin-postgres-worker-recv");
-            let work = receiver.recv_timeout(Duration::from_millis(500));
-            measure.stop();
-            inc_new_counter_debug!(
-                "accountsdb-plugin-postgres-worker-recv-us",
-                measure.as_us() as usize,
-                100000,
-                100000
-            );
-            match work {
-                Ok(work) => match work {
-                    DbWorkItem::UpdateAccount(request) => {
-                        if let Err(err) = self
-                            .client
-                            .update_account(request.account, request.is_startup)
-                        {
-                            error!("Failed to update account: ({})", err);
-                            if panic_on_db_errors {
-                                abort();
-                            }
-                        }
-                    }
-                    DbWorkItem::UpdateSlot(request) => {
-                        if let Err(err) = self.client.update_slot_status(
-                            request.slot,
-                            request.parent,
-                            request.slot_status,
-                        ) {
-                            error!("Failed to update slot: ({})", err);
-                            if panic_on_db_errors {
-                                abort();
-                            }
-                        }
-                    }
-                    DbWorkItem::LogTransaction(transaction_log_info) => {
-                        if let Err(err) = self.client.log_transaction(*transaction_log_info) {
-                            error!("Failed to update transaction: ({})", err);
-                            if panic_on_db_errors {
-                                abort();
-                            }
-                        }
-                    }
-                },
-                Err(err) => match err {
-                    RecvTimeoutError::Timeout => {
-                        if !self.is_startup_done && is_startup_done.load(Ordering::Relaxed) {
-                            if let Err(err) = self.client.notify_end_of_startup() {
-                                error!("Error in notifying end of startup: ({})", err);
-                                if panic_on_db_errors {
-                                    abort();
-                                }
-                            }
-                            self.is_startup_done = true;
-                            startup_done_count.fetch_add(1, Ordering::Relaxed);
-                        }
-
-                        continue;
-                    }
-                    _ => {
-                        error!("Error in receiving the item {:?}", err);
-                        if panic_on_db_errors {
-                            abort();
-                        }
-                        break;
-                    }
-                },
-            }
-        }
-        Ok(())
-    }
-}
-pub struct ParallelPostgresClient {
-    workers: Vec<JoinHandle<Result<(), AccountsDbPluginError>>>,
-    exit_worker: Arc<AtomicBool>,
-    is_startup_done: Arc<AtomicBool>,
-    startup_done_count: Arc<AtomicUsize>,
-    initialized_worker_count: Arc<AtomicUsize>,
-    sender: Sender<DbWorkItem>,
-    last_report: AtomicInterval,
-}
-
-impl ParallelPostgresClient {
-    pub fn new(config: &AccountsDbPluginPostgresConfig) -> Result<Self, AccountsDbPluginError> {
-        info!("Creating ParallelPostgresClient...");
-        let (sender, receiver) = bounded(MAX_ASYNC_REQUESTS);
-        let exit_worker = Arc::new(AtomicBool::new(false));
-        let mut workers = Vec::default();
-        let is_startup_done = Arc::new(AtomicBool::new(false));
-        let startup_done_count = Arc::new(AtomicUsize::new(0));
-        let worker_count = config.threads.unwrap_or(DEFAULT_THREADS_COUNT);
-        let initialized_worker_count = Arc::new(AtomicUsize::new(0));
-        for i in 0..worker_count {
-            let cloned_receiver = receiver.clone();
-            let exit_clone = exit_worker.clone();
-            let is_startup_done_clone = is_startup_done.clone();
-            let startup_done_count_clone = startup_done_count.clone();
-            let initialized_worker_count_clone = initialized_worker_count.clone();
-            let config = config.clone();
-            let worker = Builder::new()
-                .name(format!("worker-{}", i))
-                .spawn(move || -> Result<(), AccountsDbPluginError> {
-                    let panic_on_db_errors = *config
-                        .panic_on_db_errors
-                        .as_ref()
-                        .unwrap_or(&DEFAULT_PANIC_ON_DB_ERROR);
-                    let result = PostgresClientWorker::new(config);
-
-                    match result {
-                        Ok(mut worker) => {
-                            initialized_worker_count_clone.fetch_add(1, Ordering::Relaxed);
-                            worker.do_work(
-                                cloned_receiver,
-                                exit_clone,
-                                is_startup_done_clone,
-                                startup_done_count_clone,
-                                panic_on_db_errors,
-                            )?;
-                            Ok(())
-                        }
-                        Err(err) => {
-                            error!("Error when making connection to database: ({})", err);
-                            if panic_on_db_errors {
-                                abort();
-                            }
-                            Err(err)
-                        }
-                    }
-                })
-                .unwrap();
-
-            workers.push(worker);
-        }
-
-        info!("Created ParallelPostgresClient.");
-        Ok(Self {
-            last_report: AtomicInterval::default(),
-            workers,
-            exit_worker,
-            is_startup_done,
-            startup_done_count,
-            initialized_worker_count,
-            sender,
-        })
-    }
-
-    pub fn join(&mut self) -> thread::Result<()> {
-        self.exit_worker.store(true, Ordering::Relaxed);
-        while !self.workers.is_empty() {
-            let worker = self.workers.pop();
-            if worker.is_none() {
-                break;
-            }
-            let worker = worker.unwrap();
-            let result = worker.join().unwrap();
-            if result.is_err() {
-                error!("The worker thread has failed: {:?}", result);
-            }
-        }
-
-        Ok(())
-    }
-
-    pub fn update_account(
-        &mut self,
-        account: &ReplicaAccountInfo,
-        slot: u64,
-        is_startup: bool,
-    ) -> Result<(), AccountsDbPluginError> {
-        if self.last_report.should_update(30000) {
-            datapoint_debug!(
-                "postgres-plugin-stats",
-                ("message-queue-length", self.sender.len() as i64, i64),
-            );
-        }
-        let mut measure = Measure::start("accountsdb-plugin-posgres-create-work-item");
-        let wrk_item = DbWorkItem::UpdateAccount(Box::new(UpdateAccountRequest {
-            account: DbAccountInfo::new(account, slot),
-            is_startup,
-        }));
-
-        measure.stop();
-
-        inc_new_counter_debug!(
-            "accountsdb-plugin-posgres-create-work-item-us",
-            measure.as_us() as usize,
-            100000,
-            100000
-        );
-
-        let mut measure = Measure::start("accountsdb-plugin-posgres-send-msg");
-
-        if let Err(err) = self.sender.send(wrk_item) {
-            return Err(AccountsDbPluginError::AccountsUpdateError {
-                msg: format!(
-                    "Failed to update the account {:?}, error: {:?}",
-                    bs58::encode(account.pubkey()).into_string(),
-                    err
-                ),
-            });
-        }
-
-        measure.stop();
-        inc_new_counter_debug!(
-            "accountsdb-plugin-posgres-send-msg-us",
-            measure.as_us() as usize,
-            100000,
-            100000
-        );
-
-        Ok(())
-    }
-
-    pub fn update_slot_status(
-        &mut self,
-        slot: u64,
-        parent: Option<u64>,
-        status: SlotStatus,
-    ) -> Result<(), AccountsDbPluginError> {
-        if let Err(err) = self
-            .sender
-            .send(DbWorkItem::UpdateSlot(Box::new(UpdateSlotRequest {
-                slot,
-                parent,
-                slot_status: status,
-            })))
-        {
-            return Err(AccountsDbPluginError::SlotStatusUpdateError {
-                msg: format!("Failed to update the slot {:?}, error: {:?}", slot, err),
-            });
-        }
-        Ok(())
-    }
-
-    pub fn notify_end_of_startup(&mut self) -> Result<(), AccountsDbPluginError> {
-        info!("Notifying the end of startup");
-        // Ensure all items in the queue has been received by the workers
-        while !self.sender.is_empty() {
-            sleep(Duration::from_millis(100));
-        }
-        self.is_startup_done.store(true, Ordering::Relaxed);
-
-        // Wait for all worker threads to be done with flushing
-        while self.startup_done_count.load(Ordering::Relaxed)
-            != self.initialized_worker_count.load(Ordering::Relaxed)
-        {
-            info!(
-                "Startup done count: {}, good worker thread count: {}",
-                self.startup_done_count.load(Ordering::Relaxed),
-                self.initialized_worker_count.load(Ordering::Relaxed)
-            );
-            sleep(Duration::from_millis(100));
-        }
-
-        info!("Done with notifying the end of startup");
-        Ok(())
-    }
-}
-
-pub struct PostgresClientBuilder {}
-
-impl PostgresClientBuilder {
-    pub fn build_pararallel_postgres_client(
-        config: &AccountsDbPluginPostgresConfig,
-    ) -> Result<ParallelPostgresClient, AccountsDbPluginError> {
-        ParallelPostgresClient::new(config)
-    }
-
-    pub fn build_simple_postgres_client(
-        config: &AccountsDbPluginPostgresConfig,
-    ) -> Result<SimplePostgresClient, AccountsDbPluginError> {
-        SimplePostgresClient::new(config)
-    }
-}

accountsdb-plugin-postgres/src/transaction_selector.rs

@@ -1,194 +0,0 @@
-/// The transaction selector is responsible for filtering transactions
-/// in the plugin framework.
-use {log::*, solana_sdk::pubkey::Pubkey, std::collections::HashSet};
-
-pub(crate) struct TransactionSelector {
-    pub mentioned_addresses: HashSet<Vec<u8>>,
-    pub select_all_transactions: bool,
-    pub select_all_vote_transactions: bool,
-}
-
-#[allow(dead_code)]
-impl TransactionSelector {
-    pub fn default() -> Self {
-        Self {
-            mentioned_addresses: HashSet::default(),
-            select_all_transactions: false,
-            select_all_vote_transactions: false,
-        }
-    }
-
-    /// Create a selector based on the mentioned addresses
-    /// To select all transactions use ["*"] or ["all"]
-    /// To select all vote transactions, use ["all_votes"]
-    /// To select transactions mentioning specific addresses use ["<pubkey1>", "<pubkey2>", ...]
-    pub fn new(mentioned_addresses: &[String]) -> Self {
-        info!(
-            "Creating TransactionSelector from addresses: {:?}",
-            mentioned_addresses
-        );
-
-        let select_all_transactions = mentioned_addresses
-            .iter()
-            .any(|key| key == "*" || key == "all");
-        if select_all_transactions {
-            return Self {
-                mentioned_addresses: HashSet::default(),
-                select_all_transactions,
-                select_all_vote_transactions: true,
-            };
-        }
-        let select_all_vote_transactions = mentioned_addresses.iter().any(|key| key == "all_votes");
-        if select_all_vote_transactions {
-            return Self {
-                mentioned_addresses: HashSet::default(),
-                select_all_transactions,
-                select_all_vote_transactions: true,
-            };
-        }
-
-        let mentioned_addresses = mentioned_addresses
-            .iter()
-            .map(|key| bs58::decode(key).into_vec().unwrap())
-            .collect();
-
-        Self {
-            mentioned_addresses,
-            select_all_transactions: false,
-            select_all_vote_transactions: false,
-        }
-    }
-
-    /// Check if a transaction is of interest.
-    pub fn is_transaction_selected(
-        &self,
-        is_vote: bool,
-        mentioned_addresses: Box<dyn Iterator<Item = &Pubkey> + '_>,
-    ) -> bool {
-        if !self.is_enabled() {
-            return false;
-        }
-
-        if self.select_all_transactions || (self.select_all_vote_transactions && is_vote) {
-            return true;
-        }
-        for address in mentioned_addresses {
-            if self.mentioned_addresses.contains(address.as_ref()) {
-                return true;
-            }
-        }
-        false
-    }
-
-    /// Check if any transaction is of interest at all
-    pub fn is_enabled(&self) -> bool {
-        self.select_all_transactions
-            || self.select_all_vote_transactions
-            || !self.mentioned_addresses.is_empty()
-    }
-}
-
-#[cfg(test)]
-pub(crate) mod tests {
-    use super::*;
-
-    #[test]
-    fn test_select_transaction() {
-        let pubkey1 = Pubkey::new_unique();
-        let pubkey2 = Pubkey::new_unique();
-
-        let selector = TransactionSelector::new(&[pubkey1.to_string()]);
-
-        assert!(selector.is_enabled());
-
-        let addresses = [pubkey1];
-
-        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
-
-        let addresses = [pubkey2];
-        assert!(!selector.is_transaction_selected(false, Box::new(addresses.iter())));
-
-        let addresses = [pubkey1, pubkey2];
-        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
-    }
-
-    #[test]
-    fn test_select_all_transaction_using_wildcard() {
-        let pubkey1 = Pubkey::new_unique();
-        let pubkey2 = Pubkey::new_unique();
-
-        let selector = TransactionSelector::new(&["*".to_string()]);
-
-        assert!(selector.is_enabled());
-
-        let addresses = [pubkey1];
-
-        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
-
-        let addresses = [pubkey2];
-        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
-
-        let addresses = [pubkey1, pubkey2];
-        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
-    }
-
-    #[test]
-    fn test_select_all_transaction_all() {
-        let pubkey1 = Pubkey::new_unique();
-        let pubkey2 = Pubkey::new_unique();
-
-        let selector = TransactionSelector::new(&["all".to_string()]);
-
-        assert!(selector.is_enabled());
-
-        let addresses = [pubkey1];
-
-        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
-
-        let addresses = [pubkey2];
-        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
-
-        let addresses = [pubkey1, pubkey2];
-        assert!(selector.is_transaction_selected(false, Box::new(addresses.iter())));
-    }
-
-    #[test]
-    fn test_select_all_vote_transaction() {
-        let pubkey1 = Pubkey::new_unique();
-        let pubkey2 = Pubkey::new_unique();
-
-        let selector = TransactionSelector::new(&["all_votes".to_string()]);
-
-        assert!(selector.is_enabled());
-
-        let addresses = [pubkey1];
-
-        assert!(!selector.is_transaction_selected(false, Box::new(addresses.iter())));
-
-        let addresses = [pubkey2];
-        assert!(selector.is_transaction_selected(true, Box::new(addresses.iter())));
-
-        let addresses = [pubkey1, pubkey2];
-        assert!(selector.is_transaction_selected(true, Box::new(addresses.iter())));
-    }
-
-    #[test]
-    fn test_select_no_transaction() {
-        let pubkey1 = Pubkey::new_unique();
-        let pubkey2 = Pubkey::new_unique();
-
-        let selector = TransactionSelector::new(&[]);
-
-        assert!(!selector.is_enabled());
-
-        let addresses = [pubkey1];
-
-        assert!(!selector.is_transaction_selected(false, Box::new(addresses.iter())));
-
-        let addresses = [pubkey2];
-        assert!(!selector.is_transaction_selected(true, Box::new(addresses.iter())));
-
-        let addresses = [pubkey1, pubkey2];
-        assert!(!selector.is_transaction_selected(true, Box::new(addresses.iter())));
-    }
-}

banking-bench/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2021"
+edition = "2018"
 name = "solana-banking-bench"
-version = "1.10.0"
+version = "1.6.28"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,21 +10,20 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-crossbeam-channel = "0.5"
-log = "0.4.14"
+crossbeam-channel = "0.4"
+log = "0.4.11"
 rand = "0.7.0"
-rayon = "1.5.1"
-solana-core = { path = "../core", version = "=1.10.0" }
-solana-gossip = { path = "../gossip", version = "=1.10.0" }
-solana-ledger = { path = "../ledger", version = "=1.10.0" }
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-perf = { path = "../perf", version = "=1.10.0" }
-solana-poh = { path = "../poh", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-streamer = { path = "../streamer", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
+rayon = "1.5.0"
+solana-core = { path = "../core", version = "=1.6.28" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.28" }
+solana-streamer = { path = "../streamer", version = "=1.6.28" }
+solana-perf = { path = "../perf", version = "=1.6.28" }
+solana-ledger = { path = "../ledger", version = "=1.6.28" }
+solana-logger = { path = "../logger", version = "=1.6.28" }
+solana-runtime = { path = "../runtime", version = "=1.6.28" }
+solana-measure = { path = "../measure", version = "=1.6.28" }
+solana-sdk = { path = "../sdk", version = "=1.6.28" }
+solana-version = { path = "../version", version = "=1.6.28" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

banking-bench/src/main.rs

@@ -1,37 +1,38 @@
 #![allow(clippy::integer_arithmetic)]
-use {
-    clap::{crate_description, crate_name, value_t, App, Arg},
-    crossbeam_channel::unbounded,
-    log::*,
-    rand::{thread_rng, Rng},
-    rayon::prelude::*,
-    solana_core::banking_stage::BankingStage,
-    solana_gossip::cluster_info::{ClusterInfo, Node},
-    solana_ledger::{
+use clap::{crate_description, crate_name, value_t, App, Arg};
+use crossbeam_channel::unbounded;
+use log::*;
+use rand::{thread_rng, Rng};
+use rayon::prelude::*;
+use solana_core::{
+    banking_stage::{create_test_recorder, BankingStage},
+    cluster_info::ClusterInfo,
+    cluster_info::Node,
+    poh_recorder::PohRecorder,
+    poh_recorder::WorkingBankEntry,
+};
+use solana_ledger::{
     blockstore::Blockstore,
     genesis_utils::{create_genesis_config, GenesisConfigInfo},
     get_tmp_ledger_path,
-    },
-    solana_measure::measure::Measure,
-    solana_perf::packet::to_packet_batches,
-    solana_poh::poh_recorder::{create_test_recorder, PohRecorder, WorkingBankEntry},
-    solana_runtime::{
+};
+use solana_measure::measure::Measure;
+use solana_perf::packet::to_packets_chunked;
+use solana_runtime::{
     accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks,
-        cost_model::CostModel,
-    },
-    solana_sdk::{
+};
+use solana_sdk::{
     hash::Hash,
-        signature::{Keypair, Signature},
+    signature::Keypair,
+    signature::Signature,
     system_transaction,
     timing::{duration_as_us, timestamp},
     transaction::Transaction,
-    },
-    solana_streamer::socket::SocketAddrSpace,
-    std::{
-        sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex, RwLock},
+};
+use std::{
+    sync::{atomic::Ordering, mpsc::Receiver, Arc, Mutex},
     thread::sleep,
     time::{Duration, Instant},
-    },
 };
 
 fn check_txs(
@@ -77,7 +78,7 @@ fn make_accounts_txs(
         .into_par_iter()
         .map(|_| {
             let mut new = dummy.clone();
-            let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
+            let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
             if !same_payer {
                 new.message.account_keys[0] = solana_sdk::pubkey::new_rand();
             }
@@ -168,9 +169,8 @@ fn main() {
 
     let (verified_sender, verified_receiver) = unbounded();
     let (vote_sender, vote_receiver) = unbounded();
-    let (tpu_vote_sender, tpu_vote_receiver) = unbounded();
     let (replay_vote_sender, _replay_vote_receiver) = unbounded();
-    let bank0 = Bank::new_for_benches(&genesis_config);
+    let bank0 = Bank::new(&genesis_config);
     let mut bank_forks = BankForks::new(bank0);
     let mut bank = bank_forks.working_bank();
 
@@ -189,7 +189,7 @@ fn main() {
             genesis_config.hash(),
         );
         // Ignore any pesky duplicate signature errors in the case we are using single-payer
-        let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
+        let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
         fund.signatures = vec![Signature::new(&sig[0..64])];
         let x = bank.process_transaction(&fund);
         x.unwrap();
@@ -199,20 +199,19 @@ fn main() {
     if !skip_sanity {
         //sanity check, make sure all the transactions can execute sequentially
         transactions.iter().for_each(|tx| {
-            let res = bank.process_transaction(tx);
+            let res = bank.process_transaction(&tx);
             assert!(res.is_ok(), "sanity test transactions error: {:?}", res);
         });
         bank.clear_signatures();
         //sanity check, make sure all the transactions can execute in parallel
-
-        let res = bank.process_transactions(transactions.iter());
+        let res = bank.process_transactions(&transactions);
         for r in res {
             assert!(r.is_ok(), "sanity parallel execution error: {:?}", r);
         }
         bank.clear_signatures();
     }
 
-    let mut verified: Vec<_> = to_packet_batches(&transactions, packets_per_chunk);
+    let mut verified: Vec<_> = to_packets_chunked(&transactions, packets_per_chunk);
     let ledger_path = get_tmp_ledger_path!();
     {
         let blockstore = Arc::new(
@@ -220,21 +219,15 @@ fn main() {
         );
         let (exit, poh_recorder, poh_service, signal_receiver) =
             create_test_recorder(&bank, &blockstore, None);
-        let cluster_info = ClusterInfo::new(
-            Node::new_localhost().info,
-            Arc::new(Keypair::new()),
-            SocketAddrSpace::Unspecified,
-        );
+        let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
         let cluster_info = Arc::new(cluster_info);
         let banking_stage = BankingStage::new(
             &cluster_info,
             &poh_recorder,
             verified_receiver,
-            tpu_vote_receiver,
             vote_receiver,
             None,
             replay_vote_sender,
-            Arc::new(RwLock::new(CostModel::default())),
         );
         poh_recorder.lock().unwrap().set_bank(&bank);
 
@@ -314,10 +307,11 @@ fn main() {
                 tx_total_us += duration_as_us(&now.elapsed());
 
                 let mut poh_time = Measure::start("poh_time");
-                poh_recorder
-                    .lock()
-                    .unwrap()
-                    .reset(bank.clone(), Some((bank.slot(), bank.slot() + 1)));
+                poh_recorder.lock().unwrap().reset(
+                    bank.last_blockhash(),
+                    bank.slot(),
+                    Some((bank.slot(), bank.slot() + 1)),
+                );
                 poh_time.stop();
 
                 let mut new_bank_time = Measure::start("new_bank");
@@ -361,10 +355,10 @@ fn main() {
             if bank.slot() > 0 && bank.slot() % 16 == 0 {
                 for tx in transactions.iter_mut() {
                     tx.message.recent_blockhash = bank.last_blockhash();
-                    let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen::<u8>()).collect();
+                    let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
                     tx.signatures[0] = Signature::new(&sig[0..64]);
                 }
-                verified = to_packet_batches(&transactions.clone(), packets_per_chunk);
+                verified = to_packets_chunked(&transactions.clone(), packets_per_chunk);
             }
 
             start += chunk_len;
@@ -386,7 +380,6 @@ fn main() {
         );
 
         drop(verified_sender);
-        drop(tpu_vote_sender);
         drop(vote_sender);
         exit.store(true, Ordering::Relaxed);
         banking_stage.join().unwrap();

banks-client/Cargo.toml

@@ -1,27 +1,30 @@
 [package]
 name = "solana-banks-client"
-version = "1.10.0"
+version = "1.6.28"
 description = "Solana banks client"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-banks-client"
-edition = "2021"
+edition = "2018"
 
 [dependencies]
-borsh = "0.9.1"
+bincode = "1.3.1"
+borsh = "0.9.0"
+borsh-derive = "0.9.0"
 futures = "0.3"
-solana-banks-interface = { path = "../banks-interface", version = "=1.10.0" }
-solana-program = { path = "../sdk/program", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-tarpc = { version = "0.26.2", features = ["full"] }
+mio = "0.7.6"
+solana-banks-interface = { path = "../banks-interface", version = "=1.6.28" }
+solana-program = { path = "../sdk/program", version = "=1.6.28" }
+solana-sdk = { path = "../sdk", version = "=1.6.28" }
+tarpc = { version = "0.24.1", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 
 [dev-dependencies]
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-banks-server = { path = "../banks-server", version = "=1.10.0" }
+solana-runtime = { path = "../runtime", version = "=1.6.28" }
+solana-banks-server = { path = "../banks-server", version = "=1.6.28" }
 
 [lib]
 crate-type = ["lib"]

banks-client/src/lib.rs

@@ -5,33 +5,31 @@
 //! but they are undocumented, may change over time, and are generally more
 //! cumbersome to use.
 
+use borsh::BorshDeserialize;
+use futures::{future::join_all, Future, FutureExt};
 pub use solana_banks_interface::{BanksClient as TarpcClient, TransactionStatus};
-use {
-    borsh::BorshDeserialize,
-    futures::{future::join_all, Future, FutureExt},
-    solana_banks_interface::{BanksRequest, BanksResponse},
-    solana_program::{
+use solana_banks_interface::{BanksRequest, BanksResponse};
+use solana_program::{
     clock::Slot, fee_calculator::FeeCalculator, hash::Hash, program_pack::Pack, pubkey::Pubkey,
-        rent::Rent, sysvar::Sysvar,
-    },
-    solana_sdk::{
+    rent::Rent, sysvar,
+};
+use solana_sdk::{
     account::{from_account, Account},
     commitment_config::CommitmentLevel,
-        message::Message,
     signature::Signature,
     transaction::{self, Transaction},
     transport,
-    },
-    std::io::{self, Error, ErrorKind},
-    tarpc::{
-        client::{self, NewClient, RequestDispatch},
-        context::{self, Context},
-        serde_transport::tcp,
-        ClientMessage, Response, Transport,
-    },
-    tokio::{net::ToSocketAddrs, time::Duration},
-    tokio_serde::formats::Bincode,
 };
+use std::io::{self, Error, ErrorKind};
+use tarpc::{
+    client::{self, channel::RequestDispatch, NewClient},
+    context::{self, Context},
+    rpc::{ClientMessage, Response},
+    serde_transport::tcp,
+    Transport,
+};
+use tokio::{net::ToSocketAddrs, time::Duration};
+use tokio_serde::formats::Bincode;
 
 // This exists only for backward compatibility
 pub trait BanksClientExt {}
@@ -61,16 +59,11 @@ impl BanksClient {
         self.inner.send_transaction_with_context(ctx, transaction)
     }
 
-    #[deprecated(
-        since = "1.9.0",
-        note = "Please use `get_fee_for_message` or `is_blockhash_valid` instead"
-    )]
     pub fn get_fees_with_commitment_and_context(
         &mut self,
         ctx: Context,
         commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, u64)>> + '_ {
-        #[allow(deprecated)]
+    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, Slot)>> + '_ {
         self.inner
             .get_fees_with_commitment_and_context(ctx, commitment)
     }
@@ -92,14 +85,6 @@ impl BanksClient {
         self.inner.get_slot_with_context(ctx, commitment)
     }
 
-    pub fn get_block_height_with_context(
-        &mut self,
-        ctx: Context,
-        commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Slot>> + '_ {
-        self.inner.get_block_height_with_context(ctx, commitment)
-    }
-
     pub fn process_transaction_with_commitment_and_context(
         &mut self,
         ctx: Context,
@@ -133,38 +118,28 @@ impl BanksClient {
     /// Return the fee parameters associated with a recent, rooted blockhash. The cluster
     /// will use the transaction's blockhash to look up these same fee parameters and
     /// use them to calculate the transaction fee.
-    #[deprecated(
-        since = "1.9.0",
-        note = "Please use `get_fee_for_message` or `is_blockhash_valid` instead"
-    )]
     pub fn get_fees(
         &mut self,
-    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, u64)>> + '_ {
-        #[allow(deprecated)]
+    ) -> impl Future<Output = io::Result<(FeeCalculator, Hash, Slot)>> + '_ {
         self.get_fees_with_commitment_and_context(context::current(), CommitmentLevel::default())
     }
 
-    /// Return the cluster Sysvar
-    pub fn get_sysvar<T: Sysvar>(&mut self) -> impl Future<Output = io::Result<T>> + '_ {
-        self.get_account(T::id()).map(|result| {
-            let sysvar = result?
-                .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Sysvar not present"))?;
-            from_account::<T, _>(&sysvar)
-                .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Failed to deserialize sysvar"))
-        })
-    }
-
     /// Return the cluster rent
     pub fn get_rent(&mut self) -> impl Future<Output = io::Result<Rent>> + '_ {
-        self.get_sysvar::<Rent>()
+        self.get_account(sysvar::rent::id()).map(|result| {
+            let rent_sysvar = result?
+                .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "Rent sysvar not present"))?;
+            from_account::<Rent, _>(&rent_sysvar).ok_or_else(|| {
+                io::Error::new(io::ErrorKind::Other, "Failed to deserialize Rent sysvar")
+            })
+        })
     }
 
     /// Return a recent, rooted blockhash from the server. The cluster will only accept
     /// transactions with a blockhash that has not yet expired. Use the `get_fees`
     /// method to get both a blockhash and the blockhash's last valid slot.
-    #[deprecated(since = "1.9.0", note = "Please use `get_latest_blockhash` instead")]
     pub fn get_recent_blockhash(&mut self) -> impl Future<Output = io::Result<Hash>> + '_ {
-        self.get_latest_blockhash()
+        self.get_fees().map(|result| Ok(result?.1))
     }
 
     /// Send a transaction and return after the transaction has been rejected or
@@ -217,18 +192,12 @@ impl BanksClient {
         self.process_transactions_with_commitment(transactions, CommitmentLevel::default())
     }
 
-    /// Return the most recent rooted slot. All transactions at or below this slot
-    /// are said to be finalized. The cluster will not fork to a higher slot.
+    /// Return the most recent rooted slot height. All transactions at or below this height
+    /// are said to be finalized. The cluster will not fork to a higher slot height.
     pub fn get_root_slot(&mut self) -> impl Future<Output = io::Result<Slot>> + '_ {
         self.get_slot_with_context(context::current(), CommitmentLevel::default())
     }
 
-    /// Return the most recent rooted block height. All transactions at or below this height
-    /// are said to be finalized. The cluster will not fork to a higher block height.
-    pub fn get_root_block_height(&mut self) -> impl Future<Output = io::Result<Slot>> + '_ {
-        self.get_block_height_with_context(context::current(), CommitmentLevel::default())
-    }
-
     /// Return the account at the given address at the slot corresponding to the given
     /// commitment level. If the account is not found, None is returned.
     pub fn get_account_with_commitment(
@@ -324,41 +293,6 @@ impl BanksClient {
         // Convert Vec<Result<_, _>> to Result<Vec<_>>
         statuses.into_iter().collect()
     }
-
-    pub fn get_latest_blockhash(&mut self) -> impl Future<Output = io::Result<Hash>> + '_ {
-        self.get_latest_blockhash_with_commitment(CommitmentLevel::default())
-            .map(|result| {
-                result?
-                    .map(|x| x.0)
-                    .ok_or_else(|| io::Error::new(io::ErrorKind::Other, "account not found"))
-            })
-    }
-
-    pub fn get_latest_blockhash_with_commitment(
-        &mut self,
-        commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Option<(Hash, u64)>>> + '_ {
-        self.get_latest_blockhash_with_commitment_and_context(context::current(), commitment)
-    }
-
-    pub fn get_latest_blockhash_with_commitment_and_context(
-        &mut self,
-        ctx: Context,
-        commitment: CommitmentLevel,
-    ) -> impl Future<Output = io::Result<Option<(Hash, u64)>>> + '_ {
-        self.inner
-            .get_latest_blockhash_with_commitment_and_context(ctx, commitment)
-    }
-
-    pub fn get_fee_for_message_with_commitment_and_context(
-        &mut self,
-        ctx: Context,
-        commitment: CommitmentLevel,
-        message: Message,
-    ) -> impl Future<Output = io::Result<Option<u64>>> + '_ {
-        self.inner
-            .get_fee_for_message_with_commitment_and_context(ctx, commitment, message)
-    }
 }
 
 pub async fn start_client<C>(transport: C) -> io::Result<BanksClient>
@@ -366,31 +300,29 @@ where
     C: Transport<ClientMessage<BanksRequest>, Response<BanksResponse>> + Send + 'static,
 {
     Ok(BanksClient {
-        inner: TarpcClient::new(client::Config::default(), transport).spawn(),
+        inner: TarpcClient::new(client::Config::default(), transport).spawn()?,
     })
 }
 
 pub async fn start_tcp_client<T: ToSocketAddrs>(addr: T) -> io::Result<BanksClient> {
     let transport = tcp::connect(addr, Bincode::default).await?;
     Ok(BanksClient {
-        inner: TarpcClient::new(client::Config::default(), transport).spawn(),
+        inner: TarpcClient::new(client::Config::default(), transport).spawn()?,
     })
 }
 
 #[cfg(test)]
 mod tests {
-    use {
-        super::*,
-        solana_banks_server::banks_server::start_local_server,
-        solana_runtime::{
+    use super::*;
+    use solana_banks_server::banks_server::start_local_server;
+    use solana_runtime::{
         bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache,
         genesis_utils::create_genesis_config,
-        },
-        solana_sdk::{message::Message, signature::Signer, system_instruction},
-        std::sync::{Arc, RwLock},
-        tarpc::transport,
-        tokio::{runtime::Runtime, time::sleep},
     };
+    use solana_sdk::{message::Message, signature::Signer, system_instruction};
+    use std::sync::{Arc, RwLock};
+    use tarpc::transport;
+    use tokio::{runtime::Runtime, time::sleep};
 
     #[test]
     fn test_banks_client_new() {
@@ -405,7 +337,7 @@ mod tests {
         // `runtime.block_on()` just once, to run all the async code.
 
         let genesis = create_genesis_config(10);
-        let bank = Bank::new_for_tests(&genesis.genesis_config);
+        let bank = Bank::new(&genesis.genesis_config);
         let slot = bank.slot();
         let block_commitment_cache = Arc::new(RwLock::new(
             BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
@@ -418,12 +350,10 @@ mod tests {
         let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport =
-                start_local_server(bank_forks, block_commitment_cache, Duration::from_millis(1))
-                    .await;
+            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
             let mut banks_client = start_client(client_transport).await?;
 
-            let recent_blockhash = banks_client.get_latest_blockhash().await?;
+            let recent_blockhash = banks_client.get_recent_blockhash().await?;
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
             banks_client.process_transaction(transaction).await.unwrap();
             assert_eq!(banks_client.get_balance(bob_pubkey).await?, 1);
@@ -438,7 +368,7 @@ mod tests {
         // server-side functionality is available to the client.
 
         let genesis = create_genesis_config(10);
-        let bank = Bank::new_for_tests(&genesis.genesis_config);
+        let bank = Bank::new(&genesis.genesis_config);
         let slot = bank.slot();
         let block_commitment_cache = Arc::new(RwLock::new(
             BlockCommitmentCache::new_for_tests_with_slots(slot, slot),
@@ -447,18 +377,13 @@ mod tests {
 
         let mint_pubkey = &genesis.mint_keypair.pubkey();
         let bob_pubkey = solana_sdk::pubkey::new_rand();
-        let instruction = system_instruction::transfer(mint_pubkey, &bob_pubkey, 1);
-        let message = Message::new(&[instruction], Some(mint_pubkey));
+        let instruction = system_instruction::transfer(&mint_pubkey, &bob_pubkey, 1);
+        let message = Message::new(&[instruction], Some(&mint_pubkey));
 
         Runtime::new()?.block_on(async {
-            let client_transport =
-                start_local_server(bank_forks, block_commitment_cache, Duration::from_millis(1))
-                    .await;
+            let client_transport = start_local_server(bank_forks, block_commitment_cache).await;
             let mut banks_client = start_client(client_transport).await?;
-            let (recent_blockhash, last_valid_block_height) = banks_client
-                .get_latest_blockhash_with_commitment(CommitmentLevel::default())
-                .await?
-                .unwrap();
+            let (_, recent_blockhash, last_valid_slot) = banks_client.get_fees().await?;
             let transaction = Transaction::new(&[&genesis.mint_keypair], message, recent_blockhash);
             let signature = transaction.signatures[0];
             banks_client.send_transaction(transaction).await?;
@@ -466,8 +391,8 @@ mod tests {
             let mut status = banks_client.get_transaction_status(signature).await?;
 
             while status.is_none() {
-                let root_block_height = banks_client.get_root_block_height().await?;
-                if root_block_height > last_valid_block_height {
+                let root_slot = banks_client.get_root_slot().await?;
+                if root_slot > last_valid_slot {
                     break;
                 }
                 sleep(Duration::from_millis(100)).await;

banks-interface/Cargo.toml

@@ -1,18 +1,22 @@
 [package]
 name = "solana-banks-interface"
-version = "1.10.0"
+version = "1.6.28"
 description = "Solana banks RPC interface"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-banks-interface"
-edition = "2021"
+edition = "2018"
 
 [dependencies]
-serde = { version = "1.0.131", features = ["derive"] }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-tarpc = { version = "0.26.2", features = ["full"] }
+mio = "0.7.6"
+serde = { version = "1.0.122", features = ["derive"] }
+solana-sdk = { path = "../sdk", version = "=1.6.28" }
+tarpc = { version = "0.24.1", features = ["full"] }
+
+[dev-dependencies]
+tokio = { version = "1", features = ["full"] }
 
 [lib]
 crate-type = ["lib"]

banks-interface/src/lib.rs

@@ -1,18 +1,13 @@
-#![allow(deprecated)]
-
-use {
-    serde::{Deserialize, Serialize},
-    solana_sdk::{
+use serde::{Deserialize, Serialize};
+use solana_sdk::{
     account::Account,
     clock::Slot,
     commitment_config::CommitmentLevel,
     fee_calculator::FeeCalculator,
     hash::Hash,
-        message::Message,
     pubkey::Pubkey,
     signature::Signature,
     transaction::{self, Transaction, TransactionError},
-    },
 };
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -33,17 +28,12 @@ pub struct TransactionStatus {
 #[tarpc::service]
 pub trait Banks {
     async fn send_transaction_with_context(transaction: Transaction);
-    #[deprecated(
-        since = "1.9.0",
-        note = "Please use `get_fee_for_message_with_commitment_and_context` instead"
-    )]
     async fn get_fees_with_commitment_and_context(
         commitment: CommitmentLevel,
     ) -> (FeeCalculator, Hash, Slot);
     async fn get_transaction_status_with_context(signature: Signature)
         -> Option<TransactionStatus>;
     async fn get_slot_with_context(commitment: CommitmentLevel) -> Slot;
-    async fn get_block_height_with_context(commitment: CommitmentLevel) -> u64;
     async fn process_transaction_with_commitment_and_context(
         transaction: Transaction,
         commitment: CommitmentLevel,
@@ -52,22 +42,12 @@ pub trait Banks {
         address: Pubkey,
         commitment: CommitmentLevel,
     ) -> Option<Account>;
-    async fn get_latest_blockhash_with_context() -> Hash;
-    async fn get_latest_blockhash_with_commitment_and_context(
-        commitment: CommitmentLevel,
-    ) -> Option<(Hash, u64)>;
-    async fn get_fee_for_message_with_commitment_and_context(
-        commitment: CommitmentLevel,
-        message: Message,
-    ) -> Option<u64>;
 }
 
 #[cfg(test)]
 mod tests {
-    use {
-        super::*,
-        tarpc::{client, transport},
-    };
+    use super::*;
+    use tarpc::{client, transport};
 
     #[test]
     fn test_banks_client_new() {

banks-server/Cargo.toml

@@ -1,22 +1,24 @@
 [package]
 name = "solana-banks-server"
-version = "1.10.0"
+version = "1.6.28"
 description = "Solana banks server"
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 documentation = "https://docs.rs/solana-banks-server"
-edition = "2021"
+edition = "2018"
 
 [dependencies]
-bincode = "1.3.3"
+bincode = "1.3.1"
 futures = "0.3"
-solana-banks-interface = { path = "../banks-interface", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-send-transaction-service = { path = "../send-transaction-service", version = "=1.10.0" }
-tarpc = { version = "0.26.2", features = ["full"] }
+log = "0.4.11"
+mio = "0.7.6"
+solana-banks-interface = { path = "../banks-interface", version = "=1.6.28" }
+solana-runtime = { path = "../runtime", version = "=1.6.28" }
+solana-sdk = { path = "../sdk", version = "=1.6.28" }
+solana-metrics = { path = "../metrics", version = "=1.6.28" }
+tarpc = { version = "0.24.1", features = ["full"] }
 tokio = { version = "1", features = ["full"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 tokio-stream = "0.1"

banks-server/src/banks_server.rs

@@ -1,28 +1,24 @@
-use {
-    bincode::{deserialize, serialize},
-    futures::{future, prelude::stream::StreamExt},
-    solana_banks_interface::{
+use crate::send_transaction_service::{SendTransactionService, TransactionInfo};
+use bincode::{deserialize, serialize};
+use futures::{
+    future,
+    prelude::stream::{self, StreamExt},
+};
+use solana_banks_interface::{
     Banks, BanksRequest, BanksResponse, TransactionConfirmationStatus, TransactionStatus,
-    },
-    solana_runtime::{bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache},
-    solana_sdk::{
+};
+use solana_runtime::{bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache};
+use solana_sdk::{
     account::Account,
     clock::Slot,
     commitment_config::CommitmentLevel,
-        feature_set::FeatureSet,
     fee_calculator::FeeCalculator,
     hash::Hash,
-        message::{Message, SanitizedMessage},
     pubkey::Pubkey,
     signature::Signature,
     transaction::{self, Transaction},
-    },
-    solana_send_transaction_service::{
-        send_transaction_service::{SendTransactionService, TransactionInfo},
-        tpu_info::NullTpuInfo,
-    },
-    std::{
-        convert::TryFrom,
+};
+use std::{
     io,
     net::{Ipv4Addr, SocketAddr},
     sync::{
@@ -31,24 +27,22 @@ use {
     },
     thread::Builder,
     time::Duration,
-    },
-    tarpc::{
-        context::Context,
-        serde_transport::tcp,
-        server::{self, Channel, Incoming},
-        transport::{self, channel::UnboundedChannel},
-        ClientMessage, Response,
-    },
-    tokio::time::sleep,
-    tokio_serde::formats::Bincode,
 };
+use tarpc::{
+    context::Context,
+    rpc::{transport::channel::UnboundedChannel, ClientMessage, Response},
+    serde_transport::tcp,
+    server::{self, Channel, Handler},
+    transport,
+};
+use tokio::time::sleep;
+use tokio_serde::formats::Bincode;
 
 #[derive(Clone)]
 struct BanksServer {
     bank_forks: Arc<RwLock<BankForks>>,
     block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
     transaction_sender: Sender<TransactionInfo>,
-    poll_signature_status_sleep_duration: Duration,
 }
 
 impl BanksServer {
@@ -60,13 +54,11 @@ impl BanksServer {
         bank_forks: Arc<RwLock<BankForks>>,
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
         transaction_sender: Sender<TransactionInfo>,
-        poll_signature_status_sleep_duration: Duration,
     ) -> Self {
         Self {
             bank_forks,
             block_commitment_cache,
             transaction_sender,
-            poll_signature_status_sleep_duration,
         }
     }
 
@@ -81,7 +73,7 @@ impl BanksServer {
                 .map(|info| deserialize(&info.wire_transaction).unwrap())
                 .collect();
             let bank = bank_forks.read().unwrap().working_bank();
-            let _ = bank.try_process_transactions(transactions.iter());
+            let _ = bank.process_transactions(&transactions);
         }
     }
 
@@ -89,7 +81,6 @@ impl BanksServer {
     fn new_loopback(
         bank_forks: Arc<RwLock<BankForks>>,
         block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
-        poll_signature_status_sleep_duration: Duration,
     ) -> Self {
         let (transaction_sender, transaction_receiver) = channel();
         let bank = bank_forks.read().unwrap().working_bank();
@@ -104,12 +95,7 @@ impl BanksServer {
             .name("solana-bank-forks-client".to_string())
             .spawn(move || Self::run(server_bank_forks, transaction_receiver))
             .unwrap();
-        Self::new(
-            bank_forks,
-            block_commitment_cache,
-            transaction_sender,
-            poll_signature_status_sleep_duration,
-        )
+        Self::new(bank_forks, block_commitment_cache, transaction_sender)
     }
 
     fn slot(&self, commitment: CommitmentLevel) -> Slot {
@@ -127,16 +113,16 @@ impl BanksServer {
         self,
         signature: &Signature,
         blockhash: &Hash,
-        last_valid_block_height: u64,
+        last_valid_slot: Slot,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
         let mut status = self
             .bank(commitment)
             .get_signature_status_with_blockhash(signature, blockhash);
         while status.is_none() {
-            sleep(self.poll_signature_status_sleep_duration).await;
+            sleep(Duration::from_millis(200)).await;
             let bank = self.bank(commitment);
-            if bank.block_height() > last_valid_block_height {
+            if bank.slot() > last_valid_slot {
                 break;
             }
             status = bank.get_signature_status_with_blockhash(signature, blockhash);
@@ -147,11 +133,11 @@ impl BanksServer {
 
 fn verify_transaction(
     transaction: &Transaction,
-    feature_set: &Arc<FeatureSet>,
+    libsecp256k1_0_5_upgrade_enabled: bool,
 ) -> transaction::Result<()> {
     if let Err(err) = transaction.verify() {
         Err(err)
-    } else if let Err(err) = transaction.verify_precompiles(feature_set) {
+    } else if let Err(err) = transaction.verify_precompiles(libsecp256k1_0_5_upgrade_enabled) {
         Err(err)
     } else {
         Ok(())
@@ -162,21 +148,16 @@ fn verify_transaction(
 impl Banks for BanksServer {
     async fn send_transaction_with_context(self, _: Context, transaction: Transaction) {
         let blockhash = &transaction.message.recent_blockhash;
-        let last_valid_block_height = self
+        let last_valid_slot = self
             .bank_forks
             .read()
             .unwrap()
             .root_bank()
-            .get_blockhash_last_valid_block_height(blockhash)
+            .get_blockhash_last_valid_slot(&blockhash)
             .unwrap();
         let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
-        let info = TransactionInfo::new(
-            signature,
-            serialize(&transaction).unwrap(),
-            last_valid_block_height,
-            None,
-            None,
-        );
+        let info =
+            TransactionInfo::new(signature, serialize(&transaction).unwrap(), last_valid_slot);
         self.transaction_sender.send(info).unwrap();
     }
 
@@ -184,18 +165,11 @@ impl Banks for BanksServer {
         self,
         _: Context,
         commitment: CommitmentLevel,
-    ) -> (FeeCalculator, Hash, u64) {
+    ) -> (FeeCalculator, Hash, Slot) {
         let bank = self.bank(commitment);
-        let blockhash = bank.last_blockhash();
-        let lamports_per_signature = bank.get_lamports_per_signature();
-        let last_valid_block_height = bank
-            .get_blockhash_last_valid_block_height(&blockhash)
-            .unwrap();
-        (
-            FeeCalculator::new(lamports_per_signature),
-            blockhash,
-            last_valid_block_height,
-        )
+        let (blockhash, fee_calculator) = bank.last_blockhash_with_fee_calculator();
+        let last_valid_slot = bank.get_blockhash_last_valid_slot(&blockhash).unwrap();
+        (fee_calculator, blockhash, last_valid_slot)
     }
 
     async fn get_transaction_status_with_context(
@@ -238,35 +212,32 @@ impl Banks for BanksServer {
         self.slot(commitment)
     }
 
-    async fn get_block_height_with_context(self, _: Context, commitment: CommitmentLevel) -> u64 {
-        self.bank(commitment).block_height()
-    }
-
     async fn process_transaction_with_commitment_and_context(
         self,
         _: Context,
         transaction: Transaction,
         commitment: CommitmentLevel,
     ) -> Option<transaction::Result<()>> {
-        if let Err(err) = verify_transaction(&transaction, &self.bank(commitment).feature_set) {
+        if let Err(err) = verify_transaction(
+            &transaction,
+            self.bank(commitment).libsecp256k1_0_5_upgrade_enabled(),
+        ) {
             return Some(Err(err));
         }
 
         let blockhash = &transaction.message.recent_blockhash;
-        let last_valid_block_height = self
-            .bank(commitment)
-            .get_blockhash_last_valid_block_height(blockhash)
+        let last_valid_slot = self
+            .bank_forks
+            .read()
+            .unwrap()
+            .root_bank()
+            .get_blockhash_last_valid_slot(blockhash)
             .unwrap();
         let signature = transaction.signatures.get(0).cloned().unwrap_or_default();
-        let info = TransactionInfo::new(
-            signature,
-            serialize(&transaction).unwrap(),
-            last_valid_block_height,
-            None,
-            None,
-        );
+        let info =
+            TransactionInfo::new(signature, serialize(&transaction).unwrap(), last_valid_slot);
         self.transaction_sender.send(info).unwrap();
-        self.poll_signature_status(&signature, blockhash, last_valid_block_height, commitment)
+        self.poll_signature_status(&signature, blockhash, last_valid_slot, commitment)
             .await
     }
 
@@ -279,47 +250,17 @@ impl Banks for BanksServer {
         let bank = self.bank(commitment);
         bank.get_account(&address).map(Account::from)
     }
-
-    async fn get_latest_blockhash_with_context(self, _: Context) -> Hash {
-        let bank = self.bank(CommitmentLevel::default());
-        bank.last_blockhash()
-    }
-
-    async fn get_latest_blockhash_with_commitment_and_context(
-        self,
-        _: Context,
-        commitment: CommitmentLevel,
-    ) -> Option<(Hash, u64)> {
-        let bank = self.bank(commitment);
-        let blockhash = bank.last_blockhash();
-        let last_valid_block_height = bank.get_blockhash_last_valid_block_height(&blockhash)?;
-        Some((blockhash, last_valid_block_height))
-    }
-
-    async fn get_fee_for_message_with_commitment_and_context(
-        self,
-        _: Context,
-        commitment: CommitmentLevel,
-        message: Message,
-    ) -> Option<u64> {
-        let bank = self.bank(commitment);
-        let sanitized_message = SanitizedMessage::try_from(message).ok()?;
-        bank.get_fee_for_message(&sanitized_message)
-    }
 }
 
 pub async fn start_local_server(
     bank_forks: Arc<RwLock<BankForks>>,
     block_commitment_cache: Arc<RwLock<BlockCommitmentCache>>,
-    poll_signature_status_sleep_duration: Duration,
 ) -> UnboundedChannel<Response<BanksResponse>, ClientMessage<BanksRequest>> {
-    let banks_server = BanksServer::new_loopback(
-        bank_forks,
-        block_commitment_cache,
-        poll_signature_status_sleep_duration,
-    );
+    let banks_server = BanksServer::new_loopback(bank_forks, block_commitment_cache);
     let (client_transport, server_transport) = transport::channel::unbounded();
-    let server = server::BaseChannel::with_defaults(server_transport).execute(banks_server.serve());
+    let server = server::new(server::Config::default())
+        .incoming(stream::once(future::ready(server_transport)))
+        .respond_with(banks_server.serve());
     tokio::spawn(server);
     client_transport
 }
@@ -348,22 +289,11 @@ pub async fn start_tcp_server(
         .map(move |chan| {
             let (sender, receiver) = channel();
 
-            SendTransactionService::new::<NullTpuInfo>(
-                tpu_addr,
-                &bank_forks,
-                None,
-                receiver,
-                5_000,
-                0,
-            );
+            SendTransactionService::new(tpu_addr, &bank_forks, receiver);
 
-            let server = BanksServer::new(
-                bank_forks.clone(),
-                block_commitment_cache.clone(),
-                sender,
-                Duration::from_millis(200),
-            );
-            chan.execute(server.serve())
+            let server =
+                BanksServer::new(bank_forks.clone(), block_commitment_cache.clone(), sender);
+            chan.respond_with(server.serve()).execute()
         })
         // Max 10 channels.
         .buffer_unordered(10)

banks-server/src/lib.rs

@@ -1,3 +1,7 @@
 #![allow(clippy::integer_arithmetic)]
 pub mod banks_server;
 pub mod rpc_banks_service;
+pub mod send_transaction_service;
+
+#[macro_use]
+extern crate solana_metrics;

banks-server/src/rpc_banks_service.rs

@@ -1,23 +1,21 @@
 //! The `rpc_banks_service` module implements the Solana Banks RPC API.
 
-use {
-    crate::banks_server::start_tcp_server,
-    futures::{future::FutureExt, pin_mut, prelude::stream::StreamExt, select},
-    solana_runtime::{bank_forks::BankForks, commitment::BlockCommitmentCache},
-    std::{
+use crate::banks_server::start_tcp_server;
+use futures::{future::FutureExt, pin_mut, prelude::stream::StreamExt, select};
+use solana_runtime::{bank_forks::BankForks, commitment::BlockCommitmentCache};
+use std::{
     net::SocketAddr,
     sync::{
         atomic::{AtomicBool, Ordering},
         Arc, RwLock,
     },
     thread::{self, Builder, JoinHandle},
-    },
-    tokio::{
+};
+use tokio::{
     runtime::Runtime,
     time::{self, Duration},
-    },
-    tokio_stream::wrappers::IntervalStream,
 };
+use tokio_stream::wrappers::IntervalStream;
 
 pub struct RpcBanksService {
     thread_hdl: JoinHandle<()>,
@@ -103,11 +101,12 @@ impl RpcBanksService {
 
 #[cfg(test)]
 mod tests {
-    use {super::*, solana_runtime::bank::Bank};
+    use super::*;
+    use solana_runtime::bank::Bank;
 
     #[test]
     fn test_rpc_banks_server_exit() {
-        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::default_for_tests())));
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(Bank::default())));
         let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
         let exit = Arc::new(AtomicBool::new(false));
         let addr = "127.0.0.1:0".parse().unwrap();

banks-server/src/send_transaction_service.rs

@@ -0,0 +1,343 @@
+// TODO: Merge this implementation with the one at `core/src/send_transaction_service.rs`
+use log::*;
+use solana_metrics::{datapoint_warn, inc_new_counter_info};
+use solana_runtime::{bank::Bank, bank_forks::BankForks};
+use solana_sdk::{clock::Slot, signature::Signature};
+use std::{
+    collections::HashMap,
+    net::{SocketAddr, UdpSocket},
+    sync::{
+        mpsc::{Receiver, RecvTimeoutError},
+        Arc, RwLock,
+    },
+    thread::{self, Builder, JoinHandle},
+    time::{Duration, Instant},
+};
+
+/// Maximum size of the transaction queue
+const MAX_TRANSACTION_QUEUE_SIZE: usize = 10_000; // This seems like a lot but maybe it needs to be bigger one day
+
+pub struct SendTransactionService {
+    thread: JoinHandle<()>,
+}
+
+pub struct TransactionInfo {
+    pub signature: Signature,
+    pub wire_transaction: Vec<u8>,
+    pub last_valid_slot: Slot,
+}
+
+impl TransactionInfo {
+    pub fn new(signature: Signature, wire_transaction: Vec<u8>, last_valid_slot: Slot) -> Self {
+        Self {
+            signature,
+            wire_transaction,
+            last_valid_slot,
+        }
+    }
+}
+
+#[derive(Default, Debug, PartialEq)]
+struct ProcessTransactionsResult {
+    rooted: u64,
+    expired: u64,
+    retried: u64,
+    failed: u64,
+    retained: u64,
+}
+
+impl SendTransactionService {
+    pub fn new(
+        tpu_address: SocketAddr,
+        bank_forks: &Arc<RwLock<BankForks>>,
+        receiver: Receiver<TransactionInfo>,
+    ) -> Self {
+        let thread = Self::retry_thread(receiver, bank_forks.clone(), tpu_address);
+        Self { thread }
+    }
+
+    fn retry_thread(
+        receiver: Receiver<TransactionInfo>,
+        bank_forks: Arc<RwLock<BankForks>>,
+        tpu_address: SocketAddr,
+    ) -> JoinHandle<()> {
+        let mut last_status_check = Instant::now();
+        let mut transactions = HashMap::new();
+        let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+
+        Builder::new()
+            .name("send-tx-svc".to_string())
+            .spawn(move || loop {
+                match receiver.recv_timeout(Duration::from_secs(1)) {
+                    Err(RecvTimeoutError::Disconnected) => break,
+                    Err(RecvTimeoutError::Timeout) => {}
+                    Ok(transaction_info) => {
+                        Self::send_transaction(
+                            &send_socket,
+                            &tpu_address,
+                            &transaction_info.wire_transaction,
+                        );
+                        if transactions.len() < MAX_TRANSACTION_QUEUE_SIZE {
+                            transactions.insert(transaction_info.signature, transaction_info);
+                        } else {
+                            datapoint_warn!("send_transaction_service-queue-overflow");
+                        }
+                    }
+                }
+
+                if Instant::now().duration_since(last_status_check).as_secs() >= 5 {
+                    if !transactions.is_empty() {
+                        datapoint_info!(
+                            "send_transaction_service-queue-size",
+                            ("len", transactions.len(), i64)
+                        );
+                        let bank_forks = bank_forks.read().unwrap();
+                        let root_bank = bank_forks.root_bank();
+                        let working_bank = bank_forks.working_bank();
+
+                        let _result = Self::process_transactions(
+                            &working_bank,
+                            &root_bank,
+                            &send_socket,
+                            &tpu_address,
+                            &mut transactions,
+                        );
+                    }
+                    last_status_check = Instant::now();
+                }
+            })
+            .unwrap()
+    }
+
+    fn process_transactions(
+        working_bank: &Arc<Bank>,
+        root_bank: &Arc<Bank>,
+        send_socket: &UdpSocket,
+        tpu_address: &SocketAddr,
+        transactions: &mut HashMap<Signature, TransactionInfo>,
+    ) -> ProcessTransactionsResult {
+        let mut result = ProcessTransactionsResult::default();
+
+        transactions.retain(|signature, transaction_info| {
+            if root_bank.has_signature(signature) {
+                info!("Transaction is rooted: {}", signature);
+                result.rooted += 1;
+                inc_new_counter_info!("send_transaction_service-rooted", 1);
+                false
+            } else if transaction_info.last_valid_slot < root_bank.slot() {
+                info!("Dropping expired transaction: {}", signature);
+                result.expired += 1;
+                inc_new_counter_info!("send_transaction_service-expired", 1);
+                false
+            } else {
+                match working_bank.get_signature_status_slot(signature) {
+                    None => {
+                        // Transaction is unknown to the working bank, it might have been
+                        // dropped or landed in another fork.  Re-send it
+                        info!("Retrying transaction: {}", signature);
+                        result.retried += 1;
+                        inc_new_counter_info!("send_transaction_service-retry", 1);
+                        Self::send_transaction(
+                            &send_socket,
+                            &tpu_address,
+                            &transaction_info.wire_transaction,
+                        );
+                        true
+                    }
+                    Some((_slot, status)) => {
+                        if status.is_err() {
+                            info!("Dropping failed transaction: {}", signature);
+                            result.failed += 1;
+                            inc_new_counter_info!("send_transaction_service-failed", 1);
+                            false
+                        } else {
+                            result.retained += 1;
+                            true
+                        }
+                    }
+                }
+            }
+        });
+
+        result
+    }
+
+    fn send_transaction(
+        send_socket: &UdpSocket,
+        tpu_address: &SocketAddr,
+        wire_transaction: &[u8],
+    ) {
+        if let Err(err) = send_socket.send_to(wire_transaction, tpu_address) {
+            warn!("Failed to send transaction to {}: {:?}", tpu_address, err);
+        }
+    }
+
+    pub fn join(self) -> thread::Result<()> {
+        self.thread.join()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use solana_sdk::{
+        genesis_config::create_genesis_config, pubkey::Pubkey, signature::Signer,
+        system_transaction,
+    };
+    use std::sync::mpsc::channel;
+
+    #[test]
+    fn service_exit() {
+        let tpu_address = "127.0.0.1:0".parse().unwrap();
+        let bank = Bank::default();
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
+        let (sender, receiver) = channel();
+
+        let send_tranaction_service =
+            SendTransactionService::new(tpu_address, &bank_forks, receiver);
+
+        drop(sender);
+        send_tranaction_service.join().unwrap();
+    }
+
+    #[test]
+    fn process_transactions() {
+        let (genesis_config, mint_keypair) = create_genesis_config(4);
+        let bank = Bank::new(&genesis_config);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(bank)));
+        let send_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let tpu_address = "127.0.0.1:0".parse().unwrap();
+
+        let root_bank = Arc::new(Bank::new_from_parent(
+            &bank_forks.read().unwrap().working_bank(),
+            &Pubkey::default(),
+            1,
+        ));
+        let rooted_signature = root_bank
+            .transfer(1, &mint_keypair, &mint_keypair.pubkey())
+            .unwrap();
+
+        let working_bank = Arc::new(Bank::new_from_parent(&root_bank, &Pubkey::default(), 2));
+
+        let non_rooted_signature = working_bank
+            .transfer(2, &mint_keypair, &mint_keypair.pubkey())
+            .unwrap();
+
+        let failed_signature = {
+            let blockhash = working_bank.last_blockhash();
+            let transaction =
+                system_transaction::transfer(&mint_keypair, &Pubkey::default(), 1, blockhash);
+            let signature = transaction.signatures[0];
+            working_bank.process_transaction(&transaction).unwrap_err();
+            signature
+        };
+
+        let mut transactions = HashMap::new();
+
+        info!("Expired transactions are dropped..");
+        transactions.insert(
+            Signature::default(),
+            TransactionInfo::new(Signature::default(), vec![], root_bank.slot() - 1),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                expired: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Rooted transactions are dropped...");
+        transactions.insert(
+            rooted_signature,
+            TransactionInfo::new(rooted_signature, vec![], working_bank.slot()),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                rooted: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Failed transactions are dropped...");
+        transactions.insert(
+            failed_signature,
+            TransactionInfo::new(failed_signature, vec![], working_bank.slot()),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert!(transactions.is_empty());
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                failed: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+
+        info!("Non-rooted transactions are kept...");
+        transactions.insert(
+            non_rooted_signature,
+            TransactionInfo::new(non_rooted_signature, vec![], working_bank.slot()),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert_eq!(transactions.len(), 1);
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                retained: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+        transactions.clear();
+
+        info!("Unknown transactions are retried...");
+        transactions.insert(
+            Signature::default(),
+            TransactionInfo::new(Signature::default(), vec![], working_bank.slot()),
+        );
+        let result = SendTransactionService::process_transactions(
+            &working_bank,
+            &root_bank,
+            &send_socket,
+            &tpu_address,
+            &mut transactions,
+        );
+        assert_eq!(transactions.len(), 1);
+        assert_eq!(
+            result,
+            ProcessTransactionsResult {
+                retried: 1,
+                ..ProcessTransactionsResult::default()
+            }
+        );
+    }
+}

bench-exchange/.gitignore

@@ -0,0 +1,4 @@
+/target/
+/config/
+/config-local/
+/farf/

bench-exchange/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+edition = "2018"
+name = "solana-bench-exchange"
+version = "1.6.28"
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+publish = false
+
+[dependencies]
+clap = "2.33.1"
+itertools = "0.9.0"
+log = "0.4.11"
+num-derive = "0.3"
+num-traits = "0.2"
+rand = "0.7.0"
+rayon = "1.5.0"
+serde_json = "1.0.56"
+serde_yaml = "0.8.13"
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.28" }
+solana-core = { path = "../core", version = "=1.6.28" }
+solana-genesis = { path = "../genesis", version = "=1.6.28" }
+solana-client = { path = "../client", version = "=1.6.28" }
+solana-faucet = { path = "../faucet", version = "=1.6.28" }
+solana-exchange-program = { path = "../programs/exchange", version = "=1.6.28" }
+solana-logger = { path = "../logger", version = "=1.6.28" }
+solana-metrics = { path = "../metrics", version = "=1.6.28" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.28" }
+solana-runtime = { path = "../runtime", version = "=1.6.28" }
+solana-sdk = { path = "../sdk", version = "=1.6.28" }
+solana-version = { path = "../version", version = "=1.6.28" }
+
+[dev-dependencies]
+solana-local-cluster = { path = "../local-cluster", version = "=1.6.28" }
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]

bench-exchange/README.md

@@ -0,0 +1,479 @@
+# token-exchange
+Solana Token Exchange Bench
+
+If you can't wait; jump to [Running the exchange](#Running-the-exchange) to
+learn how to start and interact with the exchange.
+
+### Table of Contents
+[Overview](#Overview)<br>
+[Premise](#Premise)<br>
+[Exchange startup](#Exchange-startup)<br>
+[Order Requests](#Trade-requests)<br>
+[Order Cancellations](#Trade-cancellations)<br>
+[Trade swap](#Trade-swap)<br>
+[Exchange program operations](#Exchange-program-operations)<br>
+[Quotes and OHLCV](#Quotes-and-OHLCV)<br>
+[Investor strategies](#Investor-strategies)<br>
+[Running the exchange](#Running-the-exchange)<br>
+
+## Overview
+
+An exchange is a marketplace where one asset can be traded for another.  This
+demo demonstrates one way to host an exchange on the Solana blockchain by
+emulating a currency exchange.
+
+The assets are virtual tokens held by investors who may post order requests to
+the exchange.  A Matcher monitors the exchange and posts swap requests for
+matching orders.  All the transactions can execute concurrently.
+
+## Premise
+
+- Exchange
+  -  An exchange is a marketplace where one asset can be traded for another.
+     The exchange in this demo is the on-chain program that implements the
+     tokens and the policies for trading those tokens.
+- Token
+  - A virtual asset that can be owned, traded, and holds virtual intrinsic value
+    compared to other assets.  There are four types of tokens in this demo, A,
+    B, C, D.  Each one may be traded for another.
+- Token account
+  - An account owned by the exchange that holds a quantity of one type of token.
+- Account request
+  - A request to create a token account
+- Token request
+  - A request to deposit tokens of a particular type into a token account.
+- Asset pair
+  - A struct with fields Base and Quote, representing the two assets which make up a 
+    trading pair,  which themselves are Tokens. The Base or 'primary' asset is the
+    numerator and the Quote is the denominator for pricing purposes.
+- Order side
+  - Describes which side of the market an investor wants to place a trade on. Options
+    are "Bid" or "Ask", where a bid represents an offer to purchase the Base asset of
+    the AssetPair for a sum of the Quote Asset and an Ask is an offer to sell Base asset
+    for the Quote asset.
+- Price ratio
+  -  An expression of the relative prices of two tokens. Calculated with the Base
+     Asset as the numerator and the Quote Asset as the denominator. Ratios are 
+     represented as fixed point numbers.  The fixed point scaler is defined in
+     [exchange_state.rs](https://github.com/solana-labs/solana/blob/c2fdd1362a029dcf89c8907c562d2079d977df11/programs/exchange_api/src/exchange_state.rs#L7)
+- Order request
+  - A Solana transaction sent by a trader to the exchange to submit an order. 
+    Order requests are made up of the token pair, the order side (bid or ask),
+    quantity of the primary token, the price ratio, and the two token accounts
+    to be credited/deducted.  An example trade request looks like "T AB 5 2" 
+    which reads "Exchange 5 A tokens to B tokens at a price ratio of 1:2"  A fulfilled trade would result in 5 A tokens
+    deducted and 10 B tokens credited to the trade initiator's token accounts.
+    Successful order requests result in an order.
+- Order
+  - The result of a successful order request.  orders are stored in
+    accounts owned by the submitter of the order request.  They can only be
+    canceled by their owner but can be used by anyone in a trade swap.  They
+    contain the same information as the order request.
+- Price spread
+  - The difference between the two matching orders. The spread is the
+    profit of the Matcher initiating the swap request.
+- Match requirements
+  - Policies that result in a successful trade swap.
+- Match request
+  - A request to fill two complementary orders (bid/ask), resulting if successful,
+    in a trade being created.
+- Trade
+  - A successful trade is created from two matching orders that meet
+    swap requirements which are submitted in a Match Request by a Matcher and
+    executed by the exchange. A trade may not wholly satisfy one or both of the
+    orders in which case the orders are adjusted appropriately. Upon execution,
+    tokens are distributed to the traders' accounts and any overlap or 
+    "negative spread" between orders is deposited into the Matcher's profit
+    account.  All successful trades are recorded in the data of a new solana 
+    account for posterity.
+- Investor
+  - Individual investors who hold a number of tokens and wish to trade them on
+    the exchange.  Investors operate as Solana thin clients who own a set of
+    accounts containing tokens and/or order requests.  Investors post
+    transactions to the exchange in order to request tokens and post or cancel
+    order requests.
+- Matcher
+  - An agent who facilitates trading between investors.  Matchers operate as
+    Solana thin clients who monitor all the orders looking for a trade
+    match.  Once found, the Matcher issues a swap request to the exchange.
+    Matchers are the engine of the exchange and are rewarded for their efforts by
+    accumulating the price spreads of the swaps they initiate.  Matchers also
+    provide current bid/ask price and OHLCV (Open, High, Low, Close, Volume)
+    information on demand via a public network port.
+- Transaction fees
+  - Solana transaction fees are paid for by the transaction submitters who are
+    the Investors and Matchers.
+
+## Exchange startup
+
+The exchange is up and running when it reaches a state where it can take
+investors' trades and Matchers' match requests.  To achieve this state the
+following must occur in order:
+
+- Start the Solana blockchain
+- Start the  thin-client
+- The Matcher subscribes to change notifications for all the accounts owned by
+  the exchange program id.  The subscription is managed via Solana's JSON RPC
+  interface.
+- The Matcher starts responding to queries for bid/ask price and OHLCV
+
+The Matcher responding successfully to price and OHLCV requests is the signal to
+the investors that trades submitted after that point will be analyzed.  <!--This
+is not ideal, and instead investors should be able to submit trades at any time,
+and the Matcher could come and go without missing a trade.  One way to achieve
+this is for the Matcher to read the current state of all accounts looking for all
+open orders.-->
+
+Investors will initially query the exchange to discover their current balance
+for each type of token.  If the investor does not already have an account for
+each type of token, they will submit account requests.  Matcher as well will
+request accounts to hold the tokens they earn by initiating trade swaps.
+
+```rust
+/// Supported token types
+pub enum Token {
+    A,
+    B,
+    C,
+    D,
+}
+
+/// Supported token pairs
+pub enum TokenPair {
+    AB,
+    AC,
+    AD,
+    BC,
+    BD,
+    CD,
+}
+
+pub enum ExchangeInstruction {
+    /// New token account
+    /// key 0 - Signer
+    /// key 1 - New token account
+    AccountRequest,
+}
+
+/// Token accounts are populated with this structure
+pub struct TokenAccountInfo {
+    /// Investor who owns this account
+    pub owner: Pubkey,
+    /// Current number of tokens this account holds
+    pub tokens: Tokens,
+}
+```
+
+For this demo investors or Matcher can request more tokens from the exchange at
+any time by submitting token requests. In non-demos, an exchange of this type
+would provide another way to exchange a 3rd party asset into tokens.
+
+To request tokens, investors submit transfer requests:
+
+```rust
+pub enum ExchangeInstruction {
+    /// Transfer tokens between two accounts
+    /// key 0 - Account to transfer tokens to
+    /// key 1 - Account to transfer tokens from.  This can be the exchange program itself,
+    ///         the exchange has a limitless number of tokens it can transfer.
+    TransferRequest(Token, u64),
+}
+```
+
+## Order Requests
+
+When an investor decides to exchange a token of one type for another, they
+submit a transaction to the Solana Blockchain containing an order request, which,
+if successful, is turned into an order.  orders do not expire but are
+cancellable. <!-- orders should have a timestamp to enable trade
+expiration -->  When an order is created, tokens are deducted from a token
+account and the order acts as an escrow.  The tokens are held until the
+order is fulfilled or canceled. If the direction is `To`, then the number
+of `tokens` are deducted from the primary account, if `From` then `tokens`
+multiplied by `price` are deducted from the secondary account.  orders are
+no longer valid when the number of `tokens` goes to zero, at which point they
+can no longer be used. <!-- Could support refilling orders, so order
+accounts are refilled rather than accumulating -->
+
+```rust
+/// Direction of the exchange between two tokens in a pair
+pub enum Direction {
+    /// Trade first token type (primary) in the pair 'To' the second
+    To,
+    /// Trade first token type in the pair 'From' the second (secondary)
+    From,
+}
+
+pub struct OrderRequestInfo {
+    /// Direction of trade
+    pub direction: Direction,
+
+    /// Token pair to trade
+    pub pair: TokenPair,
+
+    /// Number of tokens to exchange; refers to the primary or the secondary depending on the direction
+    pub tokens: u64,
+
+    /// The price ratio the primary price over the secondary price.  The primary price is fixed
+    /// and equal to the variable `SCALER`.
+    pub price: u64,
+
+    /// Token account to deposit tokens on successful swap
+    pub dst_account: Pubkey,
+}
+
+pub enum ExchangeInstruction {
+    /// order request
+    /// key 0 - Signer
+    /// key 1 - Account in which to record the swap
+    /// key 2 - Token account associated with this trade
+    TradeRequest(TradeRequestInfo),
+}
+
+/// Trade accounts are populated with this structure
+pub struct TradeOrderInfo {
+    /// Owner of the order
+    pub owner: Pubkey,
+    /// Direction of the exchange
+    pub direction: Direction,
+    /// Token pair indicating two tokens to exchange, first is primary
+    pub pair: TokenPair,
+    /// Number of tokens to exchange; primary or secondary depending on direction
+    pub tokens: u64,
+    /// Scaled price of the secondary token given the primary is equal to the scale value
+    /// If scale is 1 and price is 2 then ratio is 1:2 or 1 primary token for 2 secondary tokens
+    pub price: u64,
+    /// account which the tokens were source from.  The trade account holds the tokens in escrow
+    /// until either one or more part of a swap or the trade is canceled.
+    pub src_account: Pubkey,
+    /// account which the tokens the tokens will be deposited into on a successful trade
+    pub dst_account: Pubkey,
+}
+```
+
+## Order cancellations
+
+An investor may cancel a trade at anytime, but only trades they own.  If the
+cancellation is successful, any tokens held in escrow are returned to the
+account from which they came.
+
+```rust
+pub enum ExchangeInstruction {
+    /// order cancellation
+    /// key 0 - Signer
+    /// key 1 -order to cancel
+    TradeCancellation,
+}
+```
+
+## Trade swaps
+
+The Matcher is monitoring the accounts assigned to the exchange program and
+building a trade-order table.  The order table is used to identify
+matching orders which could be fulfilled.  When a match is found the
+Matcher should issue a swap request.  Swap requests may not satisfy the entirety
+of either order, but the exchange will greedily fulfill it.  Any leftover tokens
+in either account will keep the order valid for further swap requests in
+the future.
+
+Matching orders are defined by the following swap requirements:
+
+- Opposite polarity (one `To` and one `From`)
+- Operate on the same token pair
+- The price ratio of the `From` order is greater than or equal to the `To` order
+- There are sufficient tokens to perform the trade
+
+Orders can be written in the following format:
+
+`investor direction pair quantity price-ratio`
+
+For example:
+
+- `1 T AB 2 1`
+  - Investor 1 wishes to exchange 2 A tokens to B tokens at a ratio of 1 A to 1
+    B
+- `2 F AC 6 1.2`
+  - Investor 2 wishes to exchange A tokens from 6 B tokens at a ratio of 1 A
+    from 1.2 B
+
+An order table could look something like the following. Notice how the columns
+are sorted low to high and high to low, respectively.  Prices are dramatic and
+whole for clarity.
+
+|Row| To          | From       |
+|---|-------------|------------|
+| 1 | 1 T AB 2 4  | 2 F AB 2 8 |
+| 2 | 1 T AB 1 4  | 2 F AB 2 8 |
+| 3 | 1 T AB 6 6  | 2 F AB 2 7 |
+| 4 | 1 T AB 2 8  | 2 F AB 3 6 |
+| 5 | 1 T AB 2 10 | 2 F AB 1 5 |
+
+As part of a successful swap request, the exchange will credit tokens to the
+Matcher's account equal to the difference in the price ratios or the two orders.
+These tokens are considered the Matcher's profit for initiating the trade.
+
+The Matcher would initiate the following swap on the order table above:
+
+  - Row 1, To:   Investor 1 trades 2 A tokens to 8 B tokens
+  - Row 1, From: Investor 2 trades 2 A tokens from 8 B tokens
+  - Matcher takes 8 B tokens as profit
+
+Both row 1 trades are fully realized, table becomes:
+
+|Row| To          | From       |
+|---|-------------|------------|
+| 1 | 1 T AB 1 4  | 2 F AB 2 8 |
+| 2 | 1 T AB 6 6  | 2 F AB 2 7 |
+| 3 | 1 T AB 2 8  | 2 F AB 3 6 |
+| 4 | 1 T AB 2 10 | 2 F AB 1 5 |
+
+The Matcher would initiate the following swap:
+
+  - Row 1, To:   Investor 1 trades 1 A token to 4 B tokens
+  - Row 1, From: Investor 2 trades 1 A token from 4 B tokens
+  - Matcher takes 4 B tokens as profit
+
+Row 1 From is not fully realized, table becomes:
+
+|Row| To          | From       |
+|---|-------------|------------|
+| 1 | 1 T AB 6 6  | 2 F AB 1 8 |
+| 2 | 1 T AB 2 8  | 2 F AB 2 7 |
+| 3 | 1 T AB 2 10 | 2 F AB 3 6 |
+| 4 |             | 2 F AB 1 5 |
+
+The Matcher would initiate the following swap:
+
+  - Row 1, To:   Investor 1 trades 1 A token to 6 B tokens
+  - Row 1, From: Investor 2 trades 1 A token from 6 B tokens
+  - Matcher takes 2 B tokens as profit
+
+Row 1 To is now fully realized, table becomes:
+
+|Row| To          | From       |
+|---|-------------|------------|
+| 1 | 1 T AB 5 6  | 2 F AB 2 7 |
+| 2 | 1 T AB 2 8  | 2 F AB 3 5 |
+| 3 | 1 T AB 2 10 | 2 F AB 1 5 |
+
+The Matcher would initiate the following last swap:
+
+  - Row 1, To:   Investor 1 trades 2 A token to 12 B tokens
+  - Row 1, From: Investor 2 trades 2 A token from 12 B tokens
+  - Matcher takes 2 B tokens as profit
+
+Table becomes:
+
+|Row| To          | From       |
+|---|-------------|------------|
+| 1 | 1 T AB 3 6  | 2 F AB 3 5 |
+| 2 | 1 T AB 2 8  | 2 F AB 1 5 |
+| 3 | 1 T AB 2 10 |            |
+
+At this point the lowest To's price is larger than the largest From's price so
+no more swaps would be initiated until new orders came in.
+
+```rust
+pub enum ExchangeInstruction {
+    /// Trade swap request
+    /// key 0 - Signer
+    /// key 1 - Account in which to record the swap
+    /// key 2 - 'To' order
+    /// key 3 - `From` order
+    /// key 4 - Token account associated with the To Trade
+    /// key 5 - Token account associated with From trade
+    /// key 6 - Token account in which to deposit the Matcher profit from the swap.
+    SwapRequest,
+}
+
+/// Swap accounts are populated with this structure
+pub struct TradeSwapInfo {
+    /// Pair swapped
+    pub pair: TokenPair,
+    /// `To` order
+    pub to_trade_order: Pubkey,
+    /// `From` order
+    pub from_trade_order: Pubkey,
+    /// Number of primary tokens exchanged
+    pub primary_tokens: u64,
+    /// Price the primary tokens were exchanged for
+    pub primary_price: u64,
+    /// Number of secondary tokens exchanged
+    pub secondary_tokens: u64,
+    /// Price the secondary tokens were exchanged for
+    pub secondary_price: u64,
+}
+```
+
+## Exchange program operations
+
+Putting all the commands together from above, the following operations will be
+supported by the on-chain exchange program:
+
+```rust
+pub enum ExchangeInstruction {
+    /// New token account
+    /// key 0 - Signer
+    /// key 1 - New token account
+    AccountRequest,
+
+    /// Transfer tokens between two accounts
+    /// key 0 - Account to transfer tokens to
+    /// key 1 - Account to transfer tokens from.  This can be the exchange program itself,
+    ///         the exchange has a limitless number of tokens it can transfer.
+    TransferRequest(Token, u64),
+
+    /// order request
+    /// key 0 - Signer
+    /// key 1 - Account in which to record the swap
+    /// key 2 - Token account associated with this trade
+    TradeRequest(TradeRequestInfo),
+
+    /// order cancellation
+    /// key 0 - Signer
+    /// key 1 -order to cancel
+    TradeCancellation,
+
+    /// Trade swap request
+    /// key 0 - Signer
+    /// key 1 - Account in which to record the swap
+    /// key 2 - 'To' order
+    /// key 3 - `From` order
+    /// key 4 - Token account associated with the To Trade
+    /// key 5 - Token account associated with From trade
+    /// key 6 - Token account in which to deposit the Matcher profit from the swap.
+    SwapRequest,
+}
+```
+
+## Quotes and OHLCV
+
+The Matcher will provide current bid/ask price quotes based on trade actively and
+also provide OHLCV based on some time window.  The details of how the bid/ask
+price quotes are calculated are yet to be decided.
+
+## Investor strategies
+
+To make a compelling demo, the investors needs to provide interesting trade
+behavior.  Something as simple as a randomly twiddled baseline would be a
+minimum starting point.
+
+## Running the exchange
+
+The exchange bench posts trades and swaps matches as fast as it can.  
+
+You might want to bump the duration up
+to 60 seconds and the batch size to 1000 for better numbers.  You can modify those
+in client_demo/src/demo.rs::test_exchange_local_cluster.
+
+The following command runs the bench:
+
+```bash
+$ RUST_LOG=solana_bench_exchange=info cargo test --release -- --nocapture test_exchange_local_cluster
+```
+
+To also see the cluster messages:
+
+```bash
+$ RUST_LOG=solana_bench_exchange=info,solana=info cargo test --release -- --nocapture test_exchange_local_cluster
+```

bench-exchange/src/cli.rs

@@ -0,0 +1,221 @@
+use clap::{crate_description, crate_name, value_t, App, Arg, ArgMatches};
+use solana_core::gen_keys::GenKeys;
+use solana_faucet::faucet::FAUCET_PORT;
+use solana_sdk::signature::{read_keypair_file, Keypair};
+use std::net::SocketAddr;
+use std::process::exit;
+use std::time::Duration;
+
+pub struct Config {
+    pub entrypoint_addr: SocketAddr,
+    pub faucet_addr: SocketAddr,
+    pub identity: Keypair,
+    pub threads: usize,
+    pub num_nodes: usize,
+    pub duration: Duration,
+    pub transfer_delay: u64,
+    pub fund_amount: u64,
+    pub batch_size: usize,
+    pub chunk_size: usize,
+    pub account_groups: usize,
+    pub client_ids_and_stake_file: String,
+    pub write_to_client_file: bool,
+    pub read_from_client_file: bool,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            entrypoint_addr: SocketAddr::from(([127, 0, 0, 1], 8001)),
+            faucet_addr: SocketAddr::from(([127, 0, 0, 1], FAUCET_PORT)),
+            identity: Keypair::new(),
+            num_nodes: 1,
+            threads: 4,
+            duration: Duration::new(u64::max_value(), 0),
+            transfer_delay: 0,
+            fund_amount: 100_000,
+            batch_size: 100,
+            chunk_size: 100,
+            account_groups: 100,
+            client_ids_and_stake_file: String::new(),
+            write_to_client_file: false,
+            read_from_client_file: false,
+        }
+    }
+}
+
+pub fn build_args<'a, 'b>(version: &'b str) -> App<'a, 'b> {
+    App::new(crate_name!())
+        .about(crate_description!())
+        .version(version)
+        .arg(
+            Arg::with_name("entrypoint")
+                .short("n")
+                .long("entrypoint")
+                .value_name("HOST:PORT")
+                .takes_value(true)
+                .required(false)
+                .default_value("127.0.0.1:8001")
+                .help("Cluster entry point; defaults to 127.0.0.1:8001"),
+        )
+        .arg(
+            Arg::with_name("faucet")
+                .short("d")
+                .long("faucet")
+                .value_name("HOST:PORT")
+                .takes_value(true)
+                .required(false)
+                .default_value("127.0.0.1:9900")
+                .help("Location of the faucet; defaults to 127.0.0.1:9900"),
+        )
+        .arg(
+            Arg::with_name("identity")
+                .short("i")
+                .long("identity")
+                .value_name("PATH")
+                .takes_value(true)
+                .help("File containing a client identity (keypair)"),
+        )
+        .arg(
+            Arg::with_name("threads")
+                .long("threads")
+                .value_name("<threads>")
+                .takes_value(true)
+                .required(false)
+                .default_value("1")
+                .help("Number of threads submitting transactions"),
+        )
+        .arg(
+            Arg::with_name("num-nodes")
+                .long("num-nodes")
+                .value_name("NUM")
+                .takes_value(true)
+                .required(false)
+                .default_value("1")
+                .help("Wait for NUM nodes to converge"),
+        )
+        .arg(
+            Arg::with_name("duration")
+                .long("duration")
+                .value_name("SECS")
+                .takes_value(true)
+                .default_value("60")
+                .help("Seconds to run benchmark, then exit; default is forever"),
+        )
+        .arg(
+            Arg::with_name("transfer-delay")
+                .long("transfer-delay")
+                .value_name("<delay>")
+                .takes_value(true)
+                .required(false)
+                .default_value("0")
+                .help("Delay between each chunk"),
+        )
+        .arg(
+            Arg::with_name("fund-amount")
+                .long("fund-amount")
+                .value_name("<fund>")
+                .takes_value(true)
+                .required(false)
+                .default_value("100000")
+                .help("Number of lamports to fund to each signer"),
+        )
+        .arg(
+            Arg::with_name("batch-size")
+                .long("batch-size")
+                .value_name("<batch>")
+                .takes_value(true)
+                .required(false)
+                .default_value("1000")
+                .help("Number of transactions before the signer rolls over"),
+        )
+        .arg(
+            Arg::with_name("chunk-size")
+                .long("chunk-size")
+                .value_name("<cunk>")
+                .takes_value(true)
+                .required(false)
+                .default_value("500")
+                .help("Number of transactions to generate and send at a time"),
+        )
+        .arg(
+            Arg::with_name("account-groups")
+                .long("account-groups")
+                .value_name("<groups>")
+                .takes_value(true)
+                .required(false)
+                .default_value("10")
+                .help("Number of account groups to cycle for each batch"),
+        )
+        .arg(
+            Arg::with_name("write-client-keys")
+                .long("write-client-keys")
+                .value_name("FILENAME")
+                .takes_value(true)
+                .help("Generate client keys and stakes and write the list to YAML file"),
+        )
+        .arg(
+            Arg::with_name("read-client-keys")
+                .long("read-client-keys")
+                .value_name("FILENAME")
+                .takes_value(true)
+                .help("Read client keys and stakes from the YAML file"),
+        )
+}
+
+#[allow(clippy::field_reassign_with_default)]
+pub fn extract_args(matches: &ArgMatches) -> Config {
+    let mut args = Config::default();
+
+    args.entrypoint_addr = solana_net_utils::parse_host_port(
+        matches.value_of("entrypoint").unwrap(),
+    )
+    .unwrap_or_else(|e| {
+        eprintln!("failed to parse entrypoint address: {}", e);
+        exit(1)
+    });
+
+    args.faucet_addr = solana_net_utils::parse_host_port(matches.value_of("faucet").unwrap())
+        .unwrap_or_else(|e| {
+            eprintln!("failed to parse faucet address: {}", e);
+            exit(1)
+        });
+
+    if matches.is_present("identity") {
+        args.identity = read_keypair_file(matches.value_of("identity").unwrap())
+            .expect("can't read client identity");
+    } else {
+        args.identity = {
+            let seed = [42_u8; 32];
+            let mut rnd = GenKeys::new(seed);
+            rnd.gen_keypair()
+        };
+    }
+    args.threads = value_t!(matches.value_of("threads"), usize).expect("Failed to parse threads");
+    args.num_nodes =
+        value_t!(matches.value_of("num-nodes"), usize).expect("Failed to parse num-nodes");
+    let duration = value_t!(matches.value_of("duration"), u64).expect("Failed to parse duration");
+    args.duration = Duration::from_secs(duration);
+    args.transfer_delay =
+        value_t!(matches.value_of("transfer-delay"), u64).expect("Failed to parse transfer-delay");
+    args.fund_amount =
+        value_t!(matches.value_of("fund-amount"), u64).expect("Failed to parse fund-amount");
+    args.batch_size =
+        value_t!(matches.value_of("batch-size"), usize).expect("Failed to parse batch-size");
+    args.chunk_size =
+        value_t!(matches.value_of("chunk-size"), usize).expect("Failed to parse chunk-size");
+    args.account_groups = value_t!(matches.value_of("account-groups"), usize)
+        .expect("Failed to parse account-groups");
+
+    if let Some(s) = matches.value_of("write-client-keys") {
+        args.write_to_client_file = true;
+        args.client_ids_and_stake_file = s.to_string();
+    }
+
+    if let Some(s) = matches.value_of("read-client-keys") {
+        assert!(!args.write_to_client_file);
+        args.read_from_client_file = true;
+        args.client_ids_and_stake_file = s.to_string();
+    }
+    args
+}

bench-exchange/src/lib.rs

@@ -0,0 +1,3 @@
+pub mod bench;
+pub mod cli;
+mod order_book;

bench-exchange/src/main.rs

@@ -0,0 +1,83 @@
+#![allow(clippy::integer_arithmetic)]
+pub mod bench;
+mod cli;
+pub mod order_book;
+
+use crate::bench::{airdrop_lamports, create_client_accounts_file, do_bench_exchange, Config};
+use log::*;
+use solana_core::gossip_service::{discover_cluster, get_multi_client};
+use solana_sdk::signature::Signer;
+
+fn main() {
+    solana_logger::setup();
+    solana_metrics::set_panic_hook("bench-exchange");
+
+    let matches = cli::build_args(solana_version::version!()).get_matches();
+    let cli_config = cli::extract_args(&matches);
+
+    let cli::Config {
+        entrypoint_addr,
+        faucet_addr,
+        identity,
+        threads,
+        num_nodes,
+        duration,
+        transfer_delay,
+        fund_amount,
+        batch_size,
+        chunk_size,
+        account_groups,
+        client_ids_and_stake_file,
+        write_to_client_file,
+        read_from_client_file,
+        ..
+    } = cli_config;
+
+    let config = Config {
+        identity,
+        threads,
+        duration,
+        transfer_delay,
+        fund_amount,
+        batch_size,
+        chunk_size,
+        account_groups,
+        client_ids_and_stake_file,
+        read_from_client_file,
+    };
+
+    if write_to_client_file {
+        create_client_accounts_file(
+            &config.client_ids_and_stake_file,
+            config.batch_size,
+            config.account_groups,
+            config.fund_amount,
+        );
+    } else {
+        info!("Connecting to the cluster");
+        let nodes = discover_cluster(&entrypoint_addr, num_nodes).unwrap_or_else(|_| {
+            panic!("Failed to discover nodes");
+        });
+
+        let (client, num_clients) = get_multi_client(&nodes);
+
+        info!("{} nodes found", num_clients);
+        if num_clients < num_nodes {
+            panic!("Error: Insufficient nodes discovered");
+        }
+
+        if !read_from_client_file {
+            info!("Funding keypair: {}", config.identity.pubkey());
+
+            let accounts_in_groups = batch_size * account_groups;
+            const NUM_SIGNERS: u64 = 2;
+            airdrop_lamports(
+                &client,
+                &faucet_addr,
+                &config.identity,
+                fund_amount * (accounts_in_groups + 1) as u64 * NUM_SIGNERS,
+            );
+        }
+        do_bench_exchange(vec![client], config);
+    }
+}

bench-exchange/src/order_book.rs

@@ -0,0 +1,134 @@
+use itertools::EitherOrBoth::{Both, Left, Right};
+use itertools::Itertools;
+use log::*;
+use solana_exchange_program::exchange_state::*;
+use solana_sdk::pubkey::Pubkey;
+use std::cmp::Ordering;
+use std::collections::BinaryHeap;
+use std::{error, fmt};
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct ToOrder {
+    pub pubkey: Pubkey,
+    pub info: OrderInfo,
+}
+
+impl Ord for ToOrder {
+    fn cmp(&self, other: &Self) -> Ordering {
+        other.info.price.cmp(&self.info.price)
+    }
+}
+impl PartialOrd for ToOrder {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct FromOrder {
+    pub pubkey: Pubkey,
+    pub info: OrderInfo,
+}
+
+impl Ord for FromOrder {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.info.price.cmp(&other.info.price)
+    }
+}
+impl PartialOrd for FromOrder {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+#[derive(Default)]
+pub struct OrderBook {
+    // TODO scale to x token types
+    to_ab: BinaryHeap<ToOrder>,
+    from_ab: BinaryHeap<FromOrder>,
+}
+impl fmt::Display for OrderBook {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        writeln!(
+            f,
+            "+-Order Book--------------------------+-------------------------------------+"
+        )?;
+        for (i, it) in self
+            .to_ab
+            .iter()
+            .zip_longest(self.from_ab.iter())
+            .enumerate()
+        {
+            match it {
+                Both(to, from) => writeln!(
+                    f,
+                    "| T AB {:8} for {:8}/{:8} | F AB {:8} for {:8}/{:8} |{}",
+                    to.info.tokens,
+                    SCALER,
+                    to.info.price,
+                    from.info.tokens,
+                    SCALER,
+                    from.info.price,
+                    i
+                )?,
+                Left(to) => writeln!(
+                    f,
+                    "| T AB {:8} for {:8}/{:8} |                                     |{}",
+                    to.info.tokens, SCALER, to.info.price, i
+                )?,
+                Right(from) => writeln!(
+                    f,
+                    "|                                     | F AB {:8} for {:8}/{:8} |{}",
+                    from.info.tokens, SCALER, from.info.price, i
+                )?,
+            }
+        }
+        write!(
+            f,
+            "+-------------------------------------+-------------------------------------+"
+        )?;
+        Ok(())
+    }
+}
+
+impl OrderBook {
+    // TODO
+    // pub fn cancel(&mut self, pubkey: Pubkey) -> Result<(), Box<dyn error::Error>> {
+    //     Ok(())
+    // }
+    pub fn push(&mut self, pubkey: Pubkey, info: OrderInfo) -> Result<(), Box<dyn error::Error>> {
+        check_trade(info.side, info.tokens, info.price)?;
+        match info.side {
+            OrderSide::Ask => {
+                self.to_ab.push(ToOrder { pubkey, info });
+            }
+            OrderSide::Bid => {
+                self.from_ab.push(FromOrder { pubkey, info });
+            }
+        }
+        Ok(())
+    }
+    pub fn pop(&mut self) -> Option<(ToOrder, FromOrder)> {
+        if let Some(pair) = Self::pop_pair(&mut self.to_ab, &mut self.from_ab) {
+            return Some(pair);
+        }
+        None
+    }
+    pub fn get_num_outstanding(&self) -> (usize, usize) {
+        (self.to_ab.len(), self.from_ab.len())
+    }
+
+    fn pop_pair(
+        to_ab: &mut BinaryHeap<ToOrder>,
+        from_ab: &mut BinaryHeap<FromOrder>,
+    ) -> Option<(ToOrder, FromOrder)> {
+        let to = to_ab.peek()?;
+        let from = from_ab.peek()?;
+        if from.info.price < to.info.price {
+            debug!("Trade not viable");
+            return None;
+        }
+        let to = to_ab.pop()?;
+        let from = from_ab.pop()?;
+        Some((to, from))
+    }
+}

bench-exchange/tests/bench_exchange.rs

@@ -0,0 +1,115 @@
+use log::*;
+use solana_bench_exchange::bench::{airdrop_lamports, do_bench_exchange, Config};
+use solana_core::{
+    gossip_service::{discover_cluster, get_multi_client},
+    validator::ValidatorConfig,
+};
+use solana_exchange_program::{
+    exchange_processor::process_instruction, id, solana_exchange_program,
+};
+use solana_faucet::faucet::run_local_faucet_with_port;
+use solana_local_cluster::{
+    local_cluster::{ClusterConfig, LocalCluster},
+    validator_configs::make_identical_validator_configs,
+};
+use solana_runtime::{bank::Bank, bank_client::BankClient};
+use solana_sdk::{
+    genesis_config::create_genesis_config,
+    signature::{Keypair, Signer},
+};
+use std::{process::exit, sync::mpsc::channel, time::Duration};
+
+#[test]
+#[ignore]
+fn test_exchange_local_cluster() {
+    solana_logger::setup();
+
+    const NUM_NODES: usize = 1;
+
+    let config = Config {
+        identity: Keypair::new(),
+        duration: Duration::from_secs(1),
+        fund_amount: 100_000,
+        threads: 1,
+        transfer_delay: 20, // 15
+        batch_size: 100,    // 1000
+        chunk_size: 10,     // 200
+        account_groups: 1,  // 10
+        ..Config::default()
+    };
+    let Config {
+        fund_amount,
+        batch_size,
+        account_groups,
+        ..
+    } = config;
+    let accounts_in_groups = batch_size * account_groups;
+
+    let cluster = LocalCluster::new(&mut ClusterConfig {
+        node_stakes: vec![100_000; NUM_NODES],
+        cluster_lamports: 100_000_000_000_000,
+        validator_configs: make_identical_validator_configs(&ValidatorConfig::default(), NUM_NODES),
+        native_instruction_processors: [solana_exchange_program!()].to_vec(),
+        ..ClusterConfig::default()
+    });
+
+    let faucet_keypair = Keypair::new();
+    cluster.transfer(
+        &cluster.funding_keypair,
+        &faucet_keypair.pubkey(),
+        2_000_000_000_000,
+    );
+
+    let (addr_sender, addr_receiver) = channel();
+    run_local_faucet_with_port(faucet_keypair, addr_sender, Some(1_000_000_000_000), 0);
+    let faucet_addr = addr_receiver
+        .recv_timeout(Duration::from_secs(2))
+        .expect("run_local_faucet")
+        .expect("faucet_addr");
+
+    info!("Connecting to the cluster");
+    let nodes =
+        discover_cluster(&cluster.entry_point_info.gossip, NUM_NODES).unwrap_or_else(|err| {
+            error!("Failed to discover {} nodes: {:?}", NUM_NODES, err);
+            exit(1);
+        });
+
+    let (client, num_clients) = get_multi_client(&nodes);
+
+    info!("clients: {}", num_clients);
+    assert!(num_clients >= NUM_NODES);
+
+    const NUM_SIGNERS: u64 = 2;
+    airdrop_lamports(
+        &client,
+        &faucet_addr,
+        &config.identity,
+        fund_amount * (accounts_in_groups + 1) as u64 * NUM_SIGNERS,
+    );
+
+    do_bench_exchange(vec![client], config);
+}
+
+#[test]
+fn test_exchange_bank_client() {
+    solana_logger::setup();
+    let (genesis_config, identity) = create_genesis_config(100_000_000_000_000);
+    let mut bank = Bank::new(&genesis_config);
+    bank.add_builtin("exchange_program", id(), process_instruction);
+    let clients = vec![BankClient::new(bank)];
+
+    do_bench_exchange(
+        clients,
+        Config {
+            identity,
+            duration: Duration::from_secs(1),
+            fund_amount: 100_000,
+            threads: 1,
+            transfer_delay: 20, // 0;
+            batch_size: 100,    // 1500;
+            chunk_size: 10,     // 1500;
+            account_groups: 1,  // 50;
+            ..Config::default()
+        },
+    );
+}

bench-streamer/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2021"
+edition = "2018"
 name = "solana-bench-streamer"
-version = "1.10.0"
+version = "1.6.28"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,11 +10,11 @@ publish = false
 
 [dependencies]
 clap = "2.33.1"
-solana-clap-utils = { path = "../clap-utils", version = "=1.10.0" }
-solana-streamer = { path = "../streamer", version = "=1.10.0" }
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.28" }
+solana-streamer = { path = "../streamer", version = "=1.6.28" }
+solana-logger = { path = "../logger", version = "=1.6.28" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.28" }
+solana-version = { path = "../version", version = "=1.6.28" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/src/main.rs

@@ -1,38 +1,32 @@
 #![allow(clippy::integer_arithmetic)]
-use {
-    clap::{crate_description, crate_name, App, Arg},
-    solana_streamer::{
-        packet::{Packet, PacketBatch, PacketBatchRecycler, PACKET_DATA_SIZE},
-        streamer::{receiver, PacketBatchReceiver},
-    },
-    std::{
-        cmp::max,
-        net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
-        sync::{
-            atomic::{AtomicBool, AtomicUsize, Ordering},
-            mpsc::channel,
-            Arc,
-        },
-        thread::{sleep, spawn, JoinHandle, Result},
-        time::{Duration, SystemTime},
-    },
-};
+use clap::{crate_description, crate_name, App, Arg};
+use solana_streamer::packet::{Packet, Packets, PacketsRecycler, PACKET_DATA_SIZE};
+use solana_streamer::streamer::{receiver, PacketReceiver};
+use std::cmp::max;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::mpsc::channel;
+use std::sync::Arc;
+use std::thread::sleep;
+use std::thread::{spawn, JoinHandle, Result};
+use std::time::Duration;
+use std::time::SystemTime;
 
 fn producer(addr: &SocketAddr, exit: Arc<AtomicBool>) -> JoinHandle<()> {
     let send = UdpSocket::bind("0.0.0.0:0").unwrap();
-    let mut packet_batch = PacketBatch::default();
-    packet_batch.packets.resize(10, Packet::default());
-    for w in packet_batch.packets.iter_mut() {
+    let mut msgs = Packets::default();
+    msgs.packets.resize(10, Packet::default());
+    for w in msgs.packets.iter_mut() {
         w.meta.size = PACKET_DATA_SIZE;
-        w.meta.set_addr(addr);
+        w.meta.set_addr(&addr);
     }
-    let packet_batch = Arc::new(packet_batch);
+    let msgs = Arc::new(msgs);
     spawn(move || loop {
         if exit.load(Ordering::Relaxed) {
             return;
         }
         let mut num = 0;
-        for p in &packet_batch.packets {
+        for p in &msgs.packets {
             let a = p.meta.addr();
             assert!(p.meta.size <= PACKET_DATA_SIZE);
             send.send_to(&p.data[..p.meta.size], &a).unwrap();
@@ -42,14 +36,14 @@ fn producer(addr: &SocketAddr, exit: Arc<AtomicBool>) -> JoinHandle<()> {
     })
 }
 
-fn sink(exit: Arc<AtomicBool>, rvs: Arc<AtomicUsize>, r: PacketBatchReceiver) -> JoinHandle<()> {
+fn sink(exit: Arc<AtomicBool>, rvs: Arc<AtomicUsize>, r: PacketReceiver) -> JoinHandle<()> {
     spawn(move || loop {
         if exit.load(Ordering::Relaxed) {
             return;
         }
         let timer = Duration::new(1, 0);
-        if let Ok(packet_batch) = r.recv_timeout(timer) {
-            rvs.fetch_add(packet_batch.packets.len(), Ordering::Relaxed);
+        if let Ok(msgs) = r.recv_timeout(timer) {
+            rvs.fetch_add(msgs.packets.len(), Ordering::Relaxed);
         }
     })
 }
@@ -81,7 +75,7 @@ fn main() -> Result<()> {
 
     let mut read_channels = Vec::new();
     let mut read_threads = Vec::new();
-    let recycler = PacketBatchRecycler::default();
+    let recycler = PacketsRecycler::new_without_limit("bench-streamer-recycler-shrink-stats");
     for _ in 0..num_sockets {
         let read = solana_net_utils::bind_to(ip_addr, port, false).unwrap();
         read.set_read_timeout(Some(Duration::new(1, 0))).unwrap();
@@ -98,7 +92,6 @@ fn main() -> Result<()> {
             recycler.clone(),
             "bench-streamer-test",
             1,
-            true,
         ));
     }
 

bench-tps/Cargo.toml

@@ -1,36 +1,36 @@
 [package]
 authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-edition = "2021"
+edition = "2018"
 name = "solana-bench-tps"
-version = "1.10.0"
+version = "1.6.28"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 publish = false
 
 [dependencies]
+bincode = "1.3.1"
 clap = "2.33.1"
-log = "0.4.14"
-rayon = "1.5.1"
-serde_json = "1.0.72"
-serde_yaml = "0.8.21"
-solana-core = { path = "../core", version = "=1.10.0" }
-solana-genesis = { path = "../genesis", version = "=1.10.0" }
-solana-client = { path = "../client", version = "=1.10.0" }
-solana-faucet = { path = "../faucet", version = "=1.10.0" }
-solana-gossip = { path = "../gossip", version = "=1.10.0" }
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-metrics = { path = "../metrics", version = "=1.10.0" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-solana-net-utils = { path = "../net-utils", version = "=1.10.0" }
-solana-runtime = { path = "../runtime", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-solana-streamer = { path = "../streamer", version = "=1.10.0" }
-solana-version = { path = "../version", version = "=1.10.0" }
+log = "0.4.11"
+rayon = "1.5.0"
+serde_json = "1.0.56"
+serde_yaml = "0.8.13"
+solana-clap-utils = { path = "../clap-utils", version = "=1.6.28" }
+solana-core = { path = "../core", version = "=1.6.28" }
+solana-genesis = { path = "../genesis", version = "=1.6.28" }
+solana-client = { path = "../client", version = "=1.6.28" }
+solana-faucet = { path = "../faucet", version = "=1.6.28" }
+solana-logger = { path = "../logger", version = "=1.6.28" }
+solana-metrics = { path = "../metrics", version = "=1.6.28" }
+solana-measure = { path = "../measure", version = "=1.6.28" }
+solana-net-utils = { path = "../net-utils", version = "=1.6.28" }
+solana-runtime = { path = "../runtime", version = "=1.6.28" }
+solana-sdk = { path = "../sdk", version = "=1.6.28" }
+solana-version = { path = "../version", version = "=1.6.28" }
 
 [dev-dependencies]
-serial_test = "0.5.1"
-solana-local-cluster = { path = "../local-cluster", version = "=1.10.0" }
+serial_test = "0.4.0"
+solana-local-cluster = { path = "../local-cluster", version = "=1.6.28" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/src/bench.rs

@@ -1,26 +1,25 @@
-use {
-    crate::cli::Config,
-    log::*,
-    rayon::prelude::*,
-    solana_client::perf_utils::{sample_txs, SampleStats},
-    solana_core::gen_keys::GenKeys,
-    solana_faucet::faucet::request_airdrop_transaction,
-    solana_measure::measure::Measure,
-    solana_metrics::{self, datapoint_info},
-    solana_sdk::{
+use crate::cli::Config;
+use log::*;
+use rayon::prelude::*;
+use solana_client::perf_utils::{sample_txs, SampleStats};
+use solana_core::gen_keys::GenKeys;
+use solana_faucet::faucet::request_airdrop_transaction;
+use solana_measure::measure::Measure;
+use solana_metrics::{self, datapoint_info};
+use solana_sdk::{
     client::Client,
-        clock::{DEFAULT_MS_PER_SLOT, DEFAULT_S_PER_SLOT, MAX_PROCESSING_AGE},
+    clock::{DEFAULT_S_PER_SLOT, MAX_PROCESSING_AGE},
     commitment_config::CommitmentConfig,
+    fee_calculator::FeeCalculator,
     hash::Hash,
-        instruction::{AccountMeta, Instruction},
     message::Message,
     pubkey::Pubkey,
     signature::{Keypair, Signer},
     system_instruction, system_transaction,
     timing::{duration_as_ms, duration_as_s, duration_as_us, timestamp},
     transaction::Transaction,
-    },
-    std::{
+};
+use std::{
     collections::{HashSet, VecDeque},
     net::SocketAddr,
     process::exit,
@@ -30,7 +29,6 @@ use {
     },
     thread::{sleep, Builder, JoinHandle},
     time::{Duration, Instant},
-    },
 };
 
 // The point at which transactions become "too old", in seconds.
@@ -47,12 +45,14 @@ pub type Result<T> = std::result::Result<T, BenchTpsError>;
 
 pub type SharedTransactions = Arc<RwLock<VecDeque<Vec<(Transaction, u64)>>>>;
 
-fn get_latest_blockhash<T: Client>(client: &T) -> Hash {
+fn get_recent_blockhash<T: Client>(client: &T) -> (Hash, FeeCalculator) {
     loop {
-        match client.get_latest_blockhash_with_commitment(CommitmentConfig::processed()) {
-            Ok((blockhash, _)) => return blockhash,
+        match client.get_recent_blockhash_with_commitment(CommitmentConfig::processed()) {
+            Ok((blockhash, fee_calculator, _last_valid_slot)) => {
+                return (blockhash, fee_calculator)
+            }
             Err(err) => {
-                info!("Couldn't get last blockhash: {:?}", err);
+                info!("Couldn't get recent blockhash: {:?}", err);
                 sleep(Duration::from_secs(1));
             }
         };
@@ -239,19 +239,19 @@ where
 
     let shared_txs: SharedTransactions = Arc::new(RwLock::new(VecDeque::new()));
 
-    let blockhash = Arc::new(RwLock::new(get_latest_blockhash(client.as_ref())));
+    let recent_blockhash = Arc::new(RwLock::new(get_recent_blockhash(client.as_ref()).0));
     let shared_tx_active_thread_count = Arc::new(AtomicIsize::new(0));
     let total_tx_sent_count = Arc::new(AtomicUsize::new(0));
 
     let blockhash_thread = {
         let exit_signal = exit_signal.clone();
-        let blockhash = blockhash.clone();
+        let recent_blockhash = recent_blockhash.clone();
         let client = client.clone();
         let id = id.pubkey();
         Builder::new()
             .name("solana-blockhash-poller".to_string())
             .spawn(move || {
-                poll_blockhash(&exit_signal, &blockhash, &client, &id);
+                poll_blockhash(&exit_signal, &recent_blockhash, &client, &id);
             })
             .unwrap()
     };
@@ -271,7 +271,7 @@ where
     let start = Instant::now();
 
     generate_chunked_transfers(
-        blockhash,
+        recent_blockhash,
         &shared_txs,
         shared_tx_active_thread_count,
         source_keypair_chunks,
@@ -391,22 +391,6 @@ fn generate_txs(
     }
 }
 
-fn get_new_latest_blockhash<T: Client>(client: &Arc<T>, blockhash: &Hash) -> Option<Hash> {
-    let start = Instant::now();
-    while start.elapsed().as_secs() < 5 {
-        if let Ok(new_blockhash) = client.get_latest_blockhash() {
-            if new_blockhash != *blockhash {
-                return Some(new_blockhash);
-            }
-        }
-        debug!("Got same blockhash ({:?}), will retry...", blockhash);
-
-        // Retry ~twice during a slot
-        sleep(Duration::from_millis(DEFAULT_MS_PER_SLOT / 2));
-    }
-    None
-}
-
 fn poll_blockhash<T: Client>(
     exit_signal: &Arc<AtomicBool>,
     blockhash: &Arc<RwLock<Hash>>,
@@ -418,7 +402,7 @@ fn poll_blockhash<T: Client>(
     loop {
         let blockhash_updated = {
             let old_blockhash = *blockhash.read().unwrap();
-            if let Some(new_blockhash) = get_new_latest_blockhash(client, &old_blockhash) {
+            if let Ok((new_blockhash, _fee)) = client.get_new_blockhash(&old_blockhash) {
                 *blockhash.write().unwrap() = new_blockhash;
                 blockhash_last_updated = Instant::now();
                 true
@@ -556,16 +540,16 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
                 self.len(),
             );
 
-            let blockhash = get_latest_blockhash(client.as_ref());
+            let (blockhash, _fee_calculator) = get_recent_blockhash(client.as_ref());
 
             // re-sign retained to_fund_txes with updated blockhash
             self.sign(blockhash);
-            self.send(client);
+            self.send(&client);
 
             // Sleep a few slots to allow transactions to process
             sleep(Duration::from_secs(1));
 
-            self.verify(client, to_lamports);
+            self.verify(&client, to_lamports);
 
             // retry anything that seems to have dropped through cracks
             //  again since these txs are all or nothing, they're fine to
@@ -580,7 +564,7 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
         let to_fund_txs: Vec<(&Keypair, Transaction)> = to_fund
             .par_iter()
             .map(|(k, t)| {
-                let instructions = system_instruction::transfer_many(&k.pubkey(), t);
+                let instructions = system_instruction::transfer_many(&k.pubkey(), &t);
                 let message = Message::new(&instructions, Some(&k.pubkey()));
                 (*k, Transaction::new_unsigned(message))
             })
@@ -633,7 +617,7 @@ impl<'a> FundingTransactions<'a> for Vec<(&'a Keypair, Transaction)> {
                         return None;
                     }
 
-                    let verified = if verify_funding_transfer(&client, tx, to_lamports) {
+                    let verified = if verify_funding_transfer(&client, &tx, to_lamports) {
                         verified_txs.fetch_add(1, Ordering::Relaxed);
                         Some(k.pubkey())
                     } else {
@@ -748,8 +732,8 @@ pub fn airdrop_lamports<T: Client>(
             id.pubkey(),
         );
 
-        let blockhash = get_latest_blockhash(client);
-        match request_airdrop_transaction(faucet_addr, &id.pubkey(), airdrop_amount, blockhash) {
+        let (blockhash, _fee_calculator) = get_recent_blockhash(client);
+        match request_airdrop_transaction(&faucet_addr, &id.pubkey(), airdrop_amount, blockhash) {
             Ok(transaction) => {
                 let mut tries = 0;
                 loop {
@@ -906,16 +890,8 @@ pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
     //   pay for the transaction fees in a new run.
     let enough_lamports = 8 * lamports_per_account / 10;
     if first_keypair_balance < enough_lamports || last_keypair_balance < enough_lamports {
-        let single_sig_message = Message::new_with_blockhash(
-            &[Instruction::new_with_bytes(
-                Pubkey::new_unique(),
-                &[],
-                vec![AccountMeta::new(Pubkey::new_unique(), true)],
-            )],
-            None,
-            &client.get_latest_blockhash().unwrap(),
-        );
-        let max_fee = client.get_fee_for_message(&single_sig_message).unwrap();
+        let fee_rate_governor = client.get_fee_rate_governor().unwrap();
+        let max_fee = fee_rate_governor.max_lamports_per_signature;
         let extra_fees = extra * max_fee;
         let total_keypairs = keypairs.len() as u64 + 1; // Add one for funding keypair
         let total = lamports_per_account * total_keypairs + extra_fees;
@@ -948,19 +924,17 @@ pub fn generate_and_fund_keypairs<T: 'static + Client + Send + Sync>(
 
 #[cfg(test)]
 mod tests {
-    use {
-        super::*,
-        solana_runtime::{bank::Bank, bank_client::BankClient},
-        solana_sdk::{
-            client::SyncClient, fee_calculator::FeeRateGovernor,
-            genesis_config::create_genesis_config,
-        },
-    };
+    use super::*;
+    use solana_runtime::bank::Bank;
+    use solana_runtime::bank_client::BankClient;
+    use solana_sdk::client::SyncClient;
+    use solana_sdk::fee_calculator::FeeRateGovernor;
+    use solana_sdk::genesis_config::create_genesis_config;
 
     #[test]
     fn test_bench_tps_bank_client() {
         let (genesis_config, id) = create_genesis_config(10_000);
-        let bank = Bank::new_for_tests(&genesis_config);
+        let bank = Bank::new(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
 
         let config = Config {
@@ -981,7 +955,7 @@ mod tests {
     #[test]
     fn test_bench_tps_fund_keys() {
         let (genesis_config, id) = create_genesis_config(10_000);
-        let bank = Bank::new_for_tests(&genesis_config);
+        let bank = Bank::new(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
         let keypair_count = 20;
         let lamports = 20;
@@ -1004,7 +978,7 @@ mod tests {
         let (mut genesis_config, id) = create_genesis_config(10_000);
         let fee_rate_governor = FeeRateGovernor::new(11, 0);
         genesis_config.fee_rate_governor = fee_rate_governor;
-        let bank = Bank::new_for_tests(&genesis_config);
+        let bank = Bank::new(&genesis_config);
         let client = Arc::new(BankClient::new(bank));
         let keypair_count = 20;
         let lamports = 20;

bench-tps/src/cli.rs

@@ -1,13 +1,11 @@
-use {
-    clap::{crate_description, crate_name, App, Arg, ArgMatches},
-    solana_faucet::faucet::FAUCET_PORT,
-    solana_sdk::{
-        fee_calculator::FeeRateGovernor,
+use clap::{crate_description, crate_name, App, Arg, ArgMatches};
+use solana_faucet::faucet::FAUCET_PORT;
+use solana_sdk::fee_calculator::FeeRateGovernor;
+use solana_sdk::{
     pubkey::Pubkey,
     signature::{read_keypair_file, Keypair},
-    },
-    std::{net::SocketAddr, process::exit, time::Duration},
 };
+use std::{net::SocketAddr, process::exit, time::Duration};
 
 const NUM_LAMPORTS_PER_ACCOUNT_DEFAULT: u64 = solana_sdk::native_token::LAMPORTS_PER_SOL;
 

bench-tps/src/main.rs

@@ -1,20 +1,13 @@
 #![allow(clippy::integer_arithmetic)]
-use {
-    log::*,
-    solana_bench_tps::{
-        bench::{do_bench_tps, generate_and_fund_keypairs, generate_keypairs},
-        cli,
-    },
-    solana_genesis::Base64Account,
-    solana_gossip::gossip_service::{discover_cluster, get_client, get_multi_client},
-    solana_sdk::{
-        fee_calculator::FeeRateGovernor,
-        signature::{Keypair, Signer},
-        system_program,
-    },
-    solana_streamer::socket::SocketAddrSpace,
-    std::{collections::HashMap, fs::File, io::prelude::*, path::Path, process::exit, sync::Arc},
-};
+use log::*;
+use solana_bench_tps::bench::{do_bench_tps, generate_and_fund_keypairs, generate_keypairs};
+use solana_bench_tps::cli;
+use solana_core::gossip_service::{discover_cluster, get_client, get_multi_client};
+use solana_genesis::Base64Account;
+use solana_sdk::fee_calculator::FeeRateGovernor;
+use solana_sdk::signature::{Keypair, Signer};
+use solana_sdk::system_program;
+use std::{collections::HashMap, fs::File, io::prelude::*, path::Path, process::exit, sync::Arc};
 
 /// Number of signatures for all transactions in ~1 week at ~100K TPS
 pub const NUM_SIGNATURES_FOR_TXS: u64 = 100_000 * 60 * 60 * 24 * 7;
@@ -46,7 +39,7 @@ fn main() {
     let keypair_count = *tx_count * keypair_multiplier;
     if *write_to_client_file {
         info!("Generating {} keypairs", keypair_count);
-        let (keypairs, _) = generate_keypairs(id, keypair_count as u64);
+        let (keypairs, _) = generate_keypairs(&id, keypair_count as u64);
         let num_accounts = keypairs.len() as u64;
         let max_fee =
             FeeRateGovernor::new(*target_lamports_per_signature, 0).max_lamports_per_signature;
@@ -75,14 +68,13 @@ fn main() {
     }
 
     info!("Connecting to the cluster");
-    let nodes = discover_cluster(entrypoint_addr, *num_nodes, SocketAddrSpace::Unspecified)
-        .unwrap_or_else(|err| {
+    let nodes = discover_cluster(&entrypoint_addr, *num_nodes).unwrap_or_else(|err| {
         eprintln!("Failed to discover {} nodes: {:?}", num_nodes, err);
         exit(1);
     });
 
     let client = if *multi_client {
-        let (client, num_clients) = get_multi_client(&nodes, &SocketAddrSpace::Unspecified);
+        let (client, num_clients) = get_multi_client(&nodes);
         if nodes.len() < num_clients {
             eprintln!(
                 "Error: Insufficient nodes discovered.  Expecting {} or more",
@@ -96,7 +88,7 @@ fn main() {
         let mut target_client = None;
         for node in nodes {
             if node.id == *target_node {
-                target_client = Some(Arc::new(get_client(&[node], &SocketAddrSpace::Unspecified)));
+                target_client = Some(Arc::new(get_client(&[node])));
                 break;
             }
         }
@@ -105,7 +97,7 @@ fn main() {
             exit(1);
         })
     } else {
-        Arc::new(get_client(&nodes, &SocketAddrSpace::Unspecified))
+        Arc::new(get_client(&nodes))
     };
 
     let keypairs = if *read_from_client_file {
@@ -143,7 +135,7 @@ fn main() {
         generate_and_fund_keypairs(
             client.clone(),
             Some(*faucet_addr),
-            id,
+            &id,
             keypair_count,
             *num_lamports_per_account,
         )

bench-tps/tests/bench_tps.rs

@@ -1,24 +1,20 @@
 #![allow(clippy::integer_arithmetic)]
-use {
-    serial_test::serial,
-    solana_bench_tps::{
+use serial_test::serial;
+use solana_bench_tps::{
     bench::{do_bench_tps, generate_and_fund_keypairs},
     cli::Config,
-    },
-    solana_client::thin_client::create_client,
-    solana_core::validator::ValidatorConfig,
-    solana_faucet::faucet::run_local_faucet_with_port,
-    solana_gossip::cluster_info::VALIDATOR_PORT_RANGE,
-    solana_local_cluster::{
+};
+use solana_client::thin_client::create_client;
+use solana_core::{cluster_info::VALIDATOR_PORT_RANGE, validator::ValidatorConfig};
+use solana_faucet::faucet::run_local_faucet_with_port;
+use solana_local_cluster::{
     local_cluster::{ClusterConfig, LocalCluster},
     validator_configs::make_identical_validator_configs,
-    },
-    solana_sdk::signature::{Keypair, Signer},
-    solana_streamer::socket::SocketAddrSpace,
-    std::{
+};
+use solana_sdk::signature::{Keypair, Signer};
+use std::{
     sync::{mpsc::channel, Arc},
     time::Duration,
-    },
 };
 
 fn test_bench_tps_local_cluster(config: Config) {
@@ -26,19 +22,13 @@ fn test_bench_tps_local_cluster(config: Config) {
 
     solana_logger::setup();
     const NUM_NODES: usize = 1;
-    let cluster = LocalCluster::new(
-        &mut ClusterConfig {
+    let cluster = LocalCluster::new(&mut ClusterConfig {
         node_stakes: vec![999_990; NUM_NODES],
         cluster_lamports: 200_000_000,
-            validator_configs: make_identical_validator_configs(
-                &ValidatorConfig::default(),
-                NUM_NODES,
-            ),
+        validator_configs: make_identical_validator_configs(&ValidatorConfig::default(), NUM_NODES),
         native_instruction_processors,
         ..ClusterConfig::default()
-        },
-        SocketAddrSpace::Unspecified,
-    );
+    });
 
     let faucet_keypair = Keypair::new();
     cluster.transfer(

book/src/proposals/handle-duplicate-block.md

@@ -1,88 +0,0 @@
-# Leader Duplicate Block Slashing
-
-This design describes how the cluster slashes leaders that produce duplicate
-blocks.
-
-Leaders that produce multiple blocks for the same slot increase the number of
-potential forks that the cluster has to resolve.
-
-## Primitives
-1. gossip_root: Nodes now gossip their current root
-2. gossip_duplicate_slots: Nodes can gossip up to `N` duplicate slot proofs.
-3. `DUPLICATE_THRESHOLD`: The minimum percentage of stake that needs to vote on a fork with version `X` of a duplicate slot, in order for that fork to become votable.
-
-## Protocol
-1. When WindowStage detects a duplicate slot proof `P`, it checks the new `gossip_root` to see if `<= 1/3` of the nodes have rooted a slot `S >= P`. If so, it pushes a proof to `gossip_duplicate_slots` to gossip. WindowStage then signals ReplayStage about this duplicate slot `S`. These proofs can be purged from gossip once the validator sees > 2/3 of people gossiping roots `R > S`.
-
-2. When ReplayStage receives the signal for a duplicate slot `S` from `1)` above, the validator monitors gossip and replay waiting for`>= DUPLICATE_THRESHOLD` votes for the same hash which implies the same version of the slot. If this conditon is met for some version with hash `H` of slot `S`, this is then known as the `duplicate_confirmed` version of the slot.
-
-Before a duplicate slot `S` is `duplicate_confirmed`, it's first excluded from the vote candidate set in the fork choice rules. In addition, ReplayStage also resets PoH to the *latest* ancestor of the *earliest* `non-duplicate/confirmed_duplicate_slot`, so that block generation can start happening on the earliest known *safe* block.
-
-Some notes about the `DUPLICATE_THRESHOLD`. In the cases below, assume `DUPLICATE_THRESHOLD = 52`:
-
-a) If less than `2 * DUPLICATE_THRESHOLD - 1` percentage of the network is malicious, then there can only be one such `duplicate_confirmed` version of the slot. With `DUPLICATE_THRESHOLD = 52`, this is
-a malcious tolerance of `4%`
-
-b) The liveness of the network is at most `1 - DUPLICATE_THRESHOLD - SWITCH_THRESHOLD`. This is because if you need at least `SWITCH_THRESHOLD` percentage of the stake voting on a different fork in order to switch off of a duplicate fork that has `< DUPLICATE_THRESHOLD` stake voting on it, and is *not* `duplicate_confirmed`. For `DUPLICATE_THRESHOLD = 52` and `DUPLICATE_THRESHOLD = 38`, this implies a liveness tolerance of `10%`.
-
-For example in the situation below, validators that voted on `2` can't vote any further on fork `2` because it's been removed from fork choice. Now slot 6 better have enough stake for a switching proof, or the network halts.
-
-```text
-    |-------- 2 (51% voted, then detected this slot was a duplicate and removed this slot from fork choice)
-0---|
-    |---------- 6 (39%)
-
-```
-
-3. Switching proofs need to be extended to allow including vote hashes from different versions of the same same slot (detected through 1). Right now this is not supported since switching proofs can
-only be built using votes from banks in BankForks, and two different versions of the same slot cannot
-simultaneously exist in BankForks. For instance:
-
-```text
-    |-------- 2
-    |
-0------------- 1 ------ 2'
-    |
-    |---------- 6
-
-```
-
-Imagine each version of slot 2 and 2' have `DUPLICATE_THRESHOLD / 2` of the votes on them, so neither duplicate can be confirmed. At most slot 6 has `1 - DUPLICATE_THRESHOLD / 2` of the votes
-on it, which is less than the switching threshold. Thus, in order for validators voting on `2` or `2'` to switch to slot 6, and make progress, they need to incorporate votes from the other version of the slot into their switching proofs.
-
-
-### The repair problem.
-Now what happens if one of the following occurs:
-
-1) Due to network blips/latencies, some validators fail to observe the gossip votes before they are overwritten by newer votes? Then some validators may conclude a slot `S` is `duplicate_confirmed` while others don't.
-
-2) Due to lockouts, no version of duplicate slot `S` reaches `duplicate_confirmed` status, but one of its descendants may reach `duplicate_confirmed` after those lockouts expire, which by definition, means `S` is also `duplicate_confirmed`.
-
-3) People who are catching up and don't see the votes in gossip encounter a dup block and can't make progress.
-
-We assume that given a network is eventually stable, if at least one correct validator observed `S` is `duplicate_confirmed`, then if `S` is part of the heaviest fork, then eventually all validators will observe some descendant of `S` is duplicate confirmed.
-
-This problem we need to solve is modeled simply by the below scenario:
-
-```text
-1 -> 2 (duplicate) -> 3 -> 4 (duplicate)
-```
-Assume the following:
-
-1. Due to gossiping duplciate proofs, we assume everyone will eventually see duplicate proofs for 2 and 4, so everyone agrees to remove them from fork choice until they are `duplicate_confirmed`.
-
-2. Due to lockouts, `> DUPLICATE_THRESHOLD` of the stake votes on 4, but not 2. This means at least `DUPLICATE_THRESHOLD` of people have the "correct" version of both slots 2 and 4.
-
-3. However, the remaining `1-DUPLICATE_THRESHOLD` of people have wrong version of 2. This means in replay, their slot 3 will be marked dead, *even though the faulty slot is 2*. The goal is to get these people on the right fork again.
-
-Possible solution:
-
-1. Change `EpochSlots` to signal when a bank is frozen, not when a slot is complete. If we see > `DUPLICATE_THRESHOLD` have frozen the dead slot 3, then we attempt recovery. Note this does not mean that all `DUPLICATE_THRESHOLD` have frozen the same version of the bank, it's just a signal to us that something may be wrong with our version of the bank.
-
-2. Recovery takes the form of a special repair request, `RepairDuplicateConfirmed(dead_slot, Vec<(Slot, Hash)>)`, which specifies a  dead slot, and then a vector of `(slot, hash)` of `N` of its latest parents.
-
-3. The repairer sees this request and responds with the correct hash only if any element of the `(slot, hash)` vector is both `duplicate_confirmed` and the hash doesn't match the requester's hash in the vector.
-
-4. Once the requester sees the "correct" hash is different than their frozen hash, they dump the block so that they can accept a new block, and ask the network for the block with the correct hash.
-
-Of course the repairer might lie to you, and you'll get the wrong version of the block, in which case you'll end up with another dead block and repeat the procedure.

bucket_map/Cargo.toml

@@ -1,29 +0,0 @@
-[package]
-name = "solana-bucket-map"
-version = "1.10.0"
-description = "solana-bucket-map"
-homepage = "https://solana.com/"
-documentation = "https://docs.rs/solana-bucket-map"
-readme = "../README.md"
-repository = "https://github.com/solana-labs/solana"
-authors = ["Solana Maintainers <maintainers@solana.foundation>"]
-license = "Apache-2.0"
-edition = "2021"
-
-[dependencies]
-rayon = "1.5.0"
-solana-logger = { path = "../logger", version = "=1.10.0" }
-solana-sdk = { path = "../sdk", version = "=1.10.0" }
-memmap2 = "0.5.0"
-log = { version = "0.4.11" }
-solana-measure = { path = "../measure", version = "=1.10.0" }
-rand = "0.7.0"
-fs_extra = "1.2.0"
-tempfile = "3.2.0"
-
-[lib]
-crate-type = ["lib"]
-name = "solana_bucket_map"
-
-[[bench]]
-name = "bucket_map"

bucket_map/benches/bucket_map.rs

@@ -1,77 +0,0 @@
-#![feature(test)]
-
-macro_rules! DEFINE_NxM_BENCH {
-    ($i:ident, $n:literal, $m:literal) => {
-        mod $i {
-            use super::*;
-
-            #[bench]
-            fn bench_insert_baseline_hashmap(bencher: &mut Bencher) {
-                do_bench_insert_baseline_hashmap(bencher, $n, $m);
-            }
-
-            #[bench]
-            fn bench_insert_bucket_map(bencher: &mut Bencher) {
-                do_bench_insert_bucket_map(bencher, $n, $m);
-            }
-        }
-    };
-}
-
-extern crate test;
-use {
-    rayon::prelude::*,
-    solana_bucket_map::bucket_map::{BucketMap, BucketMapConfig},
-    solana_sdk::pubkey::Pubkey,
-    std::{collections::hash_map::HashMap, sync::RwLock},
-    test::Bencher,
-};
-
-type IndexValue = u64;
-
-DEFINE_NxM_BENCH!(dim_01x02, 1, 2);
-DEFINE_NxM_BENCH!(dim_02x04, 2, 4);
-DEFINE_NxM_BENCH!(dim_04x08, 4, 8);
-DEFINE_NxM_BENCH!(dim_08x16, 8, 16);
-DEFINE_NxM_BENCH!(dim_16x32, 16, 32);
-DEFINE_NxM_BENCH!(dim_32x64, 32, 64);
-
-/// Benchmark insert with Hashmap as baseline for N threads inserting M keys each
-fn do_bench_insert_baseline_hashmap(bencher: &mut Bencher, n: usize, m: usize) {
-    let index = RwLock::new(HashMap::new());
-    (0..n).into_iter().into_par_iter().for_each(|i| {
-        let key = Pubkey::new_unique();
-        index
-            .write()
-            .unwrap()
-            .insert(key, vec![(i, IndexValue::default())]);
-    });
-    bencher.iter(|| {
-        (0..n).into_iter().into_par_iter().for_each(|_| {
-            for j in 0..m {
-                let key = Pubkey::new_unique();
-                index
-                    .write()
-                    .unwrap()
-                    .insert(key, vec![(j, IndexValue::default())]);
-            }
-        })
-    });
-}
-
-/// Benchmark insert with BucketMap with N buckets for N threads inserting M keys each
-fn do_bench_insert_bucket_map(bencher: &mut Bencher, n: usize, m: usize) {
-    let index = BucketMap::new(BucketMapConfig::new(n));
-    (0..n).into_iter().into_par_iter().for_each(|i| {
-        let key = Pubkey::new_unique();
-        index.update(&key, |_| Some((vec![(i, IndexValue::default())], 0)));
-    });
-    bencher.iter(|| {
-        (0..n).into_iter().into_par_iter().for_each(|_| {
-            for j in 0..m {
-                let key = Pubkey::new_unique();
-                index.update(&key, |_| Some((vec![(j, IndexValue::default())], 0)));
-            }
-        })
-    });
-}

bucket_map/src/bucket.rs

@@ -1,491 +0,0 @@
-use {
-    crate::{
-        bucket_item::BucketItem,
-        bucket_map::BucketMapError,
-        bucket_stats::BucketMapStats,
-        bucket_storage::{BucketStorage, Uid, DEFAULT_CAPACITY_POW2, UID_UNLOCKED},
-        index_entry::IndexEntry,
-        MaxSearch, RefCount,
-    },
-    rand::{thread_rng, Rng},
-    solana_measure::measure::Measure,
-    solana_sdk::pubkey::Pubkey,
-    std::{
-        collections::hash_map::DefaultHasher,
-        hash::{Hash, Hasher},
-        marker::PhantomData,
-        ops::RangeBounds,
-        path::PathBuf,
-        sync::{
-            atomic::{AtomicUsize, Ordering},
-            Arc, Mutex,
-        },
-    },
-};
-
-#[derive(Default)]
-pub struct ReallocatedItems {
-    // Some if the index was reallocated
-    // u64 is random associated with the new index
-    pub index: Option<(u64, BucketStorage)>,
-    // Some for a data bucket reallocation
-    // u64 is data bucket index
-    pub data: Option<(u64, BucketStorage)>,
-}
-
-#[derive(Default)]
-pub struct Reallocated {
-    /// > 0 if reallocations are encoded
-    pub active_reallocations: AtomicUsize,
-
-    /// actual reallocated bucket
-    /// mutex because bucket grow code runs with a read lock
-    pub items: Mutex<ReallocatedItems>,
-}
-
-impl Reallocated {
-    /// specify that a reallocation has occurred
-    pub fn add_reallocation(&self) {
-        assert_eq!(
-            0,
-            self.active_reallocations.fetch_add(1, Ordering::Relaxed),
-            "Only 1 reallocation can occur at a time"
-        );
-    }
-    /// Return true IFF a reallocation has occurred.
-    /// Calling this takes conceptual ownership of the reallocation encoded in the struct.
-    pub fn get_reallocated(&self) -> bool {
-        self.active_reallocations
-            .compare_exchange(1, 0, Ordering::Acquire, Ordering::Relaxed)
-            .is_ok()
-    }
-}
-
-// >= 2 instances of BucketStorage per 'bucket' in the bucket map. 1 for index, >= 1 for data
-pub struct Bucket<T> {
-    drives: Arc<Vec<PathBuf>>,
-    //index
-    pub index: BucketStorage,
-    //random offset for the index
-    random: u64,
-    //storage buckets to store SlotSlice up to a power of 2 in len
-    pub data: Vec<BucketStorage>,
-    _phantom: PhantomData<T>,
-    stats: Arc<BucketMapStats>,
-
-    pub reallocated: Reallocated,
-}
-
-impl<T: Clone + Copy> Bucket<T> {
-    pub fn new(
-        drives: Arc<Vec<PathBuf>>,
-        max_search: MaxSearch,
-        stats: Arc<BucketMapStats>,
-    ) -> Self {
-        let index = BucketStorage::new(
-            Arc::clone(&drives),
-            1,
-            std::mem::size_of::<IndexEntry>() as u64,
-            max_search,
-            Arc::clone(&stats.index),
-        );
-        Self {
-            random: thread_rng().gen(),
-            drives,
-            index,
-            data: vec![],
-            _phantom: PhantomData::default(),
-            stats,
-            reallocated: Reallocated::default(),
-        }
-    }
-
-    pub fn bucket_len(&self) -> u64 {
-        self.index.used.load(Ordering::Relaxed)
-    }
-
-    pub fn keys(&self) -> Vec<Pubkey> {
-        let mut rv = vec![];
-        for i in 0..self.index.capacity() {
-            if self.index.uid(i) == UID_UNLOCKED {
-                continue;
-            }
-            let ix: &IndexEntry = self.index.get(i);
-            rv.push(ix.key);
-        }
-        rv
-    }
-
-    pub fn items_in_range<R>(&self, range: &Option<&R>) -> Vec<BucketItem<T>>
-    where
-        R: RangeBounds<Pubkey>,
-    {
-        let mut result = Vec::with_capacity(self.index.used.load(Ordering::Relaxed) as usize);
-        for i in 0..self.index.capacity() {
-            let ii = i % self.index.capacity();
-            if self.index.uid(ii) == UID_UNLOCKED {
-                continue;
-            }
-            let ix: &IndexEntry = self.index.get(ii);
-            let key = ix.key;
-            if range.map(|r| r.contains(&key)).unwrap_or(true) {
-                let val = ix.read_value(self);
-                result.push(BucketItem {
-                    pubkey: key,
-                    ref_count: ix.ref_count(),
-                    slot_list: val.map(|(v, _ref_count)| v.to_vec()).unwrap_or_default(),
-                });
-            }
-        }
-        result
-    }
-
-    pub fn find_entry(&self, key: &Pubkey) -> Option<(&IndexEntry, u64)> {
-        Self::bucket_find_entry(&self.index, key, self.random)
-    }
-
-    fn find_entry_mut(&self, key: &Pubkey) -> Option<(&mut IndexEntry, u64)> {
-        Self::bucket_find_entry_mut(&self.index, key, self.random)
-    }
-
-    fn bucket_find_entry_mut<'a>(
-        index: &'a BucketStorage,
-        key: &Pubkey,
-        random: u64,
-    ) -> Option<(&'a mut IndexEntry, u64)> {
-        let ix = Self::bucket_index_ix(index, key, random);
-        for i in ix..ix + index.max_search() {
-            let ii = i % index.capacity();
-            if index.uid(ii) == UID_UNLOCKED {
-                continue;
-            }
-            let elem: &mut IndexEntry = index.get_mut(ii);
-            if elem.key == *key {
-                return Some((elem, ii));
-            }
-        }
-        None
-    }
-
-    fn bucket_find_entry<'a>(
-        index: &'a BucketStorage,
-        key: &Pubkey,
-        random: u64,
-    ) -> Option<(&'a IndexEntry, u64)> {
-        let ix = Self::bucket_index_ix(index, key, random);
-        for i in ix..ix + index.max_search() {
-            let ii = i % index.capacity();
-            if index.uid(ii) == UID_UNLOCKED {
-                continue;
-            }
-            let elem: &IndexEntry = index.get(ii);
-            if elem.key == *key {
-                return Some((elem, ii));
-            }
-        }
-        None
-    }
-
-    fn bucket_create_key(
-        index: &BucketStorage,
-        key: &Pubkey,
-        elem_uid: Uid,
-        random: u64,
-    ) -> Result<u64, BucketMapError> {
-        let ix = Self::bucket_index_ix(index, key, random);
-        for i in ix..ix + index.max_search() {
-            let ii = i as u64 % index.capacity();
-            if index.uid(ii) != UID_UNLOCKED {
-                continue;
-            }
-            index.allocate(ii, elem_uid).unwrap();
-            let mut elem: &mut IndexEntry = index.get_mut(ii);
-            elem.key = *key;
-            // These will be overwritten after allocation by callers.
-            // Since this part of the mmapped file could have previously been used by someone else, there can be garbage here.
-            elem.ref_count = 0;
-            elem.storage_offset = 0;
-            elem.storage_capacity_when_created_pow2 = 0;
-            elem.num_slots = 0;
-            //debug!(                "INDEX ALLOC {:?} {} {} {}",                key, ii, index.capacity, elem_uid            );
-            return Ok(ii);
-        }
-        Err(BucketMapError::IndexNoSpace(index.capacity_pow2))
-    }
-
-    pub fn addref(&mut self, key: &Pubkey) -> Option<RefCount> {
-        let (elem, _) = self.find_entry_mut(key)?;
-        elem.ref_count += 1;
-        Some(elem.ref_count)
-    }
-
-    pub fn unref(&mut self, key: &Pubkey) -> Option<RefCount> {
-        let (elem, _) = self.find_entry_mut(key)?;
-        elem.ref_count -= 1;
-        Some(elem.ref_count)
-    }
-
-    fn create_key(&self, key: &Pubkey) -> Result<u64, BucketMapError> {
-        Self::bucket_create_key(&self.index, key, IndexEntry::key_uid(key), self.random)
-    }
-
-    pub fn read_value(&self, key: &Pubkey) -> Option<(&[T], RefCount)> {
-        //debug!("READ_VALUE: {:?}", key);
-        let (elem, _) = self.find_entry(key)?;
-        elem.read_value(self)
-    }
-
-    pub fn try_write(
-        &mut self,
-        key: &Pubkey,
-        data: &[T],
-        ref_count: u64,
-    ) -> Result<(), BucketMapError> {
-        let best_fit_bucket = IndexEntry::data_bucket_from_num_slots(data.len() as u64);
-        if self.data.get(best_fit_bucket as usize).is_none() {
-            // fail early if the data bucket we need doesn't exist - we don't want the index entry partially allocated
-            return Err(BucketMapError::DataNoSpace((best_fit_bucket, 0)));
-        }
-        let index_entry = self.find_entry_mut(key);
-        let (elem, elem_ix) = match index_entry {
-            None => {
-                let ii = self.create_key(key)?;
-                let elem: &mut IndexEntry = self.index.get_mut(ii);
-                (elem, ii)
-            }
-            Some(res) => res,
-        };
-        elem.ref_count = ref_count;
-        let elem_uid = self.index.uid(elem_ix);
-        let bucket_ix = elem.data_bucket_ix();
-        let current_bucket = &self.data[bucket_ix as usize];
-        if best_fit_bucket == bucket_ix && elem.num_slots > 0 {
-            // in place update
-            let elem_loc = elem.data_loc(current_bucket);
-            let slice: &mut [T] = current_bucket.get_mut_cell_slice(elem_loc, data.len() as u64);
-            assert!(current_bucket.uid(elem_loc) == elem_uid);
-            elem.num_slots = data.len() as u64;
-            slice.clone_from_slice(data);
-            Ok(())
-        } else {
-            // need to move the allocation to a best fit spot
-            let best_bucket = &self.data[best_fit_bucket as usize];
-            let cap_power = best_bucket.capacity_pow2;
-            let cap = best_bucket.capacity();
-            let pos = thread_rng().gen_range(0, cap);
-            for i in pos..pos + self.index.max_search() {
-                let ix = i % cap;
-                if best_bucket.uid(ix) == UID_UNLOCKED {
-                    let elem_loc = elem.data_loc(current_bucket);
-                    if elem.num_slots > 0 {
-                        current_bucket.free(elem_loc, elem_uid);
-                    }
-                    elem.storage_offset = ix;
-                    elem.storage_capacity_when_created_pow2 = best_bucket.capacity_pow2;
-                    elem.num_slots = data.len() as u64;
-                    //debug!(                        "DATA ALLOC {:?} {} {} {}",                        key, elem.data_location, best_bucket.capacity, elem_uid                    );
-                    if elem.num_slots > 0 {
-                        best_bucket.allocate(ix, elem_uid).unwrap();
-                        let slice = best_bucket.get_mut_cell_slice(ix, data.len() as u64);
-                        slice.copy_from_slice(data);
-                    }
-                    return Ok(());
-                }
-            }
-            Err(BucketMapError::DataNoSpace((best_fit_bucket, cap_power)))
-        }
-    }
-
-    pub fn delete_key(&mut self, key: &Pubkey) {
-        if let Some((elem, elem_ix)) = self.find_entry(key) {
-            let elem_uid = self.index.uid(elem_ix);
-            if elem.num_slots > 0 {
-                let data_bucket = &self.data[elem.data_bucket_ix() as usize];
-                let loc = elem.data_loc(data_bucket);
-                //debug!(                    "DATA FREE {:?} {} {} {}",                    key, elem.data_location, data_bucket.capacity, elem_uid                );
-                data_bucket.free(loc, elem_uid);
-            }
-            //debug!("INDEX FREE {:?} {}", key, elem_uid);
-            self.index.free(elem_ix, elem_uid);
-        }
-    }
-
-    pub fn grow_index(&self, current_capacity_pow2: u8) {
-        if self.index.capacity_pow2 == current_capacity_pow2 {
-            let mut m = Measure::start("grow_index");
-            //debug!("GROW_INDEX: {}", current_capacity_pow2);
-            let increment = 1;
-            for i in increment.. {
-                //increasing the capacity by ^4 reduces the
-                //likelyhood of a re-index collision of 2^(max_search)^2
-                //1 in 2^32
-                let index = BucketStorage::new_with_capacity(
-                    Arc::clone(&self.drives),
-                    1,
-                    std::mem::size_of::<IndexEntry>() as u64,
-                    // *2 causes rapid growth of index buckets
-                    self.index.capacity_pow2 + i, // * 2,
-                    self.index.max_search,
-                    Arc::clone(&self.stats.index),
-                );
-                let random = thread_rng().gen();
-                let mut valid = true;
-                for ix in 0..self.index.capacity() {
-                    let uid = self.index.uid(ix);
-                    if UID_UNLOCKED != uid {
-                        let elem: &IndexEntry = self.index.get(ix);
-                        let new_ix = Self::bucket_create_key(&index, &elem.key, uid, random);
-                        if new_ix.is_err() {
-                            valid = false;
-                            break;
-                        }
-                        let new_ix = new_ix.unwrap();
-                        let new_elem: &mut IndexEntry = index.get_mut(new_ix);
-                        *new_elem = *elem;
-                        /*
-                        let dbg_elem: IndexEntry = *new_elem;
-                        assert_eq!(
-                            Self::bucket_find_entry(&index, &elem.key, random).unwrap(),
-                            (&dbg_elem, new_ix)
-                        );
-                        */
-                    }
-                }
-                if valid {
-                    let sz = index.capacity();
-                    {
-                        let mut max = self.stats.index.max_size.lock().unwrap();
-                        *max = std::cmp::max(*max, sz);
-                    }
-                    let mut items = self.reallocated.items.lock().unwrap();
-                    items.index = Some((random, index));
-                    self.reallocated.add_reallocation();
-                    break;
-                }
-            }
-            m.stop();
-            self.stats.index.resizes.fetch_add(1, Ordering::Relaxed);
-            self.stats
-                .index
-                .resize_us
-                .fetch_add(m.as_us(), Ordering::Relaxed);
-        }
-    }
-
-    pub fn apply_grow_index(&mut self, random: u64, index: BucketStorage) {
-        self.random = random;
-        self.index = index;
-    }
-
-    fn elem_size() -> u64 {
-        std::mem::size_of::<T>() as u64
-    }
-
-    pub fn apply_grow_data(&mut self, ix: usize, bucket: BucketStorage) {
-        if self.data.get(ix).is_none() {
-            for i in self.data.len()..ix {
-                // insert empty data buckets
-                self.data.push(BucketStorage::new(
-                    Arc::clone(&self.drives),
-                    1 << i,
-                    Self::elem_size(),
-                    self.index.max_search,
-                    Arc::clone(&self.stats.data),
-                ))
-            }
-            self.data.push(bucket);
-        } else {
-            self.data[ix] = bucket;
-        }
-    }
-
-    /// grow a data bucket
-    /// The application of the new bucket is deferred until the next write lock.
-    pub fn grow_data(&self, data_index: u64, current_capacity_pow2: u8) {
-        let new_bucket = BucketStorage::new_resized(
-            &self.drives,
-            self.index.max_search,
-            self.data.get(data_index as usize),
-            std::cmp::max(current_capacity_pow2 + 1, DEFAULT_CAPACITY_POW2),
-            1 << data_index,
-            Self::elem_size(),
-            &self.stats.data,
-        );
-        self.reallocated.add_reallocation();
-        let mut items = self.reallocated.items.lock().unwrap();
-        items.data = Some((data_index, new_bucket));
-    }
-
-    fn bucket_index_ix(index: &BucketStorage, key: &Pubkey, random: u64) -> u64 {
-        let uid = IndexEntry::key_uid(key);
-        let mut s = DefaultHasher::new();
-        uid.hash(&mut s);
-        //the locally generated random will make it hard for an attacker
-        //to deterministically cause all the pubkeys to land in the same
-        //location in any bucket on all validators
-        random.hash(&mut s);
-        let ix = s.finish();
-        ix % index.capacity()
-        //debug!(            "INDEX_IX: {:?} uid:{} loc: {} cap:{}",            key,            uid,            location,            index.capacity()        );
-    }
-
-    /// grow the appropriate piece. Note this takes an immutable ref.
-    /// The actual grow is set into self.reallocated and applied later on a write lock
-    pub fn grow(&self, err: BucketMapError) {
-        match err {
-            BucketMapError::DataNoSpace((data_index, current_capacity_pow2)) => {
-                //debug!("GROWING SPACE {:?}", (data_index, current_capacity_pow2));
-                self.grow_data(data_index, current_capacity_pow2);
-            }
-            BucketMapError::IndexNoSpace(current_capacity_pow2) => {
-                //debug!("GROWING INDEX {}", sz);
-                self.grow_index(current_capacity_pow2);
-            }
-        }
-    }
-
-    /// if a bucket was resized previously with a read lock, then apply that resize now
-    pub fn handle_delayed_grows(&mut self) {
-        if self.reallocated.get_reallocated() {
-            // swap out the bucket that was resized previously with a read lock
-            let mut items = ReallocatedItems::default();
-            std::mem::swap(&mut items, &mut self.reallocated.items.lock().unwrap());
-
-            if let Some((random, bucket)) = items.index.take() {
-                self.apply_grow_index(random, bucket);
-            } else {
-                // data bucket
-                let (i, new_bucket) = items.data.take().unwrap();
-                self.apply_grow_data(i as usize, new_bucket);
-            }
-        }
-    }
-
-    pub fn insert(&mut self, key: &Pubkey, value: (&[T], RefCount)) {
-        let (new, refct) = value;
-        loop {
-            let rv = self.try_write(key, new, refct);
-            match rv {
-                Ok(_) => return,
-                Err(err) => {
-                    self.grow(err);
-                    self.handle_delayed_grows();
-                }
-            }
-        }
-    }
-
-    pub fn update<F>(&mut self, key: &Pubkey, mut updatefn: F)
-    where
-        F: FnMut(Option<(&[T], RefCount)>) -> Option<(Vec<T>, RefCount)>,
-    {
-        let current = self.read_value(key);
-        let new = updatefn(current);
-        if new.is_none() {
-            self.delete_key(key);
-            return;
-        }
-        let (new, refct) = new.unwrap();
-        self.insert(key, (&new, refct));
-    }
-}

bucket_map/src/bucket_api.rs

@@ -1,148 +0,0 @@
-use {
-    crate::{
-        bucket::Bucket, bucket_item::BucketItem, bucket_map::BucketMapError,
-        bucket_stats::BucketMapStats, MaxSearch, RefCount,
-    },
-    solana_sdk::pubkey::Pubkey,
-    std::{
-        ops::RangeBounds,
-        path::PathBuf,
-        sync::{
-            atomic::{AtomicU64, Ordering},
-            Arc, RwLock, RwLockWriteGuard,
-        },
-    },
-};
-
-type LockedBucket<T> = RwLock<Option<Bucket<T>>>;
-
-pub struct BucketApi<T: Clone + Copy> {
-    drives: Arc<Vec<PathBuf>>,
-    max_search: MaxSearch,
-    pub stats: Arc<BucketMapStats>,
-
-    bucket: LockedBucket<T>,
-    count: Arc<AtomicU64>,
-}
-
-impl<T: Clone + Copy> BucketApi<T> {
-    pub fn new(
-        drives: Arc<Vec<PathBuf>>,
-        max_search: MaxSearch,
-        stats: Arc<BucketMapStats>,
-        count: Arc<AtomicU64>,
-    ) -> Self {
-        Self {
-            drives,
-            max_search,
-            stats,
-            bucket: RwLock::default(),
-            count,
-        }
-    }
-
-    /// Get the items for bucket
-    pub fn items_in_range<R>(&self, range: &Option<&R>) -> Vec<BucketItem<T>>
-    where
-        R: RangeBounds<Pubkey>,
-    {
-        self.bucket
-            .read()
-            .unwrap()
-            .as_ref()
-            .map(|bucket| bucket.items_in_range(range))
-            .unwrap_or_default()
-    }
-
-    /// Get the Pubkeys
-    pub fn keys(&self) -> Vec<Pubkey> {
-        self.bucket
-            .read()
-            .unwrap()
-            .as_ref()
-            .map_or_else(Vec::default, |bucket| bucket.keys())
-    }
-
-    /// Get the values for Pubkey `key`
-    pub fn read_value(&self, key: &Pubkey) -> Option<(Vec<T>, RefCount)> {
-        self.bucket.read().unwrap().as_ref().and_then(|bucket| {
-            bucket
-                .read_value(key)
-                .map(|(value, ref_count)| (value.to_vec(), ref_count))
-        })
-    }
-
-    pub fn bucket_len(&self) -> u64 {
-        self.bucket
-            .read()
-            .unwrap()
-            .as_ref()
-            .map(|bucket| bucket.bucket_len())
-            .unwrap_or_default()
-    }
-
-    pub fn delete_key(&self, key: &Pubkey) {
-        let mut bucket = self.get_write_bucket();
-        if let Some(bucket) = bucket.as_mut() {
-            bucket.delete_key(key)
-        }
-    }
-
-    fn get_write_bucket(&self) -> RwLockWriteGuard<Option<Bucket<T>>> {
-        let mut bucket = self.bucket.write().unwrap();
-        if bucket.is_none() {
-            *bucket = Some(Bucket::new(
-                Arc::clone(&self.drives),
-                self.max_search,
-                Arc::clone(&self.stats),
-            ));
-        } else {
-            let write = bucket.as_mut().unwrap();
-            write.handle_delayed_grows();
-            self.count.store(write.bucket_len(), Ordering::Relaxed);
-        }
-        bucket
-    }
-
-    pub fn addref(&self, key: &Pubkey) -> Option<RefCount> {
-        self.get_write_bucket()
-            .as_mut()
-            .and_then(|bucket| bucket.addref(key))
-    }
-
-    pub fn unref(&self, key: &Pubkey) -> Option<RefCount> {
-        self.get_write_bucket()
-            .as_mut()
-            .and_then(|bucket| bucket.unref(key))
-    }
-
-    pub fn insert(&self, pubkey: &Pubkey, value: (&[T], RefCount)) {
-        let mut bucket = self.get_write_bucket();
-        bucket.as_mut().unwrap().insert(pubkey, value)
-    }
-
-    pub fn grow(&self, err: BucketMapError) {
-        // grows are special - they get a read lock and modify 'reallocated'
-        // the grown changes are applied the next time there is a write lock taken
-        if let Some(bucket) = self.bucket.read().unwrap().as_ref() {
-            bucket.grow(err)
-        }
-    }
-
-    pub fn update<F>(&self, key: &Pubkey, updatefn: F)
-    where
-        F: FnMut(Option<(&[T], RefCount)>) -> Option<(Vec<T>, RefCount)>,
-    {
-        let mut bucket = self.get_write_bucket();
-        bucket.as_mut().unwrap().update(key, updatefn)
-    }
-
-    pub fn try_write(
-        &self,
-        pubkey: &Pubkey,
-        value: (&[T], RefCount),
-    ) -> Result<(), BucketMapError> {
-        let mut bucket = self.get_write_bucket();
-        bucket.as_mut().unwrap().try_write(pubkey, value.0, value.1)
-    }
-}

bucket_map/src/bucket_item.rs

@@ -1,8 +0,0 @@
-use {crate::RefCount, solana_sdk::pubkey::Pubkey};
-
-#[derive(Debug, Default, Clone)]
-pub struct BucketItem<T> {
-    pub pubkey: Pubkey,
-    pub ref_count: RefCount,
-    pub slot_list: Vec<T>,
-}

bucket_map/src/bucket_map.rs

@@ -1,529 +0,0 @@
-//! BucketMap is a mostly contention free concurrent map backed by MmapMut
-
-use {
-    crate::{bucket_api::BucketApi, bucket_stats::BucketMapStats, MaxSearch, RefCount},
-    solana_sdk::pubkey::Pubkey,
-    std::{convert::TryInto, fmt::Debug, fs, path::PathBuf, sync::Arc},
-    tempfile::TempDir,
-};
-
-#[derive(Debug, Default, Clone)]
-pub struct BucketMapConfig {
-    pub max_buckets: usize,
-    pub drives: Option<Vec<PathBuf>>,
-    pub max_search: Option<MaxSearch>,
-}
-
-impl BucketMapConfig {
-    /// Create a new BucketMapConfig
-    /// NOTE: BucketMap requires that max_buckets is a power of two
-    pub fn new(max_buckets: usize) -> BucketMapConfig {
-        BucketMapConfig {
-            max_buckets,
-            ..BucketMapConfig::default()
-        }
-    }
-}
-
-pub struct BucketMap<T: Clone + Copy + Debug> {
-    buckets: Vec<Arc<BucketApi<T>>>,
-    drives: Arc<Vec<PathBuf>>,
-    max_buckets_pow2: u8,
-    pub stats: Arc<BucketMapStats>,
-    pub temp_dir: Option<TempDir>,
-}
-
-impl<T: Clone + Copy + Debug> Drop for BucketMap<T> {
-    fn drop(&mut self) {
-        if self.temp_dir.is_none() {
-            BucketMap::<T>::erase_previous_drives(&self.drives);
-        }
-    }
-}
-
-impl<T: Clone + Copy + Debug> std::fmt::Debug for BucketMap<T> {
-    fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        Ok(())
-    }
-}
-
-#[derive(Debug)]
-pub enum BucketMapError {
-    /// (bucket_index, current_capacity_pow2)
-    DataNoSpace((u64, u8)),
-    /// current_capacity_pow2
-    IndexNoSpace(u8),
-}
-
-impl<T: Clone + Copy + Debug> BucketMap<T> {
-    pub fn new(config: BucketMapConfig) -> Self {
-        assert_ne!(
-            config.max_buckets, 0,
-            "Max number of buckets must be non-zero"
-        );
-        assert!(
-            config.max_buckets.is_power_of_two(),
-            "Max number of buckets must be a power of two"
-        );
-        // this should be <= 1 << DEFAULT_CAPACITY or we end up searching the same items over and over - probably not a big deal since it is so small anyway
-        const MAX_SEARCH: MaxSearch = 32;
-        let max_search = config.max_search.unwrap_or(MAX_SEARCH);
-
-        if let Some(drives) = config.drives.as_ref() {
-            Self::erase_previous_drives(drives);
-        }
-        let mut temp_dir = None;
-        let drives = config.drives.unwrap_or_else(|| {
-            temp_dir = Some(TempDir::new().unwrap());
-            vec![temp_dir.as_ref().unwrap().path().to_path_buf()]
-        });
-        let drives = Arc::new(drives);
-
-        let mut per_bucket_count = Vec::with_capacity(config.max_buckets);
-        per_bucket_count.resize_with(config.max_buckets, Arc::default);
-        let stats = Arc::new(BucketMapStats {
-            per_bucket_count,
-            ..BucketMapStats::default()
-        });
-        let buckets = stats
-            .per_bucket_count
-            .iter()
-            .map(|per_bucket_count| {
-                Arc::new(BucketApi::new(
-                    Arc::clone(&drives),
-                    max_search,
-                    Arc::clone(&stats),
-                    Arc::clone(per_bucket_count),
-                ))
-            })
-            .collect();
-
-        // A simple log2 function that is correct if x is a power of two
-        let log2 = |x: usize| usize::BITS - x.leading_zeros() - 1;
-
-        Self {
-            buckets,
-            drives,
-            max_buckets_pow2: log2(config.max_buckets) as u8,
-            stats,
-            temp_dir,
-        }
-    }
-
-    fn erase_previous_drives(drives: &[PathBuf]) {
-        drives.iter().for_each(|folder| {
-            let _ = fs::remove_dir_all(&folder);
-            let _ = fs::create_dir_all(&folder);
-        })
-    }
-
-    pub fn num_buckets(&self) -> usize {
-        self.buckets.len()
-    }
-
-    /// Get the values for Pubkey `key`
-    pub fn read_value(&self, key: &Pubkey) -> Option<(Vec<T>, RefCount)> {
-        self.get_bucket(key).read_value(key)
-    }
-
-    /// Delete the Pubkey `key`
-    pub fn delete_key(&self, key: &Pubkey) {
-        self.get_bucket(key).delete_key(key);
-    }
-
-    /// Update Pubkey `key`'s value with 'value'
-    pub fn insert(&self, key: &Pubkey, value: (&[T], RefCount)) {
-        self.get_bucket(key).insert(key, value)
-    }
-
-    /// Update Pubkey `key`'s value with 'value'
-    pub fn try_insert(&self, key: &Pubkey, value: (&[T], RefCount)) -> Result<(), BucketMapError> {
-        self.get_bucket(key).try_write(key, value)
-    }
-
-    /// Update Pubkey `key`'s value with function `updatefn`
-    pub fn update<F>(&self, key: &Pubkey, updatefn: F)
-    where
-        F: FnMut(Option<(&[T], RefCount)>) -> Option<(Vec<T>, RefCount)>,
-    {
-        self.get_bucket(key).update(key, updatefn)
-    }
-
-    pub fn get_bucket(&self, key: &Pubkey) -> &Arc<BucketApi<T>> {
-        self.get_bucket_from_index(self.bucket_ix(key))
-    }
-
-    pub fn get_bucket_from_index(&self, ix: usize) -> &Arc<BucketApi<T>> {
-        &self.buckets[ix]
-    }
-
-    /// Get the bucket index for Pubkey `key`
-    pub fn bucket_ix(&self, key: &Pubkey) -> usize {
-        if self.max_buckets_pow2 > 0 {
-            let location = read_be_u64(key.as_ref());
-            (location >> (u64::BITS - self.max_buckets_pow2 as u32)) as usize
-        } else {
-            0
-        }
-    }
-
-    /// Increment the refcount for Pubkey `key`
-    pub fn addref(&self, key: &Pubkey) -> Option<RefCount> {
-        let ix = self.bucket_ix(key);
-        let bucket = &self.buckets[ix];
-        bucket.addref(key)
-    }
-
-    /// Decrement the refcount for Pubkey `key`
-    pub fn unref(&self, key: &Pubkey) -> Option<RefCount> {
-        let ix = self.bucket_ix(key);
-        let bucket = &self.buckets[ix];
-        bucket.unref(key)
-    }
-}
-
-/// Look at the first 8 bytes of the input and reinterpret them as a u64
-fn read_be_u64(input: &[u8]) -> u64 {
-    assert!(input.len() >= std::mem::size_of::<u64>());
-    u64::from_be_bytes(input[0..std::mem::size_of::<u64>()].try_into().unwrap())
-}
-
-#[cfg(test)]
-mod tests {
-    use {
-        super::*,
-        rand::{thread_rng, Rng},
-        std::{collections::HashMap, sync::RwLock},
-    };
-
-    #[test]
-    fn bucket_map_test_insert() {
-        let key = Pubkey::new_unique();
-        let config = BucketMapConfig::new(1 << 1);
-        let index = BucketMap::new(config);
-        index.update(&key, |_| Some((vec![0], 0)));
-        assert_eq!(index.read_value(&key), Some((vec![0], 0)));
-    }
-
-    #[test]
-    fn bucket_map_test_insert2() {
-        for pass in 0..3 {
-            let key = Pubkey::new_unique();
-            let config = BucketMapConfig::new(1 << 1);
-            let index = BucketMap::new(config);
-            let bucket = index.get_bucket(&key);
-            if pass == 0 {
-                index.insert(&key, (&[0], 0));
-            } else {
-                let result = index.try_insert(&key, (&[0], 0));
-                assert!(result.is_err());
-                assert_eq!(index.read_value(&key), None);
-                if pass == 2 {
-                    // another call to try insert again - should still return an error
-                    let result = index.try_insert(&key, (&[0], 0));
-                    assert!(result.is_err());
-                    assert_eq!(index.read_value(&key), None);
-                }
-                bucket.grow(result.unwrap_err());
-                let result = index.try_insert(&key, (&[0], 0));
-                assert!(result.is_ok());
-            }
-            assert_eq!(index.read_value(&key), Some((vec![0], 0)));
-        }
-    }
-
-    #[test]
-    fn bucket_map_test_update2() {
-        let key = Pubkey::new_unique();
-        let config = BucketMapConfig::new(1 << 1);
-        let index = BucketMap::new(config);
-        index.insert(&key, (&[0], 0));
-        assert_eq!(index.read_value(&key), Some((vec![0], 0)));
-        index.insert(&key, (&[1], 0));
-        assert_eq!(index.read_value(&key), Some((vec![1], 0)));
-    }
-
-    #[test]
-    fn bucket_map_test_update() {
-        let key = Pubkey::new_unique();
-        let config = BucketMapConfig::new(1 << 1);
-        let index = BucketMap::new(config);
-        index.update(&key, |_| Some((vec![0], 0)));
-        assert_eq!(index.read_value(&key), Some((vec![0], 0)));
-        index.update(&key, |_| Some((vec![1], 0)));
-        assert_eq!(index.read_value(&key), Some((vec![1], 0)));
-    }
-
-    #[test]
-    fn bucket_map_test_update_to_0_len() {
-        solana_logger::setup();
-        let key = Pubkey::new_unique();
-        let config = BucketMapConfig::new(1 << 1);
-        let index = BucketMap::new(config);
-        index.update(&key, |_| Some((vec![0], 1)));
-        assert_eq!(index.read_value(&key), Some((vec![0], 1)));
-        // sets len to 0, updates in place
-        index.update(&key, |_| Some((vec![], 1)));
-        assert_eq!(index.read_value(&key), Some((vec![], 1)));
-        // sets len to 0, doesn't update in place - finds a new place, which causes us to no longer have an allocation in data
-        index.update(&key, |_| Some((vec![], 2)));
-        assert_eq!(index.read_value(&key), Some((vec![], 2)));
-        // sets len to 1, doesn't update in place - finds a new place
-        index.update(&key, |_| Some((vec![1], 2)));
-        assert_eq!(index.read_value(&key), Some((vec![1], 2)));
-    }
-
-    #[test]
-    fn bucket_map_test_delete() {
-        let config = BucketMapConfig::new(1 << 1);
-        let index = BucketMap::new(config);
-        for i in 0..10 {
-            let key = Pubkey::new_unique();
-            assert_eq!(index.read_value(&key), None);
-
-            index.update(&key, |_| Some((vec![i], 0)));
-            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
-
-            index.delete_key(&key);
-            assert_eq!(index.read_value(&key), None);
-
-            index.update(&key, |_| Some((vec![i], 0)));
-            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
-            index.delete_key(&key);
-        }
-    }
-
-    #[test]
-    fn bucket_map_test_delete_2() {
-        let config = BucketMapConfig::new(1 << 2);
-        let index = BucketMap::new(config);
-        for i in 0..100 {
-            let key = Pubkey::new_unique();
-            assert_eq!(index.read_value(&key), None);
-
-            index.update(&key, |_| Some((vec![i], 0)));
-            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
-
-            index.delete_key(&key);
-            assert_eq!(index.read_value(&key), None);
-
-            index.update(&key, |_| Some((vec![i], 0)));
-            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
-            index.delete_key(&key);
-        }
-    }
-
-    #[test]
-    fn bucket_map_test_n_drives() {
-        let config = BucketMapConfig::new(1 << 2);
-        let index = BucketMap::new(config);
-        for i in 0..100 {
-            let key = Pubkey::new_unique();
-            index.update(&key, |_| Some((vec![i], 0)));
-            assert_eq!(index.read_value(&key), Some((vec![i], 0)));
-        }
-    }
-    #[test]
-    fn bucket_map_test_grow_read() {
-        let config = BucketMapConfig::new(1 << 2);
-        let index = BucketMap::new(config);
-        let keys: Vec<Pubkey> = (0..100).into_iter().map(|_| Pubkey::new_unique()).collect();
-        for k in 0..keys.len() {
-            let key = &keys[k];
-            let i = read_be_u64(key.as_ref());
-            index.update(key, |_| Some((vec![i], 0)));
-            assert_eq!(index.read_value(key), Some((vec![i], 0)));
-            for (ix, key) in keys.iter().enumerate() {
-                let i = read_be_u64(key.as_ref());
-                //debug!("READ: {:?} {}", key, i);
-                let expected = if ix <= k { Some((vec![i], 0)) } else { None };
-                assert_eq!(index.read_value(key), expected);
-            }
-        }
-    }
-
-    #[test]
-    fn bucket_map_test_n_delete() {
-        let config = BucketMapConfig::new(1 << 2);
-        let index = BucketMap::new(config);
-        let keys: Vec<Pubkey> = (0..20).into_iter().map(|_| Pubkey::new_unique()).collect();
-        for key in keys.iter() {
-            let i = read_be_u64(key.as_ref());
-            index.update(key, |_| Some((vec![i], 0)));
-            assert_eq!(index.read_value(key), Some((vec![i], 0)));
-        }
-        for key in keys.iter() {
-            let i = read_be_u64(key.as_ref());
-            //debug!("READ: {:?} {}", key, i);
-            assert_eq!(index.read_value(key), Some((vec![i], 0)));
-        }
-        for k in 0..keys.len() {
-            let key = &keys[k];
-            index.delete_key(key);
-            assert_eq!(index.read_value(key), None);
-            for key in keys.iter().skip(k + 1) {
-                let i = read_be_u64(key.as_ref());
-                assert_eq!(index.read_value(key), Some((vec![i], 0)));
-            }
-        }
-    }
-
-    #[test]
-    fn hashmap_compare() {
-        use std::sync::Mutex;
-        solana_logger::setup();
-        let maps = (0..2)
-            .into_iter()
-            .map(|max_buckets_pow2| {
-                let config = BucketMapConfig::new(1 << max_buckets_pow2);
-                BucketMap::new(config)
-            })
-            .collect::<Vec<_>>();
-        let hash_map = RwLock::new(HashMap::<Pubkey, (Vec<(usize, usize)>, RefCount)>::new());
-        let max_slot_list_len = 3;
-        let all_keys = Mutex::new(vec![]);
-
-        let gen_rand_value = || {
-            let count = thread_rng().gen_range(0, max_slot_list_len);
-            let v = (0..count)
-                .into_iter()
-                .map(|x| (x as usize, x as usize /*thread_rng().gen::<usize>()*/))
-                .collect::<Vec<_>>();
-            let rc = thread_rng().gen::<RefCount>();
-            (v, rc)
-        };
-
-        let get_key = || {
-            let mut keys = all_keys.lock().unwrap();
-            if keys.is_empty() {
-                return None;
-            }
-            let len = keys.len();
-            Some(keys.remove(thread_rng().gen_range(0, len)))
-        };
-        let return_key = |key| {
-            let mut keys = all_keys.lock().unwrap();
-            keys.push(key);
-        };
-
-        let verify = || {
-            let mut maps = maps
-                .iter()
-                .map(|map| {
-                    let mut r = vec![];
-                    for bin in 0..map.num_buckets() {
-                        r.append(
-                            &mut map.buckets[bin]
-                                .items_in_range(&None::<&std::ops::RangeInclusive<Pubkey>>),
-                        );
-                    }
-                    r
-                })
-                .collect::<Vec<_>>();
-            let hm = hash_map.read().unwrap();
-            for (k, v) in hm.iter() {
-                for map in maps.iter_mut() {
-                    for i in 0..map.len() {
-                        if k == &map[i].pubkey {
-                            assert_eq!(map[i].slot_list, v.0);
-                            assert_eq!(map[i].ref_count, v.1);
-                            map.remove(i);
-                            break;
-                        }
-                    }
-                }
-            }
-            for map in maps.iter() {
-                assert!(map.is_empty());
-            }
-        };
-        let mut initial = 100; // put this many items in to start
-
-        // do random operations: insert, update, delete, add/unref in random order
-        // verify consistency between hashmap and all bucket maps
-        for i in 0..10000 {
-            if initial > 0 {
-                initial -= 1;
-            }
-            if initial > 0 || thread_rng().gen_range(0, 5) == 0 {
-                // insert
-                let k = solana_sdk::pubkey::new_rand();
-                let v = gen_rand_value();
-                hash_map.write().unwrap().insert(k, v.clone());
-                let insert = thread_rng().gen_range(0, 2) == 0;
-                maps.iter().for_each(|map| {
-                    if insert {
-                        map.insert(&k, (&v.0, v.1))
-                    } else {
-                        map.update(&k, |current| {
-                            assert!(current.is_none());
-                            Some(v.clone())
-                        })
-                    }
-                });
-                return_key(k);
-            }
-            if thread_rng().gen_range(0, 10) == 0 {
-                // update
-                if let Some(k) = get_key() {
-                    let hm = hash_map.read().unwrap();
-                    let (v, rc) = gen_rand_value();
-                    let v_old = hm.get(&k);
-                    let insert = thread_rng().gen_range(0, 2) == 0;
-                    maps.iter().for_each(|map| {
-                        if insert {
-                            map.insert(&k, (&v, rc))
-                        } else {
-                            map.update(&k, |current| {
-                                assert_eq!(current, v_old.map(|(v, rc)| (&v[..], *rc)), "{}", k);
-                                Some((v.clone(), rc))
-                            })
-                        }
-                    });
-                    drop(hm);
-                    hash_map.write().unwrap().insert(k, (v, rc));
-                    return_key(k);
-                }
-            }
-            if thread_rng().gen_range(0, 20) == 0 {
-                // delete
-                if let Some(k) = get_key() {
-                    let mut hm = hash_map.write().unwrap();
-                    hm.remove(&k);
-                    maps.iter().for_each(|map| {
-                        map.delete_key(&k);
-                    });
-                }
-            }
-            if thread_rng().gen_range(0, 10) == 0 {
-                // add/unref
-                if let Some(k) = get_key() {
-                    let mut inc = thread_rng().gen_range(0, 2) == 0;
-                    let mut hm = hash_map.write().unwrap();
-                    let (v, mut rc) = hm.get(&k).map(|(v, rc)| (v.to_vec(), *rc)).unwrap();
-                    if !inc && rc == 0 {
-                        // can't decrement rc=0
-                        inc = true;
-                    }
-                    rc = if inc { rc + 1 } else { rc - 1 };
-                    hm.insert(k, (v.to_vec(), rc));
-                    maps.iter().for_each(|map| {
-                        if thread_rng().gen_range(0, 2) == 0 {
-                            map.update(&k, |current| Some((current.unwrap().0.to_vec(), rc)))
-                        } else if inc {
-                            map.addref(&k);
-                        } else {
-                            map.unref(&k);
-                        }
-                    });
-
-                    return_key(k);
-                }
-            }
-            if i % 1000 == 0 {
-                verify();
-            }
-        }
-        verify();
-    }
-}

bucket_map/src/bucket_stats.rs

@@ -1,18 +0,0 @@
-use std::sync::{atomic::AtomicU64, Arc, Mutex};
-
-#[derive(Debug, Default)]
-pub struct BucketStats {
-    pub resizes: AtomicU64,
-    pub max_size: Mutex<u64>,
-    pub resize_us: AtomicU64,
-    pub new_file_us: AtomicU64,
-    pub flush_file_us: AtomicU64,
-    pub mmap_us: AtomicU64,
-}
-
-#[derive(Debug, Default)]
-pub struct BucketMapStats {
-    pub index: Arc<BucketStats>,
-    pub data: Arc<BucketStats>,
-    pub per_bucket_count: Vec<Arc<AtomicU64>>,
-}

bucket_map/src/bucket_storage.rs

@@ -1,343 +0,0 @@
-use {
-    crate::{bucket_stats::BucketStats, MaxSearch},
-    memmap2::MmapMut,
-    rand::{thread_rng, Rng},
-    solana_measure::measure::Measure,
-    std::{
-        fs::{remove_file, OpenOptions},
-        io::{Seek, SeekFrom, Write},
-        path::PathBuf,
-        sync::{
-            atomic::{AtomicU64, Ordering},
-            Arc,
-        },
-    },
-};
-
-/*
-1	2
-2	4
-3	8
-4	16
-5	32
-6	64
-7	128
-8	256
-9	512
-10	1,024
-11	2,048
-12	4,096
-13	8,192
-14	16,384
-23  8,388,608
-24  16,777,216
-*/
-pub const DEFAULT_CAPACITY_POW2: u8 = 5;
-
-/// A Header UID of 0 indicates that the header is unlocked
-pub(crate) const UID_UNLOCKED: Uid = 0;
-
-pub(crate) type Uid = u64;
-
-#[repr(C)]
-struct Header {
-    lock: AtomicU64,
-}
-
-impl Header {
-    fn try_lock(&self, uid: Uid) -> bool {
-        Ok(UID_UNLOCKED)
-            == self
-                .lock
-                .compare_exchange(UID_UNLOCKED, uid, Ordering::AcqRel, Ordering::Relaxed)
-    }
-    fn unlock(&self) -> Uid {
-        self.lock.swap(UID_UNLOCKED, Ordering::Release)
-    }
-    fn uid(&self) -> Uid {
-        self.lock.load(Ordering::Acquire)
-    }
-}
-
-pub struct BucketStorage {
-    path: PathBuf,
-    mmap: MmapMut,
-    pub cell_size: u64,
-    pub capacity_pow2: u8,
-    pub used: AtomicU64,
-    pub stats: Arc<BucketStats>,
-    pub max_search: MaxSearch,
-}
-
-#[derive(Debug)]
-pub enum BucketStorageError {
-    AlreadyAllocated,
-}
-
-impl Drop for BucketStorage {
-    fn drop(&mut self) {
-        let _ = remove_file(&self.path);
-    }
-}
-
-impl BucketStorage {
-    pub fn new_with_capacity(
-        drives: Arc<Vec<PathBuf>>,
-        num_elems: u64,
-        elem_size: u64,
-        capacity_pow2: u8,
-        max_search: MaxSearch,
-        stats: Arc<BucketStats>,
-    ) -> Self {
-        let cell_size = elem_size * num_elems + std::mem::size_of::<Header>() as u64;
-        let (mmap, path) = Self::new_map(&drives, cell_size as usize, capacity_pow2, &stats);
-        Self {
-            path,
-            mmap,
-            cell_size,
-            used: AtomicU64::new(0),
-            capacity_pow2,
-            stats,
-            max_search,
-        }
-    }
-
-    pub fn max_search(&self) -> u64 {
-        self.max_search as u64
-    }
-
-    pub fn new(
-        drives: Arc<Vec<PathBuf>>,
-        num_elems: u64,
-        elem_size: u64,
-        max_search: MaxSearch,
-        stats: Arc<BucketStats>,
-    ) -> Self {
-        Self::new_with_capacity(
-            drives,
-            num_elems,
-            elem_size,
-            DEFAULT_CAPACITY_POW2,
-            max_search,
-            stats,
-        )
-    }
-
-    pub fn uid(&self, ix: u64) -> Uid {
-        assert!(ix < self.capacity(), "bad index size");
-        let ix = (ix * self.cell_size) as usize;
-        let hdr_slice: &[u8] = &self.mmap[ix..ix + std::mem::size_of::<Header>()];
-        unsafe {
-            let hdr = hdr_slice.as_ptr() as *const Header;
-            return hdr.as_ref().unwrap().uid();
-        }
-    }
-
-    pub fn allocate(&self, ix: u64, uid: Uid) -> Result<(), BucketStorageError> {
-        assert!(ix < self.capacity(), "allocate: bad index size");
-        assert!(UID_UNLOCKED != uid, "allocate: bad uid");
-        let mut e = Err(BucketStorageError::AlreadyAllocated);
-        let ix = (ix * self.cell_size) as usize;
-        //debug!("ALLOC {} {}", ix, uid);
-        let hdr_slice: &[u8] = &self.mmap[ix..ix + std::mem::size_of::<Header>()];
-        unsafe {
-            let hdr = hdr_slice.as_ptr() as *const Header;
-            if hdr.as_ref().unwrap().try_lock(uid) {
-                e = Ok(());
-                self.used.fetch_add(1, Ordering::Relaxed);
-            }
-        };
-        e
-    }
-
-    pub fn free(&self, ix: u64, uid: Uid) {
-        assert!(ix < self.capacity(), "bad index size");
-        assert!(UID_UNLOCKED != uid, "free: bad uid");
-        let ix = (ix * self.cell_size) as usize;
-        //debug!("FREE {} {}", ix, uid);
-        let hdr_slice: &[u8] = &self.mmap[ix..ix + std::mem::size_of::<Header>()];
-        unsafe {
-            let hdr = hdr_slice.as_ptr() as *const Header;
-            //debug!("FREE uid: {}", hdr.as_ref().unwrap().uid());
-            let previous_uid = hdr.as_ref().unwrap().unlock();
-            assert_eq!(
-                previous_uid, uid,
-                "free: unlocked a header with a differet uid: {}",
-                previous_uid
-            );
-            self.used.fetch_sub(1, Ordering::Relaxed);
-        }
-    }
-
-    pub fn get<T: Sized>(&self, ix: u64) -> &T {
-        assert!(ix < self.capacity(), "bad index size");
-        let start = (ix * self.cell_size) as usize + std::mem::size_of::<Header>();
-        let end = start + std::mem::size_of::<T>();
-        let item_slice: &[u8] = &self.mmap[start..end];
-        unsafe {
-            let item = item_slice.as_ptr() as *const T;
-            &*item
-        }
-    }
-
-    pub fn get_empty_cell_slice<T: Sized>(&self) -> &[T] {
-        let len = 0;
-        let item_slice: &[u8] = &self.mmap[0..0];
-        unsafe {
-            let item = item_slice.as_ptr() as *const T;
-            std::slice::from_raw_parts(item, len as usize)
-        }
-    }
-
-    pub fn get_cell_slice<T: Sized>(&self, ix: u64, len: u64) -> &[T] {
-        assert!(ix < self.capacity(), "bad index size");
-        let ix = self.cell_size * ix;
-        let start = ix as usize + std::mem::size_of::<Header>();
-        let end = start + std::mem::size_of::<T>() * len as usize;
-        //debug!("GET slice {} {}", start, end);
-        let item_slice: &[u8] = &self.mmap[start..end];
-        unsafe {
-            let item = item_slice.as_ptr() as *const T;
-            std::slice::from_raw_parts(item, len as usize)
-        }
-    }
-
-    #[allow(clippy::mut_from_ref)]
-    pub fn get_mut<T: Sized>(&self, ix: u64) -> &mut T {
-        assert!(ix < self.capacity(), "bad index size");
-        let start = (ix * self.cell_size) as usize + std::mem::size_of::<Header>();
-        let end = start + std::mem::size_of::<T>();
-        let item_slice: &[u8] = &self.mmap[start..end];
-        unsafe {
-            let item = item_slice.as_ptr() as *mut T;
-            &mut *item
-        }
-    }
-
-    #[allow(clippy::mut_from_ref)]
-    pub fn get_mut_cell_slice<T: Sized>(&self, ix: u64, len: u64) -> &mut [T] {
-        assert!(ix < self.capacity(), "bad index size");
-        let ix = self.cell_size * ix;
-        let start = ix as usize + std::mem::size_of::<Header>();
-        let end = start + std::mem::size_of::<T>() * len as usize;
-        //debug!("GET mut slice {} {}", start, end);
-        let item_slice: &[u8] = &self.mmap[start..end];
-        unsafe {
-            let item = item_slice.as_ptr() as *mut T;
-            std::slice::from_raw_parts_mut(item, len as usize)
-        }
-    }
-
-    fn new_map(
-        drives: &[PathBuf],
-        cell_size: usize,
-        capacity_pow2: u8,
-        stats: &BucketStats,
-    ) -> (MmapMut, PathBuf) {
-        let mut measure_new_file = Measure::start("measure_new_file");
-        let capacity = 1u64 << capacity_pow2;
-        let r = thread_rng().gen_range(0, drives.len());
-        let drive = &drives[r];
-        let pos = format!("{}", thread_rng().gen_range(0, u128::MAX),);
-        let file = drive.join(pos);
-        let mut data = OpenOptions::new()
-            .read(true)
-            .write(true)
-            .create(true)
-            .open(file.clone())
-            .map_err(|e| {
-                panic!(
-                    "Unable to create data file {} in current dir({:?}): {:?}",
-                    file.display(),
-                    std::env::current_dir(),
-                    e
-                );
-            })
-            .unwrap();
-
-        // Theoretical performance optimization: write a zero to the end of
-        // the file so that we won't have to resize it later, which may be
-        // expensive.
-        //debug!("GROWING file {}", capacity * cell_size as u64);
-        data.seek(SeekFrom::Start(capacity * cell_size as u64 - 1))
-            .unwrap();
-        data.write_all(&[0]).unwrap();
-        data.seek(SeekFrom::Start(0)).unwrap();
-        measure_new_file.stop();
-        let mut measure_flush = Measure::start("measure_flush");
-        data.flush().unwrap(); // can we skip this?
-        measure_flush.stop();
-        let mut measure_mmap = Measure::start("measure_mmap");
-        let res = (unsafe { MmapMut::map_mut(&data).unwrap() }, file);
-        measure_mmap.stop();
-        stats
-            .new_file_us
-            .fetch_add(measure_new_file.as_us(), Ordering::Relaxed);
-        stats
-            .flush_file_us
-            .fetch_add(measure_flush.as_us(), Ordering::Relaxed);
-        stats
-            .mmap_us
-            .fetch_add(measure_mmap.as_us(), Ordering::Relaxed);
-        res
-    }
-
-    /// copy contents from 'old_bucket' to 'self'
-    fn copy_contents(&mut self, old_bucket: &Self) {
-        let mut m = Measure::start("grow");
-        let old_cap = old_bucket.capacity();
-        let old_map = &old_bucket.mmap;
-
-        let increment = self.capacity_pow2 - old_bucket.capacity_pow2;
-        let index_grow = 1 << increment;
-        (0..old_cap as usize).into_iter().for_each(|i| {
-            let old_ix = i * old_bucket.cell_size as usize;
-            let new_ix = old_ix * index_grow;
-            let dst_slice: &[u8] = &self.mmap[new_ix..new_ix + old_bucket.cell_size as usize];
-            let src_slice: &[u8] = &old_map[old_ix..old_ix + old_bucket.cell_size as usize];
-
-            unsafe {
-                let dst = dst_slice.as_ptr() as *mut u8;
-                let src = src_slice.as_ptr() as *const u8;
-                std::ptr::copy_nonoverlapping(src, dst, old_bucket.cell_size as usize);
-            };
-        });
-        m.stop();
-        self.stats.resizes.fetch_add(1, Ordering::Relaxed);
-        self.stats.resize_us.fetch_add(m.as_us(), Ordering::Relaxed);
-    }
-
-    /// allocate a new bucket, copying data from 'bucket'
-    pub fn new_resized(
-        drives: &Arc<Vec<PathBuf>>,
-        max_search: MaxSearch,
-        bucket: Option<&Self>,
-        capacity_pow_2: u8,
-        num_elems: u64,
-        elem_size: u64,
-        stats: &Arc<BucketStats>,
-    ) -> Self {
-        let mut new_bucket = Self::new_with_capacity(
-            Arc::clone(drives),
-            num_elems,
-            elem_size,
-            capacity_pow_2,
-            max_search,
-            Arc::clone(stats),
-        );
-        if let Some(bucket) = bucket {
-            new_bucket.copy_contents(bucket);
-        }
-        let sz = new_bucket.capacity();
-        {
-            let mut max = new_bucket.stats.max_size.lock().unwrap();
-            *max = std::cmp::max(*max, sz);
-        }
-        new_bucket
-    }
-
-    /// Return the number of cells currently allocated
-    pub fn capacity(&self) -> u64 {
-        1 << self.capacity_pow2
-    }
-}

bucket_map/src/index_entry.rs

@@ -1,67 +0,0 @@
-use {
-    crate::{
-        bucket::Bucket,
-        bucket_storage::{BucketStorage, Uid},
-        RefCount,
-    },
-    solana_sdk::{clock::Slot, pubkey::Pubkey},
-    std::{
-        collections::hash_map::DefaultHasher,
-        fmt::Debug,
-        hash::{Hash, Hasher},
-    },
-};
-
-#[repr(C)]
-#[derive(Debug, Copy, Clone, PartialEq)]
-// one instance of this per item in the index
-// stored in the index bucket
-pub struct IndexEntry {
-    pub key: Pubkey, // can this be smaller if we have reduced the keys into buckets already?
-    pub ref_count: RefCount, // can this be smaller? Do we ever need more than 4B refcounts?
-    pub storage_offset: u64, // smaller? since these are variably sized, this could get tricky. well, actually accountinfo is not variable sized...
-    // if the bucket doubled, the index can be recomputed using create_bucket_capacity_pow2
-    pub storage_capacity_when_created_pow2: u8, // see data_location
-    pub num_slots: Slot, // can this be smaller? epoch size should ~ be the max len. this is the num elements in the slot list
-}
-
-impl IndexEntry {
-    pub fn data_bucket_from_num_slots(num_slots: Slot) -> u64 {
-        (num_slots as f64).log2().ceil() as u64 // use int log here?
-    }
-
-    pub fn data_bucket_ix(&self) -> u64 {
-        Self::data_bucket_from_num_slots(self.num_slots)
-    }
-
-    pub fn ref_count(&self) -> RefCount {
-        self.ref_count
-    }
-
-    // This function maps the original data location into an index in the current bucket storage.
-    // This is coupled with how we resize bucket storages.
-    pub fn data_loc(&self, storage: &BucketStorage) -> u64 {
-        self.storage_offset << (storage.capacity_pow2 - self.storage_capacity_when_created_pow2)
-    }
-
-    pub fn read_value<'a, T>(&self, bucket: &'a Bucket<T>) -> Option<(&'a [T], RefCount)> {
-        let data_bucket_ix = self.data_bucket_ix();
-        let data_bucket = &bucket.data[data_bucket_ix as usize];
-        let slice = if self.num_slots > 0 {
-            let loc = self.data_loc(data_bucket);
-            let uid = Self::key_uid(&self.key);
-            assert_eq!(uid, bucket.data[data_bucket_ix as usize].uid(loc));
-            bucket.data[data_bucket_ix as usize].get_cell_slice(loc, self.num_slots)
-        } else {
-            // num_slots is 0. This means we don't have an actual allocation.
-            // can we trust that the data_bucket is even safe?
-            bucket.data[data_bucket_ix as usize].get_empty_cell_slice()
-        };
-        Some((slice, self.ref_count))
-    }
-    pub fn key_uid(key: &Pubkey) -> Uid {
-        let mut s = DefaultHasher::new();
-        key.hash(&mut s);
-        s.finish().max(1u64)
-    }
-}

bucket_map/src/lib.rs

@@ -1,11 +0,0 @@
-#![allow(clippy::integer_arithmetic)]
-mod bucket;
-pub mod bucket_api;
-mod bucket_item;
-pub mod bucket_map;
-mod bucket_stats;
-mod bucket_storage;
-mod index_entry;
-
-pub type MaxSearch = u8;
-pub type RefCount = u64;

bucket_map/tests/bucket_map.rs

@@ -1,48 +0,0 @@
-use {
-    rayon::prelude::*,
-    solana_bucket_map::bucket_map::{BucketMap, BucketMapConfig},
-    solana_measure::measure::Measure,
-    solana_sdk::pubkey::Pubkey,
-    std::path::PathBuf,
-};
-#[test]
-#[ignore]
-fn bucket_map_test_mt() {
-    let threads = 4096;
-    let items = 4096;
-    let tmpdir1 = std::env::temp_dir().join("bucket_map_test_mt");
-    let tmpdir2 = PathBuf::from("/mnt/data/0").join("bucket_map_test_mt");
-    let paths: Vec<PathBuf> = [tmpdir1, tmpdir2]
-        .iter()
-        .filter(|x| std::fs::create_dir_all(x).is_ok())
-        .cloned()
-        .collect();
-    assert!(!paths.is_empty());
-    let index = BucketMap::new(BucketMapConfig {
-        max_buckets: 1 << 12,
-        drives: Some(paths.clone()),
-        ..BucketMapConfig::default()
-    });
-    (0..threads).into_iter().into_par_iter().for_each(|_| {
-        let key = Pubkey::new_unique();
-        index.update(&key, |_| Some((vec![0u64], 0)));
-    });
-    let mut timer = Measure::start("bucket_map_test_mt");
-    (0..threads).into_iter().into_par_iter().for_each(|_| {
-        for _ in 0..items {
-            let key = Pubkey::new_unique();
-            let ix: u64 = index.bucket_ix(&key) as u64;
-            index.update(&key, |_| Some((vec![ix], 0)));
-            assert_eq!(index.read_value(&key), Some((vec![ix], 0)));
-        }
-    });
-    timer.stop();
-    println!("time: {}ns per item", timer.as_ns() / (threads * items));
-    let mut total = 0;
-    for tmpdir in paths.iter() {
-        let folder_size = fs_extra::dir::get_size(tmpdir).unwrap();
-        total += folder_size;
-        std::fs::remove_dir_all(tmpdir).unwrap();
-    }
-    println!("overhead: {}bytes per item", total / (threads * items));
-}

ci/buildkite-pipeline.sh

@@ -137,7 +137,7 @@ all_test_steps() {
              ^ci/test-coverage.sh \
              ^scripts/coverage.sh \
       ; then
-    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 40
+    command_step coverage ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_nightly_docker_image ci/test-coverage.sh" 30
     wait_step
   else
     annotate --style info --context test-coverage \
@@ -226,19 +226,6 @@ EOF
     annotate --style info \
       "downstream-projects skipped as no relevant files were modified"
   fi
-
-  # Wasm support
-  if affects \
-             ^ci/test-wasm.sh \
-             ^ci/test-stable.sh \
-             ^sdk/ \
-      ; then
-    command_step wasm ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-wasm.sh" 20
-  else
-    annotate --style info \
-      "wasm skipped as no relevant files were modified"
-  fi
-
   # Benches...
   if affects \
              .rs$ \
@@ -256,7 +243,7 @@ EOF
 
   command_step "local-cluster" \
     ". ci/rust-version.sh; ci/docker-run.sh \$\$rust_stable_docker_image ci/test-local-cluster.sh" \
-    50
+    45
 }
 
 pull_or_push_steps() {
@@ -275,7 +262,7 @@ pull_or_push_steps() {
     all_test_steps
   fi
 
-  # web3.js, explorer and docs changes run on Travis or Github actions...
+  # web3.js, explorer and docs changes run on Travis...
 }
 
 

ci/buildkite-secondary.yml

@@ -7,6 +7,11 @@ steps:
       - "queue=release-build"
     timeout_in_minutes: 60
     name: "publish tarball"
+  - command: "ci/publish-bpf-sdk.sh"
+    agents:
+      - "queue=release-build"
+    timeout_in_minutes: 5
+    name: "publish bpf sdk"
   - wait
   - command: "sdk/docker-solana/build.sh"
     agents:

ci/dependabot-pr.sh

@@ -23,7 +23,7 @@ echo --- "(FAILING) Backpropagating dependabot-triggered Cargo.lock updates"
 name="dependabot-buildkite"
 api_base="https://api.github.com/repos/solana-labs/solana/pulls"
 pr_num=$(echo "$BUILDKITE_BRANCH" | grep -Eo '[0-9]+')
-branch=$(curl -s "$api_base/$pr_num" | python3 -c 'import json,sys;print(json.load(sys.stdin)["head"]["ref"])')
+branch=$(curl -s "$api_base/$pr_num" | python -c 'import json,sys;print json.load(sys.stdin)["head"]["ref"]')
 
 git add :**/Cargo.lock
 EMAIL="dependabot-buildkite@noreply.solana.com" \

ci/do-audit.sh

@@ -46,11 +46,11 @@ cargo_audit_ignores=(
   # https://github.com/paritytech/jsonrpc/issues/605
   --ignore RUSTSEC-2021-0079
 
-  # chrono: Potential segfault in `localtime_r` invocations
+  # tar: Links in archive can create arbitrary directories
   #
-  # Blocked due to no safe upgrade
-  # https://github.com/chronotope/chrono/issues/499
-  --ignore RUSTSEC-2020-0159
+  # Blocked on `tar` releasing safe upgrade
+  # https://github.com/alexcrichton/tar-rs/issues/238
+  --ignore RUSTSEC-2021-0080
 
 )
 scripts/cargo-for-all-lock-files.sh stable audit "${cargo_audit_ignores[@]}"

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.57.0
+FROM solanalabs/rust:1.51.0
 ARG date
 
 RUN set -x \

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.57.0
+FROM rust:1.51.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0
@@ -11,34 +11,28 @@ RUN set -x \
  && apt-get install apt-transport-https \
  && echo deb https://apt.buildkite.com/buildkite-agent stable main > /etc/apt/sources.list.d/buildkite-agent.list \
  && apt-key adv --no-tty --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 \
- && curl -fsSL https://deb.nodesource.com/setup_current.x | bash - \
  && apt update \
  && apt install -y \
       buildkite-agent \
-      clang \
+      clang-7 \
       cmake \
       lcov \
       libudev-dev \
+      libclang-common-7-dev \
       mscgen \
-      nodejs \
       net-tools \
       rsync \
       sudo \
       golang \
       unzip \
       \
- && apt remove -y libcurl4-openssl-dev \
  && rm -rf /var/lib/apt/lists/* \
- && node --version \
- && npm --version \
  && rustup component add rustfmt \
  && rustup component add clippy \
- && rustup target add wasm32-unknown-unknown \
  && cargo install cargo-audit \
+ && cargo install svgbob_cli \
  && cargo install mdbook \
  && cargo install mdbook-linkcheck \
- && cargo install svgbob_cli \
- && cargo install wasm-pack \
  && rustc --version \
  && cargo --version \
  && curl -OL https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP \

ci/publish-bpf-sdk.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -e
+
+cd "$(dirname "$0")/.."
+eval "$(ci/channel-info.sh)"
+
+if [[ -n "$CI_TAG" ]]; then
+  CHANNEL_OR_TAG=$CI_TAG
+else
+  CHANNEL_OR_TAG=$CHANNEL
+fi
+
+(
+  set -x
+  sdk/bpf/scripts/package.sh
+  [[ -f bpf-sdk.tar.bz2 ]]
+)
+
+source ci/upload-ci-artifact.sh
+echo --- AWS S3 Store
+if [[ -z $CHANNEL_OR_TAG ]]; then
+  echo Skipped
+else
+  upload-s3-artifact "/solana/bpf-sdk.tar.bz2" "s3://solana-sdk/$CHANNEL_OR_TAG/bpf-sdk.tar.bz2"
+fi
+
+exit 0

ci/run-sanity.sh

@@ -7,7 +7,7 @@ source multinode-demo/common.sh
 
 rm -rf config/run/init-completed config/ledger config/snapshot-ledger
 
-SOLANA_RUN_SH_VALIDATOR_ARGS="--snapshot-interval-slots 200" timeout 120 ./scripts/run.sh &
+timeout 120 ./run.sh &
 pid=$!
 
 attempts=20
@@ -16,8 +16,6 @@ while [[ ! -f config/run/init-completed ]]; do
   if ((--attempts == 0)); then
      echo "Error: validator failed to boot"
      exit 1
-  else
-    echo "Checking init"
   fi
 done
 
@@ -26,7 +24,6 @@ snapshot_slot=1
 # wait a bit longer than snapshot_slot
 while [[ $($solana_cli --url http://localhost:8899 slot --commitment processed) -le $((snapshot_slot + 1)) ]]; do
   sleep 1
-  echo "Checking slot"
 done
 
 $solana_validator --ledger config/ledger exit --force || true

ci/rust-version.sh

@@ -18,13 +18,13 @@
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.57.0
+  stable_version=1.51.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2021-12-03
+  nightly_version=2021-04-18
 fi
 
 

ci/setup-new-buildkite-agent/setup-buildkite.sh

@@ -76,7 +76,7 @@ RestartForceExitStatus=SIGPIPE
 TimeoutStartSec=10
 TimeoutStopSec=0
 KillMode=process
-LimitNOFILE=1000000
+LimitNOFILE=700000
 
 [Install]
 WantedBy=multi-user.target

ci/setup-new-buildkite-agent/setup-limits.sh

@@ -8,5 +8,5 @@ source "$HERE"/utils.sh
 ensure_env || exit 1
 
 # Allow more files to be opened by a user
-echo "* - nofile 1000000" > /etc/security/limits.d/90-solana-nofiles.conf
+echo "* - nofile 700000" > /etc/security/limits.d/90-solana-nofiles.conf