Revert "Add AVX2 runtime checks (#10033)" (#10167) (#10168)

This reverts commit cf8eb7700b.

(cherry picked from commit 486168b796)

Co-authored-by: Michael Vines <mvines@gmail.com>

Cargo.toml

@@ -53,6 +53,7 @@ members = [
     "transaction-status",
     "upload-perf",
     "net-utils",
+    "version",
     "vote-signer",
     "cli",
     "rayon-threadlimit",

archiver-lib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-archiver-lib"
-version = "1.0.19"
+version = "1.0.23"
 description = "Solana Archiver Library"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,26 +11,26 @@ edition = "2018"
 [dependencies]
 bincode = "1.2.1"
 crossbeam-channel = "0.3"
-ed25519-dalek = "=1.0.0-pre.1"
+ed25519-dalek = "=1.0.0-pre.3"
 log = "0.4.8"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
-solana-client = { path = "../client", version = "1.0.19" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.19" }
+rand = "0.7.0"
+rand_chacha = "0.2.2"
+solana-client = { path = "../client", version = "1.0.23" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.23" }
 thiserror = "1.0"
 serde = "1.0.104"
 serde_json = "1.0.46"
 serde_derive = "1.0.103"
-solana-net-utils = { path = "../net-utils", version = "1.0.19" }
-solana-chacha = { path = "../chacha", version = "1.0.19" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.19" }
-solana-ledger = { path = "../ledger", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-perf = { path = "../perf", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
-solana-core = { path = "../core", version = "1.0.19" }
-solana-archiver-utils = { path = "../archiver-utils", version = "1.0.19" }
-solana-metrics = { path = "../metrics", version = "1.0.19" }
+solana-net-utils = { path = "../net-utils", version = "1.0.23" }
+solana-chacha = { path = "../chacha", version = "1.0.23" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.0.23" }
+solana-ledger = { path = "../ledger", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-perf = { path = "../perf", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
+solana-core = { path = "../core", version = "1.0.23" }
+solana-archiver-utils = { path = "../archiver-utils", version = "1.0.23" }
+solana-metrics = { path = "../metrics", version = "1.0.23" }
 
 [dev-dependencies]
 hex = "0.4.0"

archiver-lib/src/archiver.rs

@@ -1,7 +1,7 @@
 use crate::result::ArchiverError;
 use crossbeam_channel::unbounded;
-use rand::{thread_rng, Rng, SeedableRng};
-use rand_chacha::ChaChaRng;
+use rand::{thread_rng, Rng};
+use rand_chacha::{rand_core::SeedableRng, ChaChaRng};
 use solana_archiver_utils::sample_file;
 use solana_chacha::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE};
 use solana_client::{
@@ -13,8 +13,7 @@ use solana_core::{
     contact_info::ContactInfo,
     gossip_service::GossipService,
     packet::{limited_deserialize, PACKET_DATA_SIZE},
-    repair_service,
-    repair_service::{RepairService, RepairSlotRange, RepairStrategy},
+    repair_service::{self, RepairService, RepairSlotRange, RepairStats, RepairStrategy},
     serve_repair::ServeRepair,
     shred_fetch_stage::ShredFetchStage,
     sigverify_stage::{DisabledSigVerifier, SigVerifyStage},
@@ -839,13 +838,14 @@ impl Archiver {
                 repair_service::MAX_REPAIR_LENGTH,
                 &repair_slot_range,
             );
+            let mut repair_stats = RepairStats::default();
             //iter over the repairs and send them
             if let Ok(repairs) = repairs {
                 let reqs: Vec<_> = repairs
                     .into_iter()
                     .filter_map(|repair_request| {
                         serve_repair
-                            .map_repair_request(&repair_request)
+                            .map_repair_request(&repair_request, &mut repair_stats, Some(0))
                             .map(|result| ((archiver_info.gossip, result), repair_request))
                             .ok()
                     })

archiver-lib/src/result.rs

@@ -1,4 +1,3 @@
-use serde_json;
 use solana_client::client_error;
 use solana_ledger::blockstore;
 use solana_sdk::transport;

archiver-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-archiver-utils"
-version = "1.0.19"
+version = "1.0.23"
 description = "Solana Archiver Utils"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,13 +10,13 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-rand = "0.6.5"
-solana-chacha = { path = "../chacha", version = "1.0.19" }
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.19" }
-solana-ledger = { path = "../ledger", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-perf = { path = "../perf", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
+rand = "0.7.0"
+solana-chacha = { path = "../chacha", version = "1.0.23" }
+solana-chacha-sys = { path = "../chacha-sys", version = "1.0.23" }
+solana-ledger = { path = "../ledger", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-perf = { path = "../perf", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
 
 [dev-dependencies]
 hex = "0.4.0"

archiver/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-archiver"
-version = "1.0.19"
+version = "1.0.23"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,13 +10,13 @@ homepage = "https://solana.com/"
 [dependencies]
 clap = "2.33.0"
 console = "0.9.2"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.19" }
-solana-core = { path = "../core", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-metrics = { path = "../metrics", version = "1.0.19" }
-solana-archiver-lib = { path = "../archiver-lib", version = "1.0.19" }
-solana-net-utils = { path = "../net-utils", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.23" }
+solana-core = { path = "../core", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-metrics = { path = "../metrics", version = "1.0.23" }
+solana-archiver-lib = { path = "../archiver-lib", version = "1.0.23" }
+solana-net-utils = { path = "../net-utils", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
 
 
 [package.metadata.docs.rs]

banking-bench/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-banking-bench"
-version = "1.0.19"
+version = "1.0.23"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,13 +10,13 @@ homepage = "https://solana.com/"
 [dependencies]
 log = "0.4.6"
 rayon = "1.2.0"
-solana-core = { path = "../core", version = "1.0.19" }
-solana-ledger = { path = "../ledger", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-runtime = { path = "../runtime", version = "1.0.19" }
-solana-measure = { path = "../measure", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
-rand = "0.6.5"
+solana-core = { path = "../core", version = "1.0.23" }
+solana-ledger = { path = "../ledger", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-runtime = { path = "../runtime", version = "1.0.23" }
+solana-measure = { path = "../measure", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
+rand = "0.7.0"
 crossbeam-channel = "0.3"
 
 [package.metadata.docs.rs]

banking-bench/src/main.rs

@@ -246,7 +246,7 @@ fn main() {
                 poh_recorder.lock().unwrap().set_bank(&bank);
                 assert!(poh_recorder.lock().unwrap().bank().is_some());
                 if bank.slot() > 32 {
-                    bank_forks.set_root(root, &None);
+                    bank_forks.set_root(root, &None, None);
                     root += 1;
                 }
                 debug!(

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "1.0.19"
+version = "1.0.23"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,24 +14,24 @@ itertools = "0.8.2"
 log = "0.4.8"
 num-derive = "0.3"
 num-traits = "0.2"
-rand = "0.6.5"
+rand = "0.7.0"
 rayon = "1.2.0"
 serde_json = "1.0.46"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.19" }
-solana-core = { path = "../core", version = "1.0.19" }
-solana-genesis = { path = "../genesis", version = "1.0.19" }
-solana-client = { path = "../client", version = "1.0.19" }
-solana-faucet = { path = "../faucet", version = "1.0.19" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-metrics = { path = "../metrics", version = "1.0.19" }
-solana-net-utils = { path = "../net-utils", version = "1.0.19" }
-solana-runtime = { path = "../runtime", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.23" }
+solana-core = { path = "../core", version = "1.0.23" }
+solana-genesis = { path = "../genesis", version = "1.0.23" }
+solana-client = { path = "../client", version = "1.0.23" }
+solana-faucet = { path = "../faucet", version = "1.0.23" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-metrics = { path = "../metrics", version = "1.0.23" }
+solana-net-utils = { path = "../net-utils", version = "1.0.23" }
+solana-runtime = { path = "../runtime", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
 
 [dev-dependencies]
-solana-local-cluster = { path = "../local-cluster", version = "1.0.19" }
+solana-local-cluster = { path = "../local-cluster", version = "1.0.23" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-streamer/Cargo.toml

@@ -2,17 +2,17 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "1.0.19"
+version = "1.0.23"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.19" }
-solana-core = { path = "../core", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-net-utils = { path = "../net-utils", version = "1.0.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.23" }
+solana-core = { path = "../core", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-net-utils = { path = "../net-utils", version = "1.0.23" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "1.0.19"
+version = "1.0.23"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,27 +14,27 @@ log = "0.4.8"
 rayon = "1.2.0"
 serde_json = "1.0.46"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.19" }
-solana-core = { path = "../core", version = "1.0.19" }
-solana-genesis = { path = "../genesis", version = "1.0.19" }
-solana-client = { path = "../client", version = "1.0.19" }
-solana-faucet = { path = "../faucet", version = "1.0.19" }
-solana-librapay = { path = "../programs/librapay", version = "1.0.19", optional = true }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-metrics = { path = "../metrics", version = "1.0.19" }
-solana-measure = { path = "../measure", version = "1.0.19" }
-solana-net-utils = { path = "../net-utils", version = "1.0.19" }
-solana-runtime = { path = "../runtime", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
-solana-move-loader-program = { path = "../programs/move_loader", version = "1.0.19", optional = true }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.23" }
+solana-core = { path = "../core", version = "1.0.23" }
+solana-genesis = { path = "../genesis", version = "1.0.23" }
+solana-client = { path = "../client", version = "1.0.23" }
+solana-faucet = { path = "../faucet", version = "1.0.23" }
+#solana-librapay = { path = "../programs/librapay", version = "1.0.20", optional = true }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-metrics = { path = "../metrics", version = "1.0.23" }
+solana-measure = { path = "../measure", version = "1.0.23" }
+solana-net-utils = { path = "../net-utils", version = "1.0.23" }
+solana-runtime = { path = "../runtime", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
+#solana-move-loader-program = { path = "../programs/move_loader", version = "1.0.20", optional = true }
 
 [dev-dependencies]
 serial_test = "0.3.2"
 serial_test_derive = "0.4.0"
-solana-local-cluster = { path = "../local-cluster", version = "1.0.19" }
+solana-local-cluster = { path = "../local-cluster", version = "1.0.23" }
 
-[features]
-move = ["solana-librapay", "solana-move-loader-program"]
+#[features]
+#move = ["solana-librapay", "solana-move-loader-program"]
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

chacha-cuda/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha-cuda"
-version = "1.0.19"
+version = "1.0.23"
 description = "Solana Chacha Cuda APIs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,12 +10,12 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-solana-archiver-utils = { path = "../archiver-utils", version = "1.0.19" }
-solana-chacha = { path = "../chacha", version = "1.0.19" }
-solana-ledger = { path = "../ledger", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-perf = { path = "../perf", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
+solana-archiver-utils = { path = "../archiver-utils", version = "1.0.23" }
+solana-chacha = { path = "../chacha", version = "1.0.23" }
+solana-ledger = { path = "../ledger", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-perf = { path = "../perf", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
 
 [dev-dependencies]
 hex-literal = "0.2.1"

chacha-sys/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha-sys"
-version = "1.0.19"
+version = "1.0.23"
 description = "Solana chacha-sys"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

chacha/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-chacha"
-version = "1.0.19"
+version = "1.0.23"
 description = "Solana Chacha APIs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,13 +10,13 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
-solana-chacha-sys = { path = "../chacha-sys", version = "1.0.19" }
-solana-ledger = { path = "../ledger", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-perf = { path = "../perf", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
+rand = "0.7.0"
+rand_chacha = "0.2.2"
+solana-chacha-sys = { path = "../chacha-sys", version = "1.0.23" }
+solana-ledger = { path = "../ledger", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-perf = { path = "../perf", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
 
 [dev-dependencies]
 hex-literal = "0.2.1"

ci/buildkite-secondary.yml

@@ -2,6 +2,16 @@
 # Build steps that run after the primary pipeline on pushes and tags.
 # Pull requests to not run these steps.
 steps:
+  - command: "ci/publish-tarball.sh"
+    timeout_in_minutes: 60
+    name: "publish tarball"
+  - command: "ci/publish-docs.sh"
+    timeout_in_minutes: 15
+    name: "publish docs"
+  - command: "ci/publish-bpf-sdk.sh"
+    timeout_in_minutes: 5
+    name: "publish bpf sdk"
+  - wait
   - command: "sdk/docker-solana/build.sh"
     timeout_in_minutes: 60
     name: "publish docker"
@@ -9,15 +19,6 @@ steps:
     timeout_in_minutes: 240
     name: "publish crate"
     branches: "!master"
-  - command: "ci/publish-bpf-sdk.sh"
-    timeout_in_minutes: 5
-    name: "publish bpf sdk"
-  - command: "ci/publish-tarball.sh"
-    timeout_in_minutes: 60
-    name: "publish tarball"
-  - command: "ci/publish-docs.sh"
-    timeout_in_minutes: 15
-    name: "publish docs"
-  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
-    name: "move"
-    timeout_in_minutes: 20
+    #  - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-move.sh"
+    #    name: "move"
+    #    timeout_in_minutes: 20

ci/docker-run.sh

@@ -49,7 +49,7 @@ else
   # ~/.cargo
   ARGS+=(--volume "$PWD:/home")
 fi
-ARGS+=(--env "CARGO_HOME=/home/.cargo")
+ARGS+=(--env "HOME=/home" --env "CARGO_HOME=/home/.cargo")
 
 # kcov tries to set the personality of the binary which docker
 # doesn't allow by default.

ci/docker-rust-nightly/Dockerfile

@@ -1,4 +1,4 @@
-FROM solanalabs/rust:1.42.0
+FROM solanalabs/rust:1.43.0
 ARG date
 
 RUN set -x \

ci/docker-rust/Dockerfile

@@ -1,6 +1,6 @@
 # Note: when the rust version is changed also modify
 # ci/rust-version.sh to pick up the new image tag
-FROM rust:1.42.0
+FROM rust:1.43.0
 
 # Add Google Protocol Buffers for Libra's metrics library.
 ENV PROTOC_VERSION 3.8.0

ci/publish-tarball.sh

@@ -71,7 +71,7 @@ echo --- Creating release tarball
   export CHANNEL
 
   source ci/rust-version.sh stable
-  scripts/cargo-install-all.sh +"$rust_stable" --use-move solana-release
+  scripts/cargo-install-all.sh +"$rust_stable" solana-release
 
   tar cvf solana-release-$TARGET.tar solana-release
   bzip2 solana-release-$TARGET.tar

ci/rust-version.sh

@@ -1,28 +1,30 @@
 #
 # This file maintains the rust versions for use by CI.
 #
-# Build with stable rust, updating the stable toolchain if necessary:
-#   $ source ci/rust-version.sh stable
-#   $ cargo +"$rust_stable" build
-#
-# Build with nightly rust, updating the nightly toolchain if necessary:
-#   $ source ci/rust-version.sh nightly
-#   $ cargo +"$rust_nightly" build
-#
 # Obtain the environment variables without any automatic toolchain updating:
 #   $ source ci/rust-version.sh
 #
+# Obtain the environment variables updating both stable and nightly, only stable, or
+# only nightly:
+#   $ source ci/rust-version.sh all
+#   $ source ci/rust-version.sh stable
+#   $ source ci/rust-version.sh nightly
+
+# Then to build with either stable or nightly:
+#   $ cargo +"$rust_stable" build
+#   $ cargo +"$rust_nightly" build
+#
 
 if [[ -n $RUST_STABLE_VERSION ]]; then
   stable_version="$RUST_STABLE_VERSION"
 else
-  stable_version=1.42.0
+  stable_version=1.43.0
 fi
 
 if [[ -n $RUST_NIGHTLY_VERSION ]]; then
   nightly_version="$RUST_NIGHTLY_VERSION"
 else
-  nightly_version=2020-03-12
+  nightly_version=2020-04-23
 fi
 
 
@@ -51,6 +53,10 @@ export rust_nightly_docker_image=solanalabs/rust-nightly:"$nightly_version"
   nightly)
      rustup_install "$rust_nightly"
     ;;
+  all)
+     rustup_install "$rust_stable"
+     rustup_install "$rust_nightly"
+    ;;
   *)
     echo "Note: ignoring unknown argument: $1"
     ;;

ci/test-bench.sh

@@ -25,7 +25,7 @@ source ci/_
 source ci/upload-ci-artifact.sh
 
 eval "$(ci/channel-info.sh)"
-source ci/rust-version.sh nightly
+source ci/rust-version.sh all
 
 set -o pipefail
 export RUST_BACKTRACE=1

ci/test-stable.sh

@@ -47,7 +47,7 @@ echo "Executing $testName"
 case $testName in
 test-stable)
   _ cargo +"$rust_stable" test --jobs "$NPROC" --all --exclude solana-local-cluster ${V:+--verbose} -- --nocapture
-  _ cargo +"$rust_stable" test --manifest-path bench-tps/Cargo.toml --features=move ${V:+--verbose} test_bench_tps_local_cluster_move -- --nocapture
+  #_ cargo +"$rust_stable" test --manifest-path bench-tps/Cargo.toml --features=move ${V:+--verbose} test_bench_tps_local_cluster_move -- --nocapture
   ;;
 test-stable-perf)
   ci/affects-files.sh \

clap-utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-clap-utils"
-version = "1.0.19"
+version = "1.0.23"
 description = "Solana utilities for the clap"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -11,8 +11,8 @@ edition = "2018"
 [dependencies]
 clap = "2.33.0"
 rpassword = "4.0"
-solana-remote-wallet = { path = "../remote-wallet", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
 thiserror = "1.0.11"
 tiny-bip39 = "0.7.0"
 url = "2.1.0"

clap-utils/src/commitment.rs

@@ -0,0 +1,18 @@
+use crate::ArgConstant;
+use clap::Arg;
+
+pub const COMMITMENT_ARG: ArgConstant<'static> = ArgConstant {
+    name: "commitment",
+    long: "commitment",
+    help: "Return information at the selected commitment level",
+};
+
+pub fn commitment_arg<'a, 'b>() -> Arg<'a, 'b> {
+    Arg::with_name(COMMITMENT_ARG.name)
+        .long(COMMITMENT_ARG.long)
+        .takes_value(true)
+        .possible_values(&["recent", "root", "max"])
+        .default_value("recent")
+        .value_name("COMMITMENT_LEVEL")
+        .help(COMMITMENT_ARG.help)
+}

clap-utils/src/input_parsers.rs

@@ -7,6 +7,7 @@ use clap::ArgMatches;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
     clock::UnixTimestamp,
+    commitment_config::CommitmentConfig,
     native_token::sol_to_lamports,
     pubkey::Pubkey,
     signature::{read_keypair_file, Keypair, Signature, Signer},
@@ -162,6 +163,15 @@ pub fn lamports_of_sol(matches: &ArgMatches<'_>, name: &str) -> Option<u64> {
     value_of(matches, name).map(sol_to_lamports)
 }
 
+pub fn commitment_of(matches: &ArgMatches<'_>, name: &str) -> Option<CommitmentConfig> {
+    matches.value_of(name).map(|value| match value {
+        "max" => CommitmentConfig::max(),
+        "recent" => CommitmentConfig::recent(),
+        "root" => CommitmentConfig::root(),
+        _ => CommitmentConfig::default(),
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

clap-utils/src/lib.rs

@@ -42,6 +42,7 @@ impl std::fmt::Debug for DisplayError {
     }
 }
 
+pub mod commitment;
 pub mod input_parsers;
 pub mod input_validators;
 pub mod keypair;

cli-config/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli-config"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.19"
+version = "1.0.23"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"

cli/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-cli"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.19"
+version = "1.0.23"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -26,28 +26,29 @@ reqwest = { version = "0.10.1", default-features = false, features = ["blocking"
 serde = "1.0.104"
 serde_derive = "1.0.103"
 serde_json = "1.0.46"
-solana-budget-program = { path = "../programs/budget", version = "1.0.19" }
-solana-clap-utils = { path = "../clap-utils", version = "1.0.19" }
-solana-cli-config = { path = "../cli-config", version = "1.0.19" }
-solana-client = { path = "../client", version = "1.0.19" }
-solana-config-program = { path = "../programs/config", version = "1.0.19" }
-solana-faucet = { path = "../faucet", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-net-utils = { path = "../net-utils", version = "1.0.19" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.0.19" }
-solana-runtime = { path = "../runtime", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
-solana-stake-program = { path = "../programs/stake", version = "1.0.19" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.19" }
-solana-vote-signer = { path = "../vote-signer", version = "1.0.19" }
+solana-budget-program = { path = "../programs/budget", version = "1.0.23" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.23" }
+solana-cli-config = { path = "../cli-config", version = "1.0.23" }
+solana-client = { path = "../client", version = "1.0.23" }
+solana-config-program = { path = "../programs/config", version = "1.0.23" }
+solana-faucet = { path = "../faucet", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-net-utils = { path = "../net-utils", version = "1.0.23" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.0.23" }
+solana-runtime = { path = "../runtime", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
+solana-stake-program = { path = "../programs/stake", version = "1.0.23" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.23" }
+solana-transaction-status = { path = "../transaction-status", version = "1.0.23" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.23" }
+solana-vote-signer = { path = "../vote-signer", version = "1.0.23" }
 titlecase = "1.1.0"
 thiserror = "1.0.11"
 url = "2.1.1"
 
 [dev-dependencies]
-solana-core = { path = "../core", version = "1.0.19" }
-solana-budget-program = { path = "../programs/budget", version = "1.0.19" }
+solana-core = { path = "../core", version = "1.0.23" }
+solana-budget-program = { path = "../programs/budget", version = "1.0.23" }
 tempfile = "3.1.0"
 
 [[bin]]

cli/src/cli.rs

@@ -538,7 +538,7 @@ impl CliConfig<'_> {
         if !self.signers.is_empty() {
             self.signers[0].try_pubkey()
         } else {
-            Err(SignerError::CustomError(
+            Err(SignerError::Custom(
                 "Default keypair must be set if pubkey arg not provided".to_string(),
             ))
         }
@@ -1155,12 +1155,48 @@ fn process_balance(
     }
 }
 
-fn process_confirm(rpc_client: &RpcClient, signature: &Signature) -> ProcessResult {
-    match rpc_client.get_signature_status(&signature) {
+fn process_confirm(
+    rpc_client: &RpcClient,
+    config: &CliConfig,
+    signature: &Signature,
+) -> ProcessResult {
+    match rpc_client.get_signature_status_with_commitment_and_history(
+        &signature,
+        CommitmentConfig::max(),
+        true,
+    ) {
         Ok(status) => {
             if let Some(result) = status {
                 match result {
-                    Ok(_) => Ok("Confirmed".to_string()),
+                    Ok(_) => {
+                        if config.verbose {
+                            match rpc_client.get_confirmed_transaction(
+                                signature,
+                                solana_transaction_status::TransactionEncoding::Binary,
+                            ) {
+                                Ok(confirmed_transaction) => {
+                                    println!("\nTransaction:");
+                                    crate::display::println_transaction(
+                                        &confirmed_transaction
+                                            .transaction
+                                            .transaction
+                                            .decode()
+                                            .expect("Successful decode"),
+                                        &confirmed_transaction.transaction.meta,
+                                        "  ",
+                                    );
+                                    println!();
+                                    Ok(format!("Confirmed in slot {}", confirmed_transaction.slot))
+                                }
+                                Err(err) => Ok(format!(
+                                    "Confirmed. Unable to get confirmed transaction details: {}",
+                                    err
+                                )),
+                            }
+                        } else {
+                            Ok("Confirmed".to_string())
+                        }
+                    }
                     Err(err) => Ok(format!("Transaction failed with error: {}", err)),
                 }
             } else {
@@ -2034,7 +2070,7 @@ pub fn process_command(config: &CliConfig) -> ProcessResult {
         // Cancel a contract by contract Pubkey
         CliCommand::Cancel(pubkey) => process_cancel(&rpc_client, config, &pubkey),
         // Confirm the last client transaction by signature
-        CliCommand::Confirm(signature) => process_confirm(&rpc_client, signature),
+        CliCommand::Confirm(signature) => process_confirm(&rpc_client, config, signature),
         // If client has positive balance, pay lamports to another address
         CliCommand::Pay(PayCommand {
             lamports,
@@ -2183,7 +2219,7 @@ where
         Err(err) => {
             if let ClientErrorKind::TransactionError(TransactionError::InstructionError(
                 _,
-                InstructionError::CustomError(code),
+                InstructionError::Custom(code),
             )) = err.kind()
             {
                 if let Some(specific_error) = E::decode_custom_error_to_enum(*code) {

cli/src/cluster_query.rs

@@ -8,7 +8,12 @@ use crate::{
 use clap::{value_t, value_t_or_exit, App, Arg, ArgMatches, SubCommand};
 use console::{style, Emoji};
 use indicatif::{ProgressBar, ProgressStyle};
-use solana_clap_utils::{input_parsers::*, input_validators::*, keypair::signer_from_path};
+use solana_clap_utils::{
+    commitment::{commitment_arg, COMMITMENT_ARG},
+    input_parsers::*,
+    input_validators::*,
+    keypair::signer_from_path,
+};
 use solana_client::{
     pubsub_client::{PubsubClient, SlotInfoMessage},
     rpc_client::RpcClient,
@@ -91,14 +96,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
             SubCommand::with_name("epoch-info")
             .about("Get information about the current epoch")
             .alias("get-epoch-info")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return information at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("genesis-hash")
@@ -108,37 +106,16 @@ impl ClusterQuerySubCommands for App<'_, '_> {
         .subcommand(
             SubCommand::with_name("slot").about("Get current slot")
             .alias("get-slot")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return slot at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("epoch").about("Get current epoch")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return epoch at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("transaction-count").about("Get current transaction count")
             .alias("get-transaction-count")
-            .arg(
-                Arg::with_name("confirmed")
-                    .long("confirmed")
-                    .takes_value(false)
-                    .help(
-                        "Return count at maximum-lockout commitment level",
-                    ),
-            ),
+            .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("ping")
@@ -178,14 +155,7 @@ impl ClusterQuerySubCommands for App<'_, '_> {
                         .default_value("15")
                         .help("Wait up to timeout seconds for transaction confirmation"),
                 )
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Wait until the transaction is confirmed at maximum-lockout commitment level",
-                        ),
-                ),
+                .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("live-slots")
@@ -236,20 +206,13 @@ impl ClusterQuerySubCommands for App<'_, '_> {
             SubCommand::with_name("validators")
                 .about("Show summary information about the current validators")
                 .alias("show-validators")
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Return information at maximum-lockout commitment level",
-                        ),
-                )
                 .arg(
                     Arg::with_name("lamports")
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
-                ),
+                )
+                .arg(commitment_arg()),
         )
     }
 }
@@ -282,11 +245,7 @@ pub fn parse_cluster_ping(
         None
     };
     let timeout = Duration::from_secs(value_t_or_exit!(matches, "timeout", u64));
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::Ping {
             lamports,
@@ -313,11 +272,7 @@ pub fn parse_get_block_time(matches: &ArgMatches<'_>) -> Result<CliCommandInfo,
 }
 
 pub fn parse_get_epoch_info(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetEpochInfo { commitment_config },
         signers: vec![],
@@ -325,11 +280,7 @@ pub fn parse_get_epoch_info(matches: &ArgMatches<'_>) -> Result<CliCommandInfo,
 }
 
 pub fn parse_get_slot(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetSlot { commitment_config },
         signers: vec![],
@@ -337,11 +288,7 @@ pub fn parse_get_slot(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliErr
 }
 
 pub fn parse_get_epoch(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetEpoch { commitment_config },
         signers: vec![],
@@ -349,11 +296,7 @@ pub fn parse_get_epoch(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliEr
 }
 
 pub fn parse_get_transaction_count(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::GetTransactionCount { commitment_config },
         signers: vec![],
@@ -379,11 +322,7 @@ pub fn parse_show_stakes(
 
 pub fn parse_show_validators(matches: &ArgMatches<'_>) -> Result<CliCommandInfo, CliError> {
     let use_lamports_unit = matches.is_present("lamports");
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
 
     Ok(CliCommandInfo {
         command: CliCommand::ShowValidators {
@@ -454,25 +393,35 @@ pub fn process_catchup(
         }
 
         let slot_distance = rpc_slot as i64 - node_slot as i64;
+        let slots_per_second =
+            (previous_slot_distance - slot_distance) as f64 / f64::from(sleep_interval);
+        let time_remaining = (slot_distance as f64 / slots_per_second).round();
+        let time_remaining = if !time_remaining.is_normal() || time_remaining <= 0.0 {
+            "".to_string()
+        } else {
+            format!(
+                ". Time remaining: {}",
+                humantime::format_duration(Duration::from_secs_f64(time_remaining))
+            )
+        };
+
         progress_bar.set_message(&format!(
-            "Validator is {} slots away (us:{} them:{}){}",
+            "{} slots behind (us:{} them:{}){}",
             slot_distance,
             node_slot,
             rpc_slot,
             if previous_rpc_slot == std::u64::MAX {
                 "".to_string()
             } else {
-                let slots_per_second =
-                    (previous_slot_distance - slot_distance) as f64 / f64::from(sleep_interval);
-
                 format!(
-                    " and {} at {:.1} slots/second",
+                    ", {} at {:.1} slots/second{}",
                     if slots_per_second < 0.0 {
                         "falling behind"
                     } else {
                         "gaining"
                     },
                     slots_per_second,
+                    time_remaining
                 )
             }
         ));
@@ -1346,7 +1295,8 @@ mod tests {
             "2",
             "-t",
             "3",
-            "--confirmed",
+            "--commitment",
+            "max",
         ]);
         assert_eq!(
             parse_command(&test_ping, &default_keypair_file, None).unwrap(),
@@ -1356,7 +1306,7 @@ mod tests {
                     interval: Duration::from_secs(1),
                     count: Some(2),
                     timeout: Duration::from_secs(3),
-                    commitment_config: CommitmentConfig::default(),
+                    commitment_config: CommitmentConfig::max(),
                 },
                 signers: vec![default_keypair.into()],
             }

cli/src/display.rs

@@ -1,6 +1,11 @@
 use crate::cli::SettingType;
 use console::style;
-use solana_sdk::hash::Hash;
+use solana_sdk::{
+    hash::Hash, native_token::lamports_to_sol, program_utils::limited_deserialize,
+    transaction::Transaction,
+};
+use solana_transaction_status::RpcTransactionStatusMeta;
+use std::io;
 
 // Pretty print a "name value"
 pub fn println_name_value(name: &str, value: &str) {
@@ -49,3 +54,140 @@ pub fn println_signers(
     }
     println!();
 }
+
+pub fn write_transaction<W: io::Write>(
+    w: &mut W,
+    transaction: &Transaction,
+    transaction_status: &Option<RpcTransactionStatusMeta>,
+    prefix: &str,
+) -> io::Result<()> {
+    let message = &transaction.message;
+    writeln!(
+        w,
+        "{}Recent Blockhash: {:?}",
+        prefix, message.recent_blockhash
+    )?;
+    for (signature_index, signature) in transaction.signatures.iter().enumerate() {
+        writeln!(
+            w,
+            "{}Signature {}: {:?}",
+            prefix, signature_index, signature
+        )?;
+    }
+    writeln!(w, "{}{:?}", prefix, message.header)?;
+    for (account_index, account) in message.account_keys.iter().enumerate() {
+        writeln!(w, "{}Account {}: {:?}", prefix, account_index, account)?;
+    }
+    for (instruction_index, instruction) in message.instructions.iter().enumerate() {
+        let program_pubkey = message.account_keys[instruction.program_id_index as usize];
+        writeln!(w, "{}Instruction {}", prefix, instruction_index)?;
+        writeln!(
+            w,
+            "{}  Program: {} ({})",
+            prefix, program_pubkey, instruction.program_id_index
+        )?;
+        for (account_index, account) in instruction.accounts.iter().enumerate() {
+            let account_pubkey = message.account_keys[*account as usize];
+            writeln!(
+                w,
+                "{}  Account {}: {} ({})",
+                prefix, account_index, account_pubkey, account
+            )?;
+        }
+
+        let mut raw = true;
+        if program_pubkey == solana_vote_program::id() {
+            if let Ok(vote_instruction) = limited_deserialize::<
+                solana_vote_program::vote_instruction::VoteInstruction,
+            >(&instruction.data)
+            {
+                writeln!(w, "{}  {:?}", prefix, vote_instruction)?;
+                raw = false;
+            }
+        } else if program_pubkey == solana_stake_program::id() {
+            if let Ok(stake_instruction) = limited_deserialize::<
+                solana_stake_program::stake_instruction::StakeInstruction,
+            >(&instruction.data)
+            {
+                writeln!(w, "{}  {:?}", prefix, stake_instruction)?;
+                raw = false;
+            }
+        } else if program_pubkey == solana_sdk::system_program::id() {
+            if let Ok(system_instruction) = limited_deserialize::<
+                solana_sdk::system_instruction::SystemInstruction,
+            >(&instruction.data)
+            {
+                writeln!(w, "{}  {:?}", prefix, system_instruction)?;
+                raw = false;
+            }
+        }
+
+        if raw {
+            writeln!(w, "{}  Data: {:?}", prefix, instruction.data)?;
+        }
+    }
+
+    if let Some(transaction_status) = transaction_status {
+        writeln!(
+            w,
+            "{}Status: {}",
+            prefix,
+            match &transaction_status.status {
+                Ok(_) => "Ok".into(),
+                Err(err) => err.to_string(),
+            }
+        )?;
+        writeln!(
+            w,
+            "{}  Fee: {} SOL",
+            prefix,
+            lamports_to_sol(transaction_status.fee)
+        )?;
+        assert_eq!(
+            transaction_status.pre_balances.len(),
+            transaction_status.post_balances.len()
+        );
+        for (i, (pre, post)) in transaction_status
+            .pre_balances
+            .iter()
+            .zip(transaction_status.post_balances.iter())
+            .enumerate()
+        {
+            if pre == post {
+                writeln!(
+                    w,
+                    "{}  Account {} balance: {} SOL",
+                    prefix,
+                    i,
+                    lamports_to_sol(*pre)
+                )?;
+            } else {
+                writeln!(
+                    w,
+                    "{}  Account {} balance: {} SOL -> {} SOL",
+                    prefix,
+                    i,
+                    lamports_to_sol(*pre),
+                    lamports_to_sol(*post)
+                )?;
+            }
+        }
+    } else {
+        writeln!(w, "{}Status: Unavailable", prefix)?;
+    }
+
+    Ok(())
+}
+
+pub fn println_transaction(
+    transaction: &Transaction,
+    transaction_status: &Option<RpcTransactionStatusMeta>,
+    prefix: &str,
+) {
+    let mut w = Vec::new();
+    if write_transaction(&mut w, transaction, transaction_status, prefix).is_ok() {
+        if let Ok(s) = String::from_utf8(w) {
+            print!("{}", s);
+        }
+    }
+}

cli/src/vote.rs

@@ -4,7 +4,11 @@ use crate::cli::{
     ProcessResult, SignerIndex,
 };
 use clap::{value_t_or_exit, App, Arg, ArgMatches, SubCommand};
-use solana_clap_utils::{input_parsers::*, input_validators::*};
+use solana_clap_utils::{
+    commitment::{commitment_arg, COMMITMENT_ARG},
+    input_parsers::*,
+    input_validators::*,
+};
 use solana_client::rpc_client::RpcClient;
 use solana_remote_wallet::remote_wallet::RemoteWalletManager;
 use solana_sdk::{
@@ -159,14 +163,6 @@ impl VoteSubCommands for App<'_, '_> {
             SubCommand::with_name("vote-account")
                 .about("Show the contents of a vote account")
                 .alias("show-vote-account")
-                .arg(
-                    Arg::with_name("confirmed")
-                        .long("confirmed")
-                        .takes_value(false)
-                        .help(
-                            "Return information at maximum-lockout commitment level",
-                        ),
-                )
                 .arg(
                     Arg::with_name("vote_account_pubkey")
                         .index(1)
@@ -181,7 +177,8 @@ impl VoteSubCommands for App<'_, '_> {
                         .long("lamports")
                         .takes_value(false)
                         .help("Display balance in lamports instead of SOL"),
-                ),
+                )
+                .arg(commitment_arg()),
         )
         .subcommand(
             SubCommand::with_name("withdraw-from-vote-account")
@@ -322,11 +319,7 @@ pub fn parse_vote_get_account_command(
     let vote_account_pubkey =
         pubkey_of_signer(matches, "vote_account_pubkey", wallet_manager)?.unwrap();
     let use_lamports_unit = matches.is_present("lamports");
-    let commitment_config = if matches.is_present("confirmed") {
-        CommitmentConfig::default()
-    } else {
-        CommitmentConfig::recent()
-    };
+    let commitment_config = commitment_of(matches, COMMITMENT_ARG.long).unwrap();
     Ok(CliCommandInfo {
         command: CliCommand::ShowVoteAccount {
             pubkey: vote_account_pubkey,

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "1.0.19"
+version = "1.0.23"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,10 +19,10 @@ reqwest = { version = "0.10.1", default-features = false, features = ["blocking"
 serde = "1.0.104"
 serde_derive = "1.0.103"
 serde_json = "1.0.46"
-solana-net-utils = { path = "../net-utils", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
-solana-transaction-status = { path = "../transaction-status", version = "1.0.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.19" }
+solana-net-utils = { path = "../net-utils", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
+solana-transaction-status = { path = "../transaction-status", version = "1.0.23" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.23" }
 thiserror = "1.0"
 tungstenite = "0.10.1"
 url = "2.1.1"
@@ -31,7 +31,7 @@ url = "2.1.1"
 assert_matches = "1.3.0"
 jsonrpc-core = "14.0.5"
 jsonrpc-http-server = "14.0.6"
-solana-logger = { path = "../logger", version = "1.0.19" }
+solana-logger = { path = "../logger", version = "1.0.23" }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]

client/src/rpc_client.rs

@@ -16,7 +16,10 @@ use log::*;
 use serde_json::{json, Value};
 use solana_sdk::{
     account::Account,
-    clock::{Slot, UnixTimestamp, DEFAULT_TICKS_PER_SECOND, DEFAULT_TICKS_PER_SLOT},
+    clock::{
+        Slot, UnixTimestamp, DEFAULT_TICKS_PER_SECOND, DEFAULT_TICKS_PER_SLOT,
+        MAX_HASH_AGE_IN_SECONDS,
+    },
     commitment_config::CommitmentConfig,
     epoch_schedule::EpochSchedule,
     fee_calculator::{FeeCalculator, FeeRateGovernor},
@@ -27,7 +30,9 @@ use solana_sdk::{
     signers::Signers,
     transaction::{self, Transaction, TransactionError},
 };
-use solana_transaction_status::{ConfirmedBlock, TransactionEncoding, TransactionStatus};
+use solana_transaction_status::{
+    ConfirmedBlock, ConfirmedTransaction, TransactionEncoding, TransactionStatus,
+};
 use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY;
 use std::{
     error,
@@ -109,6 +114,21 @@ impl RpcClient {
         }
     }
 
+    pub fn simulate_transaction(
+        &self,
+        transaction: &Transaction,
+        sig_verify: bool,
+    ) -> RpcResult<TransactionStatus> {
+        let serialized_encoded = bs58::encode(serialize(transaction).unwrap()).into_string();
+        let response = self.send(
+            &RpcRequest::SimulateTransaction,
+            json!([serialized_encoded, { "sigVerify": sig_verify }]),
+            0,
+        )?;
+        Ok(serde_json::from_value(response)
+            .map_err(|err| ClientError::new_with_command(err.into(), "SimulateTransaction"))?)
+    }
+
     pub fn get_signature_status(
         &self,
         signature: &Signature,
@@ -147,6 +167,28 @@ impl RpcClient {
             .map(|status_meta| status_meta.status))
     }
 
+    pub fn get_signature_status_with_commitment_and_history(
+        &self,
+        signature: &Signature,
+        commitment_config: CommitmentConfig,
+        search_transaction_history: bool,
+    ) -> ClientResult<Option<transaction::Result<()>>> {
+        let signature_status = self.client.send(
+            &RpcRequest::GetSignatureStatuses,
+            json!([[signature.to_string()], {
+                "searchTransactionHistory": search_transaction_history
+            }]),
+            5,
+        )?;
+        let result: Response<Vec<Option<TransactionStatus>>> =
+            serde_json::from_value(signature_status)
+                .map_err(|err| ClientError::new_with_command(err.into(), "GetSignatureStatuses"))?;
+        Ok(result.value[0]
+            .clone()
+            .filter(|result| result.satisfies_commitment(commitment_config))
+            .map(|status_meta| status_meta.status))
+    }
+
     pub fn get_slot(&self) -> ClientResult<Slot> {
         self.get_slot_with_commitment(CommitmentConfig::default())
     }
@@ -227,6 +269,44 @@ impl RpcClient {
             .map_err(|err| ClientError::new_with_command(err.into(), "GetConfirmedBlocks"))
     }
 
+    pub fn get_confirmed_signatures_for_address(
+        &self,
+        address: &Pubkey,
+        start_slot: Slot,
+        end_slot: Slot,
+    ) -> ClientResult<Vec<Signature>> {
+        let response = self
+            .client
+            .send(
+                &RpcRequest::GetConfirmedSignaturesForAddress,
+                json!([address, start_slot, end_slot]),
+                0,
+            )
+            .map_err(|err| err.into_with_command("GetConfirmedSignaturesForAddress"))?;
+
+        serde_json::from_value(response).map_err(|err| {
+            ClientError::new_with_command(err.into(), "GetConfirmedSignaturesForAddress")
+        })
+    }
+
+    pub fn get_confirmed_transaction(
+        &self,
+        signature: &Signature,
+        encoding: TransactionEncoding,
+    ) -> ClientResult<ConfirmedTransaction> {
+        let response = self
+            .client
+            .send(
+                &RpcRequest::GetConfirmedTransaction,
+                json!([signature.to_string(), encoding]),
+                0,
+            )
+            .map_err(|err| err.into_with_command("GetConfirmedTransaction"))?;
+
+        serde_json::from_value(response)
+            .map_err(|err| ClientError::new_with_command(err.into(), "GetConfirmedTransaction"))
+    }
+
     pub fn get_block_time(&self, slot: Slot) -> ClientResult<UnixTimestamp> {
         let response = self
             .client
@@ -1029,6 +1109,7 @@ impl RpcClient {
                 }
             }
         };
+        let now = Instant::now();
         loop {
             // Return when default (max) commitment is reached
             // Failed transactions have already been eliminated, `is_some` check is sufficient
@@ -1043,7 +1124,14 @@ impl RpcClient {
                 MAX_LOCKOUT_HISTORY + 1,
             ));
             sleep(Duration::from_millis(500));
-            confirmations = self.get_num_blocks_since_signature_confirmation(&signature)?;
+            confirmations = self
+                .get_num_blocks_since_signature_confirmation(&signature)
+                .unwrap_or(confirmations);
+            if now.elapsed().as_secs() >= MAX_HASH_AGE_IN_SECONDS as u64 {
+                return Err(
+                    RpcError::ForUser("transaction not finalized. This can happen when a transaction lands in an abandoned fork. Please retry.".to_string()).into(),
+                );
+            }
         }
     }
 

client/src/rpc_config.rs

@@ -0,0 +1,28 @@
+use solana_sdk::commitment_config::CommitmentConfig;
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSignatureStatusConfig {
+    pub search_transaction_history: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSimulateTransactionConfig {
+    pub sig_verify: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum RpcLargestAccountsFilter {
+    Circulating,
+    NonCirculating,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcLargestAccountsConfig {
+    #[serde(flatten)]
+    pub commitment: Option<CommitmentConfig>,
+    pub filter: Option<RpcLargestAccountsFilter>,
+}

client/src/rpc_request.rs

@@ -11,6 +11,8 @@ pub enum RpcRequest {
     GetClusterNodes,
     GetConfirmedBlock,
     GetConfirmedBlocks,
+    GetConfirmedSignaturesForAddress,
+    GetConfirmedTransaction,
     GetEpochInfo,
     GetEpochSchedule,
     GetGenesisHash,
@@ -34,6 +36,7 @@ pub enum RpcRequest {
     RegisterNode,
     RequestAirdrop,
     SendTransaction,
+    SimulateTransaction,
     SignVote,
     GetMinimumBalanceForRentExemption,
     MinimumLedgerSlot,
@@ -51,6 +54,8 @@ impl RpcRequest {
             RpcRequest::GetClusterNodes => "getClusterNodes",
             RpcRequest::GetConfirmedBlock => "getConfirmedBlock",
             RpcRequest::GetConfirmedBlocks => "getConfirmedBlocks",
+            RpcRequest::GetConfirmedSignaturesForAddress => "getConfirmedSignaturesForAddress",
+            RpcRequest::GetConfirmedTransaction => "getConfirmedTransaction",
             RpcRequest::GetEpochInfo => "getEpochInfo",
             RpcRequest::GetEpochSchedule => "getEpochSchedule",
             RpcRequest::GetGenesisHash => "getGenesisHash",
@@ -74,6 +79,7 @@ impl RpcRequest {
             RpcRequest::RegisterNode => "registerNode",
             RpcRequest::RequestAirdrop => "requestAirdrop",
             RpcRequest::SendTransaction => "sendTransaction",
+            RpcRequest::SimulateTransaction => "simulateTransaction",
             RpcRequest::SignVote => "signVote",
             RpcRequest::GetMinimumBalanceForRentExemption => "getMinimumBalanceForRentExemption",
             RpcRequest::MinimumLedgerSlot => "minimumLedgerSlot",

client/src/rpc_response.rs

@@ -108,6 +108,8 @@ pub struct RpcContactInfo {
     pub tpu: Option<SocketAddr>,
     /// JSON RPC port
     pub rpc: Option<SocketAddr>,
+    /// Software version
+    pub version: Option<String>,
 }
 
 /// Map of leader base58 identity pubkeys to the slot indices relative to the first epoch slot
@@ -192,3 +194,19 @@ pub struct RpcStorageTurn {
     pub blockhash: String,
     pub slot: Slot,
 }
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcAccountBalance {
+    pub address: String,
+    pub lamports: u64,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct RpcSupply {
+    pub total: u64,
+    pub circulating: u64,
+    pub non_circulating: u64,
+    pub non_circulating_accounts: Vec<String>,
+}

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-core"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.19"
+version = "1.0.23"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -36,41 +36,42 @@ log = "0.4.8"
 nix = "0.17.0"
 num_cpus = "1.0.0"
 num-traits = "0.2"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
+rand = "0.7.0"
+rand_chacha = "0.2.2"
 rayon = "1.2.0"
 regex = "1.3.4"
 serde = "1.0.104"
 serde_derive = "1.0.103"
 serde_json = "1.0.46"
-solana-budget-program = { path = "../programs/budget", version = "1.0.19" }
-solana-clap-utils = { path = "../clap-utils", version = "1.0.19" }
-solana-client = { path = "../client", version = "1.0.19" }
-solana-transaction-status = { path = "../transaction-status", version = "1.0.19" }
-solana-faucet = { path = "../faucet", version = "1.0.19" }
-ed25519-dalek = "=1.0.0-pre.1"
-solana-ledger = { path = "../ledger", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.0.19" }
-solana-metrics = { path = "../metrics", version = "1.0.19" }
-solana-measure = { path = "../measure", version = "1.0.19" }
-solana-net-utils = { path = "../net-utils", version = "1.0.19" }
-solana-chacha-cuda = { path = "../chacha-cuda", version = "1.0.19" }
-solana-perf = { path = "../perf", version = "1.0.19" }
-solana-runtime = { path = "../runtime", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
-solana-stake-program = { path = "../programs/stake", version = "1.0.19" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.19" }
-solana-vote-signer = { path = "../vote-signer", version = "1.0.19" }
-solana-sys-tuner = { path = "../sys-tuner", version = "1.0.19" }
+solana-budget-program = { path = "../programs/budget", version = "1.0.23" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.23" }
+solana-client = { path = "../client", version = "1.0.23" }
+solana-transaction-status = { path = "../transaction-status", version = "1.0.23" }
+solana-faucet = { path = "../faucet", version = "1.0.23" }
+ed25519-dalek = "=1.0.0-pre.3"
+solana-ledger = { path = "../ledger", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.0.23" }
+solana-metrics = { path = "../metrics", version = "1.0.23" }
+solana-measure = { path = "../measure", version = "1.0.23" }
+solana-net-utils = { path = "../net-utils", version = "1.0.23" }
+solana-chacha-cuda = { path = "../chacha-cuda", version = "1.0.23" }
+solana-perf = { path = "../perf", version = "1.0.23" }
+solana-runtime = { path = "../runtime", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
+solana-stake-program = { path = "../programs/stake", version = "1.0.23" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.23" }
+solana-version = { path = "../version", version = "1.0.23" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.23" }
+solana-vote-signer = { path = "../vote-signer", version = "1.0.23" }
+solana-sys-tuner = { path = "../sys-tuner", version = "1.0.23" }
 tempfile = "3.1.0"
 thiserror = "1.0"
 tokio = "0.1"
 tokio-codec = "0.1"
 tokio-fs = "0.1"
 tokio-io = "0.1"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.0.19" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.0.23" }
 trees = "0.2.1"
 
 [dev-dependencies]

core/benches/cluster_info.rs

@@ -5,6 +5,7 @@ extern crate test;
 use rand::{thread_rng, Rng};
 use solana_core::cluster_info::{ClusterInfo, Node};
 use solana_core::contact_info::ContactInfo;
+use solana_ledger::shred::{Shred, NONCE_SHRED_PAYLOAD_SIZE};
 use solana_sdk::pubkey::Pubkey;
 use solana_sdk::timing::timestamp;
 use std::collections::HashMap;
@@ -20,9 +21,8 @@ fn broadcast_shreds_bench(bencher: &mut Bencher) {
     let mut cluster_info = ClusterInfo::new_with_invalid_keypair(leader_info.info.clone());
     let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
 
-    const SHRED_SIZE: usize = 1024;
     const NUM_SHREDS: usize = 32;
-    let shreds = vec![vec![0; SHRED_SIZE]; NUM_SHREDS];
+    let shreds = vec![vec![0; NONCE_SHRED_PAYLOAD_SIZE]; NUM_SHREDS];
     let seeds = vec![[0u8; 32]; NUM_SHREDS];
     let mut stakes = HashMap::new();
     const NUM_PEERS: usize = 200;

core/benches/shredder.rs

@@ -5,11 +5,11 @@ extern crate test;
 use solana_ledger::entry::{create_ticks, Entry};
 use solana_ledger::shred::{
     max_entries_per_n_shred, max_ticks_per_n_shreds, Shred, Shredder, RECOMMENDED_FEC_RATE,
-    SIZE_OF_DATA_SHRED_PAYLOAD,
+    SIZE_OF_NONCE_DATA_SHRED_PAYLOAD,
 };
 use solana_perf::test_tx;
 use solana_sdk::hash::Hash;
-use solana_sdk::signature::{Keypair, Signer};
+use solana_sdk::signature::Keypair;
 use std::sync::Arc;
 use test::Bencher;
 
@@ -29,10 +29,11 @@ fn make_large_unchained_entries(txs_per_entry: u64, num_entries: u64) -> Vec<Ent
 #[bench]
 fn bench_shredder_ticks(bencher: &mut Bencher) {
     let kp = Arc::new(Keypair::new());
-    let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD;
+    let shred_size = SIZE_OF_NONCE_DATA_SHRED_PAYLOAD;
     let num_shreds = ((1000 * 1000) + (shred_size - 1)) / shred_size;
     // ~1Mb
-    let num_ticks = max_ticks_per_n_shreds(1) * num_shreds as u64;
+    let num_ticks =
+        max_ticks_per_n_shreds(1, Some(SIZE_OF_NONCE_DATA_SHRED_PAYLOAD)) * num_shreds as u64;
     let entries = create_ticks(num_ticks, 0, Hash::default());
     bencher.iter(|| {
         let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp.clone(), 0, 0).unwrap();
@@ -43,10 +44,14 @@ fn bench_shredder_ticks(bencher: &mut Bencher) {
 #[bench]
 fn bench_shredder_large_entries(bencher: &mut Bencher) {
     let kp = Arc::new(Keypair::new());
-    let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD;
+    let shred_size = SIZE_OF_NONCE_DATA_SHRED_PAYLOAD;
     let num_shreds = ((1000 * 1000) + (shred_size - 1)) / shred_size;
     let txs_per_entry = 128;
-    let num_entries = max_entries_per_n_shred(&make_test_entry(txs_per_entry), num_shreds as u64);
+    let num_entries = max_entries_per_n_shred(
+        &make_test_entry(txs_per_entry),
+        num_shreds as u64,
+        Some(shred_size),
+    );
     let entries = make_large_unchained_entries(txs_per_entry, num_entries);
     // 1Mb
     bencher.iter(|| {
@@ -58,10 +63,10 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
 #[bench]
 fn bench_deshredder(bencher: &mut Bencher) {
     let kp = Arc::new(Keypair::new());
-    let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD;
+    let shred_size = SIZE_OF_NONCE_DATA_SHRED_PAYLOAD;
     // ~10Mb
     let num_shreds = ((10000 * 1000) + (shred_size - 1)) / shred_size;
-    let num_ticks = max_ticks_per_n_shreds(1) * num_shreds as u64;
+    let num_ticks = max_ticks_per_n_shreds(1, Some(shred_size)) * num_shreds as u64;
     let entries = create_ticks(num_ticks, 0, Hash::default());
     let shredder = Shredder::new(1, 0, RECOMMENDED_FEC_RATE, kp, 0, 0).unwrap();
     let data_shreds = shredder.entries_to_shreds(&entries, true, 0).0;
@@ -73,7 +78,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
 
 #[bench]
 fn bench_deserialize_hdr(bencher: &mut Bencher) {
-    let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD];
+    let data = vec![0; SIZE_OF_NONCE_DATA_SHRED_PAYLOAD];
 
     let shred = Shred::new_from_data(2, 1, 1, Some(&data), true, true, 0, 0, 1);
 

core/src/banking_stage.rs

@@ -1989,14 +1989,14 @@ mod tests {
                             meta.err,
                             Some(TransactionError::InstructionError(
                                 0,
-                                InstructionError::CustomError(1)
+                                InstructionError::Custom(1)
                             ))
                         );
                         assert_eq!(
                             meta.status,
                             Err(TransactionError::InstructionError(
                                 0,
-                                InstructionError::CustomError(1)
+                                InstructionError::Custom(1)
                             ))
                         );
                     } else {

core/src/broadcast_stage/standard_broadcast_run.rs

@@ -390,7 +390,7 @@ mod test {
         )));
         let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
         let mut genesis_config = create_genesis_config(10_000).genesis_config;
-        genesis_config.ticks_per_slot = max_ticks_per_n_shreds(num_shreds_per_slot) + 1;
+        genesis_config.ticks_per_slot = max_ticks_per_n_shreds(num_shreds_per_slot, None) + 1;
         let bank0 = Arc::new(Bank::new(&genesis_config));
         (
             blockstore,
@@ -484,7 +484,11 @@ mod test {
         // Interrupting the slot should cause the unfinished_slot and stats to reset
         let num_shreds = 1;
         assert!(num_shreds < num_shreds_per_slot);
-        let ticks1 = create_ticks(max_ticks_per_n_shreds(num_shreds), 0, genesis_config.hash());
+        let ticks1 = create_ticks(
+            max_ticks_per_n_shreds(num_shreds, None),
+            0,
+            genesis_config.hash(),
+        );
         let receive_results = ReceiveResults {
             entries: ticks1.clone(),
             time_elapsed: Duration::new(2, 0),

core/src/cluster_info.rs

@@ -12,8 +12,6 @@
 //! * layer 2 - Everyone else, if layer 1 is `2^10`, layer 2 should be able to fit `2^20` number of nodes.
 //!
 //! Bank needs to provide an interface for us to query the stake weight
-use crate::crds_value::CompressionType::*;
-use crate::crds_value::EpochIncompleteSlots;
 use crate::packet::limited_deserialize;
 use crate::streamer::{PacketReceiver, PacketSender};
 use crate::{
@@ -21,7 +19,10 @@ use crate::{
     crds_gossip::CrdsGossip,
     crds_gossip_error::CrdsGossipError,
     crds_gossip_pull::{CrdsFilter, CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS},
-    crds_value::{self, CrdsData, CrdsValue, CrdsValueLabel, EpochSlots, SnapshotHash, Vote},
+    crds_value::{
+        self, CrdsData, CrdsValue, CrdsValueLabel, EpochSlots, SnapshotHash, Version, Vote,
+        MAX_WALLCLOCK,
+    },
     packet::{Packet, PACKET_DATA_SIZE},
     result::{Error, Result},
     sendmmsg::{multicast, send_mmsg},
@@ -31,9 +32,9 @@ use crate::{
 use rand::distributions::{Distribution, WeightedIndex};
 use rand::SeedableRng;
 use rand_chacha::ChaChaRng;
+use solana_sdk::sanitize::{Sanitize, SanitizeError};
 
 use bincode::{serialize, serialized_size};
-use compression::prelude::*;
 use core::cmp;
 use itertools::Itertools;
 use rayon::iter::IntoParallelIterator;
@@ -87,9 +88,6 @@ const MAX_PROTOCOL_HEADER_SIZE: u64 = 214;
 /// 128MB/PACKET_DATA_SIZE
 const MAX_GOSSIP_TRAFFIC: usize = 128_000_000 / PACKET_DATA_SIZE;
 
-const NUM_BITS_PER_BYTE: u64 = 8;
-const MIN_SIZE_TO_COMPRESS_GZIP: u64 = 64;
-
 /// Keep the number of snapshot hashes a node publishes under MAX_PROTOCOL_PAYLOAD_SIZE
 pub const MAX_SNAPSHOT_HASHES: usize = 16;
 
@@ -157,6 +155,15 @@ pub struct PruneData {
     pub wallclock: u64,
 }
 
+impl Sanitize for PruneData {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        Ok(())
+    }
+}
+
 impl Signable for PruneData {
     fn pubkey(&self) -> Pubkey {
         self.pubkey
@@ -221,6 +228,20 @@ enum Protocol {
     PruneMessage(Pubkey, PruneData),
 }
 
+impl Sanitize for Protocol {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        match self {
+            Protocol::PullRequest(filter, val) => {
+                filter.sanitize()?;
+                val.sanitize()
+            }
+            Protocol::PullResponse(_, val) => val.sanitize(),
+            Protocol::PushMessage(_, val) => val.sanitize(),
+            Protocol::PruneMessage(_, val) => val.sanitize(),
+        }
+    }
+}
+
 // Rating for pull requests
 // A response table is generated as a
 // 2-d table arranged by target nodes and a
@@ -249,8 +270,8 @@ impl ClusterInfo {
                 last_timestamp_ms: 0,
             },
         };
-        let id = contact_info.id;
-        me.gossip.set_self(&id);
+        me.gossip.set_self(&contact_info.id);
+        me.gossip.set_shred_version(contact_info.shred_version);
         me.insert_self(contact_info);
         me.push_self(&HashMap::new());
         me
@@ -303,61 +324,73 @@ impl ClusterInfo {
         let now = timestamp();
         let mut spy_nodes = 0;
         let mut archivers = 0;
+        let mut different_shred_nodes = 0;
         let my_pubkey = self.my_data().id;
+        let my_shred_version = self.my_data().shred_version;
         let nodes: Vec<_> = self
             .all_peers()
             .into_iter()
-            .map(|(node, last_updated)| {
+            .filter_map(|(node, last_updated)| {
                 if Self::is_spy_node(&node) {
                     spy_nodes += 1;
                 } else if Self::is_archiver(&node) {
                     archivers += 1;
                 }
-                fn addr_to_string(default_ip: &IpAddr, addr: &SocketAddr) -> String {
-                    if ContactInfo::is_valid_address(addr) {
-                        if &addr.ip() == default_ip {
-                            addr.port().to_string()
-                        } else {
-                            addr.to_string()
-                        }
-                    } else {
-                        "none".to_string()
-                    }
-                }
 
-                let ip_addr = node.gossip.ip();
-                format!(
-                    "{:15} {:2}| {:5} | {:44} | {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {}\n",
-                    if ContactInfo::is_valid_address(&node.gossip) {
-                        ip_addr.to_string()
-                    } else {
-                        "none".to_string()
-                    },
-                    if node.id == my_pubkey { "me" } else { "" }.to_string(),
-                    now.saturating_sub(last_updated),
-                    node.id.to_string(),
-                    addr_to_string(&ip_addr, &node.gossip),
-                    addr_to_string(&ip_addr, &node.tpu),
-                    addr_to_string(&ip_addr, &node.tpu_forwards),
-                    addr_to_string(&ip_addr, &node.tvu),
-                    addr_to_string(&ip_addr, &node.tvu_forwards),
-                    addr_to_string(&ip_addr, &node.repair),
-                    addr_to_string(&ip_addr, &node.serve_repair),
-                    addr_to_string(&ip_addr, &node.storage_addr),
-                    addr_to_string(&ip_addr, &node.rpc),
-                    addr_to_string(&ip_addr, &node.rpc_pubsub),
-                    node.shred_version,
-                )
+                let node_version = self.get_node_version(&node.id);
+                if my_shred_version != 0 && (node.shred_version != 0 && node.shred_version != my_shred_version) {
+                    different_shred_nodes += 1;
+                    None
+                } else {
+                    fn addr_to_string(default_ip: &IpAddr, addr: &SocketAddr) -> String {
+                        if ContactInfo::is_valid_address(addr) {
+                            if &addr.ip() == default_ip {
+                                addr.port().to_string()
+                            } else {
+                                addr.to_string()
+                            }
+                        } else {
+                            "none".to_string()
+                        }
+                    }
+                    let ip_addr = node.gossip.ip();
+                    Some(format!(
+                        "{:15} {:2}| {:5} | {:44} |{:^15}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {:5}| {}\n",
+                        if ContactInfo::is_valid_address(&node.gossip) {
+                            ip_addr.to_string()
+                        } else {
+                            "none".to_string()
+                        },
+                        if node.id == my_pubkey { "me" } else { "" }.to_string(),
+                        now.saturating_sub(last_updated),
+                        node.id.to_string(),
+                        if let Some(node_version) = node_version {
+                            node_version.to_string()
+                        } else {
+                            "-".to_string()
+                        },
+                        addr_to_string(&ip_addr, &node.gossip),
+                        addr_to_string(&ip_addr, &node.tpu),
+                        addr_to_string(&ip_addr, &node.tpu_forwards),
+                        addr_to_string(&ip_addr, &node.tvu),
+                        addr_to_string(&ip_addr, &node.tvu_forwards),
+                        addr_to_string(&ip_addr, &node.repair),
+                        addr_to_string(&ip_addr, &node.serve_repair),
+                        addr_to_string(&ip_addr, &node.rpc),
+                        addr_to_string(&ip_addr, &node.rpc_pubsub),
+                        node.shred_version,
+                    ))
+                }
             })
             .collect();
 
         format!(
             "IP Address        |Age(ms)| Node identifier                              \
-             |Gossip| TPU  |TPUfwd| TVU  |TVUfwd|Repair|ServeR|Storag| RPC  |PubSub|ShredVer\n\
-             ------------------+-------+----------------------------------------------+\
-             ------+------+------+------+------+------+------+------+------+------+--------\n\
+             | Version       |Gossip| TPU  |TPUfwd| TVU  |TVUfwd|Repair|ServeR| RPC  |PubSub|ShredVer\n\
+             ------------------+-------+----------------------------------------------+---------------+\
+             ------+------+------+------+------+------+------+------+------+--------\n\
              {}\
-             Nodes: {}{}{}",
+             Nodes: {}{}{}{}",
             nodes.join(""),
             nodes.len() - spy_nodes - archivers,
             if archivers > 0 {
@@ -369,119 +402,29 @@ impl ClusterInfo {
                 format!("\nSpies: {}", spy_nodes)
             } else {
                 "".to_string()
+            },
+            if different_shred_nodes > 0 {
+                format!(
+                    "\nNodes with different shred version: {}",
+                    different_shred_nodes
+                )
+            } else {
+                "".to_string()
             }
         )
     }
 
-    pub fn compress_incomplete_slots(incomplete_slots: &BTreeSet<Slot>) -> EpochIncompleteSlots {
-        if !incomplete_slots.is_empty() {
-            let first_slot = incomplete_slots
-                .iter()
-                .next()
-                .expect("expected to find at least one slot");
-            let last_slot = incomplete_slots
-                .iter()
-                .next_back()
-                .expect("expected to find last slot");
-            let num_uncompressed_bits = last_slot.saturating_sub(*first_slot) + 1;
-            let num_uncompressed_bytes = if num_uncompressed_bits % NUM_BITS_PER_BYTE > 0 {
-                1
-            } else {
-                0
-            } + num_uncompressed_bits / NUM_BITS_PER_BYTE;
-            let mut uncompressed = vec![0u8; num_uncompressed_bytes as usize];
-            incomplete_slots.iter().for_each(|slot| {
-                let offset_from_first_slot = slot.saturating_sub(*first_slot);
-                let index = offset_from_first_slot / NUM_BITS_PER_BYTE;
-                let bit_index = offset_from_first_slot % NUM_BITS_PER_BYTE;
-                uncompressed[index as usize] |= 1 << bit_index;
-            });
-            if num_uncompressed_bytes >= MIN_SIZE_TO_COMPRESS_GZIP {
-                if let Ok(compressed) = uncompressed
-                    .iter()
-                    .cloned()
-                    .encode(&mut GZipEncoder::new(), Action::Finish)
-                    .collect::<std::result::Result<Vec<u8>, _>>()
-                {
-                    return EpochIncompleteSlots {
-                        first: *first_slot,
-                        compression: GZip,
-                        compressed_list: compressed,
-                    };
-                }
-            } else {
-                return EpochIncompleteSlots {
-                    first: *first_slot,
-                    compression: Uncompressed,
-                    compressed_list: uncompressed,
-                };
-            }
-        }
-        EpochIncompleteSlots::default()
-    }
-
-    fn bitmap_to_slot_list(first: Slot, bitmap: &[u8]) -> BTreeSet<Slot> {
-        let mut old_incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
-        bitmap.iter().enumerate().for_each(|(i, val)| {
-            if *val != 0 {
-                (0..8).for_each(|bit_index| {
-                    if (1 << bit_index & *val) != 0 {
-                        let slot = first + i as u64 * NUM_BITS_PER_BYTE + bit_index as u64;
-                        old_incomplete_slots.insert(slot);
-                    }
-                })
-            }
-        });
-        old_incomplete_slots
-    }
-
-    pub fn decompress_incomplete_slots(slots: &EpochIncompleteSlots) -> BTreeSet<Slot> {
-        match slots.compression {
-            Uncompressed => Self::bitmap_to_slot_list(slots.first, &slots.compressed_list),
-            GZip => {
-                if let Ok(decompressed) = slots
-                    .compressed_list
-                    .iter()
-                    .cloned()
-                    .decode(&mut GZipDecoder::new())
-                    .collect::<std::result::Result<Vec<u8>, _>>()
-                {
-                    Self::bitmap_to_slot_list(slots.first, &decompressed)
-                } else {
-                    BTreeSet::new()
-                }
-            }
-            BZip2 => {
-                if let Ok(decompressed) = slots
-                    .compressed_list
-                    .iter()
-                    .cloned()
-                    .decode(&mut BZip2Decoder::new())
-                    .collect::<std::result::Result<Vec<u8>, _>>()
-                {
-                    Self::bitmap_to_slot_list(slots.first, &decompressed)
-                } else {
-                    BTreeSet::new()
-                }
-            }
-        }
-    }
-
     pub fn push_epoch_slots(
         &mut self,
         id: Pubkey,
-        root: Slot,
+        _root: Slot,
         min: Slot,
-        slots: BTreeSet<Slot>,
-        incomplete_slots: &BTreeSet<Slot>,
+        _slots: BTreeSet<Slot>,
+        _incomplete_slots: &BTreeSet<Slot>,
     ) {
-        let compressed = Self::compress_incomplete_slots(incomplete_slots);
         let now = timestamp();
         let entry = CrdsValue::new_signed(
-            CrdsData::EpochSlots(
-                0,
-                EpochSlots::new(id, root, min, slots, vec![compressed], now),
-            ),
+            CrdsData::EpochSlots(0, EpochSlots::new(id, min, now)),
             &self.keypair,
         );
         self.gossip
@@ -618,6 +561,16 @@ impl ClusterInfo {
             .map(|x| x.value.contact_info().unwrap())
     }
 
+    pub fn get_node_version(&self, pubkey: &Pubkey) -> Option<solana_version::Version> {
+        self.gossip
+            .crds
+            .table
+            .get(&CrdsValueLabel::Version(*pubkey))
+            .map(|x| x.value.version())
+            .flatten()
+            .map(|version| version.version.clone())
+    }
+
     /// all validators that have a valid rpc port regardless of `shred_version`.
     pub fn all_rpc_peers(&self) -> Vec<ContactInfo> {
         let me = self.my_data();
@@ -1263,6 +1216,14 @@ impl ClusterInfo {
                 let mut last_contact_info_trace = timestamp();
                 let mut adopt_shred_version = obj.read().unwrap().my_data().shred_version == 0;
                 let recycler = PacketsRecycler::default();
+                {
+                    let mut obj = obj.write().unwrap();
+                    let message = CrdsValue::new_signed(
+                        CrdsData::Version(Version::new(obj.id())),
+                        &obj.keypair,
+                    );
+                    obj.push_message(message);
+                }
                 loop {
                     let start = timestamp();
                     thread_mem_usage::datapoint("solana-gossip");
@@ -1319,6 +1280,10 @@ impl ClusterInfo {
                                         entrypoint.shred_version, entrypoint.id
                                     );
                                     self_info.shred_version = entrypoint.shred_version;
+                                    obj.write()
+                                        .unwrap()
+                                        .gossip
+                                        .set_shred_version(entrypoint.shred_version);
                                     obj.write().unwrap().insert_self(self_info);
                                     adopt_shred_version = false;
                                 }
@@ -1358,6 +1323,7 @@ impl ClusterInfo {
             let from_addr = packet.meta.addr();
             limited_deserialize(&packet.data[..packet.meta.size])
                 .into_iter()
+                .filter(|r: &Protocol| r.sanitize().is_ok())
                 .for_each(|request| match request {
                     Protocol::PullRequest(filter, caller) => {
                         let start = allocated.get();
@@ -1708,7 +1674,10 @@ impl ClusterInfo {
             requests.push(more_reqs)
         }
         if num_requests >= MAX_GOSSIP_TRAFFIC {
-            warn!("Too much gossip traffic, ignoring some messages");
+            warn!(
+                "Too much gossip traffic, ignoring some messages (requests={}, max requests={})",
+                num_requests, MAX_GOSSIP_TRAFFIC
+            );
         }
         let epoch_ms;
         let stakes: HashMap<_, _> = match bank_forks {
@@ -1776,39 +1745,39 @@ impl ClusterInfo {
             .unwrap()
     }
 
-    pub fn gossip_contact_info(id: &Pubkey, gossip: SocketAddr) -> ContactInfo {
+    pub fn gossip_contact_info(id: &Pubkey, gossip: SocketAddr, shred_version: u16) -> ContactInfo {
         ContactInfo {
             id: *id,
             gossip,
             wallclock: timestamp(),
+            shred_version,
             ..ContactInfo::default()
         }
     }
 
-    pub fn spy_contact_info(id: &Pubkey) -> ContactInfo {
-        let dummy_addr = socketaddr_any!();
-
-        Self::gossip_contact_info(id, dummy_addr)
-    }
-
     /// An alternative to Spy Node that has a valid gossip address and fully participate in Gossip.
     pub fn gossip_node(
         id: &Pubkey,
         gossip_addr: &SocketAddr,
+        shred_version: u16,
     ) -> (ContactInfo, UdpSocket, Option<TcpListener>) {
         let bind_ip_addr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0));
         let (port, (gossip_socket, ip_echo)) =
             Node::get_gossip_port(gossip_addr, VALIDATOR_PORT_RANGE, bind_ip_addr);
-        let contact_info = Self::gossip_contact_info(id, SocketAddr::new(gossip_addr.ip(), port));
+        let contact_info =
+            Self::gossip_contact_info(id, SocketAddr::new(gossip_addr.ip(), port), shred_version);
 
         (contact_info, gossip_socket, Some(ip_echo))
     }
 
     /// A Node with dummy ports to spy on gossip via pull requests
-    pub fn spy_node(id: &Pubkey) -> (ContactInfo, UdpSocket, Option<TcpListener>) {
+    pub fn spy_node(
+        id: &Pubkey,
+        shred_version: u16,
+    ) -> (ContactInfo, UdpSocket, Option<TcpListener>) {
         let bind_ip_addr = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0));
         let (_, gossip_socket) = bind_in_range(bind_ip_addr, VALIDATOR_PORT_RANGE).unwrap();
-        let contact_info = Self::spy_contact_info(id);
+        let contact_info = Self::gossip_contact_info(id, socketaddr_any!(), shred_version);
 
         (contact_info, gossip_socket, None)
     }
@@ -2081,7 +2050,7 @@ impl Node {
 
 fn report_time_spent(label: &str, time: &Duration, extra: &str) {
     let count = duration_as_ms(time);
-    if count > 5 {
+    if count > 100 {
         info!("{} took: {} ms {}", label, count, extra);
     }
 }
@@ -2100,10 +2069,10 @@ mod tests {
     #[test]
     fn test_gossip_node() {
         //check that a gossip nodes always show up as spies
-        let (node, _, _) = ClusterInfo::spy_node(&Pubkey::new_rand());
+        let (node, _, _) = ClusterInfo::spy_node(&Pubkey::new_rand(), 0);
         assert!(ClusterInfo::is_spy_node(&node));
         let (node, _, _) =
-            ClusterInfo::gossip_node(&Pubkey::new_rand(), &"1.1.1.1:1111".parse().unwrap());
+            ClusterInfo::gossip_node(&Pubkey::new_rand(), &"1.1.1.1:1111".parse().unwrap(), 0);
         assert!(ClusterInfo::is_spy_node(&node));
     }
 
@@ -2111,7 +2080,7 @@ mod tests {
     fn test_cluster_spy_gossip() {
         //check that gossip doesn't try to push to invalid addresses
         let node = Node::new_localhost();
-        let (spy, _, _) = ClusterInfo::spy_node(&Pubkey::new_rand());
+        let (spy, _, _) = ClusterInfo::spy_node(&Pubkey::new_rand(), 0);
         let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
             node.info,
         )));
@@ -2500,14 +2469,7 @@ mod tests {
         }
         let value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
             0,
-            EpochSlots {
-                from: Pubkey::default(),
-                root: 0,
-                lowest: 0,
-                slots: btree_slots,
-                stash: vec![],
-                wallclock: 0,
-            },
+            EpochSlots::new(Pubkey::default(), 0, 0),
         ));
         test_split_messages(value);
     }
@@ -2519,39 +2481,19 @@ mod tests {
         let payload: Vec<CrdsValue> = vec![];
         let vec_size = serialized_size(&payload).unwrap();
         let desired_size = MAX_PROTOCOL_PAYLOAD_SIZE - vec_size;
-        let mut value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
-            0,
-            EpochSlots {
-                from: Pubkey::default(),
-                root: 0,
-                lowest: 0,
-                slots: BTreeSet::new(),
-                stash: vec![],
-                wallclock: 0,
-            },
-        ));
+        let mut value = CrdsValue::new_unsigned(CrdsData::SnapshotHashes(SnapshotHash {
+            from: Pubkey::default(),
+            hashes: vec![],
+            wallclock: 0,
+        }));
 
         let mut i = 0;
         while value.size() <= desired_size {
-            let slots = (0..i).collect::<BTreeSet<_>>();
-            if slots.len() > 200 {
-                panic!(
-                    "impossible to match size: last {:?} vs desired {:?}",
-                    serialized_size(&value).unwrap(),
-                    desired_size
-                );
-            }
-            value.data = CrdsData::EpochSlots(
-                0,
-                EpochSlots {
-                    from: Pubkey::default(),
-                    root: 0,
-                    lowest: 0,
-                    slots,
-                    stash: vec![],
-                    wallclock: 0,
-                },
-            );
+            value.data = CrdsData::SnapshotHashes(SnapshotHash {
+                from: Pubkey::default(),
+                hashes: vec![(0, Hash::default()); i],
+                wallclock: 0,
+            });
             i += 1;
         }
         let split = ClusterInfo::split_gossip_messages(vec![value.clone()]);
@@ -2681,11 +2623,9 @@ mod tests {
             node_keypair,
         );
         for i in 0..10 {
-            let mut peer_root = 5;
             let mut peer_lowest = 0;
             if i >= 5 {
                 // make these invalid for the upcoming repair request
-                peer_root = 15;
                 peer_lowest = 10;
             }
             let other_node_pubkey = Pubkey::new_rand();
@@ -2693,14 +2633,7 @@ mod tests {
             cluster_info.insert_info(other_node.clone());
             let value = CrdsValue::new_unsigned(CrdsData::EpochSlots(
                 0,
-                EpochSlots::new(
-                    other_node_pubkey,
-                    peer_root,
-                    peer_lowest,
-                    BTreeSet::new(),
-                    vec![],
-                    timestamp(),
-                ),
+                EpochSlots::new(other_node_pubkey, peer_lowest, timestamp()),
             ));
             let _ = cluster_info.gossip.crds.insert(value, timestamp());
         }
@@ -2749,6 +2682,14 @@ mod tests {
         assert_eq!(MAX_PROTOCOL_HEADER_SIZE, max_protocol_size);
     }
 
+    #[test]
+    fn test_protocol_sanitize() {
+        let mut pd = PruneData::default();
+        pd.wallclock = MAX_WALLCLOCK;
+        let msg = Protocol::PruneMessage(Pubkey::default(), pd);
+        assert_eq!(msg.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+    }
+
     // computes the maximum size for pull request blooms
     fn max_bloom_size() -> usize {
         let filter_size = serialized_size(&CrdsFilter::default())
@@ -2761,38 +2702,4 @@ mod tests {
             serialized_size(&protocol).expect("unable to serialize gossip protocol") as usize;
         PACKET_DATA_SIZE - (protocol_size - filter_size)
     }
-
-    #[test]
-    fn test_compress_incomplete_slots() {
-        let mut incomplete_slots: BTreeSet<Slot> = BTreeSet::new();
-
-        assert_eq!(
-            EpochIncompleteSlots::default(),
-            ClusterInfo::compress_incomplete_slots(&incomplete_slots)
-        );
-
-        incomplete_slots.insert(100);
-        let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
-        assert_eq!(100, compressed.first);
-        let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
-        assert_eq!(incomplete_slots, decompressed);
-
-        incomplete_slots.insert(104);
-        let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
-        assert_eq!(100, compressed.first);
-        let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
-        assert_eq!(incomplete_slots, decompressed);
-
-        incomplete_slots.insert(80);
-        let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
-        assert_eq!(80, compressed.first);
-        let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
-        assert_eq!(incomplete_slots, decompressed);
-
-        incomplete_slots.insert(10000);
-        let compressed = ClusterInfo::compress_incomplete_slots(&incomplete_slots);
-        assert_eq!(80, compressed.first);
-        let decompressed = ClusterInfo::decompress_incomplete_slots(&compressed);
-        assert_eq!(incomplete_slots, decompressed);
-    }
 }

core/src/commitment.rs

@@ -1,4 +1,5 @@
 use crate::consensus::VOTE_THRESHOLD_SIZE;
+use solana_ledger::blockstore::Blockstore;
 use solana_measure::measure::Measure;
 use solana_metrics::inc_new_counter_info;
 use solana_runtime::bank::Bank;
@@ -45,11 +46,12 @@ impl BlockCommitment {
     }
 }
 
-#[derive(Default)]
 pub struct BlockCommitmentCache {
     block_commitment: HashMap<Slot, BlockCommitment>,
+    largest_confirmed_root: Slot,
     total_stake: u64,
     bank: Arc<Bank>,
+    blockstore: Arc<Blockstore>,
     root: Slot,
 }
 
@@ -70,22 +72,41 @@ impl std::fmt::Debug for BlockCommitmentCache {
 impl BlockCommitmentCache {
     pub fn new(
         block_commitment: HashMap<Slot, BlockCommitment>,
+        largest_confirmed_root: Slot,
         total_stake: u64,
         bank: Arc<Bank>,
+        blockstore: Arc<Blockstore>,
         root: Slot,
     ) -> Self {
         Self {
             block_commitment,
+            largest_confirmed_root,
             total_stake,
             bank,
+            blockstore,
             root,
         }
     }
 
+    pub fn default_with_blockstore(blockstore: Arc<Blockstore>) -> Self {
+        Self {
+            block_commitment: HashMap::default(),
+            largest_confirmed_root: Slot::default(),
+            total_stake: u64::default(),
+            bank: Arc::new(Bank::default()),
+            blockstore,
+            root: Slot::default(),
+        }
+    }
+
     pub fn get_block_commitment(&self, slot: Slot) -> Option<&BlockCommitment> {
         self.block_commitment.get(&slot)
     }
 
+    pub fn largest_confirmed_root(&self) -> Slot {
+        self.largest_confirmed_root
+    }
+
     pub fn total_stake(&self) -> u64 {
         self.total_stake
     }
@@ -123,24 +144,27 @@ impl BlockCommitmentCache {
     }
 
     pub fn is_confirmed_rooted(&self, slot: Slot) -> bool {
-        self.get_block_commitment(slot)
-            .map(|block_commitment| {
-                (block_commitment.get_rooted_stake() as f64 / self.total_stake as f64)
-                    > VOTE_THRESHOLD_SIZE
-            })
-            .unwrap_or(false)
+        slot <= self.largest_confirmed_root()
+            && (self.blockstore.is_root(slot) || self.bank.status_cache_ancestors().contains(&slot))
     }
 
     #[cfg(test)]
-    pub fn new_for_tests() -> Self {
+    pub fn new_for_tests_with_blockstore(blockstore: Arc<Blockstore>) -> Self {
         let mut block_commitment: HashMap<Slot, BlockCommitment> = HashMap::new();
         block_commitment.insert(0, BlockCommitment::default());
         Self {
             block_commitment,
+            blockstore,
             total_stake: 42,
-            ..Self::default()
+            largest_confirmed_root: Slot::default(),
+            bank: Arc::new(Bank::default()),
+            root: Slot::default(),
         }
     }
+
+    pub(crate) fn set_get_largest_confirmed_root(&mut self, root: Slot) {
+        self.largest_confirmed_root = root;
+    }
 }
 
 pub struct CommitmentAggregationData {
@@ -159,6 +183,18 @@ impl CommitmentAggregationData {
     }
 }
 
+fn get_largest_confirmed_root(mut rooted_stake: Vec<(Slot, u64)>, total_stake: u64) -> Slot {
+    rooted_stake.sort_by(|a, b| a.0.cmp(&b.0).reverse());
+    let mut stake_sum = 0;
+    for (root, stake) in rooted_stake {
+        stake_sum += stake;
+        if (stake_sum as f64 / total_stake as f64) > VOTE_THRESHOLD_SIZE {
+            return root;
+        }
+    }
+    0
+}
+
 pub struct AggregateCommitmentService {
     t_commitment: JoinHandle<()>,
 }
@@ -216,12 +252,18 @@ impl AggregateCommitmentService {
             }
 
             let mut aggregate_commitment_time = Measure::start("aggregate-commitment-ms");
-            let block_commitment = Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
+            let (block_commitment, rooted_stake) =
+                Self::aggregate_commitment(&ancestors, &aggregation_data.bank);
+
+            let largest_confirmed_root =
+                get_largest_confirmed_root(rooted_stake, aggregation_data.total_staked);
 
             let mut new_block_commitment = BlockCommitmentCache::new(
                 block_commitment,
+                largest_confirmed_root,
                 aggregation_data.total_staked,
                 aggregation_data.bank,
+                block_commitment_cache.read().unwrap().blockstore.clone(),
                 aggregation_data.root,
             );
 
@@ -236,7 +278,10 @@ impl AggregateCommitmentService {
         }
     }
 
-    pub fn aggregate_commitment(ancestors: &[Slot], bank: &Bank) -> HashMap<Slot, BlockCommitment> {
+    pub fn aggregate_commitment(
+        ancestors: &[Slot],
+        bank: &Bank,
+    ) -> (HashMap<Slot, BlockCommitment>, Vec<(Slot, u64)>) {
         assert!(!ancestors.is_empty());
 
         // Check ancestors is sorted
@@ -245,6 +290,7 @@ impl AggregateCommitmentService {
         }
 
         let mut commitment = HashMap::new();
+        let mut rooted_stake: Vec<(Slot, u64)> = Vec::new();
         for (_, (lamports, account)) in bank.vote_accounts().into_iter() {
             if lamports == 0 {
                 continue;
@@ -257,17 +303,19 @@ impl AggregateCommitmentService {
             let vote_state = vote_state.unwrap();
             Self::aggregate_commitment_for_vote_account(
                 &mut commitment,
+                &mut rooted_stake,
                 &vote_state,
                 ancestors,
                 lamports,
             );
         }
 
-        commitment
+        (commitment, rooted_stake)
     }
 
     fn aggregate_commitment_for_vote_account(
         commitment: &mut HashMap<Slot, BlockCommitment>,
+        rooted_stake: &mut Vec<(Slot, u64)>,
         vote_state: &VoteState,
         ancestors: &[Slot],
         lamports: u64,
@@ -286,6 +334,7 @@ impl AggregateCommitmentService {
                     break;
                 }
             }
+            rooted_stake.push((root, lamports));
         }
 
         for vote in &vote_state.votes {
@@ -312,6 +361,7 @@ impl AggregateCommitmentService {
 mod tests {
     use super::*;
     use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
+    use solana_ledger::get_tmp_ledger_path;
     use solana_sdk::pubkey::Pubkey;
     use solana_stake_program::stake_state;
     use solana_vote_program::vote_state::{self, VoteStateVersions};
@@ -329,6 +379,8 @@ mod tests {
     #[test]
     fn test_get_confirmations() {
         let bank = Arc::new(Bank::default());
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         // Build BlockCommitmentCache with votes at depths 0 and 1 for 2 slots
         let mut cache0 = BlockCommitment::default();
         cache0.increase_confirmation_stake(1, 5);
@@ -346,7 +398,8 @@ mod tests {
         block_commitment.entry(0).or_insert(cache0.clone());
         block_commitment.entry(1).or_insert(cache1.clone());
         block_commitment.entry(2).or_insert(cache2.clone());
-        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 50, bank, 0);
+        let block_commitment_cache =
+            BlockCommitmentCache::new(block_commitment, 0, 50, bank, blockstore, 0);
 
         assert_eq!(block_commitment_cache.get_confirmation_count(0), Some(2));
         assert_eq!(block_commitment_cache.get_confirmation_count(1), Some(1));
@@ -354,17 +407,68 @@ mod tests {
         assert_eq!(block_commitment_cache.get_confirmation_count(3), None,);
     }
 
+    #[test]
+    fn test_is_confirmed_rooted() {
+        let bank = Arc::new(Bank::default());
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        blockstore.set_roots(&[0, 1]).unwrap();
+        // Build BlockCommitmentCache with rooted slots
+        let mut cache0 = BlockCommitment::default();
+        cache0.increase_rooted_stake(50);
+        let mut cache1 = BlockCommitment::default();
+        cache1.increase_rooted_stake(40);
+        let mut cache2 = BlockCommitment::default();
+        cache2.increase_rooted_stake(20);
+
+        let mut block_commitment = HashMap::new();
+        block_commitment.entry(1).or_insert(cache0.clone());
+        block_commitment.entry(2).or_insert(cache1.clone());
+        block_commitment.entry(3).or_insert(cache2.clone());
+        let largest_confirmed_root = 1;
+        let block_commitment_cache = BlockCommitmentCache::new(
+            block_commitment,
+            largest_confirmed_root,
+            50,
+            bank,
+            blockstore,
+            0,
+        );
+
+        assert!(block_commitment_cache.is_confirmed_rooted(0));
+        assert!(block_commitment_cache.is_confirmed_rooted(1));
+        assert!(!block_commitment_cache.is_confirmed_rooted(2));
+        assert!(!block_commitment_cache.is_confirmed_rooted(3));
+    }
+
+    #[test]
+    fn test_get_largest_confirmed_root() {
+        assert_eq!(get_largest_confirmed_root(vec![], 10), 0);
+        let mut rooted_stake = vec![];
+        rooted_stake.push((0, 5));
+        rooted_stake.push((1, 5));
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 10), 0);
+        let mut rooted_stake = vec![];
+        rooted_stake.push((1, 5));
+        rooted_stake.push((0, 10));
+        rooted_stake.push((2, 5));
+        rooted_stake.push((1, 4));
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 10), 1);
+    }
+
     #[test]
     fn test_aggregate_commitment_for_vote_account_1() {
         let ancestors = vec![3, 4, 5, 7, 9, 11];
         let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
         let lamports = 5;
         let mut vote_state = VoteState::default();
 
-        let root = ancestors.last().unwrap();
-        vote_state.root_slot = Some(*root);
+        let root = ancestors.last().unwrap().clone();
+        vote_state.root_slot = Some(root);
         AggregateCommitmentService::aggregate_commitment_for_vote_account(
             &mut commitment,
+            &mut rooted_stake,
             &vote_state,
             &ancestors,
             lamports,
@@ -375,12 +479,14 @@ mod tests {
             expected.increase_rooted_stake(lamports);
             assert_eq!(*commitment.get(&a).unwrap(), expected);
         }
+        assert_eq!(rooted_stake[0], (root, lamports));
     }
 
     #[test]
     fn test_aggregate_commitment_for_vote_account_2() {
         let ancestors = vec![3, 4, 5, 7, 9, 11];
         let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
         let lamports = 5;
         let mut vote_state = VoteState::default();
 
@@ -389,6 +495,7 @@ mod tests {
         vote_state.process_slot_vote_unchecked(*ancestors.last().unwrap());
         AggregateCommitmentService::aggregate_commitment_for_vote_account(
             &mut commitment,
+            &mut rooted_stake,
             &vote_state,
             &ancestors,
             lamports,
@@ -405,12 +512,14 @@ mod tests {
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             }
         }
+        assert_eq!(rooted_stake[0], (root, lamports));
     }
 
     #[test]
     fn test_aggregate_commitment_for_vote_account_3() {
         let ancestors = vec![3, 4, 5, 7, 9, 10, 11];
         let mut commitment = HashMap::new();
+        let mut rooted_stake = vec![];
         let lamports = 5;
         let mut vote_state = VoteState::default();
 
@@ -421,6 +530,7 @@ mod tests {
         vote_state.process_slot_vote_unchecked(ancestors[6]);
         AggregateCommitmentService::aggregate_commitment_for_vote_account(
             &mut commitment,
+            &mut rooted_stake,
             &vote_state,
             &ancestors,
             lamports,
@@ -441,6 +551,7 @@ mod tests {
                 assert_eq!(*commitment.get(&a).unwrap(), expected);
             }
         }
+        assert_eq!(rooted_stake[0], (root, lamports));
     }
 
     #[test]
@@ -450,6 +561,8 @@ mod tests {
             mut genesis_config, ..
         } = create_genesis_config(10_000);
 
+        let rooted_stake_amount = 40;
+
         let sk1 = Pubkey::new_rand();
         let pk1 = Pubkey::new_rand();
         let mut vote_account1 = vote_state::create_account(&pk1, &Pubkey::new_rand(), 0, 100);
@@ -460,12 +573,36 @@ mod tests {
         let mut vote_account2 = vote_state::create_account(&pk2, &Pubkey::new_rand(), 0, 50);
         let stake_account2 =
             stake_state::create_account(&sk2, &pk2, &vote_account2, &genesis_config.rent, 50);
+        let sk3 = Pubkey::new_rand();
+        let pk3 = Pubkey::new_rand();
+        let mut vote_account3 = vote_state::create_account(&pk3, &Pubkey::new_rand(), 0, 1);
+        let stake_account3 = stake_state::create_account(
+            &sk3,
+            &pk3,
+            &vote_account3,
+            &genesis_config.rent,
+            rooted_stake_amount,
+        );
+        let sk4 = Pubkey::new_rand();
+        let pk4 = Pubkey::new_rand();
+        let mut vote_account4 = vote_state::create_account(&pk4, &Pubkey::new_rand(), 0, 1);
+        let stake_account4 = stake_state::create_account(
+            &sk4,
+            &pk4,
+            &vote_account4,
+            &genesis_config.rent,
+            rooted_stake_amount,
+        );
 
         genesis_config.accounts.extend(vec![
             (pk1, vote_account1.clone()),
             (sk1, stake_account1),
             (pk2, vote_account2.clone()),
             (sk2, stake_account2),
+            (pk3, vote_account3.clone()),
+            (sk3, stake_account3),
+            (pk4, vote_account4.clone()),
+            (sk4, stake_account4),
         ]);
 
         // Create bank
@@ -485,7 +622,20 @@ mod tests {
         VoteState::to(&versioned, &mut vote_account2).unwrap();
         bank.store_account(&pk2, &vote_account2);
 
-        let commitment = AggregateCommitmentService::aggregate_commitment(&ancestors, &bank);
+        let mut vote_state3 = VoteState::from(&vote_account3).unwrap();
+        vote_state3.root_slot = Some(1);
+        let versioned = VoteStateVersions::Current(Box::new(vote_state3));
+        VoteState::to(&versioned, &mut vote_account3).unwrap();
+        bank.store_account(&pk3, &vote_account3);
+
+        let mut vote_state4 = VoteState::from(&vote_account4).unwrap();
+        vote_state4.root_slot = Some(2);
+        let versioned = VoteStateVersions::Current(Box::new(vote_state4));
+        VoteState::to(&versioned, &mut vote_account4).unwrap();
+        bank.store_account(&pk4, &vote_account4);
+
+        let (commitment, rooted_stake) =
+            AggregateCommitmentService::aggregate_commitment(&ancestors, &bank);
 
         for a in ancestors {
             if a <= 3 {
@@ -509,5 +659,7 @@ mod tests {
                 assert!(commitment.get(&a).is_none());
             }
         }
+        assert_eq!(rooted_stake.len(), 2);
+        assert_eq!(get_largest_confirmed_root(rooted_stake, 100), 1)
     }
 }

core/src/consensus.rs

@@ -621,7 +621,7 @@ pub mod test {
             }
             let vote = tower.new_vote_from_bank(&bank, &my_vote_pubkey).0;
             if let Some(new_root) = tower.record_bank_vote(vote) {
-                ReplayStage::handle_new_root(new_root, bank_forks, progress, &None);
+                ReplayStage::handle_new_root(new_root, bank_forks, progress, &None, None);
             }
 
             // Mark the vote for this bank under this node's pubkey so it will be

core/src/contact_info.rs

@@ -1,6 +1,8 @@
+use crate::crds_value::MAX_WALLCLOCK;
 use solana_sdk::pubkey::Pubkey;
 #[cfg(test)]
 use solana_sdk::rpc_port;
+use solana_sdk::sanitize::{Sanitize, SanitizeError};
 #[cfg(test)]
 use solana_sdk::signature::{Keypair, Signer};
 use solana_sdk::timing::timestamp;
@@ -37,6 +39,15 @@ pub struct ContactInfo {
     pub shred_version: u16,
 }
 
+impl Sanitize for ContactInfo {
+    fn sanitize(&self) -> std::result::Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        Ok(())
+    }
+}
+
 impl Ord for ContactInfo {
     fn cmp(&self, other: &Self) -> Ordering {
         self.id.cmp(&other.id)
@@ -314,4 +325,12 @@ mod tests {
         ci.rpc = socketaddr!("127.0.0.1:234");
         assert!(ci.valid_client_facing_addr().is_some());
     }
+
+    #[test]
+    fn test_sanitize() {
+        let mut ci = ContactInfo::default();
+        assert_eq!(ci.sanitize(), Ok(()));
+        ci.wallclock = MAX_WALLCLOCK;
+        assert_eq!(ci.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+    }
 }

core/src/crds_gossip.rs

@@ -20,6 +20,7 @@ pub const CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS: usize = 500;
 pub struct CrdsGossip {
     pub crds: Crds,
     pub id: Pubkey,
+    pub shred_version: u16,
     pub push: CrdsGossipPush,
     pub pull: CrdsGossipPull,
 }
@@ -29,6 +30,7 @@ impl Default for CrdsGossip {
         CrdsGossip {
             crds: Crds::default(),
             id: Pubkey::default(),
+            shred_version: 0,
             push: CrdsGossipPush::default(),
             pull: CrdsGossipPull::default(),
         }
@@ -39,6 +41,9 @@ impl CrdsGossip {
     pub fn set_self(&mut self, id: &Pubkey) {
         self.id = *id;
     }
+    pub fn set_shred_version(&mut self, shred_version: u16) {
+        self.shred_version = shred_version;
+    }
 
     /// process a push message to the network
     pub fn process_push_message(
@@ -122,6 +127,7 @@ impl CrdsGossip {
             &self.crds,
             stakes,
             &self.id,
+            self.shred_version,
             self.pull.pull_request_time.len(),
             CRDS_GOSSIP_NUM_ACTIVE,
         )
@@ -134,8 +140,14 @@ impl CrdsGossip {
         stakes: &HashMap<Pubkey, u64>,
         bloom_size: usize,
     ) -> Result<(Pubkey, Vec<CrdsFilter>, CrdsValue), CrdsGossipError> {
-        self.pull
-            .new_pull_request(&self.crds, &self.id, now, stakes, bloom_size)
+        self.pull.new_pull_request(
+            &self.crds,
+            &self.id,
+            self.shred_version,
+            now,
+            stakes,
+            bloom_size,
+        )
     }
 
     /// time when a request to `from` was initiated

core/src/crds_gossip_pull.rs

@@ -14,7 +14,6 @@ use crate::crds::Crds;
 use crate::crds_gossip::{get_stake, get_weight, CRDS_GOSSIP_DEFAULT_BLOOM_ITEMS};
 use crate::crds_gossip_error::CrdsGossipError;
 use crate::crds_value::{CrdsValue, CrdsValueLabel};
-use rand;
 use rand::distributions::{Distribution, WeightedIndex};
 use rand::Rng;
 use solana_runtime::bloom::Bloom;
@@ -37,6 +36,13 @@ pub struct CrdsFilter {
     mask_bits: u32,
 }
 
+impl solana_sdk::sanitize::Sanitize for CrdsFilter {
+    fn sanitize(&self) -> std::result::Result<(), solana_sdk::sanitize::SanitizeError> {
+        self.filter.sanitize()?;
+        Ok(())
+    }
+}
+
 impl CrdsFilter {
     pub fn new_rand(num_items: usize, max_bytes: usize) -> Self {
         let max_bits = (max_bytes * 8) as f64;
@@ -138,11 +144,12 @@ impl CrdsGossipPull {
         &self,
         crds: &Crds,
         self_id: &Pubkey,
+        self_shred_version: u16,
         now: u64,
         stakes: &HashMap<Pubkey, u64>,
         bloom_size: usize,
     ) -> Result<(Pubkey, Vec<CrdsFilter>, CrdsValue), CrdsGossipError> {
-        let options = self.pull_options(crds, &self_id, now, stakes);
+        let options = self.pull_options(crds, &self_id, self_shred_version, now, stakes);
         if options.is_empty() {
             return Err(CrdsGossipError::NoPeers);
         }
@@ -159,13 +166,20 @@ impl CrdsGossipPull {
         &self,
         crds: &'a Crds,
         self_id: &Pubkey,
+        self_shred_version: u16,
         now: u64,
         stakes: &HashMap<Pubkey, u64>,
     ) -> Vec<(f32, &'a ContactInfo)> {
         crds.table
             .values()
             .filter_map(|v| v.value.contact_info())
-            .filter(|v| v.id != *self_id && ContactInfo::is_valid_address(&v.gossip))
+            .filter(|v| {
+                v.id != *self_id
+                    && ContactInfo::is_valid_address(&v.gossip)
+                    && (self_shred_version == 0
+                        || v.shred_version == 0
+                        || self_shred_version == v.shred_version)
+            })
             .map(|item| {
                 let max_weight = f32::from(u16::max_value()) - 1.0;
                 let req_time: u64 = *self.pull_request_time.get(&item.id).unwrap_or(&0);
@@ -396,7 +410,7 @@ mod test {
             stakes.insert(id, i * 100);
         }
         let now = 1024;
-        let mut options = node.pull_options(&crds, &me.label().pubkey(), now, &stakes);
+        let mut options = node.pull_options(&crds, &me.label().pubkey(), 0, now, &stakes);
         assert!(!options.is_empty());
         options.sort_by(|(weight_l, _), (weight_r, _)| weight_r.partial_cmp(weight_l).unwrap());
         // check that the highest stake holder is also the heaviest weighted.
@@ -406,6 +420,66 @@ mod test {
         );
     }
 
+    #[test]
+    fn test_no_pulls_from_different_shred_versions() {
+        let mut crds = Crds::default();
+        let stakes = HashMap::new();
+        let node = CrdsGossipPull::default();
+
+        let gossip = socketaddr!("127.0.0.1:1234");
+
+        let me = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let spy = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 0,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_123 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_456 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 456,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+
+        crds.insert(me.clone(), 0).unwrap();
+        crds.insert(spy.clone(), 0).unwrap();
+        crds.insert(node_123.clone(), 0).unwrap();
+        crds.insert(node_456.clone(), 0).unwrap();
+
+        // shred version 123 should ignore 456 nodes
+        let options = node
+            .pull_options(&crds, &me.label().pubkey(), 123, 0, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 2);
+        assert!(options.contains(&spy.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+
+        // spy nodes will see all
+        let options = node
+            .pull_options(&crds, &spy.label().pubkey(), 0, 0, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 3);
+        assert!(options.contains(&me.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+        assert!(options.contains(&node_456.pubkey()));
+    }
+
     #[test]
     fn test_new_pull_request() {
         let mut crds = Crds::default();
@@ -416,13 +490,13 @@ mod test {
         let id = entry.label().pubkey();
         let node = CrdsGossipPull::default();
         assert_eq!(
-            node.new_pull_request(&crds, &id, 0, &HashMap::new(), PACKET_DATA_SIZE),
+            node.new_pull_request(&crds, &id, 0, 0, &HashMap::new(), PACKET_DATA_SIZE),
             Err(CrdsGossipError::NoPeers)
         );
 
         crds.insert(entry.clone(), 0).unwrap();
         assert_eq!(
-            node.new_pull_request(&crds, &id, 0, &HashMap::new(), PACKET_DATA_SIZE),
+            node.new_pull_request(&crds, &id, 0, 0, &HashMap::new(), PACKET_DATA_SIZE),
             Err(CrdsGossipError::NoPeers)
         );
 
@@ -431,7 +505,7 @@ mod test {
             0,
         )));
         crds.insert(new.clone(), 0).unwrap();
-        let req = node.new_pull_request(&crds, &id, 0, &HashMap::new(), PACKET_DATA_SIZE);
+        let req = node.new_pull_request(&crds, &id, 0, 0, &HashMap::new(), PACKET_DATA_SIZE);
         let (to, _, self_info) = req.unwrap();
         assert_eq!(to, new.label().pubkey());
         assert_eq!(self_info, entry);
@@ -466,6 +540,7 @@ mod test {
             let req = node.new_pull_request(
                 &crds,
                 &node_pubkey,
+                0,
                 u64::max_value(),
                 &HashMap::new(),
                 PACKET_DATA_SIZE,
@@ -495,6 +570,7 @@ mod test {
             &node_crds,
             &node_pubkey,
             0,
+            0,
             &HashMap::new(),
             PACKET_DATA_SIZE,
         );
@@ -567,6 +643,7 @@ mod test {
                 &node_crds,
                 &node_pubkey,
                 0,
+                0,
                 &HashMap::new(),
                 PACKET_DATA_SIZE,
             );

core/src/crds_gossip_push.rs

@@ -236,13 +236,14 @@ impl CrdsGossipPush {
         crds: &Crds,
         stakes: &HashMap<Pubkey, u64>,
         self_id: &Pubkey,
+        self_shred_version: u16,
         network_size: usize,
         ratio: usize,
     ) {
         let need = Self::compute_need(self.num_active, self.active_set.len(), ratio);
         let mut new_items = HashMap::new();
 
-        let options: Vec<_> = self.push_options(crds, &self_id, stakes);
+        let options: Vec<_> = self.push_options(crds, &self_id, self_shred_version, stakes);
         if options.is_empty() {
             return;
         }
@@ -288,13 +289,20 @@ impl CrdsGossipPush {
         &self,
         crds: &'a Crds,
         self_id: &Pubkey,
+        self_shred_version: u16,
         stakes: &HashMap<Pubkey, u64>,
     ) -> Vec<(f32, &'a ContactInfo)> {
         crds.table
             .values()
             .filter(|v| v.value.contact_info().is_some())
             .map(|v| (v.value.contact_info().unwrap(), v))
-            .filter(|(info, _)| info.id != *self_id && ContactInfo::is_valid_address(&info.gossip))
+            .filter(|(info, _)| {
+                info.id != *self_id
+                    && ContactInfo::is_valid_address(&info.gossip)
+                    && (self_shred_version == 0
+                        || info.shred_version == 0
+                        || self_shred_version == info.shred_version)
+            })
             .map(|(info, value)| {
                 let max_weight = f32::from(u16::max_value()) - 1.0;
                 let last_updated: u64 = value.local_timestamp;
@@ -510,7 +518,7 @@ mod test {
         )));
 
         assert_eq!(crds.insert(value1.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         assert!(push.active_set.get(&value1.label().pubkey()).is_some());
         let value2 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
@@ -520,7 +528,7 @@ mod test {
         assert!(push.active_set.get(&value2.label().pubkey()).is_none());
         assert_eq!(crds.insert(value2.clone(), 0), Ok(None));
         for _ in 0..30 {
-            push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+            push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
             if push.active_set.get(&value2.label().pubkey()).is_some() {
                 break;
             }
@@ -533,7 +541,7 @@ mod test {
             ));
             assert_eq!(crds.insert(value2.clone(), 0), Ok(None));
         }
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
         assert_eq!(push.active_set.len(), push.num_active);
     }
     #[test]
@@ -551,7 +559,7 @@ mod test {
             crds.insert(peer.clone(), time).unwrap();
             stakes.insert(id, i * 100);
         }
-        let mut options = push.push_options(&crds, &Pubkey::default(), &stakes);
+        let mut options = push.push_options(&crds, &Pubkey::default(), 0, &stakes);
         assert!(!options.is_empty());
         options.sort_by(|(weight_l, _), (weight_r, _)| weight_r.partial_cmp(weight_l).unwrap());
         // check that the highest stake holder is also the heaviest weighted.
@@ -560,6 +568,66 @@ mod test {
             10_000_u64
         );
     }
+
+    #[test]
+    fn test_no_pushes_to_from_different_shred_versions() {
+        let mut crds = Crds::default();
+        let stakes = HashMap::new();
+        let node = CrdsGossipPush::default();
+
+        let gossip = socketaddr!("127.0.0.1:1234");
+
+        let me = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let spy = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 0,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_123 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 123,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+        let node_456 = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo {
+            id: Pubkey::new_rand(),
+            shred_version: 456,
+            gossip: gossip.clone(),
+            ..ContactInfo::default()
+        }));
+
+        crds.insert(me.clone(), 0).unwrap();
+        crds.insert(spy.clone(), 0).unwrap();
+        crds.insert(node_123.clone(), 0).unwrap();
+        crds.insert(node_456.clone(), 0).unwrap();
+
+        // shred version 123 should ignore 456 nodes
+        let options = node
+            .push_options(&crds, &me.label().pubkey(), 123, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 2);
+        assert!(options.contains(&spy.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+
+        // spy nodes will see all
+        let options = node
+            .push_options(&crds, &spy.label().pubkey(), 0, &stakes)
+            .iter()
+            .map(|(_, c)| c.id)
+            .collect::<Vec<_>>();
+        assert_eq!(options.len(), 3);
+        assert!(options.contains(&me.pubkey()));
+        assert!(options.contains(&node_123.pubkey()));
+        assert!(options.contains(&node_456.pubkey()));
+    }
     #[test]
     fn test_new_push_messages() {
         let mut crds = Crds::default();
@@ -569,7 +637,7 @@ mod test {
             0,
         )));
         assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         let new_msg = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
             &Pubkey::new_rand(),
@@ -606,7 +674,7 @@ mod test {
             push.process_push_message(&mut crds, &Pubkey::default(), peer_3.clone(), 0),
             Ok(None)
         );
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         // push 3's contact info to 1 and 2 and 3
         let new_msg = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
@@ -628,7 +696,7 @@ mod test {
             0,
         )));
         assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         let new_msg = CrdsValue::new_unsigned(CrdsData::ContactInfo(ContactInfo::new_localhost(
             &Pubkey::new_rand(),
@@ -651,7 +719,7 @@ mod test {
             0,
         )));
         assert_eq!(crds.insert(peer.clone(), 0), Ok(None));
-        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 1, 1);
+        push.refresh_push_active_set(&crds, &HashMap::new(), &Pubkey::default(), 0, 1, 1);
 
         let mut ci = ContactInfo::new_localhost(&Pubkey::new_rand(), 0);
         ci.wallclock = 1;

core/src/crds_value.rs

@@ -1,5 +1,6 @@
 use crate::contact_info::ContactInfo;
 use bincode::{serialize, serialized_size};
+use solana_sdk::sanitize::{Sanitize, SanitizeError};
 use solana_sdk::timing::timestamp;
 use solana_sdk::{
     clock::Slot,
@@ -14,10 +15,14 @@ use std::{
     fmt,
 };
 
+pub const MAX_WALLCLOCK: u64 = 1_000_000_000_000_000;
+pub const MAX_SLOT: u64 = 1_000_000_000_000_000;
+
 pub type VoteIndex = u8;
 pub const MAX_VOTES: VoteIndex = 32;
 
 pub type EpochSlotIndex = u8;
+pub const MAX_EPOCH_SLOTS: EpochSlotIndex = 1;
 
 /// CrdsValue that is replicated across the cluster
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -26,6 +31,13 @@ pub struct CrdsValue {
     pub data: CrdsData,
 }
 
+impl Sanitize for CrdsValue {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        self.signature.sanitize()?;
+        self.data.sanitize()
+    }
+}
+
 impl Signable for CrdsValue {
     fn pubkey(&self) -> Pubkey {
         self.pubkey()
@@ -44,14 +56,8 @@ impl Signable for CrdsValue {
     }
 
     fn verify(&self) -> bool {
-        let sig_check = self
-            .get_signature()
-            .verify(&self.pubkey().as_ref(), self.signable_data().borrow());
-        let data_check = match &self.data {
-            CrdsData::Vote(ix, _) => *ix < MAX_VOTES,
-            _ => true,
-        };
-        sig_check && data_check
+        self.get_signature()
+            .verify(&self.pubkey().as_ref(), self.signable_data().borrow())
     }
 }
 
@@ -65,6 +71,7 @@ pub enum CrdsData {
     EpochSlots(EpochSlotIndex, EpochSlots),
     SnapshotHashes(SnapshotHash),
     AccountsHashes(SnapshotHash),
+    Version(Version),
 }
 
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
@@ -87,6 +94,40 @@ pub struct EpochIncompleteSlots {
     pub compressed_list: Vec<u8>,
 }
 
+impl Sanitize for EpochIncompleteSlots {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.first >= MAX_SLOT {
+            return Err(SanitizeError::InvalidValue);
+        }
+        //rest of the data doesn't matter since we no longer decompress
+        //these values
+        Ok(())
+    }
+}
+
+impl Sanitize for CrdsData {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        match self {
+            CrdsData::ContactInfo(val) => val.sanitize(),
+            CrdsData::Vote(ix, val) => {
+                if *ix >= MAX_VOTES {
+                    return Err(SanitizeError::ValueOutOfBounds);
+                }
+                val.sanitize()
+            }
+            CrdsData::SnapshotHashes(val) => val.sanitize(),
+            CrdsData::AccountsHashes(val) => val.sanitize(),
+            CrdsData::EpochSlots(ix, val) => {
+                if *ix as usize >= MAX_EPOCH_SLOTS as usize {
+                    return Err(SanitizeError::ValueOutOfBounds);
+                }
+                val.sanitize()
+            }
+            CrdsData::Version(version) => version.sanitize(),
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub struct SnapshotHash {
     pub from: Pubkey,
@@ -94,6 +135,20 @@ pub struct SnapshotHash {
     pub wallclock: u64,
 }
 
+impl Sanitize for SnapshotHash {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        for (slot, _) in &self.hashes {
+            if *slot >= MAX_SLOT {
+                return Err(SanitizeError::ValueOutOfBounds);
+            }
+        }
+        self.from.sanitize()
+    }
+}
+
 impl SnapshotHash {
     pub fn new(from: Pubkey, hashes: Vec<(Slot, Hash)>) -> Self {
         Self {
@@ -107,33 +162,47 @@ impl SnapshotHash {
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub struct EpochSlots {
     pub from: Pubkey,
-    pub root: Slot,
+    root: Slot,
     pub lowest: Slot,
-    pub slots: BTreeSet<Slot>,
-    pub stash: Vec<EpochIncompleteSlots>,
+    slots: BTreeSet<Slot>,
+    stash: Vec<EpochIncompleteSlots>,
     pub wallclock: u64,
 }
 
 impl EpochSlots {
-    pub fn new(
-        from: Pubkey,
-        root: Slot,
-        lowest: Slot,
-        slots: BTreeSet<Slot>,
-        stash: Vec<EpochIncompleteSlots>,
-        wallclock: u64,
-    ) -> Self {
+    pub fn new(from: Pubkey, lowest: Slot, wallclock: u64) -> Self {
         Self {
             from,
-            root,
+            root: 0,
             lowest,
-            slots,
-            stash,
+            slots: BTreeSet::new(),
+            stash: vec![],
             wallclock,
         }
     }
 }
 
+impl Sanitize for EpochSlots {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        if self.lowest >= MAX_SLOT {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        if self.root >= MAX_SLOT {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        for slot in &self.slots {
+            if *slot >= MAX_SLOT {
+                return Err(SanitizeError::ValueOutOfBounds);
+            }
+        }
+        self.stash.sanitize()?;
+        self.from.sanitize()
+    }
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
 pub struct Vote {
     pub from: Pubkey,
@@ -141,6 +210,16 @@ pub struct Vote {
     pub wallclock: u64,
 }
 
+impl Sanitize for Vote {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        self.from.sanitize()?;
+        self.transaction.sanitize()
+    }
+}
+
 impl Vote {
     pub fn new(from: &Pubkey, transaction: Transaction, wallclock: u64) -> Self {
         Self {
@@ -151,6 +230,33 @@ impl Vote {
     }
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+pub struct Version {
+    pub from: Pubkey,
+    pub wallclock: u64,
+    pub version: solana_version::Version,
+}
+
+impl Sanitize for Version {
+    fn sanitize(&self) -> Result<(), SanitizeError> {
+        if self.wallclock >= MAX_WALLCLOCK {
+            return Err(SanitizeError::ValueOutOfBounds);
+        }
+        self.from.sanitize()?;
+        self.version.sanitize()
+    }
+}
+
+impl Version {
+    pub fn new(from: Pubkey) -> Self {
+        Self {
+            from,
+            wallclock: timestamp(),
+            version: solana_version::Version::default(),
+        }
+    }
+}
+
 /// Type of the replicated value
 /// These are labels for values in a record that is associated with `Pubkey`
 #[derive(PartialEq, Hash, Eq, Clone, Debug)]
@@ -160,6 +266,7 @@ pub enum CrdsValueLabel {
     EpochSlots(Pubkey),
     SnapshotHashes(Pubkey),
     AccountsHashes(Pubkey),
+    Version(Pubkey),
 }
 
 impl fmt::Display for CrdsValueLabel {
@@ -170,6 +277,7 @@ impl fmt::Display for CrdsValueLabel {
             CrdsValueLabel::EpochSlots(_) => write!(f, "EpochSlots({})", self.pubkey()),
             CrdsValueLabel::SnapshotHashes(_) => write!(f, "SnapshotHashes({})", self.pubkey()),
             CrdsValueLabel::AccountsHashes(_) => write!(f, "AccountsHashes({})", self.pubkey()),
+            CrdsValueLabel::Version(_) => write!(f, "Version({})", self.pubkey()),
         }
     }
 }
@@ -182,6 +290,7 @@ impl CrdsValueLabel {
             CrdsValueLabel::EpochSlots(p) => *p,
             CrdsValueLabel::SnapshotHashes(p) => *p,
             CrdsValueLabel::AccountsHashes(p) => *p,
+            CrdsValueLabel::Version(p) => *p,
         }
     }
 }
@@ -199,7 +308,7 @@ impl CrdsValue {
         value.sign(keypair);
         value
     }
-    /// Totally unsecure unverfiable wallclock of the node that generated this message
+    /// Totally unsecure unverifiable wallclock of the node that generated this message
     /// Latest wallclock is always picked.
     /// This is used to time out push messages.
     pub fn wallclock(&self) -> u64 {
@@ -209,6 +318,7 @@ impl CrdsValue {
             CrdsData::EpochSlots(_, vote) => vote.wallclock,
             CrdsData::SnapshotHashes(hash) => hash.wallclock,
             CrdsData::AccountsHashes(hash) => hash.wallclock,
+            CrdsData::Version(version) => version.wallclock,
         }
     }
     pub fn pubkey(&self) -> Pubkey {
@@ -218,6 +328,7 @@ impl CrdsValue {
             CrdsData::EpochSlots(_, slots) => slots.from,
             CrdsData::SnapshotHashes(hash) => hash.from,
             CrdsData::AccountsHashes(hash) => hash.from,
+            CrdsData::Version(version) => version.from,
         }
     }
     pub fn label(&self) -> CrdsValueLabel {
@@ -227,6 +338,7 @@ impl CrdsValue {
             CrdsData::EpochSlots(_, _) => CrdsValueLabel::EpochSlots(self.pubkey()),
             CrdsData::SnapshotHashes(_) => CrdsValueLabel::SnapshotHashes(self.pubkey()),
             CrdsData::AccountsHashes(_) => CrdsValueLabel::AccountsHashes(self.pubkey()),
+            CrdsData::Version(_) => CrdsValueLabel::Version(self.pubkey()),
         }
     }
     pub fn contact_info(&self) -> Option<&ContactInfo> {
@@ -270,6 +382,13 @@ impl CrdsValue {
         }
     }
 
+    pub fn version(&self) -> Option<&Version> {
+        match &self.data {
+            CrdsData::Version(version) => Some(version),
+            _ => None,
+        }
+    }
+
     /// Return all the possible labels for a record identified by Pubkey.
     pub fn record_labels(key: &Pubkey) -> Vec<CrdsValueLabel> {
         let mut labels = vec![
@@ -277,6 +396,7 @@ impl CrdsValue {
             CrdsValueLabel::EpochSlots(*key),
             CrdsValueLabel::SnapshotHashes(*key),
             CrdsValueLabel::AccountsHashes(*key),
+            CrdsValueLabel::Version(*key),
         ];
         labels.extend((0..MAX_VOTES).map(|ix| CrdsValueLabel::Vote(ix, *key)));
         labels
@@ -326,7 +446,7 @@ mod test {
 
     #[test]
     fn test_labels() {
-        let mut hits = [false; 4 + MAX_VOTES as usize];
+        let mut hits = [false; 5 + MAX_VOTES as usize];
         // this method should cover all the possible labels
         for v in &CrdsValue::record_labels(&Pubkey::default()) {
             match v {
@@ -334,7 +454,8 @@ mod test {
                 CrdsValueLabel::EpochSlots(_) => hits[1] = true,
                 CrdsValueLabel::SnapshotHashes(_) => hits[2] = true,
                 CrdsValueLabel::AccountsHashes(_) => hits[3] = true,
-                CrdsValueLabel::Vote(ix, _) => hits[*ix as usize + 4] = true,
+                CrdsValueLabel::Version(_) => hits[4] = true,
+                CrdsValueLabel::Vote(ix, _) => hits[*ix as usize + 5] = true,
             }
         }
         assert!(hits.iter().all(|x| *x));
@@ -356,7 +477,7 @@ mod test {
 
         let v = CrdsValue::new_unsigned(CrdsData::EpochSlots(
             0,
-            EpochSlots::new(Pubkey::default(), 0, 0, BTreeSet::new(), vec![], 0),
+            EpochSlots::new(Pubkey::default(), 0, 0),
         ));
         assert_eq!(v.wallclock(), 0);
         let key = v.clone().epoch_slots().unwrap().from;
@@ -377,10 +498,9 @@ mod test {
             Vote::new(&keypair.pubkey(), test_tx(), timestamp()),
         ));
         verify_signatures(&mut v, &keypair, &wrong_keypair);
-        let btreeset: BTreeSet<Slot> = vec![1, 2, 3, 6, 8].into_iter().collect();
         v = CrdsValue::new_unsigned(CrdsData::EpochSlots(
             0,
-            EpochSlots::new(keypair.pubkey(), 0, 0, btreeset, vec![], timestamp()),
+            EpochSlots::new(keypair.pubkey(), 0, timestamp()),
         ));
         verify_signatures(&mut v, &keypair, &wrong_keypair);
     }
@@ -395,9 +515,21 @@ mod test {
             ),
             &keypair,
         );
-        assert!(!vote.verify());
+        assert!(vote.sanitize().is_err());
     }
 
+    #[test]
+    fn test_max_epoch_slots_index() {
+        let keypair = Keypair::new();
+        let item = CrdsValue::new_signed(
+            CrdsData::Vote(
+                MAX_VOTES,
+                Vote::new(&keypair.pubkey(), test_tx(), timestamp()),
+            ),
+            &keypair,
+        );
+        assert_eq!(item.sanitize(), Err(SanitizeError::ValueOutOfBounds));
+    }
     #[test]
     fn test_compute_vote_index_empty() {
         for i in 0..MAX_VOTES {

core/src/gossip_service.rs

@@ -75,6 +75,7 @@ pub fn discover_cluster(
         None,
         None,
         None,
+        0,
     )
 }
 
@@ -85,9 +86,11 @@ pub fn discover(
     find_node_by_pubkey: Option<Pubkey>,
     find_node_by_gossip_addr: Option<&SocketAddr>,
     my_gossip_addr: Option<&SocketAddr>,
+    my_shred_version: u16,
 ) -> std::io::Result<(Vec<ContactInfo>, Vec<ContactInfo>)> {
     let exit = Arc::new(AtomicBool::new(false));
-    let (gossip_service, ip_echo, spy_ref) = make_gossip_node(entrypoint, &exit, my_gossip_addr);
+    let (gossip_service, ip_echo, spy_ref) =
+        make_gossip_node(entrypoint, &exit, my_gossip_addr, my_shred_version);
 
     let id = spy_ref.read().unwrap().keypair.pubkey();
     info!("Entrypoint: {:?}", entrypoint);
@@ -256,12 +259,13 @@ fn make_gossip_node(
     entrypoint: Option<&SocketAddr>,
     exit: &Arc<AtomicBool>,
     gossip_addr: Option<&SocketAddr>,
+    shred_version: u16,
 ) -> (GossipService, Option<TcpListener>, Arc<RwLock<ClusterInfo>>) {
     let keypair = Arc::new(Keypair::new());
     let (node, gossip_socket, ip_echo) = if let Some(gossip_addr) = gossip_addr {
-        ClusterInfo::gossip_node(&keypair.pubkey(), gossip_addr)
+        ClusterInfo::gossip_node(&keypair.pubkey(), gossip_addr, shred_version)
     } else {
-        ClusterInfo::spy_node(&keypair.pubkey())
+        ClusterInfo::spy_node(&keypair.pubkey(), shred_version)
     };
     let mut cluster_info = ClusterInfo::new(node, keypair);
     if let Some(entrypoint) = entrypoint {

core/src/lib.rs

@@ -29,16 +29,19 @@ pub mod genesis_utils;
 pub mod gossip_service;
 pub mod ledger_cleanup_service;
 pub mod local_vote_signer_service;
+pub mod non_circulating_supply;
 pub mod packet;
 pub mod poh_recorder;
 pub mod poh_service;
 pub mod recvmmsg;
+pub mod repair_response;
 pub mod repair_service;
 pub mod replay_stage;
 mod result;
 pub mod retransmit_stage;
 pub mod rewards_recorder_service;
 pub mod rpc;
+pub mod rpc_error;
 pub mod rpc_pubsub;
 pub mod rpc_pubsub_service;
 pub mod rpc_service;

core/src/non_circulating_supply.rs

@@ -0,0 +1,192 @@
+use solana_runtime::bank::Bank;
+use solana_sdk::pubkey::Pubkey;
+use solana_stake_program::stake_state::StakeState;
+use std::{collections::HashSet, sync::Arc};
+
+pub struct NonCirculatingSupply {
+    pub lamports: u64,
+    pub accounts: Vec<Pubkey>,
+}
+
+pub fn calculate_non_circulating_supply(bank: &Arc<Bank>) -> NonCirculatingSupply {
+    debug!("Updating Bank supply, epoch: {}", bank.epoch());
+    let mut non_circulating_accounts_set: HashSet<Pubkey> = HashSet::new();
+
+    for key in non_circulating_accounts() {
+        non_circulating_accounts_set.insert(key);
+    }
+    let withdraw_authority_list = withdraw_authority();
+
+    let clock = bank.clock();
+    let stake_accounts = bank.get_program_accounts(Some(&solana_stake_program::id()));
+    for (pubkey, account) in stake_accounts.iter() {
+        let stake_account = StakeState::from(&account).unwrap_or_default();
+        match stake_account {
+            StakeState::Initialized(meta) => {
+                if meta.lockup.is_in_force(&clock, &HashSet::default())
+                    || withdraw_authority_list.contains(&meta.authorized.withdrawer)
+                {
+                    non_circulating_accounts_set.insert(*pubkey);
+                }
+            }
+            StakeState::Stake(meta, _stake) => {
+                if meta.lockup.is_in_force(&clock, &HashSet::default())
+                    || withdraw_authority_list.contains(&meta.authorized.withdrawer)
+                {
+                    non_circulating_accounts_set.insert(*pubkey);
+                }
+            }
+            _ => {}
+        }
+    }
+
+    let lamports = non_circulating_accounts_set
+        .iter()
+        .fold(0, |acc, pubkey| acc + bank.get_balance(&pubkey));
+
+    NonCirculatingSupply {
+        lamports,
+        accounts: non_circulating_accounts_set.into_iter().collect(),
+    }
+}
+
+// Mainnet-beta accounts that should be considered non-circulating
+solana_sdk::pubkeys!(
+    non_circulating_accounts,
+    [
+        "9huDUZfxoJ7wGMTffUE7vh1xePqef7gyrLJu9NApncqA",
+        "GK2zqSsXLA2rwVZk347RYhh6jJpRsCA69FjLW93ZGi3B",
+        "HCV5dGFJXRrJ3jhDYA4DCeb9TEDTwGGYXtT3wHksu2Zr",
+        "25odAafVXnd63L6Hq5Cx6xGmhKqkhE2y6UrLVuqUfWZj",
+        "14FUT96s9swbmH7ZjpDvfEDywnAYy9zaNhv4xvezySGu",
+        "HbZ5FfmKWNHC7uwk6TF1hVi6TCs7dtYfdjEcuPGgzFAg",
+        "C7C8odR8oashR5Feyrq2tJKaXL18id1dSj2zbkDGL2C2",
+        "Eyr9P5XsjK2NUKNCnfu39eqpGoiLFgVAv1LSQgMZCwiQ",
+        "DE1bawNcRJB9rVm3buyMVfr8mBEoyyu73NBovf2oXJsJ",
+        "CakcnaRDHka2gXyfbEd2d3xsvkJkqsLw2akB3zsN1D2S",
+        "7Np41oeYqPefeNQEHSv1UDhYrehxin3NStELsSKCT4K2",
+        "GdnSyH3YtwcxFvQrVVJMm1JhTS4QVX7MFsX56uJLUfiZ",
+        "Mc5XB47H3DKJHym5RLa9mPzWv5snERsF3KNv5AauXK8",
+        "7cvkjYAkUYs4W8XcXsca7cBrEGFeSUjeZmKoNBvEwyri",
+        "AG3m2bAibcY8raMt4oXEGqRHwX4FWKPPJVjZxn1LySDX",
+        "5XdtyEDREHJXXW1CTtCsVjJRjBapAwK78ZquzvnNVRrV",
+        "6yKHERk8rsbmJxvMpPuwPs1ct3hRiP7xaJF2tvnGU6nK",
+        "CHmdL15akDcJgBkY6BP3hzs98Dqr6wbdDC5p8odvtSbq",
+        "FR84wZQy3Y3j2gWz6pgETUiUoJtreMEuWfbg6573UCj9",
+        "5q54XjQ7vDx4y6KphPeE97LUNiYGtP55spjvXAWPGBuf",
+    ]
+);
+
+// Withdraw authority for autostaked accounts on mainnet-beta
+solana_sdk::pubkeys!(
+    withdraw_authority,
+    [
+        "8CUUMKYNGxdgYio5CLHRHyzMEhhVRMcqefgE6dLqnVRK",
+        "3FFaheyqtyAXZSYxDzsr5CVKvJuvZD1WE1VEsBtDbRqB",
+    ]
+);
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_sdk::{
+        account::Account, epoch_schedule::EpochSchedule, genesis_config::GenesisConfig,
+    };
+    use solana_stake_program::stake_state::{Authorized, Lockup, Meta, StakeState};
+    use std::{collections::BTreeMap, sync::Arc};
+
+    fn new_from_parent(parent: &Arc<Bank>) -> Bank {
+        Bank::new_from_parent(parent, &Pubkey::default(), parent.slot() + 1)
+    }
+
+    #[test]
+    fn test_calculate_non_circulating_supply() {
+        let mut accounts: BTreeMap<Pubkey, Account> = BTreeMap::new();
+        let balance = 10;
+        let num_genesis_accounts = 10;
+        for _ in 0..num_genesis_accounts {
+            accounts.insert(
+                Pubkey::new_rand(),
+                Account::new(balance, 0, &Pubkey::default()),
+            );
+        }
+        let non_circulating_accounts = non_circulating_accounts();
+        let num_non_circulating_accounts = non_circulating_accounts.len() as u64;
+        for key in non_circulating_accounts.clone() {
+            accounts.insert(key, Account::new(balance, 0, &Pubkey::default()));
+        }
+
+        let num_stake_accounts = 3;
+        for _ in 0..num_stake_accounts {
+            let pubkey = Pubkey::new_rand();
+            let meta = Meta {
+                authorized: Authorized::auto(&pubkey),
+                lockup: Lockup {
+                    epoch: 1,
+                    ..Lockup::default()
+                },
+                ..Meta::default()
+            };
+            let stake_account = Account::new_data_with_space(
+                balance,
+                &StakeState::Initialized(meta),
+                std::mem::size_of::<StakeState>(),
+                &solana_stake_program::id(),
+            )
+            .unwrap();
+            accounts.insert(pubkey, stake_account);
+        }
+
+        let slots_per_epoch = 32;
+        let genesis_config = GenesisConfig {
+            accounts,
+            epoch_schedule: EpochSchedule::new(slots_per_epoch),
+            ..GenesisConfig::default()
+        };
+        let mut bank = Arc::new(Bank::new(&genesis_config));
+        assert_eq!(
+            bank.capitalization(),
+            (num_genesis_accounts + num_non_circulating_accounts + num_stake_accounts) * balance
+        );
+
+        let non_circulating_supply = calculate_non_circulating_supply(&bank);
+        assert_eq!(
+            non_circulating_supply.lamports,
+            (num_non_circulating_accounts + num_stake_accounts) * balance
+        );
+        assert_eq!(
+            non_circulating_supply.accounts.len(),
+            num_non_circulating_accounts as usize + num_stake_accounts as usize
+        );
+
+        bank = Arc::new(new_from_parent(&bank));
+        let new_balance = 11;
+        for key in non_circulating_accounts {
+            bank.store_account(&key, &Account::new(new_balance, 0, &Pubkey::default()));
+        }
+        let non_circulating_supply = calculate_non_circulating_supply(&bank);
+        assert_eq!(
+            non_circulating_supply.lamports,
+            (num_non_circulating_accounts * new_balance) + (num_stake_accounts * balance)
+        );
+        assert_eq!(
+            non_circulating_supply.accounts.len(),
+            num_non_circulating_accounts as usize + num_stake_accounts as usize
+        );
+
+        // Advance bank an epoch, which should unlock stakes
+        for _ in 0..slots_per_epoch {
+            bank = Arc::new(new_from_parent(&bank));
+        }
+        assert_eq!(bank.epoch(), 1);
+        let non_circulating_supply = calculate_non_circulating_supply(&bank);
+        assert_eq!(
+            non_circulating_supply.lamports,
+            num_non_circulating_accounts * new_balance
+        );
+        assert_eq!(
+            non_circulating_supply.accounts.len(),
+            num_non_circulating_accounts as usize
+        );
+    }
+}

core/src/poh_service.rs

@@ -1,10 +1,8 @@
 //! The `poh_service` module implements a service that records the passing of
 //! "ticks", a measure of time in the PoH stream
 use crate::poh_recorder::PohRecorder;
-use core_affinity;
 use solana_sdk::clock::DEFAULT_TICKS_PER_SLOT;
 use solana_sdk::poh_config::PohConfig;
-use solana_sys_tuner;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex};
 use std::thread::{self, sleep, Builder, JoinHandle};

core/src/repair_response.rs

@@ -0,0 +1,129 @@
+use solana_ledger::{
+    blockstore::Blockstore,
+    shred::{Nonce, Shred, SIZE_OF_NONCE},
+};
+use solana_perf::packet::limited_deserialize;
+use solana_sdk::{clock::Slot, packet::Packet};
+use std::{io, net::SocketAddr};
+
+pub fn repair_response_packet(
+    blockstore: &Blockstore,
+    slot: Slot,
+    shred_index: u64,
+    dest: &SocketAddr,
+    nonce: Option<Nonce>,
+) -> Option<Packet> {
+    if Shred::is_nonce_unlocked(slot) && nonce.is_none()
+        || !Shred::is_nonce_unlocked(slot) && nonce.is_some()
+    {
+        return None;
+    }
+    let shred = blockstore
+        .get_data_shred(slot, shred_index)
+        .expect("Blockstore could not get data shred");
+    shred.map(|shred| repair_response_packet_from_shred(slot, shred, dest, nonce))
+}
+
+pub fn repair_response_packet_from_shred(
+    slot: Slot,
+    shred: Vec<u8>,
+    dest: &SocketAddr,
+    nonce: Option<Nonce>,
+) -> Packet {
+    let size_of_nonce = {
+        if Shred::is_nonce_unlocked(slot) {
+            assert!(nonce.is_some());
+            SIZE_OF_NONCE
+        } else {
+            assert!(nonce.is_none());
+            0
+        }
+    };
+    let mut packet = Packet::default();
+    packet.meta.size = shred.len() + size_of_nonce;
+    packet.meta.set_addr(dest);
+    packet.data[..shred.len()].copy_from_slice(&shred);
+    let mut wr = io::Cursor::new(&mut packet.data[shred.len()..]);
+    if let Some(nonce) = nonce {
+        bincode::serialize_into(&mut wr, &nonce).expect("Buffer not large enough to fit nonce");
+    }
+    packet
+}
+
+pub fn nonce(buf: &[u8]) -> Option<Nonce> {
+    if buf.len() < SIZE_OF_NONCE {
+        None
+    } else {
+        limited_deserialize(&buf[buf.len() - SIZE_OF_NONCE..]).ok()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use solana_ledger::{
+        shred::{Shred, Shredder, UNLOCK_NONCE_SLOT},
+        sigverify_shreds::verify_shred_cpu,
+    };
+    use solana_sdk::signature::{Keypair, Signer};
+    use std::{
+        collections::HashMap,
+        net::{IpAddr, Ipv4Addr},
+    };
+
+    fn run_test_sigverify_shred_cpu_repair(slot: Slot) {
+        solana_logger::setup();
+        let mut shred = Shred::new_from_data(
+            slot,
+            0xc0de,
+            0xdead,
+            Some(&[1, 2, 3, 4]),
+            true,
+            true,
+            0,
+            0,
+            0xc0de,
+        );
+        assert_eq!(shred.slot(), slot);
+        let keypair = Keypair::new();
+        Shredder::sign_shred(&keypair, &mut shred);
+        trace!("signature {}", shred.common_header.signature);
+        let nonce = if Shred::is_nonce_unlocked(slot) {
+            Some(9)
+        } else {
+            None
+        };
+        let mut packet = repair_response_packet_from_shred(
+            slot,
+            shred.payload,
+            &SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8080),
+            nonce,
+        );
+        packet.meta.repair = true;
+
+        let leader_slots = [(slot, keypair.pubkey().to_bytes())]
+            .iter()
+            .cloned()
+            .collect();
+        let rv = verify_shred_cpu(&packet, &leader_slots);
+        assert_eq!(rv, Some(1));
+
+        let wrong_keypair = Keypair::new();
+        let leader_slots = [(slot, wrong_keypair.pubkey().to_bytes())]
+            .iter()
+            .cloned()
+            .collect();
+        let rv = verify_shred_cpu(&packet, &leader_slots);
+        assert_eq!(rv, Some(0));
+
+        let leader_slots = HashMap::new();
+        let rv = verify_shred_cpu(&packet, &leader_slots);
+        assert_eq!(rv, None);
+    }
+
+    #[test]
+    fn test_sigverify_shred_cpu_repair() {
+        run_test_sigverify_shred_cpu_repair(UNLOCK_NONCE_SLOT);
+        run_test_sigverify_shred_cpu_repair(UNLOCK_NONCE_SLOT + 1);
+    }
+}

core/src/repair_service.rs

@@ -20,9 +20,31 @@ use std::{
     sync::{Arc, RwLock},
     thread::sleep,
     thread::{self, Builder, JoinHandle},
-    time::Duration,
+    time::{Duration, Instant},
 };
 
+#[derive(Default)]
+pub struct RepairStatsGroup {
+    pub count: u64,
+    pub min: u64,
+    pub max: u64,
+}
+
+impl RepairStatsGroup {
+    pub fn update(&mut self, slot: u64) {
+        self.count += 1;
+        self.min = std::cmp::min(self.min, slot);
+        self.max = std::cmp::max(self.max, slot);
+    }
+}
+
+#[derive(Default)]
+pub struct RepairStats {
+    pub shred: RepairStatsGroup,
+    pub highest_shred: RepairStatsGroup,
+    pub orphan: RepairStatsGroup,
+}
+
 pub const MAX_REPAIR_LENGTH: usize = 512;
 pub const REPAIR_MS: u64 = 100;
 pub const MAX_ORPHANS: usize = 5;
@@ -107,6 +129,8 @@ impl RepairService {
                 cluster_info,
             );
         }
+        let mut repair_stats = RepairStats::default();
+        let mut last_stats = Instant::now();
         loop {
             if exit.load(Ordering::Relaxed) {
                 break;
@@ -144,22 +168,34 @@ impl RepairService {
             };
 
             if let Ok(repairs) = repairs {
-                let reqs: Vec<_> = repairs
-                    .into_iter()
-                    .filter_map(|repair_request| {
-                        serve_repair
-                            .repair_request(&repair_request)
-                            .map(|result| (result, repair_request))
-                            .ok()
-                    })
-                    .collect();
-
-                for ((to, req), _) in reqs {
-                    repair_socket.send_to(&req, to).unwrap_or_else(|e| {
-                        info!("{} repair req send_to({}) error {:?}", id, to, e);
-                        0
-                    });
+                repairs.into_iter().for_each(|repair_request| {
+                    if let Ok((to, req)) =
+                        serve_repair.repair_request(&repair_request, &mut repair_stats)
+                    {
+                        repair_socket.send_to(&req, to).unwrap_or_else(|e| {
+                            info!("{} repair req send_to({}) error {:?}", id, to, e);
+                            0
+                        });
+                    }
+                });
+            }
+            if last_stats.elapsed().as_secs() > 1 {
+                let repair_total = repair_stats.shred.count
+                    + repair_stats.highest_shred.count
+                    + repair_stats.orphan.count;
+                if repair_total > 0 {
+                    datapoint_info!(
+                        "serve_repair-repair",
+                        ("repair-total", repair_total, i64),
+                        ("shred-count", repair_stats.shred.count, i64),
+                        ("highest-shred-count", repair_stats.highest_shred.count, i64),
+                        ("orphan-count", repair_stats.orphan.count, i64),
+                        ("repair-highest-slot", repair_stats.highest_shred.max, i64),
+                        ("repair-orphan", repair_stats.orphan.max, i64),
+                    );
                 }
+                repair_stats = RepairStats::default();
+                last_stats = Instant::now();
             }
             sleep(Duration::from_millis(REPAIR_MS));
         }
@@ -565,7 +601,7 @@ mod test {
             let blockstore = Blockstore::open(&blockstore_path).unwrap();
 
             let slots: Vec<u64> = vec![1, 3, 5, 7, 8];
-            let num_entries_per_slot = max_ticks_per_n_shreds(1) + 1;
+            let num_entries_per_slot = max_ticks_per_n_shreds(1, None) + 1;
 
             let shreds = make_chaining_slot_entries(&slots, num_entries_per_slot);
             for (mut slot_shreds, _) in shreds.into_iter() {

core/src/replay_stage.rs

@@ -67,7 +67,6 @@ impl Drop for Finalizer {
     }
 }
 
-#[derive(Default)]
 pub struct ReplayStageConfig {
     pub my_pubkey: Pubkey,
     pub vote_account: Pubkey,
@@ -257,13 +256,15 @@ impl ReplayStage {
                     );
 
                     let ancestors = Arc::new(bank_forks.read().unwrap().ancestors());
+                    let forks_root = bank_forks.read().unwrap().root();
                     let start = allocated.get();
                     let mut frozen_banks: Vec<_> = bank_forks
                         .read()
                         .unwrap()
                         .frozen_banks()
-                        .values()
-                        .cloned()
+                        .into_iter()
+                        .filter(|(slot, _)| *slot >= forks_root)
+                        .map(|(_, bank)| bank)
                         .collect();
                     let newly_computed_slot_stats = Self::compute_bank_stats(
                         &my_pubkey,
@@ -344,6 +345,7 @@ impl ReplayStage {
                             &accounts_hash_sender,
                             &latest_root_senders,
                             &subscriptions,
+                            &block_commitment_cache,
                         )?;
                     }
                     datapoint_debug!(
@@ -618,6 +620,7 @@ impl ReplayStage {
         accounts_hash_sender: &Option<SnapshotPackageSender>,
         latest_root_senders: &[Sender<Slot>],
         subscriptions: &Arc<RpcSubscriptions>,
+        block_commitment_cache: &Arc<RwLock<BlockCommitmentCache>>,
     ) -> Result<()> {
         if bank.is_empty() {
             inc_new_counter_info!("replay_stage-voted_empty_bank", 1);
@@ -643,7 +646,20 @@ impl ReplayStage {
             blockstore
                 .set_roots(&rooted_slots)
                 .expect("Ledger set roots failed");
-            Self::handle_new_root(new_root, &bank_forks, progress, accounts_hash_sender);
+            let largest_confirmed_root = Some(
+                block_commitment_cache
+                    .read()
+                    .unwrap()
+                    .largest_confirmed_root(),
+            );
+
+            Self::handle_new_root(
+                new_root,
+                &bank_forks,
+                progress,
+                accounts_hash_sender,
+                largest_confirmed_root,
+            );
             subscriptions.notify_roots(rooted_slots);
             latest_root_senders.iter().for_each(|s| {
                 if let Err(e) = s.send(new_root) {
@@ -979,15 +995,17 @@ impl ReplayStage {
     }
 
     pub(crate) fn handle_new_root(
-        new_root: u64,
+        new_root: Slot,
         bank_forks: &RwLock<BankForks>,
         progress: &mut HashMap<u64, ForkProgress>,
         accounts_hash_sender: &Option<SnapshotPackageSender>,
+        largest_confirmed_root: Option<Slot>,
     ) {
-        bank_forks
-            .write()
-            .unwrap()
-            .set_root(new_root, accounts_hash_sender);
+        bank_forks.write().unwrap().set_root(
+            new_root,
+            accounts_hash_sender,
+            largest_confirmed_root,
+        );
         let r_bank_forks = bank_forks.read().unwrap();
         progress.retain(|k, _| r_bank_forks.get(*k).is_some());
     }
@@ -1016,7 +1034,11 @@ impl ReplayStage {
         // Find the next slot that chains to the old slot
         let forks = forks_lock.read().unwrap();
         let frozen_banks = forks.frozen_banks();
-        let frozen_bank_slots: Vec<u64> = frozen_banks.keys().cloned().collect();
+        let frozen_bank_slots: Vec<u64> = frozen_banks
+            .keys()
+            .cloned()
+            .filter(|s| *s >= forks.root())
+            .collect();
         let next_slots = blockstore
             .get_slots_since(&frozen_bank_slots)
             .expect("Db error");
@@ -1419,7 +1441,9 @@ pub(crate) mod tests {
             let exit = Arc::new(AtomicBool::new(false));
             let subscriptions = Arc::new(RpcSubscriptions::new(
                 &exit,
-                Arc::new(RwLock::new(BlockCommitmentCache::default())),
+                Arc::new(RwLock::new(BlockCommitmentCache::default_with_blockstore(
+                    blockstore.clone(),
+                ))),
             ));
             let bank_forks = BankForks::new(0, bank0);
             bank_forks.working_bank().freeze();
@@ -1472,12 +1496,58 @@ pub(crate) mod tests {
         for i in 0..=root {
             progress.insert(i, ForkProgress::new(Hash::default()));
         }
-        ReplayStage::handle_new_root(root, &bank_forks, &mut progress, &None);
+        ReplayStage::handle_new_root(root, &bank_forks, &mut progress, &None, None);
         assert_eq!(bank_forks.read().unwrap().root(), root);
         assert_eq!(progress.len(), 1);
         assert!(progress.get(&root).is_some());
     }
 
+    #[test]
+    fn test_handle_new_root_ahead_of_largest_confirmed_root() {
+        let genesis_config = create_genesis_config(10_000).genesis_config;
+        let bank0 = Bank::new(&genesis_config);
+        let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank0)));
+        let confirmed_root = 1;
+        let fork = 2;
+        let bank1 = Bank::new_from_parent(
+            bank_forks.read().unwrap().get(0).unwrap(),
+            &Pubkey::default(),
+            confirmed_root,
+        );
+        bank_forks.write().unwrap().insert(bank1);
+        let bank2 = Bank::new_from_parent(
+            bank_forks.read().unwrap().get(confirmed_root).unwrap(),
+            &Pubkey::default(),
+            fork,
+        );
+        bank_forks.write().unwrap().insert(bank2);
+        let root = 3;
+        let root_bank = Bank::new_from_parent(
+            bank_forks.read().unwrap().get(confirmed_root).unwrap(),
+            &Pubkey::default(),
+            root,
+        );
+        bank_forks.write().unwrap().insert(root_bank);
+        let mut progress = HashMap::new();
+        for i in 0..=root {
+            progress.insert(i, ForkProgress::new(Hash::default()));
+        }
+        ReplayStage::handle_new_root(
+            root,
+            &bank_forks,
+            &mut progress,
+            &None,
+            Some(confirmed_root),
+        );
+        assert_eq!(bank_forks.read().unwrap().root(), root);
+        assert!(bank_forks.read().unwrap().get(confirmed_root).is_some());
+        assert!(bank_forks.read().unwrap().get(fork).is_none());
+        assert_eq!(progress.len(), 2);
+        assert!(progress.get(&root).is_some());
+        assert!(progress.get(&confirmed_root).is_some());
+        assert!(progress.get(&fork).is_none());
+    }
+
     #[test]
     fn test_dead_fork_transaction_error() {
         let keypair1 = Keypair::new();
@@ -1668,6 +1738,7 @@ pub(crate) mod tests {
                 ShredCommonHeader::default(),
                 data_header,
                 CodingShredHeader::default(),
+                PACKET_DATA_SIZE,
             );
             bincode::serialize_into(
                 &mut shred.payload[SIZE_OF_COMMON_SHRED_HEADER + SIZE_OF_DATA_SHRED_HEADER..],
@@ -1745,7 +1816,11 @@ pub(crate) mod tests {
             bank.store_account(&pubkey, &leader_vote_account);
         }
 
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::default_with_blockstore(blockstore.clone()),
+        ));
         let (lockouts_sender, _) = AggregateCommitmentService::new(
             &Arc::new(AtomicBool::new(false)),
             block_commitment_cache.clone(),
@@ -1950,14 +2025,14 @@ pub(crate) mod tests {
                             meta.err,
                             Some(TransactionError::InstructionError(
                                 0,
-                                InstructionError::CustomError(1)
+                                InstructionError::Custom(1)
                             ))
                         );
                         assert_eq!(
                             meta.status,
                             Err(TransactionError::InstructionError(
                                 0,
-                                InstructionError::CustomError(1)
+                                InstructionError::Custom(1)
                             ))
                         );
                     } else {

core/src/rpc_error.rs

@@ -0,0 +1,45 @@
+use jsonrpc_core::{Error, ErrorCode};
+use solana_sdk::clock::Slot;
+
+const JSON_RPC_SERVER_ERROR_0: i64 = -32000;
+const JSON_RPC_SERVER_ERROR_1: i64 = -32001;
+
+pub enum RpcCustomError {
+    NonexistentClusterRoot {
+        cluster_root: Slot,
+        node_root: Slot,
+    },
+    BlockCleanedUp {
+        slot: Slot,
+        first_available_block: Slot,
+    },
+}
+
+impl From<RpcCustomError> for Error {
+    fn from(e: RpcCustomError) -> Self {
+        match e {
+            RpcCustomError::NonexistentClusterRoot {
+                cluster_root,
+                node_root,
+            } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_0),
+                message: format!(
+                    "Cluster largest_confirmed_root {} does not exist on node. Node root: {}",
+                    cluster_root, node_root,
+                ),
+                data: None,
+            },
+            RpcCustomError::BlockCleanedUp {
+                slot,
+                first_available_block,
+            } => Self {
+                code: ErrorCode::ServerError(JSON_RPC_SERVER_ERROR_1),
+                message: format!(
+                    "Block {} cleaned up, does not exist on node. First available block: {}",
+                    slot, first_available_block,
+                ),
+                data: None,
+            },
+        }
+    }
+}

core/src/rpc_pubsub.rs

@@ -7,8 +7,13 @@ use jsonrpc_pubsub::{typed::Subscriber, Session, SubscriptionId};
 use solana_client::rpc_response::{
     Response as RpcResponse, RpcAccount, RpcKeyedAccount, RpcSignatureResult,
 };
+#[cfg(test)]
+use solana_ledger::blockstore::Blockstore;
 use solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature};
-use std::sync::{atomic, Arc};
+use std::{
+    str::FromStr,
+    sync::{atomic, Arc},
+};
 
 // Suppress needless_return due to
 //   https://github.com/paritytech/jsonrpc/blob/2d38e6424d8461cdf72e78425ce67d51af9c6586/derive/src/lib.rs#L204
@@ -118,7 +123,6 @@ pub trait RpcSolPubSub {
     fn root_unsubscribe(&self, meta: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool>;
 }
 
-#[derive(Default)]
 pub struct RpcSolPubSubImpl {
     uid: Arc<atomic::AtomicUsize>,
     subscriptions: Arc<RpcSubscriptions>,
@@ -129,9 +133,14 @@ impl RpcSolPubSubImpl {
         let uid = Arc::new(atomic::AtomicUsize::default());
         Self { uid, subscriptions }
     }
-}
 
-use std::str::FromStr;
+    #[cfg(test)]
+    fn default_with_blockstore(blockstore: Arc<Blockstore>) -> Self {
+        let uid = Arc::new(atomic::AtomicUsize::default());
+        let subscriptions = Arc::new(RpcSubscriptions::default_with_blockstore(blockstore));
+        Self { uid, subscriptions }
+    }
+}
 
 fn param<T: FromStr>(param_str: &str, thing: &str) -> Result<T> {
     param_str.parse::<T>().map_err(|_e| Error {
@@ -323,7 +332,7 @@ mod tests {
     use jsonrpc_pubsub::{PubSubHandler, Session};
     use serial_test_derive::serial;
     use solana_budget_program::{self, budget_instruction};
-    use solana_ledger::bank_forks::BankForks;
+    use solana_ledger::{bank_forks::BankForks, get_tmp_ledger_path};
     use solana_runtime::bank::Bank;
     use solana_sdk::{
         pubkey::Pubkey,
@@ -370,12 +379,16 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let rpc = RpcSolPubSubImpl {
             subscriptions: Arc::new(RpcSubscriptions::new(
                 &Arc::new(AtomicBool::new(false)),
-                Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+                Arc::new(RwLock::new(
+                    BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+                )),
             )),
-            ..RpcSolPubSubImpl::default()
+            uid: Arc::new(atomic::AtomicUsize::default()),
         };
 
         // Test signature subscriptions
@@ -416,11 +429,13 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let arc_bank = Arc::new(bank);
         let blockhash = arc_bank.last_blockhash();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
         let session = create_session();
 
         let mut io = PubSubHandler::default();
-        let rpc = RpcSolPubSubImpl::default();
+        let rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore);
         io.extend_with(rpc.to_delegate());
 
         let tx = system_transaction::transfer(&alice, &bob_pubkey, 20, blockhash);
@@ -475,13 +490,17 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
         let rpc = RpcSolPubSubImpl {
             subscriptions: Arc::new(RpcSubscriptions::new(
                 &Arc::new(AtomicBool::new(false)),
-                Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+                Arc::new(RwLock::new(
+                    BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+                )),
             )),
-            ..RpcSolPubSubImpl::default()
+            uid: Arc::new(atomic::AtomicUsize::default()),
         };
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("accountNotification");
@@ -569,9 +588,11 @@ mod tests {
     fn test_account_unsubscribe() {
         let bob_pubkey = Pubkey::new_rand();
         let session = create_session();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
 
         let mut io = PubSubHandler::default();
-        let rpc = RpcSolPubSubImpl::default();
+        let rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore);
 
         io.extend_with(rpc.to_delegate());
 
@@ -615,13 +636,17 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bob = Keypair::new();
 
-        let mut rpc = RpcSolPubSubImpl::default();
+        let mut rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore.clone());
         let exit = Arc::new(AtomicBool::new(false));
         let subscriptions = RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         );
         rpc.subscriptions = Arc::new(subscriptions);
         let session = create_session();
@@ -652,11 +677,15 @@ mod tests {
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bob = Keypair::new();
 
-        let mut rpc = RpcSolPubSubImpl::default();
+        let mut rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore.clone());
         let exit = Arc::new(AtomicBool::new(false));
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests()));
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::new_for_tests_with_blockstore(blockstore.clone()),
+        ));
 
         let subscriptions = RpcSubscriptions::new(&exit, block_commitment_cache.clone());
         rpc.subscriptions = Arc::new(subscriptions);
@@ -683,8 +712,14 @@ mod tests {
         cache0.increase_confirmation_stake(1, 10);
         let mut block_commitment = HashMap::new();
         block_commitment.entry(0).or_insert(cache0.clone());
-        let mut new_block_commitment =
-            BlockCommitmentCache::new(block_commitment, 10, bank1.clone(), 0);
+        let mut new_block_commitment = BlockCommitmentCache::new(
+            block_commitment,
+            0,
+            10,
+            bank1.clone(),
+            blockstore.clone(),
+            0,
+        );
         let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
         std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
         drop(w_block_commitment_cache);
@@ -698,7 +733,8 @@ mod tests {
         cache0.increase_confirmation_stake(2, 10);
         let mut block_commitment = HashMap::new();
         block_commitment.entry(0).or_insert(cache0.clone());
-        let mut new_block_commitment = BlockCommitmentCache::new(block_commitment, 10, bank2, 0);
+        let mut new_block_commitment =
+            BlockCommitmentCache::new(block_commitment, 0, 10, bank2, blockstore.clone(), 0);
         let mut w_block_commitment_cache = block_commitment_cache.write().unwrap();
         std::mem::swap(&mut *w_block_commitment_cache, &mut new_block_commitment);
         drop(w_block_commitment_cache);
@@ -728,7 +764,9 @@ mod tests {
     #[test]
     #[serial]
     fn test_slot_subscribe() {
-        let rpc = RpcSolPubSubImpl::default();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification");
         rpc.slot_subscribe(session, subscriber);
@@ -753,7 +791,9 @@ mod tests {
     #[test]
     #[serial]
     fn test_slot_unsubscribe() {
-        let rpc = RpcSolPubSubImpl::default();
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let rpc = RpcSolPubSubImpl::default_with_blockstore(blockstore);
         let session = create_session();
         let (subscriber, _id_receiver, receiver) = Subscriber::new_test("slotNotification");
         rpc.slot_subscribe(session, subscriber);

core/src/rpc_pubsub_service.rs

@@ -73,6 +73,7 @@ impl PubSubService {
 mod tests {
     use super::*;
     use crate::commitment::BlockCommitmentCache;
+    use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
     use std::{
         net::{IpAddr, Ipv4Addr},
         sync::RwLock,
@@ -82,9 +83,13 @@ mod tests {
     fn test_pubsub_new() {
         let pubsub_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0);
         let exit = Arc::new(AtomicBool::new(false));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let subscriptions = Arc::new(RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         ));
         let pubsub_service = PubSubService::new(&subscriptions, pubsub_addr, &exit);
         let thread = pubsub_service.thread_hdl.thread();

core/src/rpc_service.rs

@@ -44,6 +44,7 @@ struct RpcRequestMiddleware {
     snapshot_config: Option<SnapshotConfig>,
     cluster_info: Arc<RwLock<ClusterInfo>>,
     trusted_validators: Option<HashSet<Pubkey>>,
+    bank_forks: Arc<RwLock<BankForks>>,
 }
 
 impl RpcRequestMiddleware {
@@ -52,6 +53,7 @@ impl RpcRequestMiddleware {
         snapshot_config: Option<SnapshotConfig>,
         cluster_info: Arc<RwLock<ClusterInfo>>,
         trusted_validators: Option<HashSet<Pubkey>>,
+        bank_forks: Arc<RwLock<BankForks>>,
     ) -> Self {
         Self {
             ledger_path,
@@ -60,6 +62,7 @@ impl RpcRequestMiddleware {
             snapshot_config,
             cluster_info,
             trusted_validators,
+            bank_forks,
         }
     }
 
@@ -85,7 +88,7 @@ impl RpcRequestMiddleware {
             .unwrap()
     }
 
-    fn is_get_path(&self, path: &str) -> bool {
+    fn is_file_get_path(&self, path: &str) -> bool {
         match path {
             "/genesis.tar.bz2" => true,
             _ => {
@@ -98,7 +101,7 @@ impl RpcRequestMiddleware {
         }
     }
 
-    fn get(&self, path: &str) -> RequestMiddlewareAction {
+    fn process_file_get(&self, path: &str) -> RequestMiddlewareAction {
         let filename = self.ledger_path.join(
             path.split_at(1).1, // Drop leading '/' from path
         );
@@ -202,8 +205,19 @@ impl RequestMiddleware for RpcRequestMiddleware {
                 };
             }
         }
-        if self.is_get_path(request.uri().path()) {
-            self.get(request.uri().path())
+
+        if let Some(result) = process_rest(&self.bank_forks, request.uri().path()) {
+            RequestMiddlewareAction::Respond {
+                should_validate_hosts: true,
+                response: Box::new(jsonrpc_core::futures::future::ok(
+                    hyper::Response::builder()
+                        .status(hyper::StatusCode::OK)
+                        .body(hyper::Body::from(result))
+                        .unwrap(),
+                )),
+            }
+        } else if self.is_file_get_path(request.uri().path()) {
+            self.process_file_get(request.uri().path())
         } else if request.uri().path() == "/health" {
             RequestMiddlewareAction::Respond {
                 should_validate_hosts: true,
@@ -223,6 +237,26 @@ impl RequestMiddleware for RpcRequestMiddleware {
     }
 }
 
+fn process_rest(bank_forks: &Arc<RwLock<BankForks>>, path: &str) -> Option<String> {
+    match path {
+        "/v0/circulating-supply" => {
+            let r_bank_forks = bank_forks.read().unwrap();
+            let bank = r_bank_forks.root_bank();
+            let total_supply = bank.capitalization();
+            let non_circulating_supply =
+                crate::non_circulating_supply::calculate_non_circulating_supply(&bank).lamports;
+            Some(format!("{}", total_supply - non_circulating_supply))
+        }
+        "/v0/total-supply" => {
+            let r_bank_forks = bank_forks.read().unwrap();
+            let bank = r_bank_forks.root_bank();
+            let total_supply = bank.capitalization();
+            Some(format!("{}", total_supply))
+        }
+        _ => None,
+    }
+}
+
 impl JsonRpcService {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
@@ -243,7 +277,7 @@ impl JsonRpcService {
         info!("rpc configuration: {:?}", config);
         let request_processor = Arc::new(RwLock::new(JsonRpcRequestProcessor::new(
             config,
-            bank_forks,
+            bank_forks.clone(),
             block_commitment_cache,
             blockstore,
             storage_state,
@@ -268,6 +302,7 @@ impl JsonRpcService {
                     snapshot_config,
                     cluster_info.clone(),
                     trusted_validators,
+                    bank_forks.clone(),
                 );
                 let server = ServerBuilder::with_meta_extractor(
                     io,
@@ -277,7 +312,7 @@ impl JsonRpcService {
                         genesis_hash,
                     },
                 )
-                .threads(4)
+                .threads(num_cpus::get())
                 .cors(DomainsValidation::AllowOnly(vec![
                     AccessControlAllowOrigin::Any,
                 ]))
@@ -361,16 +396,18 @@ mod tests {
             solana_net_utils::find_available_port_in_range(ip_addr, (10000, 65535)).unwrap(),
         );
         let bank_forks = Arc::new(RwLock::new(BankForks::new(bank.slot(), bank)));
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
         let ledger_path = get_tmp_ledger_path!();
-        let blockstore = Blockstore::open(&ledger_path).unwrap();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::default_with_blockstore(blockstore.clone()),
+        ));
         let mut rpc_service = JsonRpcService::new(
             rpc_addr,
             JsonRpcConfig::default(),
             None,
             bank_forks,
             block_commitment_cache,
-            Arc::new(blockstore),
+            blockstore,
             cluster_info,
             Hash::default(),
             &PathBuf::from("farf"),
@@ -395,13 +432,41 @@ mod tests {
         rpc_service.join().unwrap();
     }
 
+    fn create_bank_forks() -> Arc<RwLock<BankForks>> {
+        let GenesisConfigInfo { genesis_config, .. } = create_genesis_config(10_000);
+        let bank = Bank::new(&genesis_config);
+        Arc::new(RwLock::new(BankForks::new(bank.slot(), bank)))
+    }
+
     #[test]
-    fn test_is_get_path() {
+    fn test_process_rest_api() {
+        let bank_forks = create_bank_forks();
+
+        assert_eq!(None, process_rest(&bank_forks, "not-a-supported-rest-api"));
+        assert_eq!(
+            Some("10127".to_string()),
+            process_rest(&bank_forks, "/v0/circulating-supply")
+        );
+        assert_eq!(
+            Some("10127".to_string()),
+            process_rest(&bank_forks, "/v0/total-supply")
+        );
+    }
+
+    #[test]
+    fn test_is_file_get_path() {
         let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(
             ContactInfo::default(),
         )));
+        let bank_forks = create_bank_forks();
 
-        let rrm = RpcRequestMiddleware::new(PathBuf::from("/"), None, cluster_info.clone(), None);
+        let rrm = RpcRequestMiddleware::new(
+            PathBuf::from("/"),
+            None,
+            cluster_info.clone(),
+            None,
+            bank_forks.clone(),
+        );
         let rrm_with_snapshot_config = RpcRequestMiddleware::new(
             PathBuf::from("/"),
             Some(SnapshotConfig {
@@ -411,26 +476,28 @@ mod tests {
             }),
             cluster_info,
             None,
+            bank_forks,
         );
 
-        assert!(rrm.is_get_path("/genesis.tar.bz2"));
-        assert!(!rrm.is_get_path("genesis.tar.bz2"));
+        assert!(rrm.is_file_get_path("/genesis.tar.bz2"));
+        assert!(!rrm.is_file_get_path("genesis.tar.bz2"));
 
-        assert!(!rrm.is_get_path("/snapshot.tar.bz2")); // This is a redirect
+        assert!(!rrm.is_file_get_path("/snapshot.tar.bz2")); // This is a redirect
 
-        assert!(
-            !rrm.is_get_path("/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2")
-        );
-        assert!(rrm_with_snapshot_config
-            .is_get_path("/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"));
+        assert!(!rrm.is_file_get_path(
+            "/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"
+        ));
+        assert!(rrm_with_snapshot_config.is_file_get_path(
+            "/snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"
+        ));
 
-        assert!(!rrm.is_get_path(
+        assert!(!rrm.is_file_get_path(
             "/snapshot-notaslotnumber-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2"
         ));
 
-        assert!(!rrm.is_get_path("/"));
-        assert!(!rrm.is_get_path(".."));
-        assert!(!rrm.is_get_path("🎣"));
+        assert!(!rrm.is_file_get_path("/"));
+        assert!(!rrm.is_file_get_path(".."));
+        assert!(!rrm.is_file_get_path("🎣"));
     }
 
     #[test]
@@ -439,7 +506,13 @@ mod tests {
             ContactInfo::default(),
         )));
 
-        let rm = RpcRequestMiddleware::new(PathBuf::from("/"), None, cluster_info.clone(), None);
+        let rm = RpcRequestMiddleware::new(
+            PathBuf::from("/"),
+            None,
+            cluster_info.clone(),
+            None,
+            create_bank_forks(),
+        );
         assert_eq!(rm.health_check(), "ok");
     }
 
@@ -455,6 +528,7 @@ mod tests {
             None,
             cluster_info.clone(),
             Some(trusted_validators.clone().into_iter().collect()),
+            create_bank_forks(),
         );
 
         // No account hashes for this node or any trusted validators == "behind"

core/src/rpc_subscriptions.rs

@@ -11,7 +11,7 @@ use serde::Serialize;
 use solana_client::rpc_response::{
     Response, RpcAccount, RpcKeyedAccount, RpcResponseContext, RpcSignatureResult,
 };
-use solana_ledger::bank_forks::BankForks;
+use solana_ledger::{bank_forks::BankForks, blockstore::Blockstore};
 use solana_runtime::bank::Bank;
 use solana_sdk::{
     account::Account, clock::Slot, pubkey::Pubkey, signature::Signature, transaction,
@@ -246,15 +246,6 @@ pub struct RpcSubscriptions {
     exit: Arc<AtomicBool>,
 }
 
-impl Default for RpcSubscriptions {
-    fn default() -> Self {
-        Self::new(
-            &Arc::new(AtomicBool::new(false)),
-            Arc::new(RwLock::new(BlockCommitmentCache::default())),
-        )
-    }
-}
-
 impl Drop for RpcSubscriptions {
     fn drop(&mut self) {
         self.shutdown().unwrap_or_else(|err| {
@@ -324,6 +315,15 @@ impl RpcSubscriptions {
         }
     }
 
+    pub fn default_with_blockstore(blockstore: Arc<Blockstore>) -> Self {
+        Self::new(
+            &Arc::new(AtomicBool::new(false)),
+            Arc::new(RwLock::new(BlockCommitmentCache::default_with_blockstore(
+                blockstore,
+            ))),
+        )
+    }
+
     fn check_account(
         pubkey: &Pubkey,
         bank_forks: &Arc<RwLock<BankForks>>,
@@ -624,6 +624,7 @@ pub(crate) mod tests {
     use jsonrpc_pubsub::typed::Subscriber;
     use serial_test_derive::serial;
     use solana_budget_program;
+    use solana_ledger::get_tmp_ledger_path;
     use solana_sdk::{
         signature::{Keypair, Signer},
         system_transaction,
@@ -664,6 +665,8 @@ pub(crate) mod tests {
             mint_keypair,
             ..
         } = create_genesis_config(100);
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
@@ -690,7 +693,9 @@ pub(crate) mod tests {
         let exit = Arc::new(AtomicBool::new(false));
         let subscriptions = RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         );
         subscriptions.add_account_subscription(alice.pubkey(), None, sub_id.clone(), subscriber);
 
@@ -737,6 +742,8 @@ pub(crate) mod tests {
             mint_keypair,
             ..
         } = create_genesis_config(100);
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let bank_forks = Arc::new(RwLock::new(BankForks::new(0, bank)));
@@ -763,7 +770,9 @@ pub(crate) mod tests {
         let exit = Arc::new(AtomicBool::new(false));
         let subscriptions = RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         );
         subscriptions.add_program_subscription(
             solana_budget_program::id(),
@@ -818,6 +827,8 @@ pub(crate) mod tests {
             mint_keypair,
             ..
         } = create_genesis_config(100);
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let bank = Bank::new(&genesis_config);
         let blockhash = bank.last_blockhash();
         let mut bank_forks = BankForks::new(0, bank);
@@ -856,7 +867,8 @@ pub(crate) mod tests {
         let mut block_commitment = HashMap::new();
         block_commitment.entry(0).or_insert(cache0.clone());
         block_commitment.entry(1).or_insert(cache1.clone());
-        let block_commitment_cache = BlockCommitmentCache::new(block_commitment, 10, bank1, 0);
+        let block_commitment_cache =
+            BlockCommitmentCache::new(block_commitment, 0, 10, bank1, blockstore, 0);
 
         let exit = Arc::new(AtomicBool::new(false));
         let subscriptions =
@@ -959,9 +971,13 @@ pub(crate) mod tests {
             Subscriber::new_test("slotNotification");
         let sub_id = SubscriptionId::Number(0 as u64);
         let exit = Arc::new(AtomicBool::new(false));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let subscriptions = RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         );
         subscriptions.add_slot_subscription(sub_id.clone(), subscriber);
 
@@ -1001,9 +1017,13 @@ pub(crate) mod tests {
             Subscriber::new_test("rootNotification");
         let sub_id = SubscriptionId::Number(0 as u64);
         let exit = Arc::new(AtomicBool::new(false));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
         let subscriptions = RpcSubscriptions::new(
             &exit,
-            Arc::new(RwLock::new(BlockCommitmentCache::new_for_tests())),
+            Arc::new(RwLock::new(
+                BlockCommitmentCache::new_for_tests_with_blockstore(blockstore),
+            )),
         );
         subscriptions.add_root_subscription(sub_id.clone(), subscriber);
 

core/src/serve_repair.rs

@@ -1,18 +1,21 @@
-use crate::packet::limited_deserialize;
 use crate::streamer::{PacketReceiver, PacketSender};
 use crate::{
     cluster_info::{ClusterInfo, ClusterInfoError},
     contact_info::ContactInfo,
-    packet::Packet,
+    repair_response,
+    repair_service::RepairStats,
     result::{Error, Result},
 };
 use bincode::serialize;
 use rand::{thread_rng, Rng};
-use solana_ledger::blockstore::Blockstore;
+use solana_ledger::{
+    blockstore::Blockstore,
+    shred::{Nonce, Shred},
+};
 use solana_measure::measure::Measure;
 use solana_measure::thread_mem_usage;
 use solana_metrics::{datapoint_debug, inc_new_counter_debug};
-use solana_perf::packet::{Packets, PacketsRecycler};
+use solana_perf::packet::{limited_deserialize, Packets, PacketsRecycler};
 use solana_sdk::{
     clock::Slot,
     signature::{Keypair, Signer},
@@ -28,6 +31,7 @@ use std::{
 
 /// the number of slots to respond with when responding to `Orphan` requests
 pub const MAX_ORPHAN_REPAIR_RESPONSES: usize = 10;
+pub const DEFAULT_NONCE: u32 = 42;
 
 #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
 pub enum RepairType {
@@ -46,12 +50,25 @@ impl RepairType {
     }
 }
 
+#[derive(Default)]
+pub struct ServeRepairStats {
+    pub total_packets: usize,
+    pub processed: usize,
+    pub self_repair: usize,
+    pub window_index: usize,
+    pub highest_window_index: usize,
+    pub orphan: usize,
+}
+
 /// Window protocol messages
 #[derive(Serialize, Deserialize, Debug)]
 enum RepairProtocol {
     WindowIndex(ContactInfo, u64, u64),
     HighestWindowIndex(ContactInfo, u64, u64),
     Orphan(ContactInfo, u64),
+    WindowIndexWithNonce(ContactInfo, u64, u64, Nonce),
+    HighestWindowIndexWithNonce(ContactInfo, u64, u64, Nonce),
+    OrphanWithNonce(ContactInfo, u64, Nonce),
 }
 
 #[derive(Clone)]
@@ -95,6 +112,9 @@ impl ServeRepair {
             RepairProtocol::WindowIndex(ref from, _, _) => from,
             RepairProtocol::HighestWindowIndex(ref from, _, _) => from,
             RepairProtocol::Orphan(ref from, _) => from,
+            RepairProtocol::WindowIndexWithNonce(ref from, _, _, _) => from,
+            RepairProtocol::HighestWindowIndexWithNonce(ref from, _, _, _) => from,
+            RepairProtocol::OrphanWithNonce(ref from, _, _) => from,
         }
     }
 
@@ -104,6 +124,7 @@ impl ServeRepair {
         from_addr: &SocketAddr,
         blockstore: Option<&Arc<Blockstore>>,
         request: RepairProtocol,
+        stats: &mut ServeRepairStats,
     ) -> Option<Packets> {
         let now = Instant::now();
 
@@ -111,18 +132,14 @@ impl ServeRepair {
         let my_id = me.read().unwrap().keypair.pubkey();
         let from = Self::get_repair_sender(&request);
         if from.id == my_id {
-            warn!(
-                "{}: Ignored received repair request from ME {}",
-                my_id, from.id,
-            );
-            inc_new_counter_debug!("serve_repair-handle-repair--eq", 1);
+            stats.self_repair += 1;
             return None;
         }
 
         let (res, label) = {
             match &request {
                 RepairProtocol::WindowIndex(from, slot, shred_index) => {
-                    inc_new_counter_debug!("serve_repair-request-window-index", 1);
+                    stats.window_index += 1;
                     (
                         Self::run_window_request(
                             recycler,
@@ -132,13 +149,14 @@ impl ServeRepair {
                             &me.read().unwrap().my_info,
                             *slot,
                             *shred_index,
+                            None,
                         ),
                         "WindowIndex",
                     )
                 }
 
                 RepairProtocol::HighestWindowIndex(_, slot, highest_index) => {
-                    inc_new_counter_debug!("serve_repair-request-highest-window-index", 1);
+                    stats.highest_window_index += 1;
                     (
                         Self::run_highest_window_request(
                             recycler,
@@ -146,12 +164,13 @@ impl ServeRepair {
                             blockstore,
                             *slot,
                             *highest_index,
+                            None,
                         ),
                         "HighestWindowIndex",
                     )
                 }
                 RepairProtocol::Orphan(_, slot) => {
-                    inc_new_counter_debug!("serve_repair-request-orphan", 1);
+                    stats.orphan += 1;
                     (
                         Self::run_orphan(
                             recycler,
@@ -159,10 +178,55 @@ impl ServeRepair {
                             blockstore,
                             *slot,
                             MAX_ORPHAN_REPAIR_RESPONSES,
+                            None,
                         ),
                         "Orphan",
                     )
                 }
+                RepairProtocol::WindowIndexWithNonce(_, slot, shred_index, nonce) => {
+                    stats.window_index += 1;
+                    (
+                        Self::run_window_request(
+                            recycler,
+                            from,
+                            &from_addr,
+                            blockstore,
+                            &me.read().unwrap().my_info,
+                            *slot,
+                            *shred_index,
+                            Some(*nonce),
+                        ),
+                        "WindowIndexWithNonce",
+                    )
+                }
+                RepairProtocol::HighestWindowIndexWithNonce(_, slot, highest_index, nonce) => {
+                    stats.highest_window_index += 1;
+                    (
+                        Self::run_highest_window_request(
+                            recycler,
+                            &from_addr,
+                            blockstore,
+                            *slot,
+                            *highest_index,
+                            Some(*nonce),
+                        ),
+                        "HighestWindowIndexWithNonce",
+                    )
+                }
+                RepairProtocol::OrphanWithNonce(_, slot, nonce) => {
+                    stats.orphan += 1;
+                    (
+                        Self::run_orphan(
+                            recycler,
+                            &from_addr,
+                            blockstore,
+                            *slot,
+                            MAX_ORPHAN_REPAIR_RESPONSES,
+                            Some(*nonce),
+                        ),
+                        "OrphanWithNonce",
+                    )
+                }
             }
         };
 
@@ -186,6 +250,7 @@ impl ServeRepair {
         requests_receiver: &PacketReceiver,
         response_sender: &PacketSender,
         max_packets: &mut usize,
+        stats: &mut ServeRepairStats,
     ) -> Result<()> {
         //TODO cache connections
         let timeout = Duration::new(1, 0);
@@ -202,7 +267,7 @@ impl ServeRepair {
 
         let mut time = Measure::start("repair::handle_packets");
         for reqs in reqs_v {
-            Self::handle_packets(obj, &recycler, blockstore, reqs, response_sender);
+            Self::handle_packets(obj, &recycler, blockstore, reqs, response_sender, stats);
         }
         time.stop();
         if total_packets >= *max_packets {
@@ -215,6 +280,31 @@ impl ServeRepair {
         Ok(())
     }
 
+    fn report_reset_stats(me: &Arc<RwLock<Self>>, stats: &mut ServeRepairStats) {
+        if stats.self_repair > 0 {
+            let my_id = me.read().unwrap().keypair.pubkey();
+            warn!(
+                "{}: Ignored received repair requests from ME: {}",
+                my_id, stats.self_repair,
+            );
+            inc_new_counter_debug!("serve_repair-handle-repair--eq", stats.self_repair);
+        }
+
+        debug!(
+            "repair_listener: total_packets: {} passed: {}",
+            stats.total_packets, stats.processed
+        );
+
+        inc_new_counter_debug!("serve_repair-request-window-index", stats.window_index);
+        inc_new_counter_debug!(
+            "serve_repair-request-highest-window-index",
+            stats.highest_window_index
+        );
+        inc_new_counter_debug!("serve_repair-request-orphan", stats.orphan);
+
+        *stats = ServeRepairStats::default();
+    }
+
     pub fn listen(
         me: Arc<RwLock<Self>>,
         blockstore: Option<Arc<Blockstore>>,
@@ -228,6 +318,8 @@ impl ServeRepair {
             .name("solana-repair-listen".to_string())
             .spawn(move || {
                 let mut max_packets = 1024;
+                let mut last_print = Instant::now();
+                let mut stats = ServeRepairStats::default();
                 loop {
                     let result = Self::run_listen(
                         &me,
@@ -236,6 +328,7 @@ impl ServeRepair {
                         &requests_receiver,
                         &response_sender,
                         &mut max_packets,
+                        &mut stats,
                     );
                     match result {
                         Err(Error::RecvTimeoutError(_)) | Ok(_) => {}
@@ -244,6 +337,10 @@ impl ServeRepair {
                     if exit.load(Ordering::Relaxed) {
                         return;
                     }
+                    if last_print.elapsed().as_secs() > 2 {
+                        Self::report_reset_stats(&me, &mut stats);
+                        last_print = Instant::now();
+                    }
                     thread_mem_usage::datapoint("solana-repair-listen");
                 }
             })
@@ -256,6 +353,7 @@ impl ServeRepair {
         blockstore: Option<&Arc<Blockstore>>,
         packets: Packets,
         response_sender: &PacketSender,
+        stats: &mut ServeRepairStats,
     ) {
         // iter over the packets, collect pulls separately and process everything else
         let allocated = thread_mem_usage::Allocatedp::default();
@@ -265,7 +363,9 @@ impl ServeRepair {
             limited_deserialize(&packet.data[..packet.meta.size])
                 .into_iter()
                 .for_each(|request| {
-                    let rsp = Self::handle_repair(me, recycler, &from_addr, blockstore, request);
+                    stats.processed += 1;
+                    let rsp =
+                        Self::handle_repair(me, recycler, &from_addr, blockstore, request, stats);
                     if let Some(rsp) = rsp {
                         let _ignore_disconnect = response_sender.send(rsp);
                     }
@@ -277,27 +377,59 @@ impl ServeRepair {
         });
     }
 
-    fn window_index_request_bytes(&self, slot: Slot, shred_index: u64) -> Result<Vec<u8>> {
-        let req = RepairProtocol::WindowIndex(self.my_info.clone(), slot, shred_index);
+    fn window_index_request_bytes(
+        &self,
+        slot: Slot,
+        shred_index: u64,
+        nonce: Option<Nonce>,
+    ) -> Result<Vec<u8>> {
+        let req = if let Some(nonce) = nonce {
+            RepairProtocol::WindowIndexWithNonce(self.my_info.clone(), slot, shred_index, nonce)
+        } else {
+            RepairProtocol::WindowIndex(self.my_info.clone(), slot, shred_index)
+        };
         let out = serialize(&req)?;
         Ok(out)
     }
 
-    fn window_highest_index_request_bytes(&self, slot: Slot, shred_index: u64) -> Result<Vec<u8>> {
-        let req = RepairProtocol::HighestWindowIndex(self.my_info.clone(), slot, shred_index);
+    fn window_highest_index_request_bytes(
+        &self,
+        slot: Slot,
+        shred_index: u64,
+        nonce: Option<Nonce>,
+    ) -> Result<Vec<u8>> {
+        let req = if let Some(nonce) = nonce {
+            RepairProtocol::HighestWindowIndexWithNonce(
+                self.my_info.clone(),
+                slot,
+                shred_index,
+                nonce,
+            )
+        } else {
+            RepairProtocol::HighestWindowIndex(self.my_info.clone(), slot, shred_index)
+        };
         let out = serialize(&req)?;
         Ok(out)
     }
 
-    fn orphan_bytes(&self, slot: Slot) -> Result<Vec<u8>> {
-        let req = RepairProtocol::Orphan(self.my_info.clone(), slot);
+    fn orphan_bytes(&self, slot: Slot, nonce: Option<Nonce>) -> Result<Vec<u8>> {
+        let req = if let Some(nonce) = nonce {
+            RepairProtocol::OrphanWithNonce(self.my_info.clone(), slot, nonce)
+        } else {
+            RepairProtocol::Orphan(self.my_info.clone(), slot)
+        };
         let out = serialize(&req)?;
         Ok(out)
     }
 
-    pub fn repair_request(&self, repair_request: &RepairType) -> Result<(SocketAddr, Vec<u8>)> {
+    pub fn repair_request(
+        &self,
+        repair_request: &RepairType,
+        repair_stats: &mut RepairStats,
+    ) -> Result<(SocketAddr, Vec<u8>)> {
         // find a peer that appears to be accepting replication and has the desired slot, as indicated
         // by a valid tvu port location
+        let slot = repair_request.slot();
         let valid: Vec<_> = self
             .cluster_info
             .read()
@@ -308,32 +440,38 @@ impl ServeRepair {
         }
         let n = thread_rng().gen::<usize>() % valid.len();
         let addr = valid[n].serve_repair; // send the request to the peer's serve_repair port
-        let out = self.map_repair_request(repair_request)?;
+        let nonce = if Shred::is_nonce_unlocked(slot) {
+            Some(DEFAULT_NONCE)
+        } else {
+            None
+        };
+        let out = self.map_repair_request(&repair_request, repair_stats, nonce)?;
 
         Ok((addr, out))
     }
 
-    pub fn map_repair_request(&self, repair_request: &RepairType) -> Result<Vec<u8>> {
+    pub fn map_repair_request(
+        &self,
+        repair_request: &RepairType,
+        repair_stats: &mut RepairStats,
+        nonce: Option<Nonce>,
+    ) -> Result<Vec<u8>> {
+        let slot = repair_request.slot();
+        if Shred::is_nonce_unlocked(slot) {
+            assert!(nonce.is_some());
+        }
         match repair_request {
             RepairType::Shred(slot, shred_index) => {
-                datapoint_debug!(
-                    "serve_repair-repair",
-                    ("repair-slot", *slot, i64),
-                    ("repair-ix", *shred_index, i64)
-                );
-                Ok(self.window_index_request_bytes(*slot, *shred_index)?)
+                repair_stats.shred.update(*slot);
+                Ok(self.window_index_request_bytes(*slot, *shred_index, nonce)?)
             }
             RepairType::HighestShred(slot, shred_index) => {
-                datapoint_info!(
-                    "serve_repair-repair_highest",
-                    ("repair-highest-slot", *slot, i64),
-                    ("repair-highest-ix", *shred_index, i64)
-                );
-                Ok(self.window_highest_index_request_bytes(*slot, *shred_index)?)
+                repair_stats.highest_shred.update(*slot);
+                Ok(self.window_highest_index_request_bytes(*slot, *shred_index, nonce)?)
             }
             RepairType::Orphan(slot) => {
-                datapoint_info!("serve_repair-repair_orphan", ("repair-orphan", *slot, i64));
-                Ok(self.orphan_bytes(*slot)?)
+                repair_stats.orphan.update(*slot);
+                Ok(self.orphan_bytes(*slot, nonce)?)
             }
         }
     }
@@ -346,12 +484,19 @@ impl ServeRepair {
         me: &ContactInfo,
         slot: Slot,
         shred_index: u64,
+        nonce: Option<Nonce>,
     ) -> Option<Packets> {
         if let Some(blockstore) = blockstore {
             // Try to find the requested index in one of the slots
-            let packet = Self::get_data_shred_as_packet(blockstore, slot, shred_index, from_addr);
+            let packet = repair_response::repair_response_packet(
+                blockstore,
+                slot,
+                shred_index,
+                from_addr,
+                nonce,
+            );
 
-            if let Ok(Some(packet)) = packet {
+            if let Some(packet) = packet {
                 inc_new_counter_debug!("serve_repair-window-request-ledger", 1);
                 return Some(Packets::new_with_recycler_data(
                     recycler,
@@ -379,15 +524,20 @@ impl ServeRepair {
         blockstore: Option<&Arc<Blockstore>>,
         slot: Slot,
         highest_index: u64,
+        nonce: Option<Nonce>,
     ) -> Option<Packets> {
         let blockstore = blockstore?;
         // Try to find the requested index in one of the slots
         let meta = blockstore.meta(slot).ok()??;
         if meta.received > highest_index {
             // meta.received must be at least 1 by this point
-            let packet =
-                Self::get_data_shred_as_packet(blockstore, slot, meta.received - 1, from_addr)
-                    .ok()??;
+            let packet = repair_response::repair_response_packet(
+                blockstore,
+                slot,
+                meta.received - 1,
+                from_addr,
+                nonce,
+            )?;
             return Some(Packets::new_with_recycler_data(
                 recycler,
                 "run_highest_window_request",
@@ -403,6 +553,7 @@ impl ServeRepair {
         blockstore: Option<&Arc<Blockstore>>,
         mut slot: Slot,
         max_responses: usize,
+        nonce: Option<Nonce>,
     ) -> Option<Packets> {
         let mut res = Packets::new_with_recycler(recycler.clone(), 64, "run_orphan");
         if let Some(blockstore) = blockstore {
@@ -411,9 +562,19 @@ impl ServeRepair {
                 if meta.received == 0 {
                     break;
                 }
-                let packet =
-                    Self::get_data_shred_as_packet(blockstore, slot, meta.received - 1, from_addr);
-                if let Ok(Some(packet)) = packet {
+                let nonce = if Shred::is_nonce_unlocked(slot) {
+                    nonce
+                } else {
+                    None
+                };
+                let packet = repair_response::repair_response_packet(
+                    blockstore,
+                    slot,
+                    meta.received - 1,
+                    from_addr,
+                    nonce,
+                );
+                if let Some(packet) = packet {
                     res.packets.push(packet);
                 }
                 if meta.is_parent_set() && res.packets.len() <= max_responses {
@@ -428,41 +589,31 @@ impl ServeRepair {
         }
         Some(res)
     }
-
-    fn get_data_shred_as_packet(
-        blockstore: &Arc<Blockstore>,
-        slot: Slot,
-        shred_index: u64,
-        dest: &SocketAddr,
-    ) -> Result<Option<Packet>> {
-        let data = blockstore.get_data_shred(slot, shred_index)?;
-        Ok(data.map(|data| {
-            let mut packet = Packet::default();
-            packet.meta.size = data.len();
-            packet.meta.set_addr(dest);
-            packet.data.copy_from_slice(&data);
-            packet
-        }))
-    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::result::Error;
+    use crate::{repair_response, result::Error};
     use solana_ledger::get_tmp_ledger_path;
     use solana_ledger::{
         blockstore::make_many_slot_entries,
         blockstore_processor::fill_blockstore_slot_with_ticks,
         shred::{
             max_ticks_per_n_shreds, CodingShredHeader, DataShredHeader, Shred, ShredCommonHeader,
+            NONCE_SHRED_PAYLOAD_SIZE, UNLOCK_NONCE_SLOT,
         },
     };
     use solana_sdk::{hash::Hash, pubkey::Pubkey, timing::timestamp};
 
-    /// test run_window_requestwindow requests respond with the right shred, and do not overrun
     #[test]
-    fn run_highest_window_request() {
+    fn test_run_highest_window_request() {
+        run_highest_window_request(UNLOCK_NONCE_SLOT + 3, 3, Some(9));
+        run_highest_window_request(UNLOCK_NONCE_SLOT, 3, None);
+    }
+
+    /// test run_window_request responds with the right shred, and do not overrun
+    fn run_highest_window_request(slot: Slot, num_slots: u64, nonce: Option<Nonce>) {
         let recycler = PacketsRecycler::default();
         solana_logger::setup();
         let ledger_path = get_tmp_ledger_path!();
@@ -474,41 +625,51 @@ mod tests {
                 Some(&blockstore),
                 0,
                 0,
+                nonce,
             );
             assert!(rv.is_none());
 
             let _ = fill_blockstore_slot_with_ticks(
                 &blockstore,
-                max_ticks_per_n_shreds(1) + 1,
-                2,
-                1,
+                max_ticks_per_n_shreds(1, None) + 1,
+                slot,
+                slot - num_slots + 1,
                 Hash::default(),
             );
 
+            let index = 1;
             let rv = ServeRepair::run_highest_window_request(
                 &recycler,
                 &socketaddr_any!(),
                 Some(&blockstore),
-                2,
-                1,
-            );
+                slot,
+                index,
+                nonce,
+            )
+            .expect("packets");
+
             let rv: Vec<Shred> = rv
-                .expect("packets")
                 .packets
                 .into_iter()
-                .filter_map(|b| Shred::new_from_serialized_shred(b.data.to_vec()).ok())
+                .filter_map(|b| {
+                    if nonce.is_some() {
+                        assert_eq!(repair_response::nonce(&b.data[..]), nonce);
+                    }
+                    Shred::new_from_serialized_shred(b.data.to_vec()).ok()
+                })
                 .collect();
             assert!(!rv.is_empty());
-            let index = blockstore.meta(2).unwrap().unwrap().received - 1;
+            let index = blockstore.meta(slot).unwrap().unwrap().received - 1;
             assert_eq!(rv[0].index(), index as u32);
-            assert_eq!(rv[0].slot(), 2);
+            assert_eq!(rv[0].slot(), slot);
 
             let rv = ServeRepair::run_highest_window_request(
                 &recycler,
                 &socketaddr_any!(),
                 Some(&blockstore),
-                2,
+                slot,
                 index + 1,
+                nonce,
             );
             assert!(rv.is_none());
         }
@@ -516,9 +677,14 @@ mod tests {
         Blockstore::destroy(&ledger_path).expect("Expected successful database destruction");
     }
 
-    /// test window requests respond with the right shred, and do not overrun
     #[test]
-    fn run_window_request() {
+    fn test_run_window_request() {
+        run_window_request(UNLOCK_NONCE_SLOT + 1, Some(9));
+        run_window_request(UNLOCK_NONCE_SLOT - 3, None);
+    }
+
+    /// test window requests respond with the right shred, and do not overrun
+    fn run_window_request(slot: Slot, nonce: Option<Nonce>) {
         let recycler = PacketsRecycler::default();
         solana_logger::setup();
         let ledger_path = get_tmp_ledger_path!();
@@ -545,12 +711,13 @@ mod tests {
                 &socketaddr_any!(),
                 Some(&blockstore),
                 &me,
+                slot,
                 0,
-                0,
+                nonce,
             );
             assert!(rv.is_none());
             let mut common_header = ShredCommonHeader::default();
-            common_header.slot = 2;
+            common_header.slot = slot;
             common_header.index = 1;
             let mut data_header = DataShredHeader::default();
             data_header.parent_offset = 1;
@@ -558,30 +725,37 @@ mod tests {
                 common_header,
                 data_header,
                 CodingShredHeader::default(),
+                NONCE_SHRED_PAYLOAD_SIZE,
             );
 
             blockstore
                 .insert_shreds(vec![shred_info], None, false)
                 .expect("Expect successful ledger write");
 
+            let index = 1;
             let rv = ServeRepair::run_window_request(
                 &recycler,
                 &me,
                 &socketaddr_any!(),
                 Some(&blockstore),
                 &me,
-                2,
-                1,
-            );
-            assert!(!rv.is_none());
+                slot,
+                index,
+                nonce,
+            )
+            .expect("packets");
             let rv: Vec<Shred> = rv
-                .expect("packets")
                 .packets
                 .into_iter()
-                .filter_map(|b| Shred::new_from_serialized_shred(b.data.to_vec()).ok())
+                .filter_map(|b| {
+                    if nonce.is_some() {
+                        assert_eq!(repair_response::nonce(&b.data[..]), nonce);
+                    }
+                    Shred::new_from_serialized_shred(b.data.to_vec()).ok()
+                })
                 .collect();
             assert_eq!(rv[0].index(), 1);
-            assert_eq!(rv[0].slot(), 2);
+            assert_eq!(rv[0].slot(), slot);
         }
 
         Blockstore::destroy(&ledger_path).expect("Expected successful database destruction");
@@ -592,7 +766,7 @@ mod tests {
         let me = ContactInfo::new_localhost(&Pubkey::new_rand(), timestamp());
         let cluster_info = Arc::new(RwLock::new(ClusterInfo::new_with_invalid_keypair(me)));
         let serve_repair = ServeRepair::new(cluster_info.clone());
-        let rv = serve_repair.repair_request(&RepairType::Shred(0, 0));
+        let rv = serve_repair.repair_request(&RepairType::Shred(0, 0), &mut RepairStats::default());
         assert_matches!(rv, Err(Error::ClusterInfoError(ClusterInfoError::NoPeers)));
 
         let serve_repair_addr = socketaddr!([127, 0, 0, 1], 1243);
@@ -613,7 +787,7 @@ mod tests {
         };
         cluster_info.write().unwrap().insert_info(nxt.clone());
         let rv = serve_repair
-            .repair_request(&RepairType::Shred(0, 0))
+            .repair_request(&RepairType::Shred(0, 0), &mut RepairStats::default())
             .unwrap();
         assert_eq!(nxt.serve_repair, serve_repair_addr);
         assert_eq!(rv.0, nxt.serve_repair);
@@ -640,7 +814,7 @@ mod tests {
         while !one || !two {
             //this randomly picks an option, so eventually it should pick both
             let rv = serve_repair
-                .repair_request(&RepairType::Shred(0, 0))
+                .repair_request(&RepairType::Shred(0, 0), &mut RepairStats::default())
                 .unwrap();
             if rv.0 == serve_repair_addr {
                 one = true;
@@ -653,52 +827,85 @@ mod tests {
     }
 
     #[test]
-    fn run_orphan() {
+    fn test_run_orphan() {
+        run_orphan(UNLOCK_NONCE_SLOT + 1, 3, Some(9));
+        // Test where the response will be for some slots <= UNLOCK_NONCE_SLOT,
+        // and some of the response will be for some slots > UNLOCK_NONCE_SLOT.
+        // Should not panic.
+        run_orphan(UNLOCK_NONCE_SLOT, 3, None);
+        run_orphan(UNLOCK_NONCE_SLOT, 3, Some(9));
+    }
+
+    fn run_orphan(slot: Slot, num_slots: u64, nonce: Option<Nonce>) {
         solana_logger::setup();
         let recycler = PacketsRecycler::default();
         let ledger_path = get_tmp_ledger_path!();
         {
             let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
-            let rv =
-                ServeRepair::run_orphan(&recycler, &socketaddr_any!(), Some(&blockstore), 2, 0);
+            let rv = ServeRepair::run_orphan(
+                &recycler,
+                &socketaddr_any!(),
+                Some(&blockstore),
+                slot,
+                0,
+                nonce,
+            );
             assert!(rv.is_none());
 
-            // Create slots 1, 2, 3 with 5 shreds apiece
-            let (shreds, _) = make_many_slot_entries(1, 3, 5);
+            // Create slots [slot, slot + num_slots) with 5 shreds apiece
+            let (shreds, _) = make_many_slot_entries(slot, num_slots, 5);
 
             blockstore
                 .insert_shreds(shreds, None, false)
                 .expect("Expect successful ledger write");
 
-            // We don't have slot 4, so we don't know how to service this requeset
-            let rv =
-                ServeRepair::run_orphan(&recycler, &socketaddr_any!(), Some(&blockstore), 4, 5);
+            // We don't have slot `slot + num_slots`, so we don't know how to service this request
+            let rv = ServeRepair::run_orphan(
+                &recycler,
+                &socketaddr_any!(),
+                Some(&blockstore),
+                slot + num_slots,
+                5,
+                nonce,
+            );
             assert!(rv.is_none());
 
-            // For slot 3, we should return the highest shreds from slots 3, 2, 1 respectively
-            // for this request
-            let rv: Vec<_> =
-                ServeRepair::run_orphan(&recycler, &socketaddr_any!(), Some(&blockstore), 3, 5)
-                    .expect("run_orphan packets")
-                    .packets
-                    .iter()
-                    .map(|b| b.clone())
-                    .collect();
-            let expected: Vec<_> = (1..=3)
+            // For a orphan request for `slot + num_slots - 1`, we should return the highest shreds
+            // from slots in the range [slot, slot + num_slots - 1]
+            let rv: Vec<_> = ServeRepair::run_orphan(
+                &recycler,
+                &socketaddr_any!(),
+                Some(&blockstore),
+                slot + num_slots - 1,
+                5,
+                nonce,
+            )
+            .expect("run_orphan packets")
+            .packets
+            .iter()
+            .map(|b| b.clone())
+            .collect();
+
+            // Verify responses
+            let expected: Vec<_> = (slot..slot + num_slots)
                 .rev()
-                .map(|slot| {
+                .filter_map(|slot| {
+                    let nonce = if Shred::is_nonce_unlocked(slot) {
+                        nonce
+                    } else {
+                        None
+                    };
                     let index = blockstore.meta(slot).unwrap().unwrap().received - 1;
-                    ServeRepair::get_data_shred_as_packet(
+                    repair_response::repair_response_packet(
                         &blockstore,
                         slot,
                         index,
                         &socketaddr_any!(),
+                        nonce,
                     )
-                    .unwrap()
-                    .unwrap()
                 })
                 .collect();
-            assert_eq!(rv, expected)
+            assert_eq!(rv, expected);
         }
 
         Blockstore::destroy(&ledger_path).expect("Expected successful database destruction");

core/src/storage_stage.rs

@@ -663,6 +663,7 @@ mod tests {
     use super::*;
     use crate::genesis_utils::{create_genesis_config, GenesisConfigInfo};
     use rayon::prelude::*;
+    use solana_ledger::get_tmp_ledger_path;
     use solana_runtime::bank::Bank;
     use solana_sdk::{
         hash::Hasher,
@@ -690,7 +691,11 @@ mod tests {
             &[bank.clone()],
             vec![0],
         )));
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
+        let ledger_path = get_tmp_ledger_path!();
+        let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::default_with_blockstore(blockstore),
+        ));
         let (_slot_sender, slot_receiver) = channel();
         let storage_state = StorageState::new(
             &bank.last_blockhash(),

core/src/tvu.rs

@@ -301,7 +301,9 @@ pub mod tests {
         let voting_keypair = Keypair::new();
         let storage_keypair = Arc::new(Keypair::new());
         let leader_schedule_cache = Arc::new(LeaderScheduleCache::new_from_bank(&bank));
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::default_with_blockstore(blockstore.clone()),
+        ));
         let tvu = Tvu::new(
             &voting_keypair.pubkey(),
             Some(Arc::new(voting_keypair)),
@@ -320,10 +322,7 @@ pub mod tests {
             &StorageState::default(),
             None,
             l_receiver,
-            &Arc::new(RpcSubscriptions::new(
-                &exit,
-                Arc::new(RwLock::new(BlockCommitmentCache::default())),
-            )),
+            &Arc::new(RpcSubscriptions::new(&exit, block_commitment_cache.clone())),
             &poh_recorder,
             &leader_schedule_cache,
             &exit,

core/src/validator.rs

@@ -202,7 +202,6 @@ impl Validator {
         }
 
         let bank_forks = Arc::new(RwLock::new(bank_forks));
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
 
         let mut validator_exit = ValidatorExit::default();
         let exit_ = exit.clone();
@@ -238,6 +237,9 @@ impl Validator {
         );
 
         let blockstore = Arc::new(blockstore);
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::default_with_blockstore(blockstore.clone()),
+        ));
 
         let subscriptions = Arc::new(RpcSubscriptions::new(&exit, block_commitment_cache.clone()));
 
@@ -312,6 +314,13 @@ impl Validator {
         );
 
         if config.dev_halt_at_slot.is_some() {
+            // Simulate a confirmed root to avoid RPC errors with CommitmentmentConfig::max() and
+            // to ensure RPC endpoints like getConfirmedBlock, which require a confirmed root, work
+            block_commitment_cache
+                .write()
+                .unwrap()
+                .set_get_largest_confirmed_root(bank_forks.read().unwrap().root());
+
             // Park with the RPC service running, ready for inspection!
             warn!("Validator halted");
             std::thread::park();

core/src/window_service.rs

@@ -1,31 +1,37 @@
 //! `window_service` handles the data plane incoming shreds, storing them in
 //!   blockstore and retransmitting where required
-//!
-use crate::cluster_info::ClusterInfo;
-use crate::packet::Packets;
-use crate::repair_service::{RepairService, RepairStrategy};
-use crate::result::{Error, Result};
-use crate::streamer::PacketSender;
+use crate::{
+    cluster_info::ClusterInfo,
+    packet::Packets,
+    repair_response,
+    repair_service::{RepairService, RepairStrategy},
+    result::{Error, Result},
+    serve_repair::DEFAULT_NONCE,
+    streamer::PacketSender,
+};
 use crossbeam_channel::{
     unbounded, Receiver as CrossbeamReceiver, RecvTimeoutError, Sender as CrossbeamSender,
 };
 use rayon::iter::IntoParallelRefMutIterator;
 use rayon::iter::ParallelIterator;
 use rayon::ThreadPool;
-use solana_ledger::bank_forks::BankForks;
-use solana_ledger::blockstore::{self, Blockstore, MAX_DATA_SHREDS_PER_SLOT};
-use solana_ledger::leader_schedule_cache::LeaderScheduleCache;
-use solana_ledger::shred::Shred;
+use solana_ledger::{
+    bank_forks::BankForks,
+    blockstore::{self, Blockstore, MAX_DATA_SHREDS_PER_SLOT},
+    leader_schedule_cache::LeaderScheduleCache,
+    shred::{Nonce, Shred},
+};
 use solana_metrics::{inc_new_counter_debug, inc_new_counter_error};
 use solana_rayon_threadlimit::get_thread_count;
 use solana_runtime::bank::Bank;
-use solana_sdk::pubkey::Pubkey;
-use solana_sdk::timing::duration_as_ms;
-use std::net::UdpSocket;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, RwLock};
-use std::thread::{self, Builder, JoinHandle};
-use std::time::{Duration, Instant};
+use solana_sdk::{packet::PACKET_DATA_SIZE, pubkey::Pubkey, timing::duration_as_ms};
+use std::{
+    net::{SocketAddr, UdpSocket},
+    sync::atomic::{AtomicBool, Ordering},
+    sync::{Arc, RwLock},
+    thread::{self, Builder, JoinHandle},
+    time::{Duration, Instant},
+};
 
 fn verify_shred_slot(shred: &Shred, root: u64) -> bool {
     if shred.is_data() {
@@ -102,8 +108,15 @@ fn run_check_duplicate(
     Ok(())
 }
 
+fn verify_repair(_shred: &Shred, repair_info: &Option<RepairMeta>) -> bool {
+    repair_info
+        .as_ref()
+        .map(|repair_info| repair_info.nonce == DEFAULT_NONCE)
+        .unwrap_or(true)
+}
+
 fn run_insert<F>(
-    shred_receiver: &CrossbeamReceiver<Vec<Shred>>,
+    shred_receiver: &CrossbeamReceiver<(Vec<Shred>, Vec<Option<RepairMeta>>)>,
     blockstore: &Arc<Blockstore>,
     leader_schedule_cache: &Arc<LeaderScheduleCache>,
     handle_duplicate: F,
@@ -112,12 +125,16 @@ where
     F: Fn(Shred) -> (),
 {
     let timer = Duration::from_millis(200);
-    let mut shreds = shred_receiver.recv_timeout(timer)?;
-
-    while let Ok(mut more_shreds) = shred_receiver.try_recv() {
-        shreds.append(&mut more_shreds)
+    let (mut shreds, mut repair_infos) = shred_receiver.recv_timeout(timer)?;
+    while let Ok((more_shreds, more_repair_infos)) = shred_receiver.try_recv() {
+        shreds.extend(more_shreds);
+        repair_infos.extend(more_repair_infos);
     }
 
+    assert_eq!(shreds.len(), repair_infos.len());
+    let mut i = 0;
+    shreds.retain(|shred| (verify_repair(&shred, &repair_infos[i]), i += 1).0);
+
     let blockstore_insert_metrics = blockstore.insert_shreds_handle_duplicate(
         shreds,
         Some(leader_schedule_cache),
@@ -131,7 +148,7 @@ where
 
 fn recv_window<F>(
     blockstore: &Arc<Blockstore>,
-    insert_shred_sender: &CrossbeamSender<Vec<Shred>>,
+    insert_shred_sender: &CrossbeamSender<(Vec<Shred>, Vec<Option<RepairMeta>>)>,
     my_pubkey: &Pubkey,
     verified_receiver: &CrossbeamReceiver<Vec<Packets>>,
     retransmit: &PacketSender,
@@ -155,7 +172,7 @@ where
     inc_new_counter_debug!("streamer-recv_window-recv", total_packets);
 
     let last_root = blockstore.last_root();
-    let shreds: Vec<_> = thread_pool.install(|| {
+    let (shreds, repair_infos): (Vec<_>, Vec<_>) = thread_pool.install(|| {
         packets
             .par_iter_mut()
             .flat_map(|packets| {
@@ -164,34 +181,59 @@ where
                     .iter_mut()
                     .filter_map(|packet| {
                         if packet.meta.discard {
-                            inc_new_counter_debug!("streamer-recv_window-invalid_signature", 1);
+                            inc_new_counter_debug!(
+                                "streamer-recv_window-invalid_or_unnecessary_packet",
+                                1
+                            );
                             None
-                        } else if let Ok(shred) =
-                            Shred::new_from_serialized_shred(packet.data.to_vec())
-                        {
-                            if shred_filter(&shred, last_root) {
-                                // Mark slot as dead if the current shred is on the boundary
-                                // of max shreds per slot. However, let the current shred
-                                // get retransmitted. It'll allow peer nodes to see this shred
-                                // and trigger them to mark the slot as dead.
-                                if shred.index() >= (MAX_DATA_SHREDS_PER_SLOT - 1) as u32 {
-                                    let _ = blockstore.set_dead_slot(shred.slot());
+                        } else {
+                            // shred fetch stage should be sending packets
+                            // with sufficiently large buffers. Needed to ensure
+                            // call to `new_from_serialized_shred` is safe.
+                            assert_eq!(packet.data.len(), PACKET_DATA_SIZE);
+                            let serialized_shred = packet.data.to_vec();
+                            if let Ok(shred) = Shred::new_from_serialized_shred(serialized_shred) {
+                                let repair_info = {
+                                    if packet.meta.repair && Shred::is_nonce_unlocked(shred.slot())
+                                    {
+                                        if let Some(nonce) = repair_response::nonce(&packet.data) {
+                                            let repair_info = RepairMeta {
+                                                _from_addr: packet.meta.addr(),
+                                                nonce,
+                                            };
+                                            Some(repair_info)
+                                        } else {
+                                            // If can't parse the nonce, dump the packet
+                                            return None;
+                                        }
+                                    } else {
+                                        None
+                                    }
+                                };
+                                if shred_filter(&shred, last_root) {
+                                    // Mark slot as dead if the current shred is on the boundary
+                                    // of max shreds per slot. However, let the current shred
+                                    // get retransmitted. It'll allow peer nodes to see this shred
+                                    // and trigger them to mark the slot as dead.
+                                    if shred.index() >= (MAX_DATA_SHREDS_PER_SLOT - 1) as u32 {
+                                        let _ = blockstore.set_dead_slot(shred.slot());
+                                    }
+                                    packet.meta.slot = shred.slot();
+                                    packet.meta.seed = shred.seed();
+                                    Some((shred, repair_info))
+                                } else {
+                                    packet.meta.discard = true;
+                                    None
                                 }
-                                packet.meta.slot = shred.slot();
-                                packet.meta.seed = shred.seed();
-                                Some(shred)
                             } else {
                                 packet.meta.discard = true;
                                 None
                             }
-                        } else {
-                            packet.meta.discard = true;
-                            None
                         }
                     })
                     .collect::<Vec<_>>()
             })
-            .collect()
+            .unzip()
     });
 
     trace!("{:?} shreds from packets", shreds.len());
@@ -205,7 +247,7 @@ where
         }
     }
 
-    insert_shred_sender.send(shreds)?;
+    insert_shred_sender.send((shreds, repair_infos))?;
 
     trace!(
         "Elapsed processing time in recv_window(): {}",
@@ -215,6 +257,11 @@ where
     Ok(())
 }
 
+struct RepairMeta {
+    _from_addr: SocketAddr,
+    nonce: Nonce,
+}
+
 // Implement a destructor for the window_service thread to signal it exited
 // even on panics
 struct Finalizer {
@@ -336,7 +383,7 @@ impl WindowService {
         exit: &Arc<AtomicBool>,
         blockstore: &Arc<Blockstore>,
         leader_schedule_cache: &Arc<LeaderScheduleCache>,
-        insert_receiver: CrossbeamReceiver<Vec<Shred>>,
+        insert_receiver: CrossbeamReceiver<(Vec<Shred>, Vec<Option<RepairMeta>>)>,
         duplicate_sender: CrossbeamSender<Shred>,
     ) -> JoinHandle<()> {
         let exit = exit.clone();
@@ -377,7 +424,7 @@ impl WindowService {
         id: Pubkey,
         exit: &Arc<AtomicBool>,
         blockstore: &Arc<Blockstore>,
-        insert_sender: CrossbeamSender<Vec<Shred>>,
+        insert_sender: CrossbeamSender<(Vec<Shred>, Vec<Option<RepairMeta>>)>,
         verified_receiver: CrossbeamReceiver<Vec<Packets>>,
         shred_filter: F,
         bank_forks: Option<Arc<RwLock<BankForks>>>,
@@ -483,12 +530,11 @@ mod test {
         repair_service::RepairSlotRange,
     };
     use rand::thread_rng;
-    use solana_ledger::shred::DataShredHeader;
     use solana_ledger::{
         blockstore::{make_many_slot_entries, Blockstore},
         entry::{create_ticks, Entry},
         get_tmp_ledger_path,
-        shred::Shredder,
+        shred::{DataShredHeader, Shredder, NONCE_SHRED_PAYLOAD_SIZE},
     };
     use solana_sdk::{
         clock::Slot,
@@ -562,8 +608,12 @@ mod test {
 
         // If it's a coding shred, test that slot >= root
         let (common, coding) = Shredder::new_coding_shred_header(5, 5, 5, 6, 6, 0, 0);
-        let mut coding_shred =
-            Shred::new_empty_from_header(common, DataShredHeader::default(), coding);
+        let mut coding_shred = Shred::new_empty_from_header(
+            common,
+            DataShredHeader::default(),
+            coding,
+            NONCE_SHRED_PAYLOAD_SIZE,
+        );
         Shredder::sign_shred(&leader_keypair, &mut coding_shred);
         assert_eq!(
             should_retransmit_and_persist(&coding_shred, Some(bank.clone()), &cache, &me_id, 0, 0),

core/tests/bank_forks.rs

@@ -137,7 +137,7 @@ mod tests {
             // and to allow snapshotting of bank and the purging logic on status_cache to
             // kick in
             if slot % set_root_interval == 0 || slot == last_slot - 1 {
-                bank_forks.set_root(bank.slot(), &sender);
+                bank_forks.set_root(bank.slot(), &sender, None);
             }
         }
         // Generate a snapshot package for last bank
@@ -377,9 +377,11 @@ mod tests {
                     snapshot_test_config.bank_forks.insert(new_bank);
                     current_bank = snapshot_test_config.bank_forks[new_slot].clone();
                 }
-                snapshot_test_config
-                    .bank_forks
-                    .set_root(current_bank.slot(), &snapshot_sender);
+                snapshot_test_config.bank_forks.set_root(
+                    current_bank.slot(),
+                    &snapshot_sender,
+                    None,
+                );
             }
 
             let num_old_slots = num_set_roots * *add_root_interval - MAX_CACHE_ENTRIES + 1;

core/tests/client.rs

@@ -6,6 +6,7 @@ use solana_core::{
     commitment::BlockCommitmentCache, rpc_pubsub_service::PubSubService,
     rpc_subscriptions::RpcSubscriptions, validator::TestValidator,
 };
+use solana_ledger::{blockstore::Blockstore, get_tmp_ledger_path};
 use solana_sdk::{
     commitment_config::CommitmentConfig, pubkey::Pubkey, rpc_port, signature::Signer,
     system_transaction,
@@ -39,7 +40,7 @@ fn test_rpc_client() {
 
     assert_eq!(
         client.get_version().unwrap().solana_core,
-        solana_clap_utils::version!()
+        solana_version::version!()
     );
 
     assert!(client.get_account(&bob_pubkey).is_err());
@@ -85,9 +86,13 @@ fn test_slot_subscription() {
         rpc_port::DEFAULT_RPC_PUBSUB_PORT,
     );
     let exit = Arc::new(AtomicBool::new(false));
+    let ledger_path = get_tmp_ledger_path!();
+    let blockstore = Arc::new(Blockstore::open(&ledger_path).unwrap());
     let subscriptions = Arc::new(RpcSubscriptions::new(
         &exit,
-        Arc::new(RwLock::new(BlockCommitmentCache::default())),
+        Arc::new(RwLock::new(BlockCommitmentCache::default_with_blockstore(
+            blockstore,
+        ))),
     ));
     let pubsub_service = PubSubService::new(&subscriptions, pubsub_addr, &exit);
     std::thread::sleep(Duration::from_millis(400));

core/tests/storage_stage.rs

@@ -59,7 +59,9 @@ mod tests {
             &[bank.clone()],
             vec![0],
         )));
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::default_with_blockstore(blockstore.clone()),
+        ));
         let cluster_info = test_cluster_info(&keypair.pubkey());
 
         let (bank_sender, bank_receiver) = channel();
@@ -180,7 +182,9 @@ mod tests {
             &[bank.clone()],
             vec![0],
         )));
-        let block_commitment_cache = Arc::new(RwLock::new(BlockCommitmentCache::default()));
+        let block_commitment_cache = Arc::new(RwLock::new(
+            BlockCommitmentCache::default_with_blockstore(blockstore.clone()),
+        ));
 
         let cluster_info = test_cluster_info(&keypair.pubkey());
         let (bank_sender, bank_receiver) = channel();

crate-features/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-crate-features"
-version = "1.0.19"
+version = "1.0.23"
 description = "Solana Crate Features"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,16 +12,16 @@ edition = "2018"
 backtrace = { version = "0.3.33", features = ["serialize-serde"] }
 bytes = { version = "0.4.12", features = ["either"] }
 cc = { version = "1.0.45", features = ["jobserver", "num_cpus", "parallel"]}
-curve25519-dalek = { version = "1.1.3" }
+curve25519-dalek = { version = "2" }
 either= { version = "1.5.2" }
 failure = { version = "0.1.5" }
 lazy_static = { version = "1.4.0", features = ["spin", "spin_no_std"] }
 libc = { version = "0.2.62", features = ["extra_traits"] }
-rand_chacha = { version = "0.1.1" }
+rand_chacha = { version = "0.2.2" }
 regex-syntax = { version = "0.6.12" }
 reqwest = { version = "0.9.20", default-features = false, features = ["rustls-tls"] }
 serde = { version = "1.0.100", features = ["rc"] }
-ed25519-dalek = { version = "=1.0.0-pre.1", features = ["serde"] }
+ed25519-dalek = { version = "=1.0.0-pre.3", features = ["serde"] }
 syn_0_15 = { package = "syn", version = "0.15.42", features = ["extra-traits", "fold", "full"] }
 syn_1_0 = { package = "syn", version = "1.0.3", features = ["extra-traits", "fold", "full"] }
 tokio = { version = "0.1.22",features=["bytes", "codec", "default", "fs", "io", "mio", "num_cpus", "reactor", "rt-full", "sync", "tcp", "timer", "tokio-codec", "tokio-current-thread", "tokio-executor", "tokio-io", "tokio-io", "tokio-reactor", "tokio-tcp", "tokio-tcp", "tokio-threadpool", "tokio-timer", "tokio-udp", "tokio-uds", "udp", "uds"] }

docs/src/apps/jsonrpc-api.md

@@ -31,6 +31,7 @@ To interact with a Solana node inside a JavaScript application, use the [solana-
 * [getGenesisHash](jsonrpc-api.md#getgenesishash)
 * [getIdentity](jsonrpc-api.md#getidentity)
 * [getInflation](jsonrpc-api.md#getinflation)
+* [getLargestAccounts](jsonrpc-api.md#getlargestaccounts)
 * [getLeaderSchedule](jsonrpc-api.md#getleaderschedule)
 * [getMinimumBalanceForRentExemption](jsonrpc-api.md#getminimumbalanceforrentexemption)
 * [getProgramAccounts](jsonrpc-api.md#getprogramaccounts)
@@ -42,13 +43,14 @@ To interact with a Solana node inside a JavaScript application, use the [solana-
 * [getStoragePubkeysForSlot](jsonrpc-api.md#getstoragepubkeysforslot)
 * [getStorageTurn](jsonrpc-api.md#getstorageturn)
 * [getStorageTurnRate](jsonrpc-api.md#getstorageturnrate)
+* [getSupply](jsonrpc-api.md#getsupply)
 * [getTransactionCount](jsonrpc-api.md#gettransactioncount)
-* [getTotalSupply](jsonrpc-api.md#gettotalsupply)
 * [getVersion](jsonrpc-api.md#getversion)
 * [getVoteAccounts](jsonrpc-api.md#getvoteaccounts)
 * [minimumLedgerSlot](jsonrpc-api.md#minimumledgerslot)
 * [requestAirdrop](jsonrpc-api.md#requestairdrop)
 * [sendTransaction](jsonrpc-api.md#sendtransaction)
+* [simulateTransaction](jsonrpc-api.md#simulatetransaction)
 * [setLogFilter](jsonrpc-api.md#setlogfilter)
 * [validatorExit](jsonrpc-api.md#validatorexit)
 * [Subscription Websocket](jsonrpc-api.md#subscription-websocket)
@@ -95,7 +97,8 @@ Requests can be sent in batches by sending an array of JSON-RPC request objects
 
 Solana nodes choose which bank state to query based on a commitment requirement
 set by the client. Clients may specify either:
-* `{"commitment":"max"}` - the node will query the most recent bank having reached `MAX_LOCKOUT_HISTORY` confirmations
+* `{"commitment":"max"}` - the node will query the most recent bank confirmed by the cluster as having reached `MAX_LOCKOUT_HISTORY` confirmations
+* `{"commitment":"root"}` - the node will query the most recent bank having reached `MAX_LOCKOUT_HISTORY` confirmations on this node
 * `{"commitment":"recent"}` - the node will query its most recent bank state
 
 The commitment parameter should be included as the last element in the `params` array:
@@ -195,7 +198,7 @@ The result field will be a JSON object containing:
 
 * `commitment` - commitment, comprising either:
   * `<null>` - Unknown block
-  * `<array>` - commitment, array of u64 integers logging the amount of cluster stake in lamports that has voted on the block at each depth from 0 to `MAX_LOCKOUT_HISTORY`
+  * `<array>` - commitment, array of u64 integers logging the amount of cluster stake in lamports that has voted on the block at each depth from 0 to `MAX_LOCKOUT_HISTORY` + 1
 * `totalStake` - total active stake, in lamports, of the current epoch
 
 #### Example:
@@ -205,7 +208,7 @@ The result field will be a JSON object containing:
 curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getBlockCommitment","params":[5]}' http://localhost:8899
 
 // Result
-{"jsonrpc":"2.0","result":{"commitment":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10,32],"totalStake": 42},"id":1}
+{"jsonrpc":"2.0","result":{"commitment":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10,32],"totalStake": 42},"id":1}
 ```
 
 ### getBlockTime
@@ -256,7 +259,8 @@ The result field will be an array of JSON objects, each with the following sub f
 * `pubkey: <string>` - Node public key, as base-58 encoded string
 * `gossip: <string>` - Gossip network address for the node
 * `tpu: <string>` - TPU network address for the node
-* `rpc: <string>` - JSON RPC network address for the node, or `null` if the JSON RPC service is not enabled
+* `rpc: <string>|null` - JSON RPC network address for the node, or `null` if the JSON RPC service is not enabled
+* `version: <string>|null` - The software version of the node, or `null` if the version information is not available
 
 #### Example:
 
@@ -265,7 +269,7 @@ The result field will be an array of JSON objects, each with the following sub f
 curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getClusterNodes"}' http://localhost:8899
 
 // Result
-{"jsonrpc":"2.0","result":[{"gossip":"10.239.6.48:8001","pubkey":"9QzsJf7LPLj8GkXbYT3LFDKqsj2hHG7TA3xinJHu8epQ","rpc":"10.239.6.48:8899","tpu":"10.239.6.48:8856"}],"id":1}
+{"jsonrpc":"2.0","result":[{"gossip":"10.239.6.48:8001","pubkey":"9QzsJf7LPLj8GkXbYT3LFDKqsj2hHG7TA3xinJHu8epQ","rpc":"10.239.6.48:8899","tpu":"10.239.6.48:8856"},"version":"1.0.0 c375ce1f"],"id":1}
 ```
 
 ### getConfirmedBlock
@@ -318,7 +322,7 @@ curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc": "2.0","id":1,"m
 
 #### Transaction Structure
 
-Transactions are quite different from those on other blockchains. Be sure to review [Anatomy of a Transaction](transaction.md) to learn about transactions on Solana.
+Transactions are quite different from those on other blockchains. Be sure to review [Anatomy of a Transaction](../transaction.md) to learn about transactions on Solana.
 
 The JSON structure of a transaction is defined as follows:
 
@@ -629,6 +633,32 @@ curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "m
 {"jsonrpc":"2.0","result":{"foundation":0.05,"foundationTerm":7.0,"initial":0.15,"storage":0.1,"taper":0.15,"terminal":0.015},"id":1}
 ```
 
+### getLargestAccounts
+
+Returns the 20 largest accounts, by lamport balance
+
+#### Parameters:
+
+* `<object>` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
+
+#### Results:
+
+The result will be an RpcResponse JSON object with `value` equal to an array of:
+
+* `<object>` - otherwise, a JSON object containing:
+  * `address: <string>`, base-58 encoded address of the account
+  * `lamports: <u64>`, number of lamports in the account, as a u64
+
+#### Example:
+
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getLargestAccounts"}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":{"context":{"slot":54},"value":[{"lamports":999974,"address":"99P8ZgtJYe1buSK8JXkvpLh8xPsCFuLYhz9hQFNw93WJ"},{"lamports":42,"address":"uPwWLo16MVehpyWqsLkK3Ka8nLowWvAHbBChqv2FZeL"},{"lamports":42,"address":"aYJCgU7REfu3XF8b3QhkqgqQvLizx8zxuLBHA25PzDS"},{"lamports":42,"address":"CTvHVtQ4gd4gUcw3bdVgZJJqApXE9nCbbbP4VTS5wE1D"},{"lamports":20,"address":"4fq3xJ6kfrh9RkJQsmVd5gNMvJbuSHfErywvEjNQDPxu"},{"lamports":4,"address":"AXJADheGVp9cruP8WYu46oNkRbeASngN5fPCMVGQqNHa"},{"lamports":2,"address":"8NT8yS6LiwNprgW4yM1jPPow7CwRUotddBVkrkWgYp24"},{"lamports":1,"address":"SysvarEpochSchedu1e111111111111111111111111"},{"lamports":1,"address":"11111111111111111111111111111111"},{"lamports":1,"address":"Stake11111111111111111111111111111111111111"},{"lamports":1,"address":"SysvarC1ock11111111111111111111111111111111"},{"lamports":1,"address":"StakeConfig11111111111111111111111111111111"},{"lamports":1,"address":"SysvarRent111111111111111111111111111111111"},{"lamports":1,"address":"Config1111111111111111111111111111111111111"},{"lamports":1,"address":"SysvarStakeHistory1111111111111111111111111"},{"lamports":1,"address":"SysvarRecentB1ockHashes11111111111111111111"},{"lamports":1,"address":"SysvarFees111111111111111111111111111111111"},{"lamports":1,"address":"Vote111111111111111111111111111111111111111"}]},"id":1}
+```
+
 ### getLeaderSchedule
 
 Returns the leader schedule for an epoch
@@ -911,6 +941,32 @@ curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "m
 {"jsonrpc":"2.0","result":1024,"id":1}
 ```
 
+### getSupply
+
+Returns information about the current supply.
+
+#### Parameters:
+
+* `<object>` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
+
+#### Results:
+
+The result will be an RpcResponse JSON object with `value` equal to a JSON object containing:
+
+* `total: <u64>` - Total supply in lamports
+* `circulating: <u64>` - Circulating supply in lamports
+* `nonCirculating: <u64>` - Non-circulating supply in lamports
+* `nonCirculatingAccounts: <array>` - an array of account addresses of non-circulating accounts, as strings
+
+#### Example:
+
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getSupply"}' http://localhost:8899
+// Result
+{"jsonrpc":"2.0","result":{"context":{"slot":1114},"value":{"circulating":16000,"nonCirculating":1000000,"nonCirculatingAccounts":["FEy8pTbP5fEoqMV1GdTz83byuA8EKByqYat1PKDgVAq5","9huDUZfxoJ7wGMTffUE7vh1xePqef7gyrLJu9NApncqA","3mi1GmwEE3zo2jmfDuzvjSX9ovRXsDUKHvsntpkhuLJ9","BYxEJTDerkaRWBem3XgnVcdhppktBXa2HbkHPKj2Ui4Z],total:1016000}},"id":1}
+```
+
 ### getTransactionCount
 
 Returns the current Transaction count from the ledger
@@ -933,28 +989,6 @@ curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "m
 {"jsonrpc":"2.0","result":268,"id":1}
 ```
 
-### getTotalSupply
-
-Returns the current total supply in lamports
-
-#### Parameters:
-
-* `<object>` - (optional) [Commitment](jsonrpc-api.md#configuring-state-commitment)
-
-#### Results:
-
-* `<u64>` - Total supply
-
-#### Example:
-
-```bash
-// Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getTotalSupply"}' http://localhost:8899
-
-// Result
-{"jsonrpc":"2.0","result":10126,"id":1}
-```
-
 ### getVersion
 
 Returns the current solana versions running on the node
@@ -975,7 +1009,7 @@ The result field will be a JSON object with the following fields:
 // Request
 curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getVersion"}' http://localhost:8899
 // Result
-{"jsonrpc":"2.0","result":{"solana-core": "1.0.19"},"id":1}
+{"jsonrpc":"2.0","result":{"solana-core": "1.0.23"},"id":1}
 ```
 
 ### getVoteAccounts
@@ -1071,10 +1105,34 @@ Creates new transaction
 
 ```bash
 // Request
-curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"sendTransaction", "params":["3gKEMTuxvm3DKEJc4UyiyoNz1sxwdVRW2pyDDXqaCvUjGApnsazGh2y4W92zuaSSdJhBbWLYAkZokBt4N5oW27R7zCVaLLpLxvATL2GgheEh9DmmDR1P9r1ZqirVXM2fF3z5cafmc4EtwWc1UErFdCWj1qYvy4bDGMLXRYLURxaKytEEqrxz6JXj8rUHhDpjTZeFxmC6iAW3hZr6cmaAzewQCQfiEv2HfydriwHDtN95u3Y1EF6SuXxcRqox2aTjGye2Ln9zFj4XbnAtjCmkZhR"]}' http://localhost:8899
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"sendTransaction", "params":["4hXTCkRzt9WyecNzV1XPgCDfGAZzQKNxLXgynz5QDuWWPSAZBZSHptvWRL3BjCvzUXRdKvHL2b7yGrRQcWyaqsaBCncVG7BFggS8w9snUts67BSh3EqKpXLUm5UMHfD7ZBe9GhARjbNQMLJ1QD3Spr6oMTBU6EhdB4RD8CP2xUxr2u3d6fos36PD98XS6oX8TQjLpsMwncs5DAMiD4nNnR8NBfyghGCWvCVifVwvA8B8TJxE1aiyiv2L429BCWfyzAme5sZW8rDb14NeCQHhZbtNqfXhcp2tAnaAT"]}' http://localhost:8899
 
 // Result
-{"jsonrpc":"2.0","result":"2EBVM6cB8vAAD93Ktr6Vd8p67XPbQzCJX47MpReuiCXJAtcjaxpvWpcg9Ege1Nr5Tk3a2GFrByT7WPBjdsTycY9b","id":1}
+{"jsonrpc":"2.0","result":"2id3YC2jK9G5Wo2phDx4gJVAew8DcY5NAojnVuao8rkxwPYPe8cSwE5GzhEgJA2y8fVjDEo6iR6ykBvDxrTQrtpb","id":1}
+```
+
+### simulateTransaction
+
+Simulate sending a transaction
+
+#### Parameters:
+
+* `<string>` - Transaction, as base-58 encoded string.  The transaction must have a valid blockhash, but is not required to be signed.
+* `<object>` - (optional) Configuration object containing the following field:
+  * `sigVerify: <bool>` - if true the transaction signatures will be verified (default: false)
+
+#### Results:
+
+An RpcResponse containing a TransactionStatus object
+
+#### Example:
+
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"simulateTransaction", "params":["4hXTCkRzt9WyecNzV1XPgCDfGAZzQKNxLXgynz5QDuWWPSAZBZSHptvWRL3BjCvzUXRdKvHL2b7yGrRQcWyaqsaBCncVG7BFggS8w9snUts67BSh3EqKpXLUm5UMHfD7ZBe9GhARjbNQMLJ1QD3Spr6oMTBU6EhdB4RD8CP2xUxr2u3d6fos36PD98XS6oX8TQjLpsMwncs5DAMiD4nNnR8NBfyghGCWvCVifVwvA8B8TJxE1aiyiv2L429BCWfyzAme5sZW8rDb14NeCQHhZbtNqfXhcp2tAnaAT"]}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":{"context":{"slot":218},"value":{"confirmations":0,"err":null,"slot":218,"status":{"Ok":null}}},"id":1}
 ```
 
 ### setLogFilter

docs/src/cli/usage.md

@@ -171,7 +171,7 @@ $ solana send-timestamp <PUBKEY> <PROCESS_ID> --date 2018-12-24T23:59:00
 ## Usage
 ### solana-cli
 ```text
-solana-cli 1.0.19 [channel=unknown commit=unknown]
+solana-cli 1.0.23 [channel=unknown commit=unknown]
 Blockchain, Rebuilt for Scale
 
 USAGE:

faucet/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-faucet"
-version = "1.0.19"
+version = "1.0.23"
 description = "Solana Faucet"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,10 +19,10 @@ clap = "2.33"
 log = "0.4.8"
 serde = "1.0.104"
 serde_derive = "1.0.103"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-metrics = { path = "../metrics", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-metrics = { path = "../metrics", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
 tokio = "0.1"
 tokio-codec = "0.1"
 

genesis-programs/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-genesis-programs"
-version = "1.0.19"
+version = "1.0.23"
 description = "Solana genesis programs"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,16 +10,16 @@ edition = "2018"
 
 [dependencies]
 log = { version = "0.4.8" }
-solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.0.19" }
-solana-budget-program = { path = "../programs/budget", version = "1.0.19" }
-solana-config-program = { path = "../programs/config", version = "1.0.19" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.0.19" }
-solana-runtime = { path = "../runtime", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
-solana-stake-program = { path = "../programs/stake", version = "1.0.19" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.19" }
-solana-vest-program = { path = "../programs/vest", version = "1.0.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.19" }
+solana-bpf-loader-program = { path = "../programs/bpf_loader", version = "1.0.23" }
+solana-budget-program = { path = "../programs/budget", version = "1.0.23" }
+solana-config-program = { path = "../programs/config", version = "1.0.23" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.0.23" }
+solana-runtime = { path = "../runtime", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
+solana-stake-program = { path = "../programs/stake", version = "1.0.23" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.23" }
+solana-vest-program = { path = "../programs/vest", version = "1.0.23" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.23" }
 
 [lib]
 crate-type = ["lib"]

genesis/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-genesis"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.19"
+version = "1.0.23"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -15,13 +15,13 @@ chrono = "0.4"
 serde = "1.0.104"
 serde_json = "1.0.46"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.19" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.0.19" }
-solana-ledger = { path = "../ledger", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
-solana-stake-program = { path = "../programs/stake", version = "1.0.19" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.23" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.0.23" }
+solana-ledger = { path = "../ledger", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
+solana-stake-program = { path = "../programs/stake", version = "1.0.23" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.23" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.23" }
 tempfile = "3.1.0"
 
 [[bin]]

gossip/Cargo.toml

@@ -3,19 +3,19 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-gossip"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.19"
+version = "1.0.23"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.19" }
-solana-core = { path = "../core", version = "1.0.19" }
-solana-client = { path = "../client", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-net-utils = { path = "../net-utils", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.23" }
+solana-core = { path = "../core", version = "1.0.23" }
+solana-client = { path = "../client", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-net-utils = { path = "../net-utils", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
 
 
 

gossip/src/main.rs

@@ -15,6 +15,13 @@ use std::process::exit;
 fn main() -> Result<(), Box<dyn error::Error>> {
     solana_logger::setup_with_default("solana=info");
 
+    let shred_version_arg = Arg::with_name("shred_version")
+        .long("shred-version")
+        .value_name("VERSION")
+        .takes_value(true)
+        .default_value("0")
+        .help("Filter gossip nodes by this shred version");
+
     let matches = App::new(crate_name!())
         .about(crate_description!())
         .version(solana_clap_utils::version!())
@@ -53,6 +60,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                         .default_value("5")
                         .help("Timeout in seconds"),
                 )
+                .arg(&shred_version_arg)
                 .setting(AppSettings::DisableVersion),
         )
         .subcommand(
@@ -110,6 +118,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                         .validator(is_pubkey)
                         .help("Public key of a specific node to wait for"),
                 )
+                .arg(&shred_version_arg)
                 .arg(
                     Arg::with_name("timeout")
                         .long("timeout")
@@ -167,6 +176,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
             let pubkey = matches
                 .value_of("node_pubkey")
                 .map(|pubkey_str| pubkey_str.parse::<Pubkey>().unwrap());
+            let shred_version = value_t_or_exit!(matches, "shred_version", u16);
 
             let entrypoint_addr = parse_entrypoint(&matches);
 
@@ -212,6 +222,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                 pubkey,
                 None,
                 Some(&gossip_addr),
+                shred_version,
             )?;
 
             if timeout.is_some() {
@@ -251,6 +262,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
             let all = matches.is_present("all");
             let entrypoint_addr = parse_entrypoint(&matches);
             let timeout = value_t_or_exit!(matches, "timeout", u64);
+            let shred_version = value_t_or_exit!(matches, "shred_version", u16);
             let (nodes, _archivers) = discover(
                 entrypoint_addr.as_ref(),
                 Some(1),
@@ -258,6 +270,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                 None,
                 entrypoint_addr.as_ref(),
                 None,
+                shred_version,
             )?;
 
             let rpc_addrs: Vec<_> = nodes
@@ -298,6 +311,7 @@ fn main() -> Result<(), Box<dyn error::Error>> {
                 Some(pubkey),
                 None,
                 None,
+                0,
             )?;
             let node = nodes.iter().find(|x| x.id == pubkey).unwrap();
 

install/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-install"
 description = "The solana cluster software installer"
-version = "1.0.19"
+version = "1.0.23"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -24,11 +24,11 @@ reqwest = { version = "0.10.1", default-features = false, features = ["blocking"
 serde = "1.0.104"
 serde_derive = "1.0.103"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.19" }
-solana-client = { path = "../client", version = "1.0.19" }
-solana-config-program = { path = "../programs/config", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.23" }
+solana-client = { path = "../client", version = "1.0.23" }
+solana-config-program = { path = "../programs/config", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
 semver = "0.9.0"
 tar = "0.4.26"
 tempdir = "0.3.7"

install/src/command.rs

@@ -581,7 +581,7 @@ pub fn info(config_file: &str, local_info_only: bool) -> Result<Option<UpdateMan
     if let Some(explicit_release) = &config.explicit_release {
         match explicit_release {
             ExplicitRelease::Semver(release_semver) => {
-                println_name_value(&format!("v{}Release version:", BULLET), &release_semver);
+                println_name_value(&format!("{}Release version:", BULLET), &release_semver);
                 println_name_value(
                     &format!("{}Release URL:", BULLET),
                     &github_release_download_url(release_semver),

install/src/main-install-init.rs

@@ -1,4 +1,3 @@
-use atty;
 use std::process::exit;
 
 fn press_enter() {

keygen/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-keygen"
-version = "1.0.19"
+version = "1.0.23"
 description = "Solana key generation utility"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,10 +13,10 @@ bs58 = "0.3.0"
 clap = "2.33"
 dirs = "2.0.2"
 num_cpus = "1.12.0"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.19" }
-solana-cli-config = { path = "../cli-config", version = "1.0.19" }
-solana-remote-wallet = { path = "../remote-wallet", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.23" }
+solana-cli-config = { path = "../cli-config", version = "1.0.23" }
+solana-remote-wallet = { path = "../remote-wallet", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
 tiny-bip39 = "0.7.0"
 
 [[bin]]

keygen/src/keygen.rs

@@ -1,10 +1,8 @@
 use bip39::{Language, Mnemonic, MnemonicType, Seed};
-use bs58;
 use clap::{
     crate_description, crate_name, value_t, values_t_or_exit, App, AppSettings, Arg, ArgMatches,
     SubCommand,
 };
-use num_cpus;
 use solana_clap_utils::{
     input_validators::is_derivation,
     keypair::{

ledger-tool/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-ledger-tool"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.19"
+version = "1.0.23"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,13 +14,15 @@ clap = "2.33.0"
 histogram = "*"
 serde_json = "1.0.46"
 serde_yaml = "0.8.11"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.19" }
-solana-ledger = { path = "../ledger", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-runtime = { path = "../runtime", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.19" }
-solana-stake-program = { path = "../programs/stake", version = "1.0.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.23" }
+solana-cli = { path = "../cli", version = "1.0.23" }
+solana-ledger = { path = "../ledger", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-runtime = { path = "../runtime", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.23" }
+solana-stake-program = { path = "../programs/stake", version = "1.0.23" }
+solana-transaction-status = { path = "../transaction-status", version = "1.0.23" }
 tempfile = "3.1.0"
 
 [dev-dependencies]

ledger-tool/src/main.rs

@@ -2,7 +2,6 @@ use clap::{
     crate_description, crate_name, value_t, value_t_or_exit, values_t_or_exit, App, Arg,
     ArgMatches, SubCommand,
 };
-use histogram;
 use serde_json::json;
 use solana_clap_utils::input_validators::is_slot;
 use solana_ledger::{
@@ -15,8 +14,8 @@ use solana_ledger::{
     snapshot_utils,
 };
 use solana_sdk::{
-    clock::Slot, genesis_config::GenesisConfig, native_token::lamports_to_sol,
-    program_utils::limited_deserialize, pubkey::Pubkey, shred_version::compute_shred_version,
+    clock::Slot, genesis_config::GenesisConfig, native_token::lamports_to_sol, pubkey::Pubkey,
+    shred_version::compute_shred_version,
 };
 use solana_vote_program::vote_state::VoteState;
 use std::{
@@ -83,112 +82,23 @@ fn output_slot(
                     entry.transactions.len()
                 );
                 for (transactions_index, transaction) in entry.transactions.iter().enumerate() {
-                    let message = &transaction.message;
                     println!("    Transaction {}", transactions_index);
-                    println!("      Recent Blockhash: {:?}", message.recent_blockhash);
-                    for (signature_index, signature) in transaction.signatures.iter().enumerate() {
-                        println!("      Signature {}: {:?}", signature_index, signature);
-                    }
-                    println!("      Header: {:?}", message.header);
-                    for (account_index, account) in message.account_keys.iter().enumerate() {
-                        println!("      Account {}: {:?}", account_index, account);
-                    }
-                    for (instruction_index, instruction) in message.instructions.iter().enumerate()
-                    {
-                        let program_pubkey =
-                            message.account_keys[instruction.program_id_index as usize];
-                        println!("      Instruction {}", instruction_index);
-                        println!(
-                            "        Program: {} ({})",
-                            program_pubkey, instruction.program_id_index
-                        );
-                        for (account_index, account) in instruction.accounts.iter().enumerate() {
-                            let account_pubkey = message.account_keys[*account as usize];
-                            println!(
-                                "        Account {}: {} ({})",
-                                account_index, account_pubkey, account
+                    let transaction_status = blockstore
+                        .read_transaction_status((transaction.signatures[0], slot))
+                        .unwrap_or_else(|err| {
+                            eprintln!(
+                                "Failed to read transaction status for {} at slot {}: {}",
+                                transaction.signatures[0], slot, err
                             );
-                        }
+                            None
+                        })
+                        .map(|transaction_status| transaction_status.into());
 
-                        let mut raw = true;
-                        if program_pubkey == solana_vote_program::id() {
-                            if let Ok(vote_instruction) =
-                                limited_deserialize::<
-                                    solana_vote_program::vote_instruction::VoteInstruction,
-                                >(&instruction.data)
-                            {
-                                println!("        {:?}", vote_instruction);
-                                raw = false;
-                            }
-                        } else if program_pubkey == solana_stake_program::id() {
-                            if let Ok(stake_instruction) =
-                                limited_deserialize::<
-                                    solana_stake_program::stake_instruction::StakeInstruction,
-                                >(&instruction.data)
-                            {
-                                println!("        {:?}", stake_instruction);
-                                raw = false;
-                            }
-                        } else if program_pubkey == solana_sdk::system_program::id() {
-                            if let Ok(system_instruction) =
-                                limited_deserialize::<
-                                    solana_sdk::system_instruction::SystemInstruction,
-                                >(&instruction.data)
-                            {
-                                println!("        {:?}", system_instruction);
-                                raw = false;
-                            }
-                        }
-
-                        if raw {
-                            println!("        Data: {:?}", instruction.data);
-                        }
-                    }
-                    match blockstore.read_transaction_status((transaction.signatures[0], slot)) {
-                        Ok(transaction_status) => {
-                            if let Some(transaction_status) = transaction_status {
-                                println!(
-                                    "      Status: {}",
-                                    if transaction_status.status.is_ok() {
-                                        "Ok".into()
-                                    } else {
-                                        transaction_status.status.unwrap_err().to_string()
-                                    }
-                                );
-                                println!("        Fee: {}", transaction_status.fee);
-                                assert_eq!(
-                                    transaction_status.pre_balances.len(),
-                                    transaction_status.post_balances.len()
-                                );
-                                for (i, (pre, post)) in transaction_status
-                                    .pre_balances
-                                    .iter()
-                                    .zip(transaction_status.post_balances.iter())
-                                    .enumerate()
-                                {
-                                    if pre == post {
-                                        println!(
-                                            "        Account {} balance: {} SOL",
-                                            i,
-                                            lamports_to_sol(*pre)
-                                        );
-                                    } else {
-                                        println!(
-                                            "        Account {} balance: {} SOL -> {} SOL",
-                                            i,
-                                            lamports_to_sol(*pre),
-                                            lamports_to_sol(*post)
-                                        );
-                                    }
-                                }
-                            } else {
-                                println!("      Status: Unavailable");
-                            }
-                        }
-                        Err(err) => {
-                            println!("      Status: {:?}", err);
-                        }
-                    }
+                    solana_cli::display::println_transaction(
+                        &transaction,
+                        &transaction_status,
+                        "      ",
+                    );
                 }
             }
             LedgerOutputMethod::Json => {
@@ -814,6 +724,13 @@ fn main() {
                     .takes_value(false)
                     .help("Include sysvars too"),
             )
+        ).subcommand(
+            SubCommand::with_name("capitalization")
+            .about("Print capitalization (aka, total suppy)")
+            .arg(&no_snapshot_arg)
+            .arg(&account_paths_arg)
+            .arg(&halt_at_slot_arg)
+            .arg(&hard_forks_arg)
         ).subcommand(
             SubCommand::with_name("prune")
             .about("Prune the ledger at the block height")
@@ -1137,6 +1054,83 @@ fn main() {
                 }
             }
         }
+        ("capitalization", Some(arg_matches)) => {
+            let dev_halt_at_slot = value_t!(arg_matches, "halt_at_slot", Slot).ok();
+            let process_options = ProcessOptions {
+                dev_halt_at_slot,
+                new_hard_forks: hardforks_of(arg_matches, "hard_forks"),
+                poh_verify: false,
+                ..ProcessOptions::default()
+            };
+            let genesis_config = open_genesis_config(&ledger_path);
+            match load_bank_forks(arg_matches, &ledger_path, &genesis_config, process_options) {
+                Ok((bank_forks, bank_forks_info, _leader_schedule_cache, _snapshot_hash)) => {
+                    let slot = dev_halt_at_slot.unwrap_or_else(|| {
+                        if bank_forks_info.len() > 1 {
+                            eprintln!("Error: multiple forks present");
+                            exit(1);
+                        }
+                        bank_forks_info[0].bank_slot
+                    });
+
+                    let bank = bank_forks.get(slot).unwrap_or_else(|| {
+                        eprintln!("Error: Slot {} is not available", slot);
+                        exit(1);
+                    });
+
+                    use solana_sdk::native_token::LAMPORTS_PER_SOL;
+                    use std::fmt::{Display, Formatter, Result};
+                    pub struct Sol(u64);
+
+                    impl Display for Sol {
+                        fn fmt(&self, f: &mut Formatter) -> Result {
+                            write!(
+                                f,
+                                "{}.{:09} SOL",
+                                self.0 / LAMPORTS_PER_SOL,
+                                self.0 % LAMPORTS_PER_SOL
+                            )
+                        }
+                    }
+
+                    let computed_capitalization: u64 = bank
+                        .get_program_accounts(None)
+                        .into_iter()
+                        .filter_map(|(_pubkey, account)| {
+                            if account.lamports == u64::max_value() {
+                                return None;
+                            }
+
+                            let is_specially_retained =
+                                solana_sdk::native_loader::check_id(&account.owner)
+                                    || solana_sdk::sysvar::check_id(&account.owner);
+
+                            if is_specially_retained {
+                                // specially retained accounts are ensured to exist by
+                                // alwaysing having a balance of 1 lamports, which is
+                                // outside the capitalization calculation.
+                                Some(account.lamports - 1)
+                            } else {
+                                Some(account.lamports)
+                            }
+                        })
+                        .sum();
+
+                    if bank.capitalization() != computed_capitalization {
+                        panic!(
+                            "Capitalization mismatch!?: {} != {}",
+                            bank.capitalization(),
+                            computed_capitalization
+                        );
+                    }
+                    println!("Capitalization: {}", Sol(bank.capitalization()));
+                }
+                Err(err) => {
+                    eprintln!("Failed to load ledger: {:?}", err);
+                    exit(1);
+                }
+            }
+        }
         ("purge", Some(arg_matches)) => {
             let start_slot = value_t_or_exit!(arg_matches, "start_slot", Slot);
             let end_slot = value_t!(arg_matches, "end_slot", Slot);

ledger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-ledger"
-version = "1.0.19"
+version = "1.0.23"
 description = "Solana ledger"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -21,26 +21,26 @@ itertools = "0.8.2"
 libc = "0.2.66"
 log = { version = "0.4.8" }
 num_cpus = "1.0.0"
-rand = "0.6.5"
-rand_chacha = "0.1.1"
+rand = "0.7.0"
+rand_chacha = "0.2.2"
 rayon = "1.2.0"
 reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0.1-3", features = ["simd-accel"] }
 regex = "1.3.4"
 serde = "1.0.104"
 serde_bytes = "0.11.3"
-solana-transaction-status = { path = "../transaction-status", version = "1.0.19" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-measure = { path = "../measure", version = "1.0.19" }
-solana-merkle-tree = { path = "../merkle-tree", version = "1.0.19" }
-solana-metrics = { path = "../metrics", version = "1.0.19" }
-solana-perf = { path = "../perf", version = "1.0.19" }
+solana-transaction-status = { path = "../transaction-status", version = "1.0.23" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-measure = { path = "../measure", version = "1.0.23" }
+solana-merkle-tree = { path = "../merkle-tree", version = "1.0.23" }
+solana-metrics = { path = "../metrics", version = "1.0.23" }
+solana-perf = { path = "../perf", version = "1.0.23" }
 ed25519-dalek = "1.0.0-pre.1"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.0.19" }
-solana-runtime = { path = "../runtime", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
-solana-stake-program = { path = "../programs/stake", version = "1.0.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.19" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.0.23" }
+solana-runtime = { path = "../runtime", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
+solana-stake-program = { path = "../programs/stake", version = "1.0.23" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.23" }
 symlink = "0.1.0"
 tar = "0.4.26"
 thiserror = "1.0"
@@ -50,14 +50,14 @@ lazy_static = "1.4.0"
 [dependencies.rocksdb]
 # Avoid the vendored bzip2 within rocksdb-sys that can cause linker conflicts
 # when also using the bzip2 crate
-version = "0.13.0"
+version = "0.14.0"
 default-features = false
 features = ["lz4"]
 
 [dev-dependencies]
 assert_matches = "1.3.0"
 matches = "0.1.6"
-solana-budget-program = { path = "../programs/budget", version = "1.0.19" }
+solana-budget-program = { path = "../programs/budget", version = "1.0.23" }
 
 [lib]
 crate-type = ["lib"]

ledger/src/bank_forks.rs

@@ -165,6 +165,7 @@ impl BankForks {
         &mut self,
         root: Slot,
         snapshot_package_sender: &Option<SnapshotPackageSender>,
+        largest_confirmed_root: Option<Slot>,
     ) {
         self.root = root;
         let set_root_start = Instant::now();
@@ -205,7 +206,7 @@ impl BankForks {
             }
         }
 
-        self.prune_non_root(root);
+        self.prune_non_root(root, largest_confirmed_root);
 
         inc_new_counter_info!(
             "bank-forks_set_root_ms",
@@ -221,6 +222,10 @@ impl BankForks {
         self.root
     }
 
+    pub fn root_bank(&self) -> &Arc<Bank> {
+        self.banks.get(&self.root()).expect("Root bank must exist")
+    }
+
     pub fn purge_old_snapshots(&self) {
         // Remove outdated snapshots
         let config = self.snapshot_config.as_ref().unwrap();
@@ -276,10 +281,19 @@ impl BankForks {
         Ok(())
     }
 
-    fn prune_non_root(&mut self, root: Slot) {
+    fn prune_non_root(&mut self, root: Slot, largest_confirmed_root: Option<Slot>) {
         let descendants = self.descendants();
-        self.banks
-            .retain(|slot, _| slot == &root || descendants[&root].contains(slot));
+        self.banks.retain(|slot, _| {
+            *slot == root
+                || descendants[&root].contains(slot)
+                || (*slot < root
+                    && *slot >= largest_confirmed_root.unwrap_or(root)
+                    && descendants[slot].contains(&root))
+        });
+        datapoint_debug!(
+            "bank_forks_purge_non_root",
+            ("num_banks_retained", self.banks.len(), i64),
+        );
     }
 
     pub fn set_snapshot_config(&mut self, snapshot_config: Option<SnapshotConfig>) {

ledger/src/blockstore.rs

@@ -177,7 +177,7 @@ impl Blockstore {
         fs::create_dir_all(&ledger_path)?;
         let blockstore_path = ledger_path.join(BLOCKSTORE_DIRECTORY);
 
-        adjust_ulimit_nofile();
+        adjust_ulimit_nofile()?;
 
         // Open the database
         let mut measure = Measure::start("open");
@@ -1468,7 +1468,7 @@ impl Blockstore {
     }
 
     pub fn get_first_available_block(&self) -> Result<Slot> {
-        let mut root_iterator = self.rooted_slot_iterator(0)?;
+        let mut root_iterator = self.rooted_slot_iterator(self.lowest_slot())?;
         Ok(root_iterator.next().unwrap_or_default())
     }
 
@@ -1982,10 +1982,11 @@ impl Blockstore {
         let data_shreds = data_shreds?;
         assert!(data_shreds.last().unwrap().data_complete());
 
-        let deshred_payload = Shredder::deshred(&data_shreds).map_err(|_| {
-            BlockstoreError::InvalidShredData(Box::new(bincode::ErrorKind::Custom(
-                "Could not reconstruct data block from constituent shreds".to_string(),
-            )))
+        let deshred_payload = Shredder::deshred(&data_shreds).map_err(|e| {
+            BlockstoreError::InvalidShredData(Box::new(bincode::ErrorKind::Custom(format!(
+                "Could not reconstruct data block from constituent shreds, error: {:?}",
+                e
+            ))))
         })?;
 
         debug!("{:?} shreds in last FEC set", data_shreds.len(),);
@@ -2780,10 +2781,12 @@ pub fn make_chaining_slot_entries(
 }
 
 #[cfg(not(unix))]
-fn adjust_ulimit_nofile() {}
+fn adjust_ulimit_nofile() -> Result<()> {
+    Ok(())
+}
 
 #[cfg(unix)]
-fn adjust_ulimit_nofile() {
+fn adjust_ulimit_nofile() -> Result<()> {
     // Rocks DB likes to have many open files.  The default open file descriptor limit is
     // usually not enough
     let desired_nofile = 65000;
@@ -2811,11 +2814,13 @@ fn adjust_ulimit_nofile() {
             if cfg!(target_os = "macos") {
                 error!("On mac OS you may need to run |sudo launchctl limit maxfiles 65536 200000| first");
             }
+            return Err(BlockstoreError::UnableToSetOpenFileDescriptorLimit);
         }
 
         nofile = get_nofile();
     }
     info!("Maximum open file descriptors: {}", nofile.rlim_cur);
+    Ok(())
 }
 
 #[cfg(test)]
@@ -2826,7 +2831,7 @@ pub mod tests {
         entry::{next_entry, next_entry_mut},
         genesis_utils::{create_genesis_config, GenesisConfigInfo},
         leader_schedule::{FixedSchedule, LeaderSchedule},
-        shred::{max_ticks_per_n_shreds, DataShredHeader},
+        shred::{max_ticks_per_n_shreds, DataShredHeader, NONCE_SHRED_PAYLOAD_SIZE},
     };
     use assert_matches::assert_matches;
     use bincode::serialize;
@@ -2976,7 +2981,7 @@ pub mod tests {
     #[test]
     fn test_insert_get_bytes() {
         // Create enough entries to ensure there are at least two shreds created
-        let num_entries = max_ticks_per_n_shreds(1) + 1;
+        let num_entries = max_ticks_per_n_shreds(1, None) + 1;
         assert!(num_entries > 1);
 
         let (mut shreds, _) = make_slot_entries(0, 0, num_entries);
@@ -3216,7 +3221,7 @@ pub mod tests {
     #[test]
     fn test_insert_data_shreds_basic() {
         // Create enough entries to ensure there are at least two shreds created
-        let num_entries = max_ticks_per_n_shreds(1) + 1;
+        let num_entries = max_ticks_per_n_shreds(1, None) + 1;
         assert!(num_entries > 1);
 
         let (mut shreds, entries) = make_slot_entries(0, 0, num_entries);
@@ -3263,7 +3268,7 @@ pub mod tests {
     #[test]
     fn test_insert_data_shreds_reverse() {
         let num_shreds = 10;
-        let num_entries = max_ticks_per_n_shreds(num_shreds);
+        let num_entries = max_ticks_per_n_shreds(num_shreds, None);
         let (mut shreds, entries) = make_slot_entries(0, 0, num_entries);
         let num_shreds = shreds.len() as u64;
 
@@ -3440,7 +3445,7 @@ pub mod tests {
         {
             let blockstore = Blockstore::open(&blockstore_path).unwrap();
             // Create enough entries to ensure there are at least two shreds created
-            let min_entries = max_ticks_per_n_shreds(1) + 1;
+            let min_entries = max_ticks_per_n_shreds(1, None) + 1;
             for i in 0..4 {
                 let slot = i;
                 let parent_slot = if i == 0 { 0 } else { i - 1 };
@@ -3867,7 +3872,7 @@ pub mod tests {
             let blockstore = Blockstore::open(&blockstore_path).unwrap();
             let num_slots = 15;
             // Create enough entries to ensure there are at least two shreds created
-            let entries_per_slot = max_ticks_per_n_shreds(1) + 1;
+            let entries_per_slot = max_ticks_per_n_shreds(1, None) + 1;
             assert!(entries_per_slot > 1);
 
             let (mut shreds, _) = make_many_slot_entries(0, num_slots, entries_per_slot);
@@ -4237,7 +4242,7 @@ pub mod tests {
         let gap: u64 = 10;
         assert!(gap > 3);
         // Create enough entries to ensure there are at least two shreds created
-        let num_entries = max_ticks_per_n_shreds(1) + 1;
+        let num_entries = max_ticks_per_n_shreds(1, None) + 1;
         let entries = create_ticks(num_entries, 0, Hash::default());
         let mut shreds = entries_to_test_shreds(entries, slot, 0, true, 0);
         let num_shreds = shreds.len();
@@ -4549,6 +4554,7 @@ pub mod tests {
                 shred.clone(),
                 DataShredHeader::default(),
                 coding.clone(),
+                NONCE_SHRED_PAYLOAD_SIZE,
             );
 
             // Insert a good coding shred
@@ -4581,6 +4587,7 @@ pub mod tests {
                     shred.clone(),
                     DataShredHeader::default(),
                     coding.clone(),
+                    NONCE_SHRED_PAYLOAD_SIZE,
                 );
                 let index = index_cf.get(shred.slot).unwrap().unwrap();
                 assert!(Blockstore::should_insert_coding_shred(
@@ -4596,6 +4603,7 @@ pub mod tests {
                     shred.clone(),
                     DataShredHeader::default(),
                     coding.clone(),
+                    NONCE_SHRED_PAYLOAD_SIZE,
                 );
                 let index = coding_shred.coding_header.position - 1;
                 coding_shred.set_index(index as u32);
@@ -4614,6 +4622,7 @@ pub mod tests {
                     shred.clone(),
                     DataShredHeader::default(),
                     coding.clone(),
+                    NONCE_SHRED_PAYLOAD_SIZE,
                 );
                 coding_shred.coding_header.num_coding_shreds = 0;
                 let index = index_cf.get(coding_shred.slot()).unwrap().unwrap();
@@ -4630,6 +4639,7 @@ pub mod tests {
                     shred.clone(),
                     DataShredHeader::default(),
                     coding.clone(),
+                    NONCE_SHRED_PAYLOAD_SIZE,
                 );
                 coding_shred.coding_header.num_coding_shreds = coding_shred.coding_header.position;
                 let index = index_cf.get(coding_shred.slot()).unwrap().unwrap();
@@ -4647,6 +4657,7 @@ pub mod tests {
                     shred.clone(),
                     DataShredHeader::default(),
                     coding.clone(),
+                    NONCE_SHRED_PAYLOAD_SIZE,
                 );
                 coding_shred.common_header.fec_set_index = std::u32::MAX - 1;
                 coding_shred.coding_header.num_coding_shreds = 3;
@@ -4679,6 +4690,7 @@ pub mod tests {
                     shred.clone(),
                     DataShredHeader::default(),
                     coding.clone(),
+                    NONCE_SHRED_PAYLOAD_SIZE,
                 );
                 let index = index_cf.get(coding_shred.slot()).unwrap().unwrap();
                 coding_shred.set_slot(*last_root.read().unwrap());

ledger/src/blockstore_db.rs

@@ -1,7 +1,6 @@
 use crate::blockstore_meta;
 use bincode::{deserialize, serialize};
 use byteorder::{BigEndian, ByteOrder};
-use fs_extra;
 use log::*;
 pub use rocksdb::Direction as IteratorDirection;
 use rocksdb::{
@@ -56,6 +55,7 @@ pub enum BlockstoreError {
     Serialize(#[from] Box<bincode::ErrorKind>),
     FsExtraError(#[from] fs_extra::error::Error),
     SlotCleanedUp,
+    UnableToSetOpenFileDescriptorLimit,
 }
 pub type Result<T> = std::result::Result<T, BlockstoreError>;
 
@@ -247,12 +247,12 @@ impl Rocks {
             IteratorMode::Start => RocksIteratorMode::Start,
             IteratorMode::End => RocksIteratorMode::End,
         };
-        let iter = self.0.iterator_cf(cf, iterator_mode)?;
+        let iter = self.0.iterator_cf(cf, iterator_mode);
         Ok(iter)
     }
 
     fn raw_iterator_cf(&self, cf: &ColumnFamily) -> Result<DBRawIterator> {
-        let raw_iter = self.0.raw_iterator_cf(cf)?;
+        let raw_iter = self.0.raw_iterator_cf(cf);
 
         Ok(raw_iter)
     }
@@ -783,13 +783,12 @@ where
 impl<'a> WriteBatch<'a> {
     pub fn put_bytes<C: Column + ColumnName>(&mut self, key: C::Index, bytes: &[u8]) -> Result<()> {
         self.write_batch
-            .put_cf(self.get_cf::<C>(), &C::key(key), bytes)?;
+            .put_cf(self.get_cf::<C>(), &C::key(key), bytes);
         Ok(())
     }
 
     pub fn delete<C: Column + ColumnName>(&mut self, key: C::Index) -> Result<()> {
-        self.write_batch
-            .delete_cf(self.get_cf::<C>(), &C::key(key))?;
+        self.write_batch.delete_cf(self.get_cf::<C>(), &C::key(key));
         Ok(())
     }
 
@@ -800,7 +799,7 @@ impl<'a> WriteBatch<'a> {
     ) -> Result<()> {
         let serialized_value = serialize(&value)?;
         self.write_batch
-            .put_cf(self.get_cf::<C>(), &C::key(key), &serialized_value)?;
+            .put_cf(self.get_cf::<C>(), &C::key(key), &serialized_value);
         Ok(())
     }
 
@@ -816,7 +815,7 @@ impl<'a> WriteBatch<'a> {
         to: C::Index,
     ) -> Result<()> {
         self.write_batch
-            .delete_range_cf(cf, C::key(from), C::key(to))?;
+            .delete_range_cf(cf, C::key(from), C::key(to));
         Ok(())
     }
 }
@@ -844,5 +843,9 @@ fn get_db_options() -> Options {
     options.create_missing_column_families(true);
     // A good value for this is the number of cores on the machine
     options.increase_parallelism(num_cpus::get() as i32);
+
+    // Set max total wal size to 4G.
+    options.set_max_total_wal_size(4 * 1024 * 1024 * 1024);
+
     options
 }

ledger/src/leader_schedule.rs

@@ -1,6 +1,5 @@
 use rand::distributions::{Distribution, WeightedIndex};
-use rand::SeedableRng;
-use rand_chacha::ChaChaRng;
+use rand_chacha::{rand_core::SeedableRng, ChaChaRng};
 use solana_sdk::pubkey::Pubkey;
 use std::ops::Index;
 use std::sync::Arc;

ledger/src/shred.rs

@@ -9,7 +9,7 @@ use rayon::{
     slice::ParallelSlice,
     ThreadPool,
 };
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Serialize, Serializer};
 use solana_metrics::datapoint_debug;
 use solana_perf::packet::Packet;
 use solana_rayon_threadlimit::get_thread_count;
@@ -24,25 +24,33 @@ use std::mem::size_of;
 use std::{sync::Arc, time::Instant};
 use thiserror::Error;
 
+pub type Nonce = u32;
+
 /// The following constants are computed by hand, and hardcoded.
 /// `test_shred_constants` ensures that the values are correct.
 /// Constants are used over lazy_static for performance reasons.
 pub const SIZE_OF_COMMON_SHRED_HEADER: usize = 83;
 pub const SIZE_OF_DATA_SHRED_HEADER: usize = 3;
+pub const SIZE_OF_DATA_SHRED_HEADER_SIZE_FIELD: usize = 2;
 pub const SIZE_OF_CODING_SHRED_HEADER: usize = 6;
 pub const SIZE_OF_SIGNATURE: usize = 64;
 pub const SIZE_OF_SHRED_TYPE: usize = 1;
 pub const SIZE_OF_SHRED_SLOT: usize = 8;
 pub const SIZE_OF_SHRED_INDEX: usize = 4;
+pub const SIZE_OF_NONCE: usize = 4;
 pub const SIZE_OF_DATA_SHRED_IGNORED_TAIL: usize =
     SIZE_OF_COMMON_SHRED_HEADER + SIZE_OF_CODING_SHRED_HEADER;
 pub const SIZE_OF_DATA_SHRED_PAYLOAD: usize = PACKET_DATA_SIZE
     - SIZE_OF_COMMON_SHRED_HEADER
     - SIZE_OF_DATA_SHRED_HEADER
     - SIZE_OF_DATA_SHRED_IGNORED_TAIL;
+pub const SIZE_OF_NONCE_DATA_SHRED_PAYLOAD: usize =
+    SIZE_OF_DATA_SHRED_PAYLOAD - SIZE_OF_NONCE - SIZE_OF_DATA_SHRED_HEADER_SIZE_FIELD;
 
 pub const OFFSET_OF_SHRED_SLOT: usize = SIZE_OF_SIGNATURE + SIZE_OF_SHRED_TYPE;
 pub const OFFSET_OF_SHRED_INDEX: usize = OFFSET_OF_SHRED_SLOT + SIZE_OF_SHRED_SLOT;
+pub const NONCE_SHRED_PAYLOAD_SIZE: usize = PACKET_DATA_SIZE - SIZE_OF_NONCE;
+pub const UNLOCK_NONCE_SLOT: Slot = 13_115_515;
 
 thread_local!(static PAR_THREAD_POOL: RefCell<ThreadPool> = RefCell::new(rayon::ThreadPoolBuilder::new()
                     .num_threads(get_thread_count())
@@ -107,6 +115,20 @@ pub struct ShredCommonHeader {
 pub struct DataShredHeader {
     pub parent_offset: u16,
     pub flags: u8,
+    #[serde(skip_deserializing)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(serialize_with = "option_as_u16_serialize")]
+    pub size: Option<u16>,
+}
+
+#[allow(clippy::trivially_copy_pass_by_ref)]
+fn option_as_u16_serialize<S>(x: &Option<u16>, s: S) -> std::result::Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    assert!(x.is_some());
+    let num = x.unwrap();
+    s.serialize_u16(num)
 }
 
 /// The coding shred header has FEC information
@@ -168,7 +190,8 @@ impl Shred {
         version: u16,
         fec_set_index: u32,
     ) -> Self {
-        let mut payload = vec![0; PACKET_DATA_SIZE];
+        let payload_size = Self::get_expected_payload_size_from_slot(slot);
+        let mut payload = vec![0; payload_size];
         let common_header = ShredCommonHeader {
             slot,
             index,
@@ -177,9 +200,20 @@ impl Shred {
             ..ShredCommonHeader::default()
         };
 
+        let size = if Self::is_nonce_unlocked(slot) {
+            Some(
+                (data.map(|d| d.len()).unwrap_or(0)
+                    + SIZE_OF_DATA_SHRED_HEADER
+                    + SIZE_OF_DATA_SHRED_HEADER_SIZE_FIELD
+                    + SIZE_OF_COMMON_SHRED_HEADER) as u16,
+            )
+        } else {
+            None
+        };
         let mut data_header = DataShredHeader {
             parent_offset,
             flags: reference_tick.min(SHRED_TICK_REFERENCE_MASK),
+            size,
         };
 
         if is_last_data {
@@ -198,9 +232,10 @@ impl Shred {
             &common_header,
         )
         .expect("Failed to write header into shred buffer");
+        let size_of_data_shred_header = Shredder::get_expected_data_header_size_from_slot(slot);
         Self::serialize_obj_into(
             &mut start,
-            SIZE_OF_DATA_SHRED_HEADER,
+            size_of_data_shred_header,
             &mut payload,
             &data_header,
         )
@@ -218,11 +253,21 @@ impl Shred {
         }
     }
 
-    pub fn new_from_serialized_shred(payload: Vec<u8>) -> Result<Self> {
+    pub fn new_from_serialized_shred(mut payload: Vec<u8>) -> Result<Self> {
         let mut start = 0;
         let common_header: ShredCommonHeader =
             Self::deserialize_obj(&mut start, SIZE_OF_COMMON_SHRED_HEADER, &payload)?;
 
+        let slot = common_header.slot;
+        let expected_data_size = Self::get_expected_payload_size_from_slot(slot);
+        // Safe because any payload from the network must have passed through
+        // window service,  which implies payload wll be of size
+        // PACKET_DATA_SIZE, and `expected_data_size` <= PACKET_DATA_SIZE.
+        //
+        // On the other hand, if this function is called locally, the payload size should match
+        // the `expected_data_size`.
+        assert!(payload.len() >= expected_data_size);
+        payload.truncate(expected_data_size);
         let shred = if common_header.shred_type == ShredType(CODING_SHRED) {
             let coding_header: CodingShredHeader =
                 Self::deserialize_obj(&mut start, SIZE_OF_CODING_SHRED_HEADER, &payload)?;
@@ -233,11 +278,14 @@ impl Shred {
                 payload,
             }
         } else if common_header.shred_type == ShredType(DATA_SHRED) {
+            // This doesn't need to change since we skip deserialization of the
+            // "size" field in the header for now
+            let size_of_data_shred_header = SIZE_OF_DATA_SHRED_HEADER;
             let data_header: DataShredHeader =
-                Self::deserialize_obj(&mut start, SIZE_OF_DATA_SHRED_HEADER, &payload)?;
+                Self::deserialize_obj(&mut start, size_of_data_shred_header, &payload)?;
             if u64::from(data_header.parent_offset) > common_header.slot {
                 return Err(ShredError::InvalidParentOffset {
-                    slot: common_header.slot,
+                    slot,
                     parent_offset: data_header.parent_offset,
                 });
             }
@@ -258,8 +306,10 @@ impl Shred {
         common_header: ShredCommonHeader,
         data_header: DataShredHeader,
         coding_header: CodingShredHeader,
+        payload_size: usize,
     ) -> Self {
-        let mut payload = vec![0; PACKET_DATA_SIZE];
+        assert!(payload_size == NONCE_SHRED_PAYLOAD_SIZE || payload_size == PACKET_DATA_SIZE);
+        let mut payload = vec![0; payload_size];
         let mut start = 0;
         Self::serialize_obj_into(
             &mut start,
@@ -268,10 +318,15 @@ impl Shred {
             &common_header,
         )
         .expect("Failed to write header into shred buffer");
+        let expected_data_header_size = if payload_size == NONCE_SHRED_PAYLOAD_SIZE {
+            SIZE_OF_DATA_SHRED_HEADER + SIZE_OF_DATA_SHRED_HEADER_SIZE_FIELD
+        } else {
+            SIZE_OF_DATA_SHRED_HEADER
+        };
         if common_header.shred_type == ShredType(DATA_SHRED) {
             Self::serialize_obj_into(
                 &mut start,
-                SIZE_OF_DATA_SHRED_HEADER,
+                expected_data_header_size,
                 &mut payload,
                 &data_header,
             )
@@ -293,11 +348,13 @@ impl Shred {
         }
     }
 
-    pub fn new_empty_data_shred() -> Self {
+    pub fn new_empty_data_shred(payload_size: usize) -> Self {
+        assert!(payload_size == NONCE_SHRED_PAYLOAD_SIZE || payload_size == PACKET_DATA_SIZE);
         Self::new_empty_from_header(
             ShredCommonHeader::default(),
             DataShredHeader::default(),
             CodingShredHeader::default(),
+            payload_size,
         )
     }
 
@@ -403,6 +460,18 @@ impl Shred {
         self.signature()
             .verify(pubkey.as_ref(), &self.payload[SIZE_OF_SIGNATURE..])
     }
+
+    pub fn is_nonce_unlocked(slot: Slot) -> bool {
+        slot > UNLOCK_NONCE_SLOT
+    }
+
+    fn get_expected_payload_size_from_slot(slot: Slot) -> usize {
+        if Self::is_nonce_unlocked(slot) {
+            NONCE_SHRED_PAYLOAD_SIZE
+        } else {
+            PACKET_DATA_SIZE
+        }
+    }
 }
 
 #[derive(Debug)]
@@ -467,7 +536,7 @@ impl Shredder {
 
         let now = Instant::now();
 
-        let no_header_size = SIZE_OF_DATA_SHRED_PAYLOAD;
+        let no_header_size = Self::get_expected_data_shred_payload_size_from_slot(self.slot);
         let num_shreds = (serialized_shreds.len() + no_header_size - 1) / no_header_size;
         let last_shred_index = next_shred_index + num_shreds as u32 - 1;
 
@@ -628,7 +697,8 @@ impl Shredder {
             let start_index = data_shred_batch[0].common_header.index;
 
             // All information after coding shred field in a data shred is encoded
-            let valid_data_len = PACKET_DATA_SIZE - SIZE_OF_DATA_SHRED_IGNORED_TAIL;
+            let expected_payload_size = Shred::get_expected_payload_size_from_slot(slot);
+            let valid_data_len = expected_payload_size - SIZE_OF_DATA_SHRED_IGNORED_TAIL;
             let data_ptrs: Vec<_> = data_shred_batch
                 .iter()
                 .map(|data| &data.payload[..valid_data_len])
@@ -646,8 +716,12 @@ impl Shredder {
                     i,
                     version,
                 );
-                let shred =
-                    Shred::new_empty_from_header(header, DataShredHeader::default(), coding_header);
+                let shred = Shred::new_empty_from_header(
+                    header,
+                    DataShredHeader::default(),
+                    coding_header,
+                    expected_payload_size,
+                );
                 coding_shreds.push(shred.payload);
             });
 
@@ -701,7 +775,10 @@ impl Shredder {
         expected_index: usize,
         index_found: usize,
         present: &mut [bool],
+        payload_size: usize,
     ) -> Vec<Vec<u8>> {
+        // Safe to assert because `new_from_serialized_shred` guarantees the size
+        assert!(payload_size == NONCE_SHRED_PAYLOAD_SIZE || payload_size == PACKET_DATA_SIZE);
         let end_index = index_found.saturating_sub(1);
         // The index of current shred must be within the range of shreds that are being
         // recovered
@@ -715,9 +792,9 @@ impl Shredder {
             .map(|missing| {
                 present[missing.saturating_sub(first_index_in_fec_set)] = false;
                 if missing < first_index_in_fec_set + num_data {
-                    Shred::new_empty_data_shred().payload
+                    Shred::new_empty_data_shred(payload_size).payload
                 } else {
-                    vec![0; PACKET_DATA_SIZE]
+                    vec![0; payload_size]
                 }
             })
             .collect();
@@ -732,6 +809,8 @@ impl Shredder {
         first_code_index: usize,
         slot: Slot,
     ) -> std::result::Result<Vec<Shred>, reed_solomon_erasure::Error> {
+        let expected_payload_size =
+            Self::verify_consistent_shred_payload_sizes(&"try_recovery()", &shreds)?;
         let mut recovered_data = vec![];
         let fec_set_size = num_data + num_coding;
 
@@ -751,6 +830,7 @@ impl Shredder {
                         next_expected_index,
                         index,
                         &mut present,
+                        expected_payload_size,
                     );
                     blocks.push(shred.payload);
                     next_expected_index = index + 1;
@@ -767,6 +847,7 @@ impl Shredder {
                 next_expected_index,
                 first_index + fec_set_size,
                 &mut present,
+                expected_payload_size,
             );
 
             shred_bufs.append(&mut pending_shreds);
@@ -777,7 +858,7 @@ impl Shredder {
 
             let session = Session::new(num_data, num_coding)?;
 
-            let valid_data_len = PACKET_DATA_SIZE - SIZE_OF_DATA_SHRED_IGNORED_TAIL;
+            let valid_data_len = expected_payload_size - SIZE_OF_DATA_SHRED_IGNORED_TAIL;
             let coding_block_offset = SIZE_OF_CODING_SHRED_HEADER + SIZE_OF_COMMON_SHRED_HEADER;
             let mut blocks: Vec<(&mut [u8], bool)> = shred_bufs
                 .iter_mut()
@@ -822,8 +903,11 @@ impl Shredder {
     /// Combines all shreds to recreate the original buffer
     pub fn deshred(shreds: &[Shred]) -> std::result::Result<Vec<u8>, reed_solomon_erasure::Error> {
         let num_data = shreds.len();
-        let data_shred_bufs = {
+        let expected_payload_size =
+            Self::verify_consistent_shred_payload_sizes(&"deshred()", shreds)?;
+        let (data_shred_bufs, slot) = {
             let first_index = shreds.first().unwrap().index() as usize;
+            let slot = shreds.first().unwrap().slot();
             let last_shred = shreds.last().unwrap();
             let last_index = if last_shred.data_complete() || last_shred.last_in_slot() {
                 last_shred.index() as usize
@@ -835,10 +919,32 @@ impl Shredder {
                 return Err(reed_solomon_erasure::Error::TooFewDataShards);
             }
 
-            shreds.iter().map(|shred| &shred.payload).collect()
+            (shreds.iter().map(|shred| &shred.payload).collect(), slot)
         };
 
-        Ok(Self::reassemble_payload(num_data, data_shred_bufs))
+        let expected_data_header_size = Self::get_expected_data_header_size_from_slot(slot);
+        Ok(Self::reassemble_payload(
+            num_data,
+            data_shred_bufs,
+            expected_payload_size,
+            expected_data_header_size,
+        ))
+    }
+
+    pub fn get_expected_data_shred_payload_size_from_slot(slot: Slot) -> usize {
+        if Shred::is_nonce_unlocked(slot) {
+            SIZE_OF_NONCE_DATA_SHRED_PAYLOAD
+        } else {
+            SIZE_OF_DATA_SHRED_PAYLOAD
+        }
+    }
+
+    pub fn get_expected_data_header_size_from_slot(slot: Slot) -> usize {
+        if Shred::is_nonce_unlocked(slot) {
+            SIZE_OF_DATA_SHRED_HEADER + SIZE_OF_DATA_SHRED_HEADER_SIZE_FIELD
+        } else {
+            SIZE_OF_DATA_SHRED_HEADER
+        }
     }
 
     fn get_shred_index(
@@ -854,26 +960,60 @@ impl Shredder {
         }
     }
 
-    fn reassemble_payload(num_data: usize, data_shred_bufs: Vec<&Vec<u8>>) -> Vec<u8> {
-        let valid_data_len = PACKET_DATA_SIZE - SIZE_OF_DATA_SHRED_IGNORED_TAIL;
+    fn reassemble_payload(
+        num_data: usize,
+        data_shred_bufs: Vec<&Vec<u8>>,
+        expected_payload_size: usize,
+        expected_data_header_size: usize,
+    ) -> Vec<u8> {
+        let valid_data_len = expected_payload_size - SIZE_OF_DATA_SHRED_IGNORED_TAIL;
         data_shred_bufs[..num_data]
             .iter()
             .flat_map(|data| {
-                let offset = SIZE_OF_COMMON_SHRED_HEADER + SIZE_OF_DATA_SHRED_HEADER;
+                let offset = SIZE_OF_COMMON_SHRED_HEADER + expected_data_header_size;
                 data[offset..valid_data_len].iter()
             })
             .cloned()
             .collect()
     }
+
+    fn verify_consistent_shred_payload_sizes(
+        caller: &str,
+        shreds: &[Shred],
+    ) -> std::result::Result<usize, reed_solomon_erasure::Error> {
+        if shreds.is_empty() {
+            return Err(reed_solomon_erasure::Error::TooFewShardsPresent);
+        }
+        let slot = shreds[0].slot();
+        let expected_payload_size = Shred::get_expected_payload_size_from_slot(slot);
+        for shred in shreds {
+            if shred.payload.len() != expected_payload_size {
+                error!(
+                    "{} Shreds for slot: {} are inconsistent sizes. One shred: {} Another shred: {}",
+                    caller,
+                    slot,
+                    expected_payload_size,
+                    shred.payload.len()
+                );
+                return Err(reed_solomon_erasure::Error::IncorrectShardSize);
+            }
+        }
+
+        Ok(expected_payload_size)
+    }
 }
 
-pub fn max_ticks_per_n_shreds(num_shreds: u64) -> u64 {
+pub fn max_ticks_per_n_shreds(num_shreds: u64, shred_data_size: Option<usize>) -> u64 {
     let ticks = create_ticks(1, 0, Hash::default());
-    max_entries_per_n_shred(&ticks[0], num_shreds)
+    max_entries_per_n_shred(&ticks[0], num_shreds, shred_data_size)
 }
 
-pub fn max_entries_per_n_shred(entry: &Entry, num_shreds: u64) -> u64 {
-    let shred_data_size = SIZE_OF_DATA_SHRED_PAYLOAD as u64;
+pub fn max_entries_per_n_shred(
+    entry: &Entry,
+    num_shreds: u64,
+    shred_data_size: Option<usize>,
+) -> u64 {
+    let shred_data_size = shred_data_size.unwrap_or(SIZE_OF_NONCE_DATA_SHRED_PAYLOAD) as u64;
     let vec_size = bincode::serialized_size(&vec![entry]).unwrap();
     let entry_size = bincode::serialized_size(entry).unwrap();
     let count_size = vec_size - entry_size;
@@ -891,7 +1031,8 @@ pub fn verify_test_data_shred(
     is_last_in_slot: bool,
     is_last_in_fec_set: bool,
 ) {
-    assert_eq!(shred.payload.len(), PACKET_DATA_SIZE);
+    let expected_payload_size = Shred::get_expected_payload_size_from_slot(slot);
+    assert_eq!(shred.payload.len(), expected_payload_size);
     assert!(shred.is_data());
     assert_eq!(shred.index(), index);
     assert_eq!(shred.slot(), slot);
@@ -932,6 +1073,14 @@ pub mod tests {
             SIZE_OF_DATA_SHRED_HEADER,
             serialized_size(&DataShredHeader::default()).unwrap() as usize
         );
+        let data_shred_header_with_size = DataShredHeader {
+            size: Some(1000),
+            ..DataShredHeader::default()
+        };
+        assert_eq!(
+            SIZE_OF_DATA_SHRED_HEADER + SIZE_OF_DATA_SHRED_HEADER_SIZE_FIELD,
+            serialized_size(&data_shred_header_with_size).unwrap() as usize
+        );
         assert_eq!(
             SIZE_OF_SIGNATURE,
             bincode::serialized_size(&Signature::default()).unwrap() as usize
@@ -951,17 +1100,16 @@ pub mod tests {
     }
 
     fn verify_test_code_shred(shred: &Shred, index: u32, slot: Slot, pk: &Pubkey, verify: bool) {
-        assert_eq!(shred.payload.len(), PACKET_DATA_SIZE);
+        let expected_payload_size = Shred::get_expected_payload_size_from_slot(slot);
+        assert_eq!(shred.payload.len(), expected_payload_size);
         assert!(!shred.is_data());
         assert_eq!(shred.index(), index);
         assert_eq!(shred.slot(), slot);
         assert_eq!(verify, shred.verify(pk));
     }
 
-    #[test]
-    fn test_data_shredder() {
+    fn run_test_data_shredder(slot: Slot) {
         let keypair = Arc::new(Keypair::new());
-        let slot = 0x123456789abcdef0;
 
         // Test that parent cannot be > current slot
         assert_matches!(
@@ -996,7 +1144,7 @@ pub mod tests {
             .collect();
 
         let size = serialized_size(&entries).unwrap();
-        let no_header_size = SIZE_OF_DATA_SHRED_PAYLOAD as u64;
+        let no_header_size = Shredder::get_expected_data_shred_payload_size_from_slot(slot) as u64;
         let num_expected_data_shreds = (size + no_header_size - 1) / no_header_size;
         let num_expected_coding_shreds =
             Shredder::calculate_num_coding_shreds(num_expected_data_shreds as f32, fec_rate);
@@ -1051,6 +1199,12 @@ pub mod tests {
         assert_eq!(entries, deshred_entries);
     }
 
+    #[test]
+    fn test_data_shredder() {
+        run_test_data_shredder(UNLOCK_NONCE_SLOT);
+        run_test_data_shredder(UNLOCK_NONCE_SLOT + 1);
+    }
+
     #[test]
     fn test_deserialize_shred_payload() {
         let keypair = Arc::new(Keypair::new());
@@ -1077,12 +1231,10 @@ pub mod tests {
         assert_eq!(deserialized_shred, *data_shreds.last().unwrap());
     }
 
-    #[test]
-    fn test_shred_reference_tick() {
+    fn run_test_shred_reference_tick(slot: Slot) {
         let keypair = Arc::new(Keypair::new());
-        let slot = 1;
 
-        let parent_slot = 0;
+        let parent_slot = slot - 1;
         let shredder = Shredder::new(slot, parent_slot, 0.0, keypair.clone(), 5, 0)
             .expect("Failed in creating shredder");
 
@@ -1107,6 +1259,12 @@ pub mod tests {
         assert_eq!(deserialized_shred.reference_tick(), 5);
     }
 
+    #[test]
+    fn test_shred_reference_tick() {
+        run_test_shred_reference_tick(UNLOCK_NONCE_SLOT);
+        run_test_shred_reference_tick(UNLOCK_NONCE_SLOT + 1);
+    }
+
     #[test]
     fn test_shred_reference_tick_overflow() {
         let keypair = Arc::new(Keypair::new());
@@ -1143,22 +1301,21 @@ pub mod tests {
         );
     }
 
-    #[test]
-    fn test_data_and_code_shredder() {
+    fn run_test_data_and_code_shredder(slot: Slot) {
         let keypair = Arc::new(Keypair::new());
 
-        let slot = 0x123456789abcdef0;
         // Test that FEC rate cannot be > 1.0
         assert_matches!(
             Shredder::new(slot, slot - 5, 1.001, keypair.clone(), 0, 0),
             Err(ShredError::InvalidFecRate(_))
         );
 
-        let shredder = Shredder::new(0x123456789abcdef0, slot - 5, 1.0, keypair.clone(), 0, 0)
+        let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone(), 0, 0)
             .expect("Failed in creating shredder");
 
         // Create enough entries to make > 1 shred
-        let num_entries = max_ticks_per_n_shreds(1) + 1;
+        let no_header_size = Shredder::get_expected_data_shred_payload_size_from_slot(slot);
+        let num_entries = max_ticks_per_n_shreds(1, Some(no_header_size)) + 1;
         let entries: Vec<_> = (0..num_entries)
             .map(|_| {
                 let keypair0 = Keypair::new();
@@ -1190,9 +1347,13 @@ pub mod tests {
     }
 
     #[test]
-    fn test_recovery_and_reassembly() {
+    fn test_data_and_code_shredder() {
+        run_test_data_and_code_shredder(UNLOCK_NONCE_SLOT);
+        run_test_data_and_code_shredder(UNLOCK_NONCE_SLOT + 1);
+    }
+
+    fn run_test_recovery_and_reassembly(slot: Slot) {
         let keypair = Arc::new(Keypair::new());
-        let slot = 0x123456789abcdef0;
         let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone(), 0, 0)
             .expect("Failed in creating shredder");
 
@@ -1202,7 +1363,9 @@ pub mod tests {
         let entry = Entry::new(&Hash::default(), 1, vec![tx0]);
 
         let num_data_shreds: usize = 5;
-        let num_entries = max_entries_per_n_shred(&entry, num_data_shreds as u64);
+        let no_header_size = Shredder::get_expected_data_shred_payload_size_from_slot(slot);
+        let num_entries =
+            max_entries_per_n_shred(&entry, num_data_shreds as u64, Some(no_header_size));
         let entries: Vec<_> = (0..num_entries)
             .map(|_| {
                 let keypair0 = Keypair::new();
@@ -1441,6 +1604,12 @@ pub mod tests {
         );
     }
 
+    #[test]
+    fn test_recovery_and_reassembly() {
+        run_test_recovery_and_reassembly(UNLOCK_NONCE_SLOT);
+        run_test_recovery_and_reassembly(UNLOCK_NONCE_SLOT + 1);
+    }
+
     #[test]
     fn test_shred_version() {
         let keypair = Arc::new(Keypair::new());

ledger/src/sigverify_shreds.rs

@@ -1,5 +1,5 @@
 #![allow(clippy::implicit_hasher)]
-use crate::shred::ShredType;
+use crate::shred::{Shred, ShredType, SIZE_OF_NONCE};
 use rayon::{
     iter::{
         IndexedParallelIterator, IntoParallelIterator, IntoParallelRefMutIterator, ParallelIterator,
@@ -16,9 +16,12 @@ use solana_perf::{
     sigverify::{self, batch_size, TxOffset},
 };
 use solana_rayon_threadlimit::get_thread_count;
-use solana_sdk::pubkey::Pubkey;
-use solana_sdk::signature::Signature;
-use solana_sdk::signature::{Keypair, Signer};
+use solana_sdk::{
+    clock::Slot,
+    pubkey::Pubkey,
+    signature::Signature,
+    signature::{Keypair, Signer},
+};
 use std::sync::Arc;
 use std::{collections::HashMap, mem::size_of};
 
@@ -40,13 +43,12 @@ lazy_static! {
 ///   ...
 /// }
 /// Signature is the first thing in the packet, and slot is the first thing in the signed message.
-fn verify_shred_cpu(packet: &Packet, slot_leaders: &HashMap<u64, [u8; 32]>) -> Option<u8> {
+pub fn verify_shred_cpu(packet: &Packet, slot_leaders: &HashMap<u64, [u8; 32]>) -> Option<u8> {
     let sig_start = 0;
     let sig_end = size_of::<Signature>();
     let slot_start = sig_end + size_of::<ShredType>();
     let slot_end = slot_start + size_of::<u64>();
     let msg_start = sig_end;
-    let msg_end = packet.meta.size;
     if packet.meta.discard {
         return Some(0);
     }
@@ -55,6 +57,11 @@ fn verify_shred_cpu(packet: &Packet, slot_leaders: &HashMap<u64, [u8; 32]>) -> O
         return Some(0);
     }
     let slot: u64 = limited_deserialize(&packet.data[slot_start..slot_end]).ok()?;
+    let msg_end = if packet.meta.repair && Shred::is_nonce_unlocked(slot) {
+        packet.meta.size.saturating_sub(SIZE_OF_NONCE)
+    } else {
+        packet.meta.size
+    };
     trace!("slot {}", slot);
     let pubkey = slot_leaders.get(&slot)?;
     if packet.meta.size < sig_end {
@@ -94,10 +101,10 @@ fn slot_key_data_for_gpu<
     batches: &[Packets],
     slot_keys: &HashMap<u64, T>,
     recycler_cache: &RecyclerCache,
-) -> (PinnedVec<u8>, TxOffset, usize) {
+) -> (PinnedVec<u8>, TxOffset, usize, Vec<Vec<Slot>>) {
     //TODO: mark Pubkey::default shreds as failed after the GPU returns
     assert_eq!(slot_keys.get(&std::u64::MAX), Some(&T::default()));
-    let slots: Vec<Vec<u64>> = SIGVERIFY_THREAD_POOL.install(|| {
+    let slots: Vec<Vec<Slot>> = SIGVERIFY_THREAD_POOL.install(|| {
         batches
             .into_par_iter()
             .map(|p| {
@@ -157,7 +164,7 @@ fn slot_key_data_for_gpu<
     trace!("keyvec.len: {}", keyvec.len());
     trace!("keyvec: {:?}", keyvec);
     trace!("offsets: {:?}", offsets);
-    (keyvec, offsets, num_in_packets)
+    (keyvec, offsets, num_in_packets, slots)
 }
 
 fn vec_size_in_packets(keyvec: &PinnedVec<u8>) -> usize {
@@ -177,6 +184,7 @@ fn shred_gpu_offsets(
     mut pubkeys_end: usize,
     batches: &[Packets],
     recycler_cache: &RecyclerCache,
+    slots: Option<Vec<Vec<Slot>>>,
 ) -> (TxOffset, TxOffset, TxOffset, Vec<Vec<u32>>) {
     let mut signature_offsets = recycler_cache.offsets().allocate("shred_signatures");
     signature_offsets.set_pinnable();
@@ -185,13 +193,30 @@ fn shred_gpu_offsets(
     let mut msg_sizes = recycler_cache.offsets().allocate("shred_msg_sizes");
     msg_sizes.set_pinnable();
     let mut v_sig_lens = vec![];
-    for batch in batches {
+    let mut slots_iter;
+    let mut slots_iter_ref: &mut dyn Iterator<Item = Vec<Slot>> = &mut std::iter::repeat(vec![]);
+    if let Some(slots) = slots {
+        slots_iter = slots.into_iter();
+        slots_iter_ref = &mut slots_iter;
+    }
+    for (batch, slots) in batches.iter().zip(slots_iter_ref) {
         let mut sig_lens = Vec::new();
-        for packet in &batch.packets {
+        let mut inner_slot_iter;
+        let mut inner_slot_iter_ref: &mut dyn Iterator<Item = Slot> = &mut std::iter::repeat(0);
+        if !slots.is_empty() {
+            inner_slot_iter = slots.into_iter();
+            inner_slot_iter_ref = &mut inner_slot_iter;
+        };
+
+        for (packet, slot) in batch.packets.iter().zip(inner_slot_iter_ref) {
             let sig_start = pubkeys_end;
             let sig_end = sig_start + size_of::<Signature>();
             let msg_start = sig_end;
-            let msg_end = sig_start + packet.meta.size;
+            let msg_end = if packet.meta.repair && Shred::is_nonce_unlocked(slot) {
+                sig_start + packet.meta.size.saturating_sub(SIZE_OF_NONCE)
+            } else {
+                sig_start + packet.meta.size
+            };
             signature_offsets.push(sig_start as u32);
             msg_start_offsets.push(msg_start as u32);
             let msg_size = if msg_end < msg_start {
@@ -222,7 +247,7 @@ pub fn verify_shreds_gpu(
     let mut elems = Vec::new();
     let mut rvs = Vec::new();
     let count = batch_size(batches);
-    let (pubkeys, pubkey_offsets, mut num_packets) =
+    let (pubkeys, pubkey_offsets, mut num_packets, slots) =
         slot_key_data_for_gpu(0, batches, slot_leaders, recycler_cache);
     //HACK: Pubkeys vector is passed along as a `Packets` buffer to the GPU
     //TODO: GPU needs a more opaque interface, which can handle variable sized structures for data
@@ -230,7 +255,7 @@ pub fn verify_shreds_gpu(
     trace!("num_packets: {}", num_packets);
     trace!("pubkeys_len: {}", pubkeys_len);
     let (signature_offsets, msg_start_offsets, msg_sizes, v_sig_lens) =
-        shred_gpu_offsets(pubkeys_len, batches, recycler_cache);
+        shred_gpu_offsets(pubkeys_len, batches, recycler_cache, Some(slots));
     let mut out = recycler_cache.buffer().allocate("out_buffer");
     out.set_pinnable();
     elems.push(
@@ -367,7 +392,7 @@ pub fn sign_shreds_gpu(
 
     trace!("offset: {}", offset);
     let (signature_offsets, msg_start_offsets, msg_sizes, _v_sig_lens) =
-        shred_gpu_offsets(offset, batches, recycler_cache);
+        shred_gpu_offsets(offset, batches, recycler_cache, None);
     let total_sigs = signature_offsets.len();
     let mut signatures_out = recycler_cache.buffer().allocate("ed25519 signatures");
     signatures_out.set_pinnable();
@@ -445,14 +470,12 @@ pub fn sign_shreds_gpu(
 #[cfg(test)]
 pub mod tests {
     use super::*;
-    use crate::shred::SIZE_OF_DATA_SHRED_PAYLOAD;
-    use crate::shred::{Shred, Shredder};
+    use crate::shred::{Shred, Shredder, SIZE_OF_DATA_SHRED_PAYLOAD, UNLOCK_NONCE_SLOT};
     use solana_sdk::signature::{Keypair, Signer};
-    #[test]
-    fn test_sigverify_shred_cpu() {
+
+    fn run_test_sigverify_shred_cpu(slot: Slot) {
         solana_logger::setup();
         let mut packet = Packet::default();
-        let slot = 0xdeadc0de;
         let mut shred = Shred::new_from_data(
             slot,
             0xc0de,
@@ -492,10 +515,14 @@ pub mod tests {
     }
 
     #[test]
-    fn test_sigverify_shreds_cpu() {
+    fn test_sigverify_shred_cpu() {
+        run_test_sigverify_shred_cpu(UNLOCK_NONCE_SLOT);
+        run_test_sigverify_shred_cpu(UNLOCK_NONCE_SLOT + 1);
+    }
+
+    fn run_test_sigverify_shreds_cpu(slot: Slot) {
         solana_logger::setup();
         let mut batch = [Packets::default()];
-        let slot = 0xdeadc0de;
         let mut shred = Shred::new_from_data(
             slot,
             0xc0de,
@@ -542,12 +569,16 @@ pub mod tests {
     }
 
     #[test]
-    fn test_sigverify_shreds_gpu() {
+    fn test_sigverify_shreds_cpu() {
+        run_test_sigverify_shreds_cpu(UNLOCK_NONCE_SLOT);
+        run_test_sigverify_shreds_cpu(UNLOCK_NONCE_SLOT + 1);
+    }
+
+    fn run_test_sigverify_shreds_gpu(slot: Slot) {
         solana_logger::setup();
         let recycler_cache = RecyclerCache::default();
 
         let mut batch = [Packets::default()];
-        let slot = 0xdeadc0de;
         let mut shred = Shred::new_from_data(
             slot,
             0xc0de,
@@ -603,14 +634,18 @@ pub mod tests {
     }
 
     #[test]
-    fn test_sigverify_shreds_sign_gpu() {
+    fn test_sigverify_shreds_gpu() {
+        run_test_sigverify_shreds_gpu(UNLOCK_NONCE_SLOT);
+        run_test_sigverify_shreds_gpu(UNLOCK_NONCE_SLOT + 1);
+    }
+
+    fn run_test_sigverify_shreds_sign_gpu(slot: Slot) {
         solana_logger::setup();
         let recycler_cache = RecyclerCache::default();
 
         let mut packets = Packets::default();
         let num_packets = 32;
         let num_batches = 100;
-        let slot = 0xdeadc0de;
         packets.packets.resize(num_packets, Packet::default());
         for (i, p) in packets.packets.iter_mut().enumerate() {
             let shred = Shred::new_from_data(
@@ -650,11 +685,15 @@ pub mod tests {
     }
 
     #[test]
-    fn test_sigverify_shreds_sign_cpu() {
+    fn test_sigverify_shreds_sign_gpu() {
+        run_test_sigverify_shreds_sign_gpu(UNLOCK_NONCE_SLOT);
+        run_test_sigverify_shreds_sign_gpu(UNLOCK_NONCE_SLOT + 1);
+    }
+
+    fn run_test_sigverify_shreds_sign_cpu(slot: Slot) {
         solana_logger::setup();
 
         let mut batch = [Packets::default()];
-        let slot = 0xdeadc0de;
         let keypair = Keypair::new();
         let shred = Shred::new_from_data(
             slot,
@@ -685,4 +724,10 @@ pub mod tests {
         let rv = verify_shreds_cpu(&batch, &pubkeys);
         assert_eq!(rv, vec![vec![1]]);
     }
+
+    #[test]
+    fn test_sigverify_shreds_sign_cpu() {
+        run_test_sigverify_shreds_sign_cpu(UNLOCK_NONCE_SLOT);
+        run_test_sigverify_shreds_sign_cpu(UNLOCK_NONCE_SLOT + 1);
+    }
 }

ledger/tests/shred.rs

@@ -1,16 +1,15 @@
 use solana_ledger::entry::Entry;
 use solana_ledger::shred::{
-    max_entries_per_n_shred, verify_test_data_shred, Shred, Shredder, MAX_DATA_SHREDS_PER_FEC_BLOCK,
+    max_entries_per_n_shred, verify_test_data_shred, Shred, Shredder,
+    MAX_DATA_SHREDS_PER_FEC_BLOCK, UNLOCK_NONCE_SLOT,
 };
 use solana_sdk::signature::{Keypair, Signer};
-use solana_sdk::{hash::Hash, system_transaction};
+use solana_sdk::{clock::Slot, hash::Hash, system_transaction};
 use std::convert::TryInto;
 use std::sync::Arc;
 
-#[test]
-fn test_multi_fec_block_coding() {
+fn run_test_multi_fec_block_coding(slot: Slot) {
     let keypair = Arc::new(Keypair::new());
-    let slot = 0x123456789abcdef0;
     let shredder = Shredder::new(slot, slot - 5, 1.0, keypair.clone(), 0, 0)
         .expect("Failed in creating shredder");
 
@@ -20,7 +19,8 @@ fn test_multi_fec_block_coding() {
     let keypair1 = Keypair::new();
     let tx0 = system_transaction::transfer(&keypair0, &keypair1.pubkey(), 1, Hash::default());
     let entry = Entry::new(&Hash::default(), 1, vec![tx0]);
-    let num_entries = max_entries_per_n_shred(&entry, num_data_shreds as u64);
+    let no_header_size = Shredder::get_expected_data_shred_payload_size_from_slot(slot);
+    let num_entries = max_entries_per_n_shred(&entry, num_data_shreds as u64, Some(no_header_size));
 
     let entries: Vec<_> = (0..num_entries)
         .map(|_| {
@@ -94,3 +94,9 @@ fn test_multi_fec_block_coding() {
     let result = Shredder::deshred(&all_shreds[..]).unwrap();
     assert_eq!(serialized_entries[..], result[..serialized_entries.len()]);
 }
+
+#[test]
+fn test_multi_fec_block_coding() {
+    run_test_multi_fec_block_coding(UNLOCK_NONCE_SLOT);
+    run_test_multi_fec_block_coding(UNLOCK_NONCE_SLOT + 1);
+}

local-cluster/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-local-cluster"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.19"
+version = "1.0.23"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,24 +11,24 @@ homepage = "https://solana.com/"
 [dependencies]
 itertools = "0.8.1"
 log = "0.4.8"
-rand = "0.6.5"
-solana-archiver-lib = { path = "../archiver-lib", version = "1.0.19" }
-solana-config-program = { path = "../programs/config", version = "1.0.19" }
-solana-core = { path = "../core", version = "1.0.19" }
-solana-client = { path = "../client", version = "1.0.19" }
-solana-faucet = { path = "../faucet", version = "1.0.19" }
-solana-exchange-program = { path = "../programs/exchange", version = "1.0.19" }
-solana-genesis-programs = { path = "../genesis-programs", version = "1.0.19" }
-solana-ledger = { path = "../ledger", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
-solana-runtime = { path = "../runtime", version = "1.0.19" }
-solana-sdk = { path = "../sdk", version = "1.0.19" }
-solana-stake-program = { path = "../programs/stake", version = "1.0.19" }
-solana-storage-program = { path = "../programs/storage", version = "1.0.19" }
-solana-vest-program = { path = "../programs/vest", version = "1.0.19" }
-solana-vote-program = { path = "../programs/vote", version = "1.0.19" }
+rand = "0.7.0"
+solana-archiver-lib = { path = "../archiver-lib", version = "1.0.23" }
+solana-config-program = { path = "../programs/config", version = "1.0.23" }
+solana-core = { path = "../core", version = "1.0.23" }
+solana-client = { path = "../client", version = "1.0.23" }
+solana-faucet = { path = "../faucet", version = "1.0.23" }
+solana-exchange-program = { path = "../programs/exchange", version = "1.0.23" }
+solana-genesis-programs = { path = "../genesis-programs", version = "1.0.23" }
+solana-ledger = { path = "../ledger", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
+solana-runtime = { path = "../runtime", version = "1.0.23" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
+solana-stake-program = { path = "../programs/stake", version = "1.0.23" }
+solana-storage-program = { path = "../programs/storage", version = "1.0.23" }
+solana-vest-program = { path = "../programs/vest", version = "1.0.23" }
+solana-vote-program = { path = "../programs/vote", version = "1.0.23" }
 tempfile = "3.1.0"
-solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.0.19" }
+solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.0.23" }
 
 [dev-dependencies]
 assert_matches = "1.3.0"

log-analyzer/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-log-analyzer"
 description = "The solana cluster network analysis tool"
-version = "1.0.19"
+version = "1.0.23"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,8 +14,8 @@ byte-unit = "3.0.3"
 clap = "2.33.0"
 serde = "1.0.104"
 serde_json = "1.0.46"
-solana-clap-utils = { path = "../clap-utils", version = "1.0.19" }
-solana-logger = { path = "../logger", version = "1.0.19" }
+solana-clap-utils = { path = "../clap-utils", version = "1.0.23" }
+solana-logger = { path = "../logger", version = "1.0.23" }
 
 [[bin]]
 name = "solana-log-analyzer"

logger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-logger"
-version = "1.0.19"
+version = "1.0.23"
 description = "Solana Logger"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

logger/src/lib.rs

@@ -5,7 +5,7 @@ use std::sync::{Arc, RwLock};
 
 lazy_static! {
     static ref LOGGER: Arc<RwLock<env_logger::Logger>> =
-        { Arc::new(RwLock::new(env_logger::Logger::from_default_env())) };
+        Arc::new(RwLock::new(env_logger::Logger::from_default_env()));
 }
 
 struct LoggerShim {}

measure/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-measure"
 description = "Blockchain, Rebuilt for Scale"
-version = "1.0.19"
+version = "1.0.23"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -12,8 +12,8 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.8"
-solana-sdk = { path = "../sdk", version = "1.0.19" }
-solana-metrics = { path = "../metrics", version = "1.0.19" }
+solana-sdk = { path = "../sdk", version = "1.0.23" }
+solana-metrics = { path = "../metrics", version = "1.0.23" }
 
 [target."cfg(unix)".dependencies]
 jemallocator = "0.3.2"