Display release date in the local timezone

* fix accounts_db storage.reset()

* fix compilation errors, remove unused, fix test_accounts_grow() failure

Cargo.lock

@@ -1056,11 +1056,6 @@ dependencies = [
  "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "ipnetwork"
-version = "0.12.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "itertools"
 version = "0.7.11"
@@ -1633,33 +1628,6 @@ name = "pkg-config"
 version = "0.3.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "pnet_base"
-version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "pnet_datalink"
-version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "ipnetwork 0.12.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)",
- "pnet_base 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "pnet_sys 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "pnet_sys"
-version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "predicates"
 version = "1.0.0"
@@ -2194,7 +2162,7 @@ dependencies = [
 
 [[package]]
 name = "solana"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "bs58 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2225,20 +2193,20 @@ dependencies = [
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-budget-api 0.14.0",
+ "solana-budget-api 0.14.1",
- "solana-budget-program 0.14.0",
+ "solana-budget-program 0.14.1",
- "solana-client 0.14.0",
+ "solana-client 0.14.1",
- "solana-drone 0.14.0",
+ "solana-drone 0.14.1",
- "solana-kvstore 0.14.0",
+ "solana-kvstore 0.14.1",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-metrics 0.14.0",
+ "solana-metrics 0.14.1",
- "solana-netutil 0.14.0",
+ "solana-netutil 0.14.1",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
- "solana-storage-api 0.14.0",
+ "solana-storage-api 0.14.1",
- "solana-vote-api 0.14.0",
+ "solana-vote-api 0.14.1",
- "solana-vote-program 0.14.0",
+ "solana-vote-program 0.14.1",
- "solana-vote-signer 0.14.0",
+ "solana-vote-signer 0.14.1",
  "sys-info 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2247,7 +2215,7 @@ dependencies = [
 
 [[package]]
 name = "solana-bench-exchange"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "bs58 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2261,79 +2229,79 @@ dependencies = [
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.14.0",
+ "solana 0.14.1",
- "solana-client 0.14.0",
+ "solana-client 0.14.1",
- "solana-drone 0.14.0",
+ "solana-drone 0.14.1",
- "solana-exchange-api 0.14.0",
+ "solana-exchange-api 0.14.1",
- "solana-exchange-program 0.14.0",
+ "solana-exchange-program 0.14.1",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-metrics 0.14.0",
+ "solana-metrics 0.14.1",
- "solana-netutil 0.14.0",
+ "solana-netutil 0.14.1",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
  "untrusted 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "ws 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "solana-bench-streamer"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.14.0",
+ "solana 0.14.1",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-netutil 0.14.0",
+ "solana-netutil 0.14.1",
 ]
 
 [[package]]
 name = "solana-bench-tps"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rayon 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.14.0",
+ "solana 0.14.1",
- "solana-client 0.14.0",
+ "solana-client 0.14.1",
- "solana-drone 0.14.0",
+ "solana-drone 0.14.1",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-metrics 0.14.0",
+ "solana-metrics 0.14.1",
- "solana-netutil 0.14.0",
+ "solana-netutil 0.14.1",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-bpf-programs"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "elf 0.0.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-bpfloader 0.14.0",
+ "solana-bpfloader 0.14.1",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
  "solana_rbpf 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "walkdir 2.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "solana-bpfloader"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
  "solana_rbpf 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "solana-budget-api"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2342,23 +2310,23 @@ dependencies = [
  "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-budget-program"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-budget-api 0.14.0",
+ "solana-budget-api 0.14.1",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-client"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "bs58 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2369,37 +2337,37 @@ dependencies = [
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-netutil 0.14.0",
+ "solana-netutil 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-config-api"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-config-program"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-config-api 0.14.0",
+ "solana-config-api 0.14.1",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-drone"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2408,97 +2376,97 @@ dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-metrics 0.14.0",
+ "solana-metrics 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
  "tokio 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "solana-exchange-api"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-metrics 0.14.0",
+ "solana-metrics 0.14.1",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-exchange-program"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-exchange-api 0.14.0",
+ "solana-exchange-api 0.14.1",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-failure-program"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-fullnode"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.14.0",
+ "solana 0.14.1",
- "solana-drone 0.14.0",
+ "solana-drone 0.14.1",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-metrics 0.14.0",
+ "solana-metrics 0.14.1",
- "solana-netutil 0.14.0",
+ "solana-netutil 0.14.1",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
- "solana-vote-api 0.14.0",
+ "solana-vote-api 0.14.1",
- "solana-vote-signer 0.14.0",
+ "solana-vote-signer 0.14.1",
 ]
 
 [[package]]
 name = "solana-genesis"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "hashbrown 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.14.0",
+ "solana 0.14.1",
- "solana-budget-api 0.14.0",
+ "solana-budget-api 0.14.1",
- "solana-config-api 0.14.0",
+ "solana-config-api 0.14.1",
- "solana-exchange-api 0.14.0",
+ "solana-exchange-api 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
- "solana-stake-api 0.14.0",
+ "solana-stake-api 0.14.1",
- "solana-storage-api 0.14.0",
+ "solana-storage-api 0.14.1",
- "solana-token-api 0.14.0",
+ "solana-token-api 0.14.1",
- "solana-vote-api 0.14.0",
+ "solana-vote-api 0.14.1",
 ]
 
 [[package]]
 name = "solana-gossip"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "env_logger 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.14.0",
+ "solana 0.14.1",
- "solana-client 0.14.0",
+ "solana-client 0.14.1",
- "solana-netutil 0.14.0",
+ "solana-netutil 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-install"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "bs58 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2515,10 +2483,10 @@ dependencies = [
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_yaml 0.8.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-client 0.14.0",
+ "solana-client 0.14.1",
- "solana-config-api 0.14.0",
+ "solana-config-api 0.14.1",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
  "tar 0.4.23 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2526,16 +2494,16 @@ dependencies = [
 
 [[package]]
 name = "solana-keygen"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-kvstore"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2555,70 +2523,70 @@ dependencies = [
  "assert_cmd 0.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.14.0",
+ "solana 0.14.1",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-logger"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "env_logger 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "solana-metrics"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "influx_db_client 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
  "sys-info 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "solana-netutil"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
- "ipnetwork 0.12.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "nix 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "pnet_datalink 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "reqwest 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)",
  "socket2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
+ "tokio 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "solana-noop-program"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-replicator"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.14.0",
+ "solana 0.14.1",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-netutil 0.14.0",
+ "solana-netutil 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-runtime"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "bv 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2634,15 +2602,15 @@ dependencies = [
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-metrics 0.14.0",
+ "solana-metrics 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
- "solana-vote-api 0.14.0",
+ "solana-vote-api 0.14.1",
 ]
 
 [[package]]
 name = "solana-sdk"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "bs58 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2666,55 +2634,55 @@ dependencies = [
 
 [[package]]
 name = "solana-stake-api"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-metrics 0.14.0",
+ "solana-metrics 0.14.1",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
- "solana-vote-api 0.14.0",
+ "solana-vote-api 0.14.1",
 ]
 
 [[package]]
 name = "solana-stake-program"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
- "solana-stake-api 0.14.0",
+ "solana-stake-api 0.14.1",
 ]
 
 [[package]]
 name = "solana-storage-api"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-storage-program"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
- "solana-storage-api 0.14.0",
+ "solana-storage-api 0.14.1",
 ]
 
 [[package]]
 name = "solana-token-api"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2722,18 +2690,18 @@ dependencies = [
  "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-token-program"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
- "solana-token-api 0.14.0",
+ "solana-token-api 0.14.1",
 ]
 
 [[package]]
@@ -2741,36 +2709,36 @@ name = "solana-upload-perf"
 version = "0.14.0"
 dependencies = [
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-metrics 0.14.0",
+ "solana-metrics 0.14.1",
 ]
 
 [[package]]
 name = "solana-vote-api"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-metrics 0.14.0",
+ "solana-metrics 0.14.1",
- "solana-runtime 0.14.0",
+ "solana-runtime 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-vote-program"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
- "solana-vote-api 0.14.0",
+ "solana-vote-api 0.14.1",
 ]
 
 [[package]]
 name = "solana-vote-signer"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bs58 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2780,13 +2748,13 @@ dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-metrics 0.14.0",
+ "solana-metrics 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
 ]
 
 [[package]]
 name = "solana-wallet"
-version = "0.14.0"
+version = "0.14.1"
 dependencies = [
  "bincode 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "bs58 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2796,16 +2764,16 @@ dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.14.0",
+ "solana 0.14.1",
- "solana-budget-api 0.14.0",
+ "solana-budget-api 0.14.1",
- "solana-budget-program 0.14.0",
+ "solana-budget-program 0.14.1",
- "solana-client 0.14.0",
+ "solana-client 0.14.1",
- "solana-drone 0.14.0",
+ "solana-drone 0.14.1",
- "solana-logger 0.14.0",
+ "solana-logger 0.14.1",
- "solana-netutil 0.14.0",
+ "solana-netutil 0.14.1",
- "solana-sdk 0.14.0",
+ "solana-sdk 0.14.1",
- "solana-vote-api 0.14.0",
+ "solana-vote-api 0.14.1",
- "solana-vote-signer 0.14.0",
+ "solana-vote-signer 0.14.1",
 ]
 
 [[package]]
@@ -3531,7 +3499,6 @@ dependencies = [
 "checksum indicatif 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2c60da1c9abea75996b70a931bba6c750730399005b61ccd853cee50ef3d0d0c"
 "checksum influx_db_client 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1af8df5705f0b30bcb504bafc9396d995a555c4d6bd6f9097729ad47b8a49a38"
 "checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08"
-"checksum ipnetwork 0.12.8 (registry+https://github.com/rust-lang/crates.io-index)" = "70783119ac90828aaba91eae39db32c6c1b8838deea3637e5238efa0130801ab"
 "checksum itertools 0.7.11 (registry+https://github.com/rust-lang/crates.io-index)" = "0d47946d458e94a1b7bcabbf6521ea7c037062c81f534615abcad76e84d4970d"
 "checksum itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b8467d9c1cebe26feb08c640139247fac215782d35371ade9a2136ed6085358"
 "checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b"
@@ -3595,9 +3562,6 @@ dependencies = [
 "checksum phf_generator 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)" = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662"
 "checksum phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)" = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0"
 "checksum pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "676e8eb2b1b4c9043511a9b7bea0915320d7e502b0a079fb03f9635a5252b18c"
-"checksum pnet_base 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)" = "948dbdd36f46888ada1d497703e6cae53d227ab0e8871638aba492ad1e4a76dc"
-"checksum pnet_datalink 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8d3b3dd76a11ad99d92fef54b2489f76f4045ebd5251bd1af485d55a7e13db6"
-"checksum pnet_sys 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)" = "963b9109a05c3ac370abc3fda61bff20d03743c2947942173871b9cac2b9acb0"
 "checksum predicates 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fa984b7cd021a0bf5315bcce4c4ae61d2a535db2a8d288fc7578638690a7b7c3"
 "checksum predicates-core 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "06075c3a3e92559ff8929e7a280684489ea27fe44805174c3ebd9328dcb37178"
 "checksum predicates-tree 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8e63c4859013b38a76eca2414c64911fba30def9e3202ac461a2d22831220124"

RELEASE.md

@@ -61,7 +61,7 @@ There are three release channels that map to branches as follows:
 
 ## Release Steps
 
-### Changing channels
+### Advance the Channels
 
 #### Create the new branch
 1. Pick your branch point for release on master.
@@ -84,7 +84,7 @@ There are three release channels that map to branches as follows:
 At this point, `ci/channel-info.sh` should show your freshly cut release branch as
 "BETA_CHANNEL" and the previous release branch as "STABLE_CHANNEL".
 
-### Updating channels (i.e. "making a release")
+### Make the Release
 
 We use [github's Releases UI](https://github.com/solana-labs/solana/releases) for tagging a release.
 
@@ -99,13 +99,59 @@ We use [github's Releases UI](https://github.com/solana-labs/solana/releases) fo
    release should be `<branchname>.X-rc.0`.
 1. Verify release automation:
    1. [Crates.io](https://crates.io/crates/solana) should have an updated Solana version.
-   1. ...
-1. After testnet deployment, verify that testnets are running correct software.
-   http://metrics.solana.com should show testnet running on a hash from your
-   newly created branch.
 1. Once the release has been made, update Cargo.toml on the release branch to the next
    semantic version (e.g. 0.9.0 -> 0.9.1) by running
    `./scripts/increment-cargo-version.sh patch`, then rebuild with `cargo
    build` to cause a refresh of `Cargo.lock`.
 1. Push your Cargo.toml change and the autogenerated Cargo.lock changes to the
    release branch.
+
+### Update software on testnet.solana.com
+
+The testnet running on testnet.solana.com is set to use a fixed release tag
+which is set in the Buildkite testnet-management pipeline.
+This tag needs to be updated and the testnet restarted after a new release
+tag is created.
+
+#### Update testnet schedules
+
+Go to https://buildkite.com/solana-labs and click through: Pipelines ->
+testnet-management -> Pipeline Settings -> Schedules
+Or just click here:
+https://buildkite.com/solana-labs/testnet-management/settings/schedules
+
+There are two scheduled jobs for testnet: a daily restart and an hourly sanity-or-restart. \
+https://buildkite.com/solana-labs/testnet-management/settings/schedules/0efd7856-7143-4713-8817-47e6bdb05387
+https://buildkite.com/solana-labs/testnet-management/settings/schedules/2a926646-d972-42b5-aeb9-bb6759592a53
+
+On each schedule:
+1.  Set TESTNET_TAG environment variable to the desired release tag.
+    1. Example, TESTNET_TAG=v0.13.2
+1.  Set the Build Branch to the branch that TESTNET_TAG is from.
+    1. Example: v0.13
+
+#### Restart the testnet
+
+Trigger a TESTNET_OP=create-and-start to refresh the cluster with the new version
+
+1.  Go to https://buildkite.com/solana-labs/testnet-management
+2.  Click "New Build" and use the following settings, then click "Create Build"
+    1.  Commit: HEAD
+    1.  Branch: [channel branch as set in the schedules]
+    1.  Environment Variables:
+```
+TESTNET=testnet
+TESTNET_TAG=[same value as used in TESTNET_TAG in the schedules]
+TESTNET_OP=create-and-start
+```
+
+#### Update documentation
+
+Document the new recommended version by updating
+```export SOLANA_RELEASE=[new scheduled TESTNET_TAG value]```
+in book/src/testnet-participation.md for both edge and beta channel branches.
+
+### Alert the community
+
+Notify Discord users on #validator-support that a new release for
+testnet.solana.com is available

bench-exchange/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-exchange"
-version = "0.14.0"
+version = "0.14.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -21,16 +21,16 @@ serde = "1.0.87"
 serde_derive = "1.0.87"
 serde_json = "1.0.38"
 # solana-runtime = { path = "../solana/runtime"}
-solana = { path = "../core", version = "0.14.0" }
+solana = { path = "../core", version = "0.14.1"  }
-solana-client = { path = "../client", version = "0.14.0" }
+solana-client = { path = "../client", version = "0.14.1"  }
-solana-drone = { path = "../drone", version = "0.14.0" }
+solana-drone = { path = "../drone", version = "0.14.1"  }
-solana-exchange-api = { path = "../programs/exchange_api", version = "0.14.0" }
+solana-exchange-api = { path = "../programs/exchange_api", version = "0.14.1"  }
-solana-exchange-program = { path = "../programs/exchange_program", version = "0.14.0" }
+solana-exchange-program = { path = "../programs/exchange_program", version = "0.14.1"  }
-solana-logger = { path = "../logger", version = "0.14.0" }
+solana-logger = { path = "../logger", version = "0.14.1"  }
-solana-metrics = { path = "../metrics", version = "0.14.0" }
+solana-metrics = { path = "../metrics", version = "0.14.1"  }
-solana-netutil = { path = "../netutil", version = "0.14.0" }
+solana-netutil = { path = "../netutil", version = "0.14.1"  }
-solana-runtime = { path = "../runtime", version = "0.14.0" }
+solana-runtime = { path = "../runtime", version = "0.14.1"  }
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
 ws = "0.8.0"
 untrusted = "0.6.2"
 

bench-exchange/src/cli.rs

@@ -1,10 +1,10 @@
-use std::net::SocketAddr;
-use std::time::Duration;
-
 use clap::{crate_description, crate_name, crate_version, value_t, App, Arg, ArgMatches};
 use solana::gen_keys::GenKeys;
 use solana_drone::drone::DRONE_PORT;
 use solana_sdk::signature::{read_keypair, Keypair, KeypairUtil};
+use std::net::SocketAddr;
+use std::process::exit;
+use std::time::Duration;
 
 pub struct Config {
     pub network_addr: SocketAddr,
@@ -146,16 +146,17 @@ pub fn build_args<'a, 'b>() -> App<'a, 'b> {
 pub fn extract_args<'a>(matches: &ArgMatches<'a>) -> Config {
     let mut args = Config::default();
 
-    args.network_addr = matches
+    args.network_addr = solana_netutil::parse_host_port(matches.value_of("network").unwrap())
-        .value_of("network")
+        .unwrap_or_else(|e| {
-        .unwrap()
+            eprintln!("failed to parse network address: {}", e);
-        .parse()
+            exit(1)
-        .expect("Failed to parse network");
+        });
-    args.drone_addr = matches
+
-        .value_of("drone")
+    args.drone_addr = solana_netutil::parse_host_port(matches.value_of("drone").unwrap())
-        .unwrap()
+        .unwrap_or_else(|e| {
-        .parse()
+            eprintln!("failed to parse drone address: {}", e);
-        .expect("Failed to parse drone address");
+            exit(1)
+        });
 
     if matches.is_present("identity") {
         args.identity = read_keypair(matches.value_of("identity").unwrap())

bench-streamer/Cargo.toml

@@ -2,16 +2,16 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "0.14.0"
+version = "0.14.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-solana = { path = "../core", version = "0.14.0" }
+solana = { path = "../core", version = "0.14.1"  }
-solana-logger = { path = "../logger", version = "0.14.0" }
+solana-logger = { path = "../logger", version = "0.14.1"  }
-solana-netutil = { path = "../netutil", version = "0.14.0" }
+solana-netutil = { path = "../netutil", version = "0.14.1"  }
 
 [features]
 cuda = ["solana/cuda"]

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "0.14.0"
+version = "0.14.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,14 +11,14 @@ homepage = "https://solana.com/"
 clap = "2.33.0"
 rayon = "1.0.3"
 serde_json = "1.0.39"
-solana = { path = "../core", version = "0.14.0" }
+solana = { path = "../core", version = "0.14.1"  }
-solana-client = { path = "../client", version = "0.14.0" }
+solana-client = { path = "../client", version = "0.14.1"  }
-solana-drone = { path = "../drone", version = "0.14.0" }
+solana-drone = { path = "../drone", version = "0.14.1"  }
-solana-logger = { path = "../logger", version = "0.14.0" }
+solana-logger = { path = "../logger", version = "0.14.1"  }
-solana-metrics = { path = "../metrics", version = "0.14.0" }
+solana-metrics = { path = "../metrics", version = "0.14.1"  }
-solana-netutil = { path = "../netutil", version = "0.14.0" }
+solana-netutil = { path = "../netutil", version = "0.14.1"  }
-solana-runtime = { path = "../runtime", version = "0.14.0" }
+solana-runtime = { path = "../runtime", version = "0.14.1"  }
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
 
 [features]
 cuda = ["solana/cuda"]

book/src/SUMMARY.md

@@ -20,6 +20,7 @@
   - [Ledger Replication](ledger-replication.md)
   - [Secure Vote Signing](vote-signing.md)
   - [Staking Delegation and Rewards](stake-delegation-and-rewards.md)
+  - [Performance Metrics](performance-metrics.md)
 
 - [Anatomy of a Fullnode](fullnode.md)
   - [TPU](tpu.md)

book/src/performance-metrics.md

@@ -0,0 +1,29 @@
+# Performance Metrics
+
+Solana cluster performance is measured as average number of transactions per second
+that the network can sustain (TPS). And, how long it takes for a transaction to be
+confirmed by super majority of the cluster (Confirmation Time).
+
+Each cluster node maintains various counters that are incremented on certain events.
+These counters are periodically uploaded to a cloud based database. Solana's metrics
+dashboard fetches these counters, and computes the performance metrics and displays
+it on the dashboard. 
+
+## TPS
+
+The leader node's banking stage maintains a count of transactions that it processed.
+The dashboard displays the count averaged over 2 second period in the TPS time series
+graph. The dashboard also shows per second mean, maximum and total TPS as a running
+counter.
+
+## Confirmation Time
+
+Each validator node maintains a list of active ledger forks that are visible to the node.
+A fork is considered to be frozen when the node has received and processed all entries
+corresponding to the fork. A fork is considered to be confirmed when it receives cumulative
+super majority vote, and when one of its children forks is frozen.
+
+The node assigns a timestamp to every new fork, and computes the time it took to confirm
+the fork. This time is reflected as validator confirmation time in performance metrics.
+The performance dashboard displays the average of each validator node's confirmation time
+as a time series graph. 

book/src/testnet-participation.md

@@ -109,20 +109,20 @@ Then the following command will start a new validator node.
 If this is a `solana-install`-installation:
 ```bash
 $ clear-fullnode-config.sh
-$ fullnode.sh --public-address --poll-for-new-genesis-block testnet.solana.com
+$ fullnode.sh --poll-for-new-genesis-block testnet.solana.com
 ```
 
 Alternatively, the `solana-install run` command can be used to run the validator
 node while periodically checking for and applying software updates:
 ```bash
 $ clear-fullnode-config.sh
-$ solana-install run fullnode.sh -- --public-address --poll-for-new-genesis-block testnet.solana.com
+$ solana-install run fullnode.sh -- --poll-for-new-genesis-block testnet.solana.com
 ```
 
 If you built from source:
 ```bash
 $ USE_INSTALL=1 ./multinode-demo/clear-fullnode-config.sh
-$ USE_INSTALL=1 ./multinode-demo/fullnode.sh --public-address --poll-for-new-genesis-block testnet.solana.com
+$ USE_INSTALL=1 ./multinode-demo/fullnode.sh --poll-for-new-genesis-block testnet.solana.com
 ```
 
 #### Controlling local network port allocation

ci/nits.sh

@@ -18,14 +18,14 @@ declare prints=(
 # Parts of the tree that are expected to be print free
 declare print_free_tree=(
   'core/src'
-  'drone'
+  'drone/src'
-  'metrics'
+  'metrics/src'
-  'netutil'
+  'netutil/src'
-  'runtime'
+  'runtime/src'
-  'sdk'
+  'sdk/src'
 )
 
-if _ git grep "${prints[@]/#/-e }" -- "${print_free_tree[@]}"; then
+if _ git grep --max-depth=0 "${prints[@]/#/-e }" -- "${print_free_tree[@]}"; then
     exit 1
 fi
 

ci/testnet-deploy.sh

@@ -11,15 +11,19 @@ clientNodeCount=0
 additionalFullNodeCount=10
 publicNetwork=false
 stopNetwork=false
-skipSetup=false
+reuseLedger=false
+skipCreate=false
 skipStart=false
 externalNode=false
+failOnValidatorBootupFailure=true
 tarChannelOrTag=edge
 delete=false
 enableGpu=false
 bootDiskType=""
 leaderRotation=true
 blockstreamer=false
+deployUpdateManifest=true
+fetchLogs=true
 
 usage() {
   exitcode=0
@@ -55,8 +59,12 @@ Deploys a CD testnet
    -r                   - Reuse existing node/ledger configuration from a
                           previous |start| (ie, don't run ./multinode-demo/setup.sh).
    -x                   - External node.  Default: false
+   -e                   - Skip create.  Assume the nodes have already been created
    -s                   - Skip start.  Nodes will still be created or configured, but network software will not be started.
    -S                   - Stop network software without tearing down nodes.
+   -f                   - Discard validator nodes that didn't bootup successfully
+   -w                   - Skip time-consuming "bells and whistles" that are
+                          unnecessary for a high-node count demo testnet
 
    Note: the SOLANA_METRICS_CONFIG environment variable is used to configure
          metrics
@@ -66,7 +74,7 @@ EOF
 
 zone=()
 
-while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:p:C:S" opt; do
+while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:p:C:Sfew" opt; do
   case $opt in
   h | \?)
     usage
@@ -119,7 +127,10 @@ while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:p:C:S" opt; do
     delete=true
     ;;
   r)
-    skipSetup=true
+    reuseLedger=true
+    ;;
+  e)
+    skipCreate=true
     ;;
   s)
     skipStart=true
@@ -127,12 +138,19 @@ while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:p:C:S" opt; do
   x)
     externalNode=true
     ;;
+  f)
+    failOnValidatorBootupFailure=false
+    ;;
   u)
     blockstreamer=true
     ;;
   S)
     stopNetwork=true
     ;;
+  w)
+    fetchLogs=false
+    deployUpdateManifest=false
+    ;;
   *)
     usage "Error: unhandled option: $opt"
     ;;
@@ -170,15 +188,15 @@ for val in "${zone[@]}"; do
 done
 
 if $stopNetwork; then
-  skipSetup=true
+  skipCreate=true
 fi
 
 if $delete; then
-  skipSetup=false
+  skipCreate=false
 fi
 
 # Create the network
-if ! $skipSetup; then
+if ! $skipCreate; then
   echo "--- $cloudProvider.sh delete"
   # shellcheck disable=SC2068
   time net/"$cloudProvider".sh delete ${zone_args[@]} -p "$netName" ${externalNode:+-x}
@@ -224,6 +242,10 @@ if ! $skipSetup; then
     create_args+=(-x)
   fi
 
+  if ! $failOnValidatorBootupFailure; then
+    create_args+=(-f)
+  fi
+
   time net/"$cloudProvider".sh create "${create_args[@]}"
 else
   echo "--- $cloudProvider.sh config"
@@ -236,6 +258,14 @@ else
     config_args+=(-P)
   fi
 
+  if $externalNode; then
+    config_args+=(-x)
+  fi
+
+  if ! $failOnValidatorBootupFailure; then
+    config_args+=(-f)
+  fi
+
   time net/"$cloudProvider".sh config "${config_args[@]}"
 fi
 net/init-metrics.sh -e
@@ -249,53 +279,50 @@ if $stopNetwork; then
   exit 0
 fi
 
-echo --- net.sh start
-maybeRejectExtraNodes=
-if ! $publicNetwork; then
-  maybeRejectExtraNodes="-o rejectExtraNodes"
-fi
-maybeNoValidatorSanity=
-if [[ -n $NO_VALIDATOR_SANITY ]]; then
-  maybeNoValidatorSanity="-o noValidatorSanity"
-fi
-maybeNoLedgerVerify=
-if [[ -n $NO_LEDGER_VERIFY ]]; then
-  maybeNoLedgerVerify="-o noLedgerVerify"
-fi
-
-maybeSkipSetup=
-if $skipSetup; then
-  maybeSkipSetup="-r"
-fi
-
 ok=true
 if ! $skipStart; then
   (
-    if $skipSetup; then
+    if $skipCreate; then
       # TODO: Enable rolling updates
       #op=update
       op=restart
     else
       op=start
     fi
+    echo "--- net.sh $op"
+    args=("$op" -t "$tarChannelOrTag")
+
+    if ! $publicNetwork; then
+      args+=(-o rejectExtraNodes)
+    fi
+    if [[ -n $NO_VALIDATOR_SANITY ]]; then
+      args+=(-o noValidatorSanity)
+    fi
+    if [[ -n $NO_LEDGER_VERIFY ]]; then
+      args+=(-o noLedgerVerify)
+    fi
+
+    if $reuseLedger; then
+      args+=(-r)
+    fi
+
+    if ! $failOnValidatorBootupFailure; then
+      args+=(-F)
+    fi
 
-    maybeUpdateManifestKeypairFile=
     # shellcheck disable=SC2154 # SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu comes from .buildkite/env/
-    if [[ -n $SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu ]]; then
+    if $deployUpdateManifest && [[ -n $SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu ]]; then
       echo "$SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu" > update_manifest_keypair.json
-      maybeUpdateManifestKeypairFile="-i update_manifest_keypair.json"
+      args+=(-i update_manifest_keypair.json)
     fi
 
     # shellcheck disable=SC2086 # Don't want to double quote the $maybeXYZ variables
-    time net/net.sh $op -t "$tarChannelOrTag" \
+    time net/net.sh "${args[@]}"
-      $maybeUpdateManifestKeypairFile \
-      $maybeSkipSetup \
-      $maybeRejectExtraNodes \
-      $maybeNoValidatorSanity \
-      $maybeNoLedgerVerify
   ) || ok=false
 
+  if $fetchLogs; then
     net/net.sh logs
+  fi
 fi
 
 $ok

ci/testnet-manager.sh

@@ -52,7 +52,7 @@ steps:
             value: "create-and-start"
           - label: "Create testnet, but do not start software.  If the testnet already exists it will be deleted and re-created"
             value: "create"
-          - label: "Start network software on an existing testnet.  If software is already running it will be restarted."
+          - label: "Start network software on an existing testnet.  If software is already running it will be restarted"
             value: "start"
           - label: "Stop network software without deleting testnet nodes"
             value: "stop"
@@ -62,11 +62,11 @@ steps:
             value: "sanity-or-restart"
           - label: "Sanity check only"
             value: "sanity"
-          - label: "Delete the testnet.
+          - label: "Delete the testnet"
             value: "delete"
-          - label: "Enable/unlock the testnet."
+          - label: "Enable/unlock the testnet"
             value: "enable"
-          - label: "Delete and then lock the testnet from further operation until it is re-enabled."
+          - label: "Delete and then lock the testnet from further operation until it is re-enabled"
             value: "disable"
   - command: "ci/$(basename "$0")"
     agents:
@@ -81,7 +81,40 @@ eval "$(ci/channel-info.sh)"
 
 
 EC2_ZONES=(us-west-1a sa-east-1a ap-northeast-2a eu-central-1a ca-central-1a)
-GCE_ZONES=(us-west1-b asia-east2-a europe-west4-a southamerica-east1-b us-east4-c)
+
+# GCE zones with _lots_ of quota
+GCE_ZONES=(
+  us-west1-a
+  us-central1-a
+  us-east1-b
+  europe-west4-a
+
+  us-west1-b
+  us-central1-b
+  us-east1-c
+  europe-west4-b
+
+  us-west1-c
+  us-east1-d
+  europe-west4-c
+)
+
+# GCE zones with enough quota for one CPU-only fullnode
+GCE_LOW_QUOTA_ZONES=(
+  asia-east2-a
+  asia-northeast1-b
+  asia-northeast2-b
+  asia-south1-c
+  asia-southeast1-b
+  australia-southeast1-b
+  europe-north1-a
+  europe-west2-b
+  europe-west3-c
+  europe-west6-a
+  northamerica-northeast1-a
+  southamerica-east1-b
+)
+
 case $TESTNET in
 testnet-edge|testnet-edge-perf)
   CHANNEL_OR_TAG=edge
@@ -107,7 +140,9 @@ testnet-perf)
 testnet-demo)
   CHANNEL_OR_TAG=beta
   CHANNEL_BRANCH=$BETA_CHANNEL
-  : "${GCE_NODE_COUNT:=200}"
+  : "${GCE_NODE_COUNT:=150}"
+  : "${GCE_LOW_QUOTA_NODE_COUNT:=70}"
+  : "${TESTNET_DB_HOST:=https://clocktower-f1d56615.influxcloud.net:8086}"
   ;;
 *)
   echo "Error: Invalid TESTNET=$TESTNET"
@@ -123,6 +158,10 @@ GCE_ZONE_ARGS=()
 for val in "${GCE_ZONES[@]}"; do
   GCE_ZONE_ARGS+=("-z $val")
 done
+GCE_LOW_QUOTA_ZONE_ARGS=()
+for val in "${GCE_LOW_QUOTA_ZONES[@]}"; do
+  GCE_LOW_QUOTA_ZONE_ARGS+=("-z $val")
+done
 
 if [[ -n $TESTNET_DB_HOST ]]; then
   SOLANA_METRICS_PARTIAL_CONFIG="host=$TESTNET_DB_HOST,$SOLANA_METRICS_PARTIAL_CONFIG"
@@ -151,6 +190,7 @@ steps:
         TESTNET_DB_HOST: "$TESTNET_DB_HOST"
         EC2_NODE_COUNT: "$EC2_NODE_COUNT"
         GCE_NODE_COUNT: "$GCE_NODE_COUNT"
+        GCE_LOW_QUOTA_NODE_COUNT: "$GCE_LOW_QUOTA_NODE_COUNT"
 EOF
     ) | buildkite-agent pipeline upload
     exit 0
@@ -227,7 +267,8 @@ sanity() {
       ok=true
       if [[ -n $GCE_NODE_COUNT ]]; then
         NO_LEDGER_VERIFY=1 \
-          ci/testnet-sanity.sh demo-testnet-solana-com gce "${GCE_ZONES[0]}" || ok=false
+        NO_VALIDATOR_SANITY=1 \
+          ci/testnet-sanity.sh demo-testnet-solana-com gce "${GCE_ZONES[0]}" -f || ok=false
       else
         echo "Error: no GCE nodes"
         ok=false
@@ -270,7 +311,7 @@ deploy() {
       set -x
       ci/testnet-deploy.sh -p edge-testnet-solana-com -C ec2 -z us-west-1a \
         -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0ccd4f2239886fa94 \
-        ${skipCreate:+-r} \
+        ${skipCreate:+-e} \
         ${skipStart:+-s} \
         ${maybeStop:+-S} \
         ${maybeDelete:+-D}
@@ -285,7 +326,7 @@ deploy() {
         ci/testnet-deploy.sh -p edge-perf-testnet-solana-com -C ec2 -z us-west-2b \
           -g -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
-          ${skipCreate:+-r} \
+          ${skipCreate:+-e} \
           ${skipStart:+-s} \
           ${maybeStop:+-S} \
           ${maybeDelete:+-D}
@@ -298,7 +339,7 @@ deploy() {
         ci/testnet-deploy.sh -p beta-testnet-solana-com -C ec2 -z us-west-1a \
           -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0f286cf8a0771ce35 \
           -b \
-          ${skipCreate:+-r} \
+          ${skipCreate:+-e} \
           ${skipStart:+-s} \
           ${maybeStop:+-S} \
           ${maybeDelete:+-D}
@@ -313,7 +354,7 @@ deploy() {
         ci/testnet-deploy.sh -p beta-perf-testnet-solana-com -C ec2 -z us-west-2b \
           -g -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
-          ${skipCreate:+-r} \
+          ${skipCreate:+-e} \
           ${skipStart:+-s} \
           ${maybeStop:+-S} \
           ${maybeDelete:+-D}
@@ -330,7 +371,7 @@ deploy() {
       # shellcheck disable=SC2068
       ci/testnet-deploy.sh -p testnet-solana-com -C ec2 ${EC2_ZONE_ARGS[@]} \
         -t "$CHANNEL_OR_TAG" -n "$EC2_NODE_COUNT" -c 0 -u -P -a eipalloc-0fa502bf95f6f18b2 \
-        ${skipCreate:+-r} \
+        ${skipCreate:+-e} \
         ${maybeSkipStart:+-s} \
         ${maybeStop:+-S} \
         ${maybeDelete:+-D}
@@ -339,11 +380,11 @@ deploy() {
         # shellcheck disable=SC2068
         ci/testnet-deploy.sh -p testnet-solana-com -C gce ${GCE_ZONE_ARGS[@]} \
           -t "$CHANNEL_OR_TAG" -n "$GCE_NODE_COUNT" -c 0 -P \
-          ${skipCreate:+-r} \
+          ${skipCreate:+-e} \
           ${skipStart:+-s} \
           ${maybeStop:+-S} \
           ${maybeDelete:+-D} \
-          ${EC2_NODE_COUNT:+-x}
+          -x
       fi
     )
     ;;
@@ -358,7 +399,7 @@ deploy() {
           -t "$CHANNEL_OR_TAG" -c 2 \
           -b \
           -d pd-ssd \
-          ${skipCreate:+-r} \
+          ${skipCreate:+-e} \
           ${skipStart:+-s} \
           ${maybeStop:+-S} \
           ${maybeDelete:+-D}
@@ -367,12 +408,29 @@ deploy() {
   testnet-demo)
     (
       set -x
-      if [[ -n $GCE_NODE_COUNT ]]; then
+
+      if [[ -n $GCE_LOW_QUOTA_NODE_COUNT ]] || [[ -n $skipStart ]]; then
+        maybeSkipStart="skip"
+      fi
+
       # shellcheck disable=SC2068
-        ci/testnet-deploy.sh -p testnet-demo -C gce ${GCE_ZONE_ARGS[@]} \
+      NO_LEDGER_VERIFY=1 \
-          -t "$CHANNEL_OR_TAG" -n "$GCE_NODE_COUNT" -c 1 -P -u \
+      NO_VALIDATOR_SANITY=1 \
+        ci/testnet-deploy.sh -p demo-testnet-solana-com -C gce ${GCE_ZONE_ARGS[@]} \
+          -t "$CHANNEL_OR_TAG" -n "$GCE_NODE_COUNT" -c 0 -P -u -f -w \
           -a demo-testnet-solana-com \
-          ${skipCreate:+-r} \
+          ${skipCreate:+-e} \
+          ${maybeSkipStart:+-s} \
+          ${maybeStop:+-S} \
+          ${maybeDelete:+-D}
+
+      if [[ -n $GCE_LOW_QUOTA_NODE_COUNT ]]; then
+        # shellcheck disable=SC2068
+        NO_LEDGER_VERIFY=1 \
+        NO_VALIDATOR_SANITY=1 \
+          ci/testnet-deploy.sh -p demo-testnet-solana-com2 -C gce ${GCE_LOW_QUOTA_ZONE_ARGS[@]} \
+            -t "$CHANNEL_OR_TAG" -n "$GCE_LOW_QUOTA_NODE_COUNT" -c 0 -P -f -x -w \
+            ${skipCreate:+-e} \
             ${skipStart:+-s} \
             ${maybeStop:+-S} \
             ${maybeDelete:+-D}

client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-client"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana Client"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -17,10 +17,10 @@ reqwest = "0.9.11"
 serde = "1.0.89"
 serde_derive = "1.0.88"
 serde_json = "1.0.39"
-solana-netutil = { path = "../netutil", version = "0.14.0" }
+solana-netutil = { path = "../netutil", version = "0.14.1"  }
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
 
 [dev-dependencies]
 jsonrpc-core = "10.1.0"
 jsonrpc-http-server = "10.1.0"
-solana-logger = { path = "../logger", version = "0.14.0" }
+solana-logger = { path = "../logger", version = "0.14.1"  }

core/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana"
 description = "Blockchain, Rebuilt for Scale"
-version = "0.14.0"
+version = "0.14.1"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "../README.md"
@@ -47,18 +47,18 @@ rocksdb = "0.11.0"
 serde = "1.0.89"
 serde_derive = "1.0.88"
 serde_json = "1.0.39"
-solana-budget-api = { path = "../programs/budget_api", version = "0.14.0" }
+solana-budget-api = { path = "../programs/budget_api", version = "0.14.1"  }
-solana-client = { path = "../client", version = "0.14.0" }
+solana-client = { path = "../client", version = "0.14.1"  }
-solana-drone = { path = "../drone", version = "0.14.0" }
+solana-drone = { path = "../drone", version = "0.14.1"  }
-solana-kvstore = { path = "../kvstore", version = "0.14.0" , optional = true }
+solana-kvstore = { path = "../kvstore", version = "0.14.1" , optional = true  }
-solana-logger = { path = "../logger", version = "0.14.0" }
+solana-logger = { path = "../logger", version = "0.14.1"  }
-solana-metrics = { path = "../metrics", version = "0.14.0" }
+solana-metrics = { path = "../metrics", version = "0.14.1"  }
-solana-netutil = { path = "../netutil", version = "0.14.0" }
+solana-netutil = { path = "../netutil", version = "0.14.1"  }
-solana-runtime = { path = "../runtime", version = "0.14.0" }
+solana-runtime = { path = "../runtime", version = "0.14.1"  }
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
-solana-storage-api = { path = "../programs/storage_api", version = "0.14.0" }
+solana-storage-api = { path = "../programs/storage_api", version = "0.14.1"  }
-solana-vote-api = { path = "../programs/vote_api", version = "0.14.0" }
+solana-vote-api = { path = "../programs/vote_api", version = "0.14.1"  }
-solana-vote-signer = { path = "../vote-signer", version = "0.14.0" }
+solana-vote-signer = { path = "../vote-signer", version = "0.14.1"  }
 sys-info = "0.5.6"
 tokio = "0.1"
 tokio-codec = "0.1"
@@ -67,8 +67,8 @@ untrusted = "0.6.2"
 [dev-dependencies]
 hex-literal = "0.1.4"
 matches = "0.1.6"
-solana-vote-program = { path = "../programs/vote_program", version = "0.14.0" }
+solana-vote-program = { path = "../programs/vote_program", version = "0.14.1"  }
-solana-budget-program = { path = "../programs/budget_program", version = "0.14.0" }
+solana-budget-program = { path = "../programs/budget_program", version = "0.14.1"  }
 
 [[bench]]
 name = "banking_stage"

core/src/bank_forks.rs

@@ -35,7 +35,8 @@ impl BankForks {
     pub fn ancestors(&self) -> HashMap<u64, HashSet<u64>> {
         let mut ancestors = HashMap::new();
         for bank in self.banks.values() {
-            let set = bank.parents().into_iter().map(|b| b.slot()).collect();
+            let mut set: HashSet<u64> = bank.ancestors.keys().cloned().collect();
+            set.remove(&bank.slot());
             ancestors.insert(bank.slot(), set);
         }
         ancestors
@@ -46,9 +47,11 @@ impl BankForks {
         let mut descendants = HashMap::new();
         for bank in self.banks.values() {
             let _ = descendants.entry(bank.slot()).or_insert(HashSet::new());
-            for parent in bank.parents() {
+            let mut set: HashSet<u64> = bank.ancestors.keys().cloned().collect();
+            set.remove(&bank.slot());
+            for parent in set {
                 descendants
-                    .entry(parent.slot())
+                    .entry(parent)
                     .or_insert(HashSet::new())
                     .insert(bank.slot());
             }
@@ -116,8 +119,9 @@ impl BankForks {
     }
 
     fn prune_non_root(&mut self, root: u64) {
+        let descendants = self.descendants();
         self.banks
-            .retain(|slot, bank| *slot >= root || bank.is_in_subtree_of(root))
+            .retain(|slot, _| descendants[&root].contains(slot))
     }
 }
 

core/src/blocktree_processor.rs

@@ -260,7 +260,7 @@ mod tests {
     use super::*;
     use crate::blocktree::create_new_tmp_ledger;
     use crate::blocktree::tests::entries_to_blobs;
-    use crate::entry::{create_ticks, next_entry, Entry};
+    use crate::entry::{create_ticks, next_entry, next_entry_mut, Entry};
     use solana_sdk::genesis_block::GenesisBlock;
     use solana_sdk::hash::Hash;
     use solana_sdk::instruction::InstructionError;
@@ -961,4 +961,77 @@ mod tests {
         // Should not see duplicate signature error
         assert_eq!(bank.process_transaction(&fail_tx), Ok(()));
     }
+
+    #[test]
+    #[ignore]
+    fn test_process_entries_stress() {
+        // this test throws lots of rayon threads at process_entries()
+        //  finds bugs in very low-layer stuff
+        solana_logger::setup();
+        let (genesis_block, mint_keypair) = GenesisBlock::new(1_000_000_000);
+        let mut bank = Bank::new(&genesis_block);
+
+        const NUM_TRANSFERS: usize = 100;
+        let keypairs: Vec<_> = (0..NUM_TRANSFERS * 2).map(|_| Keypair::new()).collect();
+
+        // give everybody one lamport
+        for keypair in &keypairs {
+            bank.transfer(1, &mint_keypair, &keypair.pubkey())
+                .expect("funding failed");
+        }
+
+        let mut i = 0;
+        let mut hash = bank.last_blockhash();
+        loop {
+            let entries: Vec<_> = (0..NUM_TRANSFERS)
+                .map(|i| {
+                    next_entry_mut(
+                        &mut hash,
+                        0,
+                        vec![system_transaction::transfer(
+                            &keypairs[i],
+                            &keypairs[i + NUM_TRANSFERS].pubkey(),
+                            1,
+                            bank.last_blockhash(),
+                            0,
+                        )],
+                    )
+                })
+                .collect();
+            info!("paying iteration {}", i);
+            process_entries(&bank, &entries).expect("paying failed");
+
+            let entries: Vec<_> = (0..NUM_TRANSFERS)
+                .map(|i| {
+                    next_entry_mut(
+                        &mut hash,
+                        0,
+                        vec![system_transaction::transfer(
+                            &keypairs[i + NUM_TRANSFERS],
+                            &keypairs[i].pubkey(),
+                            1,
+                            bank.last_blockhash(),
+                            0,
+                        )],
+                    )
+                })
+                .collect();
+
+            info!("refunding iteration {}", i);
+            process_entries(&bank, &entries).expect("refunding failed");
+
+            // advance to next block
+            process_entries(
+                &bank,
+                &(0..bank.ticks_per_slot())
+                    .map(|_| next_entry_mut(&mut hash, 1, vec![]))
+                    .collect::<Vec<_>>(),
+            )
+            .expect("process ticks failed");
+
+            i += 1;
+            bank = Bank::new_from_parent(&Arc::new(bank), &Pubkey::default(), i as u64);
+            bank.squash();
+        }
+    }
 }

core/src/cluster_info.rs

@@ -249,7 +249,7 @@ impl ClusterInfo {
         let nodes: Vec<_> = self
             .all_peers()
             .into_iter()
-            .map(|node| {
+            .map(|(node, last_updated)| {
                 if !ContactInfo::is_valid_address(&node.gossip) {
                     spy_nodes += 1;
                 }
@@ -266,7 +266,7 @@ impl ClusterInfo {
                      tpu:    {:20} |         |\n  \
                      rpc:    {:20} |         |\n",
                     addr_to_string(&node.gossip),
-                    now.saturating_sub(node.wallclock),
+                    now.saturating_sub(last_updated),
                     node.id,
                     if node.id == my_id { "(me)" } else { "" }.to_string(),
                     addr_to_string(&node.tpu),
@@ -347,14 +347,17 @@ impl ClusterInfo {
             .collect()
     }
 
-    // All nodes in gossip, including spy nodes
+    // All nodes in gossip (including spy nodes) and the last time we heard about them
-    pub(crate) fn all_peers(&self) -> Vec<ContactInfo> {
+    pub(crate) fn all_peers(&self) -> Vec<(ContactInfo, u64)> {
         self.gossip
             .crds
             .table
             .values()
-            .filter_map(|x| x.value.contact_info())
+            .filter_map(|x| {
-            .cloned()
+                x.value
+                    .contact_info()
+                    .map(|ci| (ci.clone(), x.local_timestamp))
+            })
             .collect()
     }
 

core/src/contact_info.rs

@@ -7,7 +7,7 @@ use solana_sdk::signature::{Keypair, KeypairUtil};
 use solana_sdk::signature::{Signable, Signature};
 use solana_sdk::timing::timestamp;
 use std::cmp::{Ord, Ordering, PartialEq, PartialOrd};
-use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::net::{IpAddr, SocketAddr};
 
 /// Structure representing a node on the network
 #[derive(Serialize, Deserialize, Clone, Debug)]
@@ -56,7 +56,7 @@ impl Eq for ContactInfo {}
 #[macro_export]
 macro_rules! socketaddr {
     ($ip:expr, $port:expr) => {
-        std::net::SocketAddr::from((Ipv4Addr::from($ip), $port))
+        std::net::SocketAddr::from((std::net::Ipv4Addr::from($ip), $port))
     };
     ($str:expr) => {{
         let a: std::net::SocketAddr = $str.parse().unwrap();

core/src/fullnode.rs

@@ -74,6 +74,7 @@ pub struct Fullnode {
     poh_service: PohService,
     tpu: Tpu,
     tvu: Tvu,
+    ip_echo_server: solana_netutil::IpEchoServer,
 }
 
 impl Fullnode {
@@ -159,6 +160,9 @@ impl Fullnode {
             ))
         };
 
+        let ip_echo_server =
+            solana_netutil::ip_echo_server(node.sockets.gossip.local_addr().unwrap().port());
+
         let subscriptions = Arc::new(RpcSubscriptions::default());
         let rpc_pubsub_service = if node.info.rpc_pubsub.port() == 0 {
             None
@@ -271,6 +275,7 @@ impl Fullnode {
             exit,
             poh_service,
             poh_recorder,
+            ip_echo_server,
         }
     }
 
@@ -330,6 +335,7 @@ impl Service for Fullnode {
         self.gossip_service.join()?;
         self.tpu.join()?;
         self.tvu.join()?;
+        self.ip_echo_server.shutdown_now();
 
         Ok(())
     }

core/src/packet.rs

@@ -618,7 +618,7 @@ mod tests {
     use solana_sdk::system_transaction;
     use std::io;
     use std::io::Write;
-    use std::net::{Ipv4Addr, SocketAddr, UdpSocket};
+    use std::net::{SocketAddr, UdpSocket};
 
     #[test]
     fn test_packets_set_addr() {

core/src/rpc.rs

@@ -294,7 +294,7 @@ impl RpcSol for RpcSolImpl {
         Ok(cluster_info
             .all_peers()
             .iter()
-            .filter_map(|contact_info| {
+            .filter_map(|(contact_info, _)| {
                 if ContactInfo::is_valid_address(&contact_info.gossip) {
                     Some(RpcContactInfo {
                         id: contact_info.id.to_string(),

drone/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-drone"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana Drone"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -20,9 +20,9 @@ clap = "2.33"
 log = "0.4.2"
 serde = "1.0.90"
 serde_derive = "1.0.90"
-solana-logger = { path = "../logger", version = "0.14.0" }
+solana-logger = { path = "../logger", version = "0.14.1"  }
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
-solana-metrics = { path = "../metrics", version = "0.14.0" }
+solana-metrics = { path = "../metrics", version = "0.14.1"  }
 tokio = "0.1"
 tokio-codec = "0.1"
 

fullnode/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-fullnode"
 description = "Blockchain, Rebuilt for Scale"
-version = "0.14.0"
+version = "0.14.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -12,15 +12,15 @@ homepage = "https://solana.com/"
 clap = "2.33.0"
 log = "0.4.2"
 serde_json = "1.0.39"
-solana = { path = "../core", version = "0.14.0" }
+solana = { path = "../core", version = "0.14.1"  }
-solana-drone = { path = "../drone", version = "0.14.0" }
+solana-drone = { path = "../drone", version = "0.14.1"  }
-solana-logger = { path = "../logger", version = "0.14.0" }
+solana-logger = { path = "../logger", version = "0.14.1"  }
-solana-netutil = { path = "../netutil", version = "0.14.0" }
+solana-netutil = { path = "../netutil", version = "0.14.1"  }
-solana-metrics = { path = "../metrics", version = "0.14.0" }
+solana-metrics = { path = "../metrics", version = "0.14.1"  }
-solana-runtime = { path = "../runtime", version = "0.14.0" }
+solana-runtime = { path = "../runtime", version = "0.14.1"  }
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
-solana-vote-api = { path = "../programs/vote_api", version = "0.14.0" }
+solana-vote-api = { path = "../programs/vote_api", version = "0.14.1"  }
-solana-vote-signer = { path = "../vote-signer", version = "0.14.0" }
+solana-vote-signer = { path = "../vote-signer", version = "0.14.1"  }
 
 [features]
 chacha = ["solana/chacha"]

fullnode/src/main.rs

@@ -5,6 +5,7 @@ use solana::contact_info::ContactInfo;
 use solana::fullnode::{Fullnode, FullnodeConfig};
 use solana::local_vote_signer_service::LocalVoteSignerService;
 use solana::service::Service;
+use solana::socketaddr;
 use solana_netutil::parse_port_range;
 use solana_sdk::signature::{read_keypair, Keypair, KeypairUtil};
 use std::fs::File;
@@ -131,16 +132,10 @@ fn main() {
                 .takes_value(true)
                 .help("Comma separated persistent accounts location"),
         )
-        .arg(
-            clap::Arg::with_name("public_address")
-                .long("public-address")
-                .takes_value(false)
-                .help("Advertise public machine address in gossip.  By default the local machine address is advertised"),
-        )
         .arg(
             clap::Arg::with_name("gossip_port")
                 .long("gossip-port")
-                .value_name("PORT")
+                .value_name("HOST:PORT")
                 .takes_value(true)
                 .help("Gossip port number for the node"),
         )
@@ -195,19 +190,14 @@ fn main() {
     let dynamic_port_range = parse_port_range(matches.value_of("dynamic_port_range").unwrap())
         .expect("invalid dynamic_port_range");
 
-    let gossip_addr = {
+    let mut gossip_addr = solana_netutil::parse_port_or_addr(
-        let mut addr = solana_netutil::parse_port_or_addr(
         matches.value_of("gossip_port"),
+        socketaddr!(
+            [127, 0, 0, 1],
             solana_netutil::find_available_port_in_range(dynamic_port_range)
-                .expect("unable to allocate gossip_port"),
+                .expect("unable to find an available gossip port")
+        ),
     );
-        if matches.is_present("public_address") {
-            addr.set_ip(solana_netutil::get_public_ip_addr().unwrap());
-        } else {
-            addr.set_ip(solana_netutil::get_ip_addr(false).unwrap());
-        }
-        addr
-    };
 
     if let Some(paths) = matches.value_of("accounts") {
         fullnode_config.account_paths = Some(paths.to_string());
@@ -215,9 +205,11 @@ fn main() {
         fullnode_config.account_paths = None;
     }
     let cluster_entrypoint = matches.value_of("network").map(|network| {
-        let gossip_addr =
+        let entrypoint_addr =
             solana_netutil::parse_host_port(network).expect("failed to parse network address");
-        ContactInfo::new_gossip_entry_point(&gossip_addr)
+        gossip_addr.set_ip(solana_netutil::get_public_ip_addr(&entrypoint_addr).unwrap());
+
+        ContactInfo::new_gossip_entry_point(&entrypoint_addr)
     });
     let (_signer_service, _signer_addr) = if let Some(signer_addr) = matches.value_of("signer") {
         (

genesis/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-genesis"
 description = "Blockchain, Rebuilt for Scale"
-version = "0.14.0"
+version = "0.14.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,19 +11,19 @@ homepage = "https://solana.com/"
 [dependencies]
 clap = "2.33.0"
 serde_json = "1.0.39"
-solana = { path = "../core", version = "0.14.0" }
+solana = { path = "../core", version = "0.14.1"  }
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
-solana-budget-api = { path = "../programs/budget_api", version = "0.14.0" }
+solana-budget-api = { path = "../programs/budget_api", version = "0.14.1"  }
-solana-stake-api = { path = "../programs/stake_api", version = "0.14.0" }
+solana-stake-api = { path = "../programs/stake_api", version = "0.14.1"  }
-solana-storage-api = { path = "../programs/storage_api", version = "0.14.0" }
+solana-storage-api = { path = "../programs/storage_api", version = "0.14.1"  }
-solana-token-api = { path = "../programs/token_api", version = "0.14.0" }
+solana-token-api = { path = "../programs/token_api", version = "0.14.1"  }
-solana-config-api = { path = "../programs/config_api", version = "0.14.0" }
+solana-config-api = { path = "../programs/config_api", version = "0.14.1"  }
-solana-exchange-api = { path = "../programs/exchange_api", version = "0.14.0" }
+solana-exchange-api = { path = "../programs/exchange_api", version = "0.14.1"  }
 
 
 [dev-dependencies]
 hashbrown = "0.3.0"
-solana-vote-api = { path = "../programs/vote_api", version = "0.14.0" }
+solana-vote-api = { path = "../programs/vote_api", version = "0.14.1"  }
 
 [features]
 cuda = ["solana/cuda"]

gossip/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-gossip"
 description = "Blockchain, Rebuilt for Scale"
-version = "0.14.0"
+version = "0.14.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,10 +11,10 @@ homepage = "https://solana.com/"
 [dependencies]
 clap = "2.33.0"
 env_logger = "0.6.1"
-solana = { path = "../core", version = "0.14.0" }
+solana = { path = "../core", version = "0.14.1"  }
-solana-client = { path = "../client", version = "0.14.0"  }
+solana-client = { path = "../client", version = "0.14.1"   }
-solana-netutil = { path = "../netutil", version = "0.14.0" }
+solana-netutil = { path = "../netutil", version = "0.14.1"  }
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
 
 [features]
 chacha = []

install/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-install"
 description = "The solana cluster software installer"
-version = "0.14.0"
+version = "0.14.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -28,10 +28,10 @@ ring = "0.13.2"
 serde = "1.0.90"
 serde_derive = "1.0.90"
 serde_yaml = "0.8.8"
-solana-client = { path = "../client", version = "0.14.0" }
+solana-client = { path = "../client", version = "0.14.1"  }
-solana-config-api = { path = "../programs/config_api", version = "0.14.0" }
+solana-config-api = { path = "../programs/config_api", version = "0.14.1"  }
-solana-logger = { path = "../logger", version = "0.14.0" }
+solana-logger = { path = "../logger", version = "0.14.1"  }
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
 tar = "0.4.23"
 tempdir = "0.3.7"
 url = "1.7.2"

install/src/command.rs

@@ -250,7 +250,10 @@ fn get_update_manifest(
 fn check_env_path_for_bin_dir(config: &Config) {
     use std::env;
 
-    let bin_dir = config.active_release_bin_dir();
+    let bin_dir = config
+        .active_release_bin_dir()
+        .canonicalize()
+        .unwrap_or_default();
     let found = match env::var_os("PATH") {
         Some(paths) => env::split_paths(&paths).any(|path| {
             if let Ok(path) = path.canonicalize() {
@@ -383,10 +386,7 @@ pub fn info(config_file: &str, local_info_only: bool) -> Result<Option<UpdateMan
 
     fn print_update_manifest(update_manifest: &UpdateManifest) {
         let when = Local.timestamp(update_manifest.timestamp_secs as i64, 0);
-        println_name_value(
+        println_name_value(&format!("{}release date", BULLET), &when.to_string());
-            &format!("{}release date", BULLET),
-            &when.format("%c").to_string(),
-        );
         println_name_value(
             &format!("{}download URL", BULLET),
             &update_manifest.download_url,

keygen/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-keygen"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana key generation utility"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,7 +15,7 @@ erasure = []
 [dependencies]
 dirs = "1.0.5"
 clap = "2.33"
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
 
 [[bin]]
 name = "solana-keygen"

kvstore/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-kvstore"
 description = "Embedded Key-Value store for solana"
-version = "0.14.0"
+version = "0.14.1"
 homepage = "https://solana.com/"
 repository = "https://github.com/solana-labs/solana"
 authors = ["Solana Maintainers <maintainers@solana.com>"]

logger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-logger"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana Logger"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

metrics/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-metrics"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana Metrics"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,7 +14,7 @@ log = "0.4.2"
 reqwest = "0.9.15"
 lazy_static = "1.3.0"
 sys-info = "0.5.6"
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
 
 [dev-dependencies]
 rand = "0.6.5"

multinode-demo/bootstrap-leader.sh

@@ -40,17 +40,18 @@ bootstrap_leader_vote_id_path="$SOLANA_CONFIG_DIR"/bootstrap-leader-vote-id.json
 bootstrap_leader_vote_id=$($solana_keygen pubkey "$bootstrap_leader_vote_id_path")
 
 trap 'kill "$pid" && wait "$pid"' INT TERM ERR
-$program \
+
-  --identity "$bootstrap_leader_id_path" \
+default_fullnode_arg --identity "$bootstrap_leader_id_path"
-  --voting-keypair "$bootstrap_leader_vote_id_path" \
+default_fullnode_arg --voting-keypair "$bootstrap_leader_vote_id_path"
-  --vote-account  "$bootstrap_leader_vote_id" \
+default_fullnode_arg --vote-account  "$bootstrap_leader_vote_id"
-  --ledger "$SOLANA_CONFIG_DIR"/bootstrap-leader-ledger \
+default_fullnode_arg --ledger "$SOLANA_CONFIG_DIR"/bootstrap-leader-ledger
-  --accounts "$SOLANA_CONFIG_DIR"/bootstrap-leader-accounts \
+default_fullnode_arg --accounts "$SOLANA_CONFIG_DIR"/bootstrap-leader-accounts
-  --gossip-port 8001 \
+default_fullnode_arg --rpc-port 8899
-  --rpc-port 8899 \
+default_fullnode_arg --rpc-drone-address 127.0.0.1:9900
-  --rpc-drone-address 127.0.0.1:9900 \
+default_fullnode_arg --gossip-port 8001
-  "${extra_fullnode_args[@]}" \
+
-  > >($bootstrap_leader_logger) 2>&1 &
+echo "$PS4 $program ${extra_fullnode_args[*]}"
+$program "${extra_fullnode_args[@]}" > >($bootstrap_leader_logger) 2>&1 &
 pid=$!
 oom_score_adj "$pid" 1000
 

multinode-demo/common.sh

@@ -121,7 +121,6 @@ Start a full node
   --label LABEL             - Append the given label to the fullnode configuration files, useful when running
                               multiple fullnodes from the same filesystem location
   --stake LAMPORTS          - Number of lamports to stake
-  --public-address          - advertise public machine address in gossip.  By default the local machine address is advertised
   --no-voting               - start node without vote signer
   --rpc-port port           - custom RPC port for this node
 

multinode-demo/extra-fullnode-args.sh

@@ -34,9 +34,6 @@ while [[ ${1:0:1} = - ]]; do
   elif [[ $1 = --init-complete-file ]]; then
     extra_fullnode_args+=("$1" "$2")
     shift 2
-  elif [[ $1 = --public-address ]]; then
-    extra_fullnode_args+=("$1")
-    shift
   elif [[ $1 = --stake ]]; then
     stake="$2"
     shift 2
@@ -64,3 +61,20 @@ done
 if [[ -n $3 ]]; then
   fullnode_usage "$@"
 fi
+
+default_fullnode_arg() {
+  declare name=$1
+  declare value=$2
+
+  for arg in "${extra_fullnode_args[@]}"; do
+    if [[ $arg = "$name" ]]; then
+      return
+    fi
+  done
+
+  if [[ -n $value ]]; then
+    extra_fullnode_args+=("$name" "$value")
+  else
+    extra_fullnode_args+=("$name")
+  fi
+}

multinode-demo/fullnode.sh

@@ -160,23 +160,21 @@ while true; do
   if ((stake)); then
     setup_vote_account "${leader_address%:*}" "$fullnode_id_path" "$fullnode_vote_id_path" "$stake"
   fi
+  set +x
 
-  $program \
+  default_fullnode_arg --identity "$fullnode_id_path"
-    --identity "$fullnode_id_path" \
+  default_fullnode_arg --voting-keypair "$fullnode_vote_id_path"
-    --voting-keypair "$fullnode_vote_id_path" \
+  default_fullnode_arg --vote-account "$fullnode_vote_id"
-    --vote-account "$fullnode_vote_id" \
+  default_fullnode_arg --network "$leader_address"
-    --network "$leader_address" \
+  default_fullnode_arg --ledger "$ledger_config_dir"
-    --ledger "$ledger_config_dir" \
+  default_fullnode_arg --accounts "$accounts_config_dir"
-    --accounts "$accounts_config_dir" \
+  default_fullnode_arg --rpc-drone-address "${leader_address%:*}:9900"
-    --rpc-drone-address "${leader_address%:*}:9900" \
+  echo "$PS4 $program ${extra_fullnode_args[*]}"
-    "${extra_fullnode_args[@]}" \
+  $program "${extra_fullnode_args[@]}" > >($fullnode_logger) 2>&1 &
-    > >($fullnode_logger) 2>&1 &
   pid=$!
   oom_score_adj "$pid" 1000
 
-  set +x
   while true; do
-
     if ! kill -0 "$pid"; then
       wait "$pid"
       exit 0

net/common.sh

@@ -19,6 +19,7 @@ mkdir -p "$netConfigDir" "$netLogDir"
 source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh
 
 configFile="$netConfigDir/config"
+geoipConfigFile="$netConfigDir/geoip.yml"
 
 entrypointIp=
 publicNetwork=
@@ -28,10 +29,13 @@ externalNodeSshKey=
 sshOptions=()
 fullnodeIpList=()
 fullnodeIpListPrivate=()
+fullnodeIpListZone=()
 clientIpList=()
 clientIpListPrivate=()
+clientIpListZone=()
 blockstreamerIpList=()
 blockstreamerIpListPrivate=()
+blockstreamerIpListZone=()
 leaderRotation=
 
 buildSshOptions() {

net/gce.sh

@@ -163,11 +163,13 @@ while getopts "h?p:Pn:c:z:gG:a:d:buxf" opt; do
     enableGpu=true
     bootstrapLeaderMachineType=$gpuBootstrapLeaderMachineType
     fullNodeMachineType=$bootstrapLeaderMachineType
+    blockstreamerMachineType=$bootstrapLeaderMachineType
     ;;
   G)
     enableGpu=true
     bootstrapLeaderMachineType="$OPTARG"
     fullNodeMachineType=$bootstrapLeaderMachineType
+    blockstreamerMachineType=$bootstrapLeaderMachineType
     ;;
   a)
     customAddress=$OPTARG
@@ -225,6 +227,7 @@ esac
 #               name     - name of the instance
 #               publicIp - The public IP address of this instance
 #               privateIp - The private IP address of this instance
+#               zone     - Zone of this instance
 #               count    - Monotonically increasing count for each
 #                          invocation of cmd, starting at 1
 #               ...      - Extra args to cmd..
@@ -240,11 +243,70 @@ cloud_ForEachInstance() {
     declare name publicIp privateIp
     IFS=: read -r name publicIp privateIp zone < <(echo "$info")
 
-    eval "$cmd" "$name" "$publicIp" "$privateIp" "$count" "$@"
+    eval "$cmd" "$name" "$publicIp" "$privateIp" "$zone" "$count" "$@"
     count=$((count + 1))
   done
 }
 
+# Given a cloud provider zone, return an approximate lat,long location for the
+# data center.  Normal geoip lookups for cloud provider IP addresses are
+# sometimes widely inaccurate.
+zoneLocation() {
+  declare zone="$1"
+  case "$zone" in
+  us-west1*)
+    echo "[45.5946, -121.1787]"
+    ;;
+  us-central1*)
+    echo "[41.2619, -95.8608]"
+    ;;
+  us-east1*)
+    echo "[33.1960, -80.0131]"
+    ;;
+  asia-east2*)
+    echo "[22.3193, 114.1694]"
+    ;;
+  asia-northeast1*)
+    echo "[35.6762, 139.6503]"
+    ;;
+  asia-northeast2*)
+    echo "[34.6937, 135.5023]"
+    ;;
+  asia-south1*)
+    echo "[19.0760, 72.8777]"
+    ;;
+  asia-southeast1*)
+    echo "[1.3404, 103.7090]"
+    ;;
+  australia-southeast1*)
+    echo "[-33.8688, 151.2093]"
+    ;;
+  europe-north1*)
+    echo "[60.5693, 27.1878]"
+    ;;
+  europe-west2*)
+    echo "[51.5074, -0.1278]"
+    ;;
+  europe-west3*)
+    echo "[50.1109, 8.6821]"
+    ;;
+  europe-west4*)
+    echo "[53.4386, 6.8355]"
+    ;;
+  europe-west6*)
+    echo "[47.3769, 8.5417]"
+    ;;
+  northamerica-northeast1*)
+    echo "[45.5017, -73.5673]"
+    ;;
+  southamerica-east1*)
+    echo "[-23.5505, -46.6333]"
+    ;;
+  *)
+    ;;
+  esac
+}
+
 prepareInstancesAndWriteConfigFile() {
   $metricsWriteDatapoint "testnet-deploy net-config-begin=1"
 
@@ -252,6 +314,7 @@ prepareInstancesAndWriteConfigFile() {
     echo "Appending to existing config file"
     echo "externalNodeSshKey=$sshPrivateKey" >> "$configFile"
   else
+    rm -f "$geoipConfigFile"
     cat >> "$configFile" <<EOF
 # autogenerated at $(date)
 netBasename=$prefix
@@ -260,6 +323,7 @@ sshPrivateKey=$sshPrivateKey
 leaderRotation=$leaderRotation
 EOF
   fi
+  touch "$geoipConfigFile"
 
   buildSshOptions
 
@@ -267,9 +331,11 @@ EOF
     declare name="$1"
     declare publicIp="$2"
     declare privateIp="$3"
+    declare zone="$4"
+    #declare index="$5"
 
-    declare failOnFailure="$5"
+    declare failOnFailure="$6"
-    declare arrayName="$6"
+    declare arrayName="$7"
 
     # This check should eventually be moved to cloud provider specific script
     if [ "$publicIp" = "TERMINATED" ] || [ "$privateIp" = "TERMINATED" ]; then
@@ -284,7 +350,7 @@ EOF
     echo "Waiting for $name to finish booting..."
     (
       set -x +e
-      for i in $(seq 1 20); do
+      for i in $(seq 1 60); do
         timeout --preserve-status --foreground 20s ssh "${sshOptions[@]}" "$publicIp" "ls -l /.instance-startup-complete"
         ret=$?
         if [[ $ret -eq 0 ]]; then
@@ -303,21 +369,21 @@ EOF
         exit 1
       fi
     else
-      echo "$arrayName+=($publicIp)  # $name" >> "$configFile"
+      {
-      echo "${arrayName}Private+=($privateIp)  # $name" >> "$configFile"
+        echo "$arrayName+=($publicIp)  # $name"
+        echo "${arrayName}Private+=($privateIp)  # $name"
+        echo "${arrayName}Zone+=($zone)  # $name"
+      } >> "$configFile"
+
+      declare latlng=
+      latlng=$(zoneLocation "$zone")
+      if [[ -n $latlng ]]; then
+        echo "$publicIp: $latlng" >> "$geoipConfigFile"
+      fi
     fi
   }
 
-  if $externalNodes; then
+  fetchPrivateKey() {
-    echo "Bootstrap leader is already configured"
-  else
-    echo "Looking for bootstrap leader instance..."
-    cloud_FindInstance "$prefix-bootstrap-leader"
-    [[ ${#instances[@]} -eq 1 ]] || {
-      echo "Unable to find bootstrap leader"
-      exit 1
-    }
-
     (
       declare nodeName
       declare nodeIp
@@ -336,8 +402,10 @@ EOF
         set -x -o pipefail
         for i in $(seq 1 30); do
           if cloud_FetchFile "$nodeName" "$nodeIp" /solana-id_ecdsa "$sshPrivateKey" "$nodeZone"; then
+            if cloud_FetchFile "$nodeName" "$nodeIp" /solana-id_ecdsa.pub "$sshPrivateKey.pub" "$nodeZone"; then
               break
             fi
+          fi
 
           sleep 1
           echo "Retry $i..."
@@ -348,6 +416,20 @@ EOF
       fi
     )
 
+  }
+
+  if $externalNodes; then
+    echo "Bootstrap leader is already configured"
+  else
+    echo "Looking for bootstrap leader instance..."
+    cloud_FindInstance "$prefix-bootstrap-leader"
+    [[ ${#instances[@]} -eq 1 ]] || {
+      echo "Unable to find bootstrap leader"
+      exit 1
+    }
+
+    fetchPrivateKey
+
     echo "fullnodeIpList=()" >> "$configFile"
     echo "fullnodeIpListPrivate=()" >> "$configFile"
     cloud_ForEachInstance recordInstanceIp true fullnodeIpList
@@ -357,11 +439,15 @@ EOF
     for zone in "${zones[@]}"; do
       echo "Looking for additional fullnode instances in $zone ..."
       cloud_FindInstances "$prefix-$zone-fullnode"
-      [[ ${#instances[@]} -gt 0 ]] || {
+      if [[ ${#instances[@]} -gt 0 ]]; then
-        echo "Unable to find additional fullnodes"
+        fetchPrivateKey
-        exit 1
-      }
         cloud_ForEachInstance recordInstanceIp "$failOnValidatorBootupFailure" fullnodeIpList
+      else
+        echo "Unable to find additional fullnodes"
+        if $failOnValidatorBootupFailure; then
+          exit 1
+        fi
+      fi
     done
   fi
 
@@ -586,29 +672,33 @@ info)
     declare nodeType=$1
     declare ip=$2
     declare ipPrivate=$3
-    printf "  %-16s | %-15s | %-15s\n" "$nodeType" "$ip" "$ipPrivate"
+    declare zone=$4
+    printf "  %-16s | %-15s | %-15s | %s\n" "$nodeType" "$ip" "$ipPrivate" "$zone"
   }
 
-  printNode "Node Type" "Public IP" "Private IP"
+  printNode "Node Type" "Public IP" "Private IP" "Zone"
-  echo "-------------------+-----------------+-----------------"
+  echo "-------------------+-----------------+-----------------+--------------"
   nodeType=bootstrap-leader
   for i in $(seq 0 $(( ${#fullnodeIpList[@]} - 1)) ); do
     ipAddress=${fullnodeIpList[$i]}
     ipAddressPrivate=${fullnodeIpListPrivate[$i]}
-    printNode $nodeType "$ipAddress" "$ipAddressPrivate"
+    zone=${fullnodeIpListZone[$i]}
+    printNode $nodeType "$ipAddress" "$ipAddressPrivate" "$zone"
     nodeType=fullnode
   done
 
   for i in $(seq 0 $(( ${#clientIpList[@]} - 1)) ); do
     ipAddress=${clientIpList[$i]}
     ipAddressPrivate=${clientIpListPrivate[$i]}
-    printNode bench-tps "$ipAddress" "$ipAddressPrivate"
+    zone=${clientIpListZone[$i]}
+    printNode bench-tps "$ipAddress" "$ipAddressPrivate" "$zone"
   done
 
   for i in $(seq 0 $(( ${#blockstreamerIpList[@]} - 1)) ); do
     ipAddress=${blockstreamerIpList[$i]}
     ipAddressPrivate=${blockstreamerIpListPrivate[$i]}
-    printNode blockstreamer "$ipAddress" "$ipAddressPrivate"
+    zone=${blockstreamerIpListZone[$i]}
+    printNode blockstreamer "$ipAddress" "$ipAddressPrivate" "$zone"
   done
   ;;
 *)

net/net.sh

@@ -51,6 +51,7 @@ Operate a configured testnet
                                         to the bench-tps client.
 
  sanity/start/update-specific options:
+   -F                   - Discard validator nodes that didn't bootup successfully
    -o noLedgerVerify    - Skip ledger verification
    -o noValidatorSanity - Skip fullnode sanity
    -o rejectExtraNodes  - Require the exact number of nodes
@@ -80,12 +81,13 @@ numBenchTpsClients=0
 numBenchExchangeClients=0
 benchTpsExtraArgs=
 benchExchangeExtraArgs=
+failOnValidatorBootupFailure=true
 
 command=$1
 [[ -n $command ]] || usage
 shift
 
-while getopts "h?T:t:o:f:rD:i:c:" opt; do
+while getopts "h?T:t:o:f:rD:i:c:F" opt; do
   case $opt in
   h | \?)
     usage
@@ -167,6 +169,9 @@ while getopts "h?T:t:o:f:rD:i:c:" opt; do
     }
     getClientTypeAndNum
     ;;
+  F)
+    failOnValidatorBootupFailure=false
+    ;;
   *)
     usage "Error: unhandled option: $opt"
     ;;
@@ -188,7 +193,7 @@ else
 fi
 
 annotate() {
-  ${BUILDKITE:-false} && {
+  [[ -z $BUILDKITE ]] || {
     buildkite-agent annotate "$@"
   }
 }
@@ -285,12 +290,12 @@ startBootstrapLeader() {
       "./solana/net/remote/remote-node.sh \
          $deployMethod \
          bootstrap-leader \
-         $publicNetwork \
          $entrypointIp \
          $((${#fullnodeIpList[@]} + ${#blockstreamerIpList[@]})) \
          \"$RUST_LOG\" \
          $skipSetup \
          $leaderRotation \
+         $failOnValidatorBootupFailure \
       "
   ) >> "$logFile" 2>&1 || {
     cat "$logFile"
@@ -313,12 +318,12 @@ startNode() {
       "./solana/net/remote/remote-node.sh \
          $deployMethod \
          $nodeType \
-         $publicNetwork \
          $entrypointIp \
          $((${#fullnodeIpList[@]} + ${#blockstreamerIpList[@]})) \
          \"$RUST_LOG\" \
          $skipSetup \
          $leaderRotation \
+         $failOnValidatorBootupFailure \
       "
   ) >> "$logFile" 2>&1 &
   declare pid=$!
@@ -442,13 +447,14 @@ start() {
 
   declare bootstrapLeader=true
   declare nodeType=fullnode
+  declare loopCount=0
   for ipAddress in "${fullnodeIpList[@]}" - "${blockstreamerIpList[@]}"; do
     if [[ $ipAddress = - ]]; then
       nodeType=blockstreamer
       continue
     fi
     if $updateNodes; then
-      stopNode "$ipAddress"
+      stopNode "$ipAddress" true
     fi
     if $bootstrapLeader; then
       SECONDS=0
@@ -460,7 +466,6 @@ start() {
       bootstrapLeader=false
       SECONDS=0
       pids=()
-      loopCount=0
     else
       startNode "$ipAddress" $nodeType
 
@@ -475,9 +480,13 @@ start() {
     declare ok=true
     wait "$pid" || ok=false
     if ! $ok; then
+      echo "+++ fullnode failed to start"
       cat "$netLogDir/fullnode-$pid.log"
-      echo ^^^ +++
+      if $failOnValidatorBootupFailure; then
         exit 1
+      else
+        echo "Failure is non-fatal"
+      fi
     fi
   done
 
@@ -488,7 +497,7 @@ start() {
 
   if $updateNodes; then
     for ipAddress in "${clientIpList[@]}"; do
-      stopNode "$ipAddress"
+      stopNode "$ipAddress" true
     done
   fi
   sanity
@@ -542,7 +551,10 @@ start() {
 
 stopNode() {
   local ipAddress=$1
+  local block=$2
+  declare logFile="$netLogDir/stop-fullnode-$ipAddress.log"
   echo "--- Stopping node: $ipAddress"
+  echo "stop log: $logFile"
   (
     set -x
     # shellcheck disable=SC2029 # It's desired that PS4 be expanded on the client side
@@ -558,17 +570,38 @@ stopNode() {
         pkill -9 \$pattern
       done
     "
-  ) || true
+  ) >> "$logFile" 2>&1 &
+
+  declare pid=$!
+  ln -sfT "stop-fullnode-$ipAddress.log" "$netLogDir/stop-fullnode-$pid.log"
+  if $block; then
+    wait $pid
+  else
+    pids+=("$pid")
+  fi
 }
 
 stop() {
   SECONDS=0
   $metricsWriteDatapoint "testnet-deploy net-stop-begin=1"
 
+  declare loopCount=0
+  pids=()
   for ipAddress in "${fullnodeIpList[@]}" "${blockstreamerIpList[@]}" "${clientIpList[@]}"; do
-    stopNode "$ipAddress"
+    stopNode "$ipAddress" false
+
+    # Stagger additional node stop time to avoid too many concurrent ssh
+    # sessions
+    ((loopCount++ % 4 == 0)) && sleep 2
   done
 
+  echo --- Waiting for nodes to finish stopping
+  for pid in "${pids[@]}"; do
+    echo -n "$pid "
+    wait "$pid" || true
+  done
+  echo
+
   $metricsWriteDatapoint "testnet-deploy net-stop-complete=1"
   echo "Stopping nodes took $SECONDS seconds"
 }

net/remote/remote-node.sh

@@ -6,12 +6,12 @@ cd "$(dirname "$0")"/../..
 set -x
 deployMethod="$1"
 nodeType="$2"
-publicNetwork="$3"
+entrypointIp="$3"
-entrypointIp="$4"
+numNodes="$4"
-numNodes="$5"
+RUST_LOG="$5"
-RUST_LOG="$6"
+skipSetup="$6"
-skipSetup="$7"
+leaderRotation="$7"
-leaderRotation="$8"
+failOnValidatorBootupFailure="$8"
 set +x
 export RUST_LOG
 
@@ -30,17 +30,18 @@ missing() {
 
 [[ -n $deployMethod ]]  || missing deployMethod
 [[ -n $nodeType ]]      || missing nodeType
-[[ -n $publicNetwork ]] || missing publicNetwork
 [[ -n $entrypointIp ]]  || missing entrypointIp
 [[ -n $numNodes ]]      || missing numNodes
 [[ -n $skipSetup ]]     || missing skipSetup
 [[ -n $leaderRotation ]] || missing leaderRotation
+[[ -n $failOnValidatorBootupFailure ]] || missing failOnValidatorBootupFailure
 
 cat > deployConfig <<EOF
 deployMethod="$deployMethod"
 entrypointIp="$entrypointIp"
 numNodes="$numNodes"
 leaderRotation=$leaderRotation
+failOnValidatorBootupFailure=$failOnValidatorBootupFailure
 EOF
 
 source net/common.sh
@@ -81,11 +82,10 @@ local|tar)
     fi
     ./multinode-demo/drone.sh > drone.log 2>&1 &
 
-    args=()
+    args=(
-    if $publicNetwork; then
+      --enable-rpc-exit
-      args+=(--public-address)
+      --gossip-port "$entrypointIp":8001
-    fi
+    )
-    args+=(--enable-rpc-exit)
 
     ./multinode-demo/bootstrap-leader.sh "${args[@]}" > bootstrap-leader.log 2>&1 &
     ln -sTf bootstrap-leader.log fullnode.log
@@ -99,9 +99,6 @@ local|tar)
     fi
 
     args=()
-    if $publicNetwork; then
-      args+=("--public-address")
-    fi
     if [[ $nodeType = blockstreamer ]]; then
       args+=(
         --blockstream /tmp/solana-blockstream.sock
@@ -135,6 +132,7 @@ local|tar)
       scp "$entrypointIp":~/solana/config-local/mint-id.json config-local/
       ./multinode-demo/drone.sh > drone.log 2>&1 &
 
+      export BLOCKEXPLORER_GEOIP_WHITELIST=$PWD/net/config/geoip.yml
       npm install @solana/blockexplorer@1
       npx solana-blockexplorer > blockexplorer.log 2>&1 &
 

net/remote/remote-sanity.sh

@@ -9,6 +9,7 @@ cd "$(dirname "$0")"/../..
 deployMethod=
 entrypointIp=
 numNodes=
+failOnValidatorBootupFailure=
 
 [[ -r deployConfig ]] || {
   echo deployConfig missing
@@ -26,6 +27,7 @@ missing() {
 [[ -n $entrypointIp ]]   || missing entrypointIp
 [[ -n $numNodes ]]       || missing numNodes
 [[ -n $leaderRotation ]] || missing leaderRotation
+[[ -n $failOnValidatorBootupFailure ]] || missing failOnValidatorBootupFailure
 
 ledgerVerify=true
 validatorSanity=true
@@ -79,7 +81,17 @@ local|tar)
   exit 1
 esac
 
-echo "+++ $entrypointIp: node count ($numNodes expected)"
+if $failOnValidatorBootupFailure; then
+  numSanityNodes="$numNodes"
+else
+  numSanityNodes=1
+  if $rejectExtraNodes; then
+    echo "rejectExtraNodes cannot be used with failOnValidatorBootupFailure"
+    exit 1
+  fi
+fi
+
+echo "+++ $entrypointIp: node count ($numSanityNodes expected)"
 (
   set -x
   $solana_keygen -o "$client_id"
@@ -90,7 +102,7 @@ echo "+++ $entrypointIp: node count ($numNodes expected)"
   fi
 
   timeout 2m $solana_gossip --network "$entrypointIp:8001" \
-    spy --$nodeArg "$numNodes" \
+    spy --$nodeArg "$numSanityNodes" \
 )
 
 echo "--- RPC API: getTransactionCount"

net/scripts/ec2-provider.sh

@@ -310,18 +310,24 @@ cloud_DeleteInstances() {
     return
   fi
 
-  declare names=("${instances[@]/:*/}")
+  # Terminate the instances
-  declare zones=("${instances[@]/*:/}")
+  for instance in "${instances[@]}"; do
+    declare name="${instance/:*/}"
+    declare zone="${instance/*:/}"
     declare region=
-  region=$(__cloud_GetRegion "${zones[0]}")
+    region=$(__cloud_GetRegion "$zone")
-
     (
       set -x
-    aws ec2 terminate-instances --region "$region" --instance-ids "${names[@]}"
+      aws ec2 terminate-instances --region "$region" --instance-ids "$name"
     )
+  done
 
   # Wait until the instances are terminated
-  for name in "${names[@]}"; do
+  for instance in "${instances[@]}"; do
+    declare name="${instance/:*/}"
+    declare zone="${instance/*:/}"
+    declare region=
+    region=$(__cloud_GetRegion "$zone")
     while true; do
       declare instanceState
       instanceState=$(\

net/scripts/gce-provider.sh

@@ -34,8 +34,10 @@ __cloud_FindInstances() {
     instances+=("$name:$publicIp:$privateIp:$zone")
   done < <(gcloud compute instances list \
              --filter "$filter" \
-             --format 'value(name,networkInterfaces[0].accessConfigs[0].natIP,networkInterfaces[0].networkIP,status,zone)')
+             --format 'value(name,networkInterfaces[0].accessConfigs[0].natIP,networkInterfaces[0].networkIP,status,zone)' \
+           | grep RUNNING)
 }
+
 #
 # cloud_FindInstances [namePrefix]
 #

net/scripts/install-earlyoom.sh

@@ -18,8 +18,8 @@ else
   apt install --quiet --yes ./earlyoom_1.2-*_amd64.deb
 
   cat > earlyoom <<OOM
-  # use the kernel OOM killer, trigger at 20% available RAM,
+  # trigger at 20% available RAM,
-  EARLYOOM_ARGS="-k -m 20"
+  EARLYOOM_ARGS="-m 20"
 OOM
   cp earlyoom /etc/default/
   rm earlyoom

netutil/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-netutil"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana Network Utilities"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,16 +9,22 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
+bincode = "1.1.3"
+clap = "2.33.0"
 log = "0.4.2"
-ipnetwork = "0.12.7"
 nix = "0.12.0"
-pnet_datalink = "0.21.0"
 rand = "0.6.1"
-reqwest = "0.9.0"
 socket2 = "0.3.8"
-
+solana-logger = { path = "../logger", version = "0.14.1"  }
-[dev-dependencies]
+tokio = "0.1"
-solana-logger = { path = "../logger", version = "0.14.0" }
 
 [lib]
 name = "solana_netutil"
+
+[[bin]]
+name = "solana-ip-address"
+path = "src/bin/ip_address.rs"
+
+[[bin]]
+name = "solana-ip-address-server"
+path = "src/bin/ip_address_server.rs"

netutil/src/bin/ip_address.rs

@@ -0,0 +1,26 @@
+use clap::{crate_version, App, Arg};
+
+fn main() {
+    solana_logger::setup();
+    let matches = App::new("solana-ip-address")
+        .version(crate_version!())
+        .arg(
+            Arg::with_name("host_port")
+                .index(1)
+                .required(true)
+                .help("Host:port to connect to"),
+        )
+        .get_matches();
+
+    let host_port = matches.value_of("host_port").unwrap();
+    let addr = solana_netutil::parse_host_port(host_port)
+        .unwrap_or_else(|_| panic!("failed to parse {}", host_port));
+
+    match solana_netutil::get_public_ip_addr(&addr) {
+        Ok(ip) => println!("{}", ip),
+        Err(err) => {
+            eprintln!("{}: {}", addr, err);
+            std::process::exit(1)
+        }
+    }
+}

netutil/src/bin/ip_address_server.rs

@@ -0,0 +1,23 @@
+use clap::{crate_version, App, Arg};
+
+fn main() {
+    solana_logger::setup();
+    let matches = App::new("solana-ip-address-server")
+        .version(crate_version!())
+        .arg(
+            Arg::with_name("port")
+                .index(1)
+                .required(true)
+                .help("TCP port to bind to"),
+        )
+        .get_matches();
+
+    let port = matches.value_of("port").unwrap();
+    let port = port
+        .parse()
+        .unwrap_or_else(|_| panic!("Unable to parse {}", port));
+    let _runtime = solana_netutil::ip_echo_server(port);
+    loop {
+        std::thread::park();
+    }
+}

netutil/src/ip_echo_server.rs

@@ -0,0 +1,44 @@
+use log::*;
+use std::net::SocketAddr;
+use tokio;
+use tokio::net::TcpListener;
+use tokio::prelude::{Future, Stream};
+use tokio::runtime::Runtime;
+
+pub type IpEchoServer = Runtime;
+
+/// Starts a simple TCP server on the given port that echos the IP address of any peer that
+/// connects.  Used by |get_public_ip_addr|
+pub fn ip_echo_server(port: u16) -> IpEchoServer {
+    let bind_addr = SocketAddr::from(([0, 0, 0, 0], port));
+    let tcp =
+        TcpListener::bind(&bind_addr).unwrap_or_else(|_| panic!("Unable to bind to {}", bind_addr));
+    info!("bound to {:?}", bind_addr);
+
+    let server = tcp
+        .incoming()
+        .map_err(|err| warn!("accept failed: {:?}", err))
+        .for_each(move |socket| {
+            let ip = socket
+                .peer_addr()
+                .and_then(|peer_addr| {
+                    bincode::serialize(&peer_addr.ip()).map_err(|err| {
+                        std::io::Error::new(
+                            std::io::ErrorKind::Other,
+                            format!("Failed to serialize: {:?}", err),
+                        )
+                    })
+                })
+                .unwrap_or_else(|_| vec![]);
+
+            let write_future = tokio::io::write_all(socket, ip)
+                .map_err(|err| warn!("write error: {:?}", err))
+                .map(|_| ());
+
+            tokio::spawn(write_future)
+        });
+
+    let mut rt = Runtime::new().expect("Failed to create Runtime");
+    rt.spawn(server);
+    rt
+}

netutil/src/lib.rs

@@ -1,14 +1,16 @@
 //! The `netutil` module assists with networking
-use log::*;
 use nix::sys::socket::setsockopt;
 use nix::sys::socket::sockopt::{ReuseAddr, ReusePort};
-use pnet_datalink as datalink;
 use rand::{thread_rng, Rng};
-use reqwest;
 use socket2::{Domain, SockAddr, Socket, Type};
 use std::io;
-use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, ToSocketAddrs, UdpSocket};
+use std::io::Read;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpListener, TcpStream, ToSocketAddrs, UdpSocket};
 use std::os::unix::io::AsRawFd;
+use std::time::Duration;
+
+mod ip_echo_server;
+pub use ip_echo_server::*;
 
 /// A data type representing a public Udp socket
 pub struct UdpSocketPair {
@@ -19,34 +21,43 @@ pub struct UdpSocketPair {
 
 pub type PortRange = (u16, u16);
 
-/// Tries to determine the public IP address of this machine
+/// Determine the public IP address of this machine by asking an ip_echo_server at the given
-pub fn get_public_ip_addr() -> Result<IpAddr, String> {
+/// address
-    let body = reqwest::get("http://ifconfig.co/ip")
+pub fn get_public_ip_addr(ip_echo_server_addr: &SocketAddr) -> Result<IpAddr, String> {
-        .map_err(|err| err.to_string())?
+    let mut data = Vec::new();
-        .text()
-        .map_err(|err| err.to_string())?;
 
-    match body.lines().next() {
+    let timeout = Duration::new(5, 0);
-        Some(ip) => Result::Ok(ip.parse().unwrap()),
+    TcpStream::connect_timeout(ip_echo_server_addr, timeout)
-        None => Result::Err("Empty response body".to_string()),
+        .and_then(|mut stream| {
-    }
+            stream
+                .set_read_timeout(Some(Duration::new(10, 0)))
+                .expect("set_read_timeout");
+            stream.read_to_end(&mut data)
+        })
+        .and_then(|_| {
+            bincode::deserialize(&data).map_err(|err| {
+                io::Error::new(
+                    io::ErrorKind::Other,
+                    format!("Failed to deserialize: {:?}", err),
+                )
+            })
+        })
+        .map_err(|err| err.to_string())
 }
 
-pub fn parse_port_or_addr(optstr: Option<&str>, default_port: u16) -> SocketAddr {
+pub fn parse_port_or_addr(optstr: Option<&str>, default_addr: SocketAddr) -> SocketAddr {
-    let daddr = SocketAddr::from(([0, 0, 0, 0], default_port));
-
     if let Some(addrstr) = optstr {
         if let Ok(port) = addrstr.parse() {
-            let mut addr = daddr;
+            let mut addr = default_addr;
             addr.set_port(port);
             addr
         } else if let Ok(addr) = addrstr.parse() {
             addr
         } else {
-            daddr
+            default_addr
         }
     } else {
-        daddr
+        default_addr
     }
 }
 
@@ -95,67 +106,6 @@ pub fn parse_host_port(host_port: &str) -> Result<SocketAddr, String> {
     }
 }
 
-fn find_eth0ish_ip_addr(
-    ifaces: &mut Vec<datalink::NetworkInterface>,
-    enable_ipv6: bool,
-) -> Option<IpAddr> {
-    // put eth0 and wifi0, etc. up front of our list of candidates
-    ifaces.sort_by(|a, b| {
-        a.name
-            .chars()
-            .last()
-            .unwrap()
-            .cmp(&b.name.chars().last().unwrap())
-    });
-
-    let mut lo = None;
-    for iface in ifaces.clone() {
-        trace!("get_ip_addr considering iface {}", iface.name);
-        for p in iface.ips {
-            trace!("  ip {}", p);
-            if p.ip().is_multicast() {
-                trace!("    multicast");
-                continue;
-            }
-            match p.ip() {
-                IpAddr::V4(addr) => {
-                    if addr.is_link_local() {
-                        trace!("    link local");
-                        continue;
-                    }
-                    if p.ip().is_loopback() {
-                        // Fall back to loopback if no better option exists
-                        // (local development and test)
-                        trace!("    loopback");
-                        lo = Some(p.ip());
-                        continue;
-                    }
-                    trace!("    picked {}", p.ip());
-                    return Some(p.ip());
-                }
-                IpAddr::V6(_addr) => {
-                    // Select an ipv6 address if requested
-                    if enable_ipv6 {
-                        if p.ip().is_loopback() {
-                            trace!("    loopback");
-                            lo = Some(p.ip());
-                            continue;
-                        }
-                        return Some(p.ip());
-                    }
-                }
-            }
-        }
-    }
-    trace!("    picked {:?}", lo);
-    lo
-}
-
-pub fn get_ip_addr(enable_ipv6: bool) -> Option<IpAddr> {
-    let mut ifaces = datalink::interfaces();
-    find_eth0ish_ip_addr(&mut ifaces, enable_ipv6)
-}
-
 fn udp_socket(reuseaddr: bool) -> io::Result<Socket> {
     let sock = Socket::new(Domain::ipv4(), Type::dgram(), None)?;
     let sock_fd = sock.as_raw_fd();
@@ -252,101 +202,16 @@ pub fn find_available_port_in_range(range: PortRange) -> io::Result<u16> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use ipnetwork::IpNetwork;
-
-    #[test]
-    fn test_find_eth0ish_ip_addr() {
-        solana_logger::setup();
-
-        macro_rules! mock_interface {
-            ($name:ident, $ip_mask:expr) => {
-                datalink::NetworkInterface {
-                    name: stringify!($name).to_string(),
-                    index: 0,
-                    mac: None,
-                    ips: vec![IpNetwork::V4($ip_mask.parse().unwrap())],
-                    flags: 0,
-                }
-            };
-        }
-
-        // loopback bad when alternatives exist
-        assert_eq!(
-            find_eth0ish_ip_addr(
-                &mut vec![
-                    mock_interface!(eth0, "192.168.137.1/8"),
-                    mock_interface!(lo, "127.0.0.1/24")
-                ],
-                false
-            ),
-            Some(mock_interface!(eth0, "192.168.137.1/8").ips[0].ip())
-        );
-        // find loopback as a last resort
-        assert_eq!(
-            find_eth0ish_ip_addr(&mut vec![mock_interface!(lo, "127.0.0.1/24")], false),
-            Some(mock_interface!(lo, "127.0.0.1/24").ips[0].ip()),
-        );
-        // multicast bad
-        assert_eq!(
-            find_eth0ish_ip_addr(&mut vec![mock_interface!(eth0, "224.0.1.5/24")], false),
-            None
-        );
-
-        // finds "wifi0"
-        assert_eq!(
-            find_eth0ish_ip_addr(
-                &mut vec![
-                    mock_interface!(eth0, "224.0.1.5/24"),
-                    mock_interface!(eth2, "192.168.137.1/8"),
-                    mock_interface!(eth3, "10.0.75.1/8"),
-                    mock_interface!(eth4, "172.22.140.113/4"),
-                    mock_interface!(lo, "127.0.0.1/24"),
-                    mock_interface!(wifi0, "192.168.1.184/8"),
-                ],
-                false
-            ),
-            Some(mock_interface!(wifi0, "192.168.1.184/8").ips[0].ip())
-        );
-        // finds "wifi0" in the middle
-        assert_eq!(
-            find_eth0ish_ip_addr(
-                &mut vec![
-                    mock_interface!(eth0, "224.0.1.5/24"),
-                    mock_interface!(eth2, "192.168.137.1/8"),
-                    mock_interface!(eth3, "10.0.75.1/8"),
-                    mock_interface!(wifi0, "192.168.1.184/8"),
-                    mock_interface!(eth4, "172.22.140.113/4"),
-                    mock_interface!(lo, "127.0.0.1/24"),
-                ],
-                false
-            ),
-            Some(mock_interface!(wifi0, "192.168.1.184/8").ips[0].ip())
-        );
-        // picks "eth2", is lowest valid "eth"
-        assert_eq!(
-            find_eth0ish_ip_addr(
-                &mut vec![
-                    mock_interface!(eth0, "224.0.1.5/24"),
-                    mock_interface!(eth2, "192.168.137.1/8"),
-                    mock_interface!(eth3, "10.0.75.1/8"),
-                    mock_interface!(eth4, "172.22.140.113/4"),
-                    mock_interface!(lo, "127.0.0.1/24"),
-                ],
-                false
-            ),
-            Some(mock_interface!(eth2, "192.168.137.1/8").ips[0].ip())
-        );
-    }
 
     #[test]
     fn test_parse_port_or_addr() {
-        let p1 = parse_port_or_addr(Some("9000"), 1);
+        let p1 = parse_port_or_addr(Some("9000"), SocketAddr::from(([1, 2, 3, 4], 1)));
         assert_eq!(p1.port(), 9000);
-        let p2 = parse_port_or_addr(Some("127.0.0.1:7000"), 1);
+        let p2 = parse_port_or_addr(Some("127.0.0.1:7000"), SocketAddr::from(([1, 2, 3, 4], 1)));
         assert_eq!(p2.port(), 7000);
-        let p2 = parse_port_or_addr(Some("hi there"), 1);
+        let p2 = parse_port_or_addr(Some("hi there"), SocketAddr::from(([1, 2, 3, 4], 1)));
         assert_eq!(p2.port(), 1);
-        let p3 = parse_port_or_addr(None, 1);
+        let p3 = parse_port_or_addr(None, SocketAddr::from(([1, 2, 3, 4], 1)));
         assert_eq!(p3.port(), 1);
     }
 

programs/bpf/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana-bpf-programs"
 description = "Blockchain, Rebuilt for Scale"
-version = "0.14.0"
+version = "0.14.1"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "README.md"
@@ -22,10 +22,10 @@ bincode = "1.1.3"
 byteorder = "1.3.1"
 elf = "0.0.10"
 solana_rbpf = "=0.1.10"
-solana-bpfloader = { path = "../bpf_loader", version = "0.14.0" }
+solana-bpfloader = { path = "../bpf_loader", version = "0.14.1"  }
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-runtime = { path = "../../runtime", version = "0.14.0" }
+solana-runtime = { path = "../../runtime", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
 
 [[bench]]
 name = "bpf_loader"

programs/bpf/rust/noop/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "solana-bpf-rust-noop"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana BPF noop program written in Rust"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

programs/bpf_loader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpfloader"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana BPF Loader"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,8 +15,8 @@ libc = "0.2.51"
 log = "0.4.2"
 solana_rbpf = "=0.1.10"
 serde = "1.0.90"
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
 
 [lib]
 name = "solana_bpf_loader"

programs/budget_api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-budget-api"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana Budget program API"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -16,10 +16,10 @@ num-derive = "0.2"
 num-traits = "0.2"
 serde = "1.0.90"
 serde_derive = "1.0.90"
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
 
 [dev-dependencies]
-solana-runtime = { path = "../../runtime", version = "0.14.0" }
+solana-runtime = { path = "../../runtime", version = "0.14.1"  }
 
 [lib]
 name = "solana_budget_api"

programs/budget_program/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-budget-program"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana budget program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,9 +10,9 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.2"
-solana-budget-api = { path = "../budget_api", version = "0.14.0" }
+solana-budget-api = { path = "../budget_api", version = "0.14.1"  }
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
 
 [lib]
 name = "solana_budget_program"

programs/config_api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-config-api"
-version = "0.14.0"
+version = "0.14.1"
 description = "config program API"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,11 +13,11 @@ bincode = "1.1.3"
 log = "0.4.2"
 serde = "1.0.90"
 serde_derive = "1.0.90"
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
 
 [dev-dependencies]
-solana-runtime = { path = "../../runtime", version = "0.14.0" }
+solana-runtime = { path = "../../runtime", version = "0.14.1"  }
 
 [lib]
 name = "solana_config_api"

programs/config_program/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-config-program"
-version = "0.14.0"
+version = "0.14.1"
 description = "config program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,9 +10,9 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.2"
-solana-config-api = { path = "../config_api", version = "0.14.0" }
+solana-config-api = { path = "../config_api", version = "0.14.1"  }
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
 
 [lib]
 name = "solana_config_program"

programs/exchange_api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-exchange-api"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana Exchange program API"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,12 +13,12 @@ log = "0.4.2"
 bincode = "1.1.3"
 serde = "1.0.90"
 serde_derive = "1.0.90"
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
-solana-metrics = { path = "../../metrics", version = "0.14.0" }
+solana-metrics = { path = "../../metrics", version = "0.14.1"  }
 
 [dev-dependencies]
-solana-runtime = { path = "../../runtime", version = "0.14.0" }
+solana-runtime = { path = "../../runtime", version = "0.14.1"  }
 
 [lib]
 name = "solana_exchange_api"

programs/exchange_program/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-exchange-program"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana exchange program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,9 +10,9 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.2"
-solana-exchange-api = { path = "../exchange_api", version = "0.14.0" }
+solana-exchange-api = { path = "../exchange_api", version = "0.14.1"  }
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
 
 [lib]
 name = "solana_exchange_program"

programs/failure_program/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-failure-program"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana failure program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,11 +9,11 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
 log = "0.4.2"
 
 [dev-dependencies]
-solana-runtime = { path = "../../runtime", version = "0.14.0" }
+solana-runtime = { path = "../../runtime", version = "0.14.1"  }
 
 [lib]
 name = "solana_failure_program"

programs/noop_program/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-noop-program"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana noop program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,12 +9,12 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
 log = "0.4.2"
 
 [dev-dependencies]
-solana-runtime = { path = "../../runtime", version = "0.14.0" }
+solana-runtime = { path = "../../runtime", version = "0.14.1"  }
 
 [lib]
 name = "solana_noop_program"

programs/stake_api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-stake-api"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana Stake program API"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,13 +13,13 @@ bincode = "1.1.3"
 log = "0.4.2"
 serde = "1.0.90"
 serde_derive = "1.0.90"
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-metrics = { path = "../../metrics", version = "0.14.0" }
+solana-metrics = { path = "../../metrics", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
-solana-vote-api = { path = "../vote_api", version = "0.14.0" }
+solana-vote-api = { path = "../vote_api", version = "0.14.1"  }
 
 [dev-dependencies]
-solana-runtime = { path = "../../runtime", version = "0.14.0" }
+solana-runtime = { path = "../../runtime", version = "0.14.1"  }
 
 [lib]
 name = "solana_stake_api"

programs/stake_program/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-stake-program"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana stake program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,9 +10,9 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.2"
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
-solana-stake-api = { path = "../stake_api", version = "0.14.0" }
+solana-stake-api = { path = "../stake_api", version = "0.14.1"  }
 
 [lib]
 name = "solana_stake_program"

programs/storage_api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-storage-api"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana Storage program API"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,11 +13,11 @@ bincode = "1.1.3"
 log = "0.4.2"
 serde = "1.0.90"
 serde_derive = "1.0.90"
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
 
 [dev-dependencies]
-solana-runtime = { path = "../../runtime", version = "0.14.0" }
+solana-runtime = { path = "../../runtime", version = "0.14.1"  }
 
 [lib]
 name = "solana_storage_api"

programs/storage_program/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-storage-program"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana storage program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,9 +10,9 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.2"
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
-solana-storage-api = { path = "../storage_api", version = "0.14.0" }
+solana-storage-api = { path = "../storage_api", version = "0.14.1"  }
 
 [lib]
 name = "solana_storage_program"

programs/token_api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-token-api"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana Token API"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -15,8 +15,8 @@ num-derive = "0.2"
 num-traits = "0.2"
 serde = "1.0.90"
 serde_derive = "1.0.90"
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
 
 [lib]
 name = "solana_token_api"

programs/token_program/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-token-program"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana token program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,9 +10,9 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.2"
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
-solana-token-api = { path = "../token_api", version = "0.14.0" }
+solana-token-api = { path = "../token_api", version = "0.14.1"  }
 
 [lib]
 name = "solana_token_program"

programs/vote_api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-vote-api"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana Vote program API"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,12 +13,12 @@ bincode = "1.1.3"
 log = "0.4.2"
 serde = "1.0.90"
 serde_derive = "1.0.90"
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-metrics = { path = "../../metrics", version = "0.14.0" }
+solana-metrics = { path = "../../metrics", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
 
 [dev-dependencies]
-solana-runtime = { path = "../../runtime", version = "0.14.0" }
+solana-runtime = { path = "../../runtime", version = "0.14.1"  }
 
 [lib]
 name = "solana_vote_api"

programs/vote_program/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-vote-program"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana vote program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,9 +10,9 @@ edition = "2018"
 
 [dependencies]
 log = "0.4.2"
-solana-logger = { path = "../../logger", version = "0.14.0" }
+solana-logger = { path = "../../logger", version = "0.14.1"  }
-solana-sdk = { path = "../../sdk", version = "0.14.0" }
+solana-sdk = { path = "../../sdk", version = "0.14.1"  }
-solana-vote-api = { path = "../vote_api", version = "0.14.0" }
+solana-vote-api = { path = "../vote_api", version = "0.14.1"  }
 
 [lib]
 name = "solana_vote_program"

replicator/Cargo.toml

@@ -2,17 +2,17 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-replicator"
-version = "0.14.0"
+version = "0.14.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.33.0"
-solana = { path = "../core", version = "0.14.0" }
+solana = { path = "../core", version = "0.14.1"  }
-solana-logger = { path = "../logger", version = "0.14.0" }
+solana-logger = { path = "../logger", version = "0.14.1"  }
-solana-netutil = { path = "../netutil", version = "0.14.0" }
+solana-netutil = { path = "../netutil", version = "0.14.1"  }
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
 
 [features]
 chacha = ["solana/chacha"]

replicator/src/main.rs

@@ -4,14 +4,14 @@ use solana::contact_info::ContactInfo;
 use solana::replicator::Replicator;
 use solana::socketaddr;
 use solana_sdk::signature::{read_keypair, Keypair, KeypairUtil};
-use std::net::Ipv4Addr;
 use std::process::exit;
 use std::sync::Arc;
 
 fn main() {
     solana_logger::setup();
 
-    let matches = App::new(crate_name!()).about(crate_description!())
+    let matches = App::new(crate_name!())
+        .about(crate_description!())
         .version(crate_version!())
         .arg(
             Arg::with_name("identity")
@@ -39,12 +39,6 @@ fn main() {
                 .required(true)
                 .help("use DIR as persistent ledger location"),
         )
-        .arg(
-            clap::Arg::with_name("public_address")
-                .long("public-address")
-                .takes_value(false)
-                .help("Advertise public machine address in gossip.  By default the local machine address is advertised"),
-        )
         .get_matches();
 
     let ledger_path = matches.value_of("ledger").unwrap();
@@ -58,13 +52,16 @@ fn main() {
         Keypair::new()
     };
 
+    let network_addr = matches
+        .value_of("network")
+        .map(|network| {
+            solana_netutil::parse_host_port(network).expect("failed to parse network address")
+        })
+        .unwrap();
+
     let gossip_addr = {
         let mut addr = socketaddr!([127, 0, 0, 1], 8700);
-        if matches.is_present("public_address") {
+        addr.set_ip(solana_netutil::get_public_ip_addr(&network_addr).unwrap());
-            addr.set_ip(solana_netutil::get_public_ip_addr().unwrap());
-        } else {
-            addr.set_ip(solana_netutil::get_ip_addr(false).unwrap());
-        }
         addr
     };
     let node =
@@ -76,13 +73,6 @@ fn main() {
         gossip_addr
     );
 
-    let network_addr = matches
-        .value_of("network")
-        .map(|network| {
-            solana_netutil::parse_host_port(network).expect("failed to parse network address")
-        })
-        .unwrap();
-
     let leader_info = ContactInfo::new_gossip_entry_point(&network_addr);
     let storage_keypair = Arc::new(Keypair::new());
     let mut replicator = Replicator::new(

runtime/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-runtime"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana runtime"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -23,10 +23,10 @@ rayon = "1.0.0"
 serde = "1.0.88"
 serde_derive = "1.0.88"
 serde_json = "1.0.38"
-solana-logger = { path = "../logger", version = "0.14.0" }
+solana-logger = { path = "../logger", version = "0.14.1"  }
-solana-metrics = { path = "../metrics", version = "0.14.0" }
+solana-metrics = { path = "../metrics", version = "0.14.1"  }
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
-solana-vote-api = { path = "../programs/vote_api", version = "0.14.0" }
+solana-vote-api = { path = "../programs/vote_api", version = "0.14.1"  }
 
 [lib]
 name = "solana_runtime"

runtime/src/accounts_db.rs

@@ -67,23 +67,12 @@ pub type AccountStorage = HashMap<usize, Arc<AccountStorageEntry>>;
 pub type InstructionAccounts = Vec<Account>;
 pub type InstructionLoaders = Vec<Vec<(Pubkey, Account)>>;
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone, Copy)]
 pub enum AccountStorageStatus {
     StorageAvailable = 0,
     StorageFull = 1,
 }
 
-impl From<usize> for AccountStorageStatus {
-    fn from(status: usize) -> Self {
-        use self::AccountStorageStatus::*;
-        match status {
-            0 => StorageAvailable,
-            1 => StorageFull,
-            _ => unreachable!(),
-        }
-    }
-}
-
 /// Persistent storage structure holding the accounts
 pub struct AccountStorageEntry {
     id: AppendVecId,
@@ -95,11 +84,10 @@ pub struct AccountStorageEntry {
 
     /// Keeps track of the number of accounts stored in a specific AppendVec.
     ///  This is periodically checked to reuse the stores that do not have
-    /// any accounts in it.
+    ///  any accounts in it
-    count: AtomicUsize,
+    /// status corresponding to the storage, lets us know that
-
+    ///  the append_vec, once maxed out, then emptied, can be reclaimed
-    /// status corresponding to the storage
+    count_and_status: RwLock<(usize, AccountStorageStatus)>,
-    status: AtomicUsize,
 }
 
 impl AccountStorageEntry {
@@ -114,28 +102,65 @@ impl AccountStorageEntry {
             id,
             fork_id,
             accounts,
-            count: AtomicUsize::new(0),
+            count_and_status: RwLock::new((0, AccountStorageStatus::StorageAvailable)),
-            status: AtomicUsize::new(AccountStorageStatus::StorageAvailable as usize),
         }
     }
 
-    pub fn set_status(&self, status: AccountStorageStatus) {
+    pub fn set_status(&self, mut status: AccountStorageStatus) {
-        self.status.store(status as usize, Ordering::Relaxed);
+        let mut count_and_status = self.count_and_status.write().unwrap();
+
+        let count = count_and_status.0;
+
+        if status == AccountStorageStatus::StorageFull && count == 0 {
+            // this case arises when the append_vec is full (store_ptrs fails),
+            //  but all accounts have already been removed from the storage
+            //
+            // the only time it's safe to call reset() on an append_vec is when
+            //  every account has been removed
+            //          **and**
+            //  the append_vec has previously been completely full
+            //
+            self.accounts.reset();
+            status = AccountStorageStatus::StorageAvailable;
         }
 
-    pub fn get_status(&self) -> AccountStorageStatus {
+        *count_and_status = (count, status);
-        self.status.load(Ordering::Relaxed).into()
+    }
+
+    pub fn status(&self) -> AccountStorageStatus {
+        self.count_and_status.read().unwrap().1
+    }
+
+    pub fn count(&self) -> usize {
+        self.count_and_status.read().unwrap().0
     }
 
     fn add_account(&self) {
-        self.count.fetch_add(1, Ordering::Relaxed);
+        let mut count_and_status = self.count_and_status.write().unwrap();
+        *count_and_status = (count_and_status.0 + 1, count_and_status.1);
     }
 
     fn remove_account(&self) {
-        if self.count.fetch_sub(1, Ordering::Relaxed) == 1 {
+        let mut count_and_status = self.count_and_status.write().unwrap();
+        let (count, mut status) = *count_and_status;
+
+        if count == 1 && status == AccountStorageStatus::StorageFull {
+            // this case arises when we remove the last account from the
+            //  storage, but we've learned from previous write attempts that
+            //  the storage is full
+            //
+            // the only time it's safe to call reset() on an append_vec is when
+            //  every account has been removed
+            //          **and**
+            //  the append_vec has previously been completely full
+            //
+            // otherwise, the storage may be in flight with a store()
+            //   call
             self.accounts.reset();
-            self.set_status(AccountStorageStatus::StorageAvailable);
+            status = AccountStorageStatus::StorageAvailable;
         }
+
+        *count_and_status = (count - 1, status);
     }
 }
 
@@ -193,7 +218,7 @@ impl AccountsDB {
 
     pub fn has_accounts(&self, fork: Fork) -> bool {
         for x in self.storage.read().unwrap().values() {
-            if x.fork_id == fork && x.count.load(Ordering::Relaxed) > 0 {
+            if x.fork_id == fork && x.count() > 0 {
                 return true;
             }
         }
@@ -254,8 +279,7 @@ impl AccountsDB {
             stores
                 .values()
                 .filter_map(|x| {
-                    if x.get_status() == AccountStorageStatus::StorageAvailable
+                    if x.status() == AccountStorageStatus::StorageAvailable && x.fork_id == fork_id
-                        && x.fork_id == fork_id
                     {
                         Some(x.clone())
                     } else {
@@ -354,7 +378,7 @@ impl AccountsDB {
         let dead_forks: HashSet<Fork> = storage
             .values()
             .filter_map(|x| {
-                if x.count.load(Ordering::Relaxed) == 0 {
+                if x.count() == 0 {
                     Some(x.fork_id)
                 } else {
                     None
@@ -363,13 +387,7 @@ impl AccountsDB {
             .collect();
         let live_forks: HashSet<Fork> = storage
             .values()
-            .filter_map(|x| {
+            .filter_map(|x| if x.count() > 0 { Some(x.fork_id) } else { None })
-                if x.count.load(Ordering::Relaxed) > 0 {
-                    Some(x.fork_id)
-                } else {
-                    None
-                }
-            })
             .collect();
         dead_forks.difference(&live_forks).cloned().collect()
     }
@@ -603,15 +621,15 @@ mod tests {
         {
             let stores = db.storage.read().unwrap();
             assert_eq!(stores.len(), 2);
-            assert_eq!(stores[&0].count.load(Ordering::Relaxed), 2);
+            assert_eq!(stores[&0].count(), 2);
-            assert_eq!(stores[&1].count.load(Ordering::Relaxed), 2);
+            assert_eq!(stores[&1].count(), 2);
         }
         db.add_root(1);
         {
             let stores = db.storage.read().unwrap();
             assert_eq!(stores.len(), 2);
-            assert_eq!(stores[&0].count.load(Ordering::Relaxed), 2);
+            assert_eq!(stores[&0].count(), 2);
-            assert_eq!(stores[&1].count.load(Ordering::Relaxed), 2);
+            assert_eq!(stores[&1].count(), 2);
         }
     }
 
@@ -685,11 +703,8 @@ mod tests {
     fn check_storage(accounts: &AccountsDB, count: usize) -> bool {
         let stores = accounts.storage.read().unwrap();
         assert_eq!(stores.len(), 1);
-        assert_eq!(
+        assert_eq!(stores[&0].status(), AccountStorageStatus::StorageAvailable);
-            stores[&0].get_status(),
+        stores[&0].count() == count
-            AccountStorageStatus::StorageAvailable
-        );
-        stores[&0].count.load(Ordering::Relaxed) == count
     }
 
     fn check_accounts(accounts: &AccountsDB, pubkeys: &Vec<Pubkey>, fork: Fork) {
@@ -779,8 +794,8 @@ mod tests {
         {
             let stores = accounts.storage.read().unwrap();
             assert_eq!(stores.len(), 1);
-            assert_eq!(stores[&0].count.load(Ordering::Relaxed), 1);
+            assert_eq!(stores[&0].count(), 1);
-            assert_eq!(stores[&0].get_status(), status[0]);
+            assert_eq!(stores[&0].status(), AccountStorageStatus::StorageAvailable);
         }
 
         let pubkey2 = Pubkey::new_rand();
@@ -789,27 +804,28 @@ mod tests {
         {
             let stores = accounts.storage.read().unwrap();
             assert_eq!(stores.len(), 2);
-            assert_eq!(stores[&0].count.load(Ordering::Relaxed), 1);
+            assert_eq!(stores[&0].count(), 1);
-            assert_eq!(stores[&0].get_status(), status[1]);
+            assert_eq!(stores[&0].status(), AccountStorageStatus::StorageFull);
-            assert_eq!(stores[&1].count.load(Ordering::Relaxed), 1);
+            assert_eq!(stores[&1].count(), 1);
-            assert_eq!(stores[&1].get_status(), status[0]);
+            assert_eq!(stores[&1].status(), AccountStorageStatus::StorageAvailable);
         }
         let ancestors = vec![(0, 0)].into_iter().collect();
         assert_eq!(accounts.load_slow(&ancestors, &pubkey1).unwrap(), account1);
         assert_eq!(accounts.load_slow(&ancestors, &pubkey2).unwrap(), account2);
 
+        // lots of stores, but 3 storages should be enough for everything
         for i in 0..25 {
             let index = i % 2;
             accounts.store(0, &[(&pubkey1, &account1)]);
             {
                 let stores = accounts.storage.read().unwrap();
                 assert_eq!(stores.len(), 3);
-                assert_eq!(stores[&0].count.load(Ordering::Relaxed), count[index]);
+                assert_eq!(stores[&0].count(), count[index]);
-                assert_eq!(stores[&0].get_status(), status[0]);
+                assert_eq!(stores[&0].status(), status[0]);
-                assert_eq!(stores[&1].count.load(Ordering::Relaxed), 1);
+                assert_eq!(stores[&1].count(), 1);
-                assert_eq!(stores[&1].get_status(), status[1]);
+                assert_eq!(stores[&1].status(), status[1]);
-                assert_eq!(stores[&2].count.load(Ordering::Relaxed), count[index ^ 1]);
+                assert_eq!(stores[&2].count(), count[index ^ 1]);
-                assert_eq!(stores[&2].get_status(), status[0]);
+                assert_eq!(stores[&2].status(), status[0]);
             }
             let ancestors = vec![(0, 0)].into_iter().collect();
             assert_eq!(accounts.load_slow(&ancestors, &pubkey1).unwrap(), account1);

runtime/src/accounts_index.rs

@@ -66,9 +66,11 @@ impl<T: Clone> AccountsIndex<T> {
         self.roots.contains(&fork)
     }
     pub fn add_root(&mut self, fork: Fork) {
-        if fork > self.last_root {
+        assert!(
+            (self.last_root == 0 && fork == 0) || (fork > self.last_root),
+            "new roots must be increasing"
+        );
         self.last_root = fork;
-        }
         self.roots.insert(fork);
     }
     /// Remove the fork when the storage for the fork is freed
@@ -156,15 +158,22 @@ mod tests {
     fn test_max_last_root() {
         let mut index = AccountsIndex::<bool>::default();
         index.add_root(1);
-        index.add_root(0);
         assert_eq!(index.last_root, 1);
     }
 
+    #[test]
+    #[should_panic]
+    fn test_max_last_root_old() {
+        let mut index = AccountsIndex::<bool>::default();
+        index.add_root(1);
+        index.add_root(0);
+    }
+
     #[test]
     fn test_cleanup_first() {
         let mut index = AccountsIndex::<bool>::default();
-        index.add_root(1);
         index.add_root(0);
+        index.add_root(1);
         index.cleanup_dead_fork(0);
         assert!(index.is_root(1));
         assert!(!index.is_root(0));
@@ -174,8 +183,8 @@ mod tests {
     fn test_cleanup_last() {
         //this behavior might be undefined, clean up should only occur on older forks
         let mut index = AccountsIndex::<bool>::default();
-        index.add_root(1);
         index.add_root(0);
+        index.add_root(1);
         index.cleanup_dead_fork(1);
         assert!(!index.is_root(1));
         assert!(index.is_root(0));

runtime/src/append_vec.rs

@@ -74,7 +74,7 @@ impl AppendVec {
             .open(file)
             .expect("Unable to open data file");
 
-        data.seek(SeekFrom::Start(size as u64)).unwrap();
+        data.seek(SeekFrom::Start((size - 1) as u64)).unwrap();
         data.write_all(&[0]).unwrap();
         data.seek(SeekFrom::Start(0)).unwrap();
         data.flush().unwrap();
@@ -153,7 +153,7 @@ impl AppendVec {
             end += val.1;
         }
 
-        if (self.file_size as usize) <= end {
+        if (self.file_size as usize) < end {
             return None;
         }
 

runtime/src/bank.rs

@@ -124,7 +124,7 @@ pub struct Bank {
     parent: RwLock<Option<Arc<Bank>>>,
 
     /// The set of parents including this bank
-    ancestors: HashMap<u64, usize>,
+    pub ancestors: HashMap<u64, usize>,
 
     /// Hash of this Bank's state. Only meaningful after freezing.
     hash: RwLock<Hash>,
@@ -293,7 +293,7 @@ impl Bank {
         *self.parent.write().unwrap() = None;
 
         let squash_accounts_start = Instant::now();
-        for p in &parents {
+        for p in parents.iter().rev() {
             // root forks cannot be purged
             self.accounts.add_root(p.slot());
         }
@@ -1026,24 +1026,6 @@ impl Bank {
         // Register a bogus executable account, which will be loaded and ignored.
         self.register_native_instruction_processor("", &program_id);
     }
-
-    pub fn is_in_subtree_of(&self, parent: u64) -> bool {
-        if self.slot() == parent {
-            return true;
-        }
-        let mut next_parent = self.parent();
-
-        while let Some(p) = next_parent {
-            if p.slot() == parent {
-                return true;
-            } else if p.slot() < parent {
-                return false;
-            }
-            next_parent = p.parent();
-        }
-
-        false
-    }
 }
 
 impl Drop for Bank {
@@ -1841,30 +1823,6 @@ mod tests {
         assert_eq!(bank.is_votable(), true);
     }
 
-    #[test]
-    fn test_is_in_subtree_of() {
-        let (genesis_block, _) = GenesisBlock::new(1);
-        let parent = Arc::new(Bank::new(&genesis_block));
-        // Bank 1
-        let bank = Arc::new(new_from_parent(&parent));
-        // Bank 2
-        let bank2 = new_from_parent(&bank);
-        // Bank 5
-        let bank5 = Bank::new_from_parent(&bank, &Pubkey::default(), 5);
-
-        // Parents of bank 2: 0 -> 1 -> 2
-        assert!(bank2.is_in_subtree_of(0));
-        assert!(bank2.is_in_subtree_of(1));
-        assert!(bank2.is_in_subtree_of(2));
-        assert!(!bank2.is_in_subtree_of(3));
-
-        // Parents of bank 5: 0 -> 1 -> 5
-        assert!(bank5.is_in_subtree_of(0));
-        assert!(bank5.is_in_subtree_of(1));
-        assert!(!bank5.is_in_subtree_of(2));
-        assert!(!bank5.is_in_subtree_of(4));
-    }
-
     #[test]
     fn test_bank_inherit_tx_count() {
         let (genesis_block, mint_keypair) = GenesisBlock::new(500);

scripts/oom-monitor.sh

@@ -30,7 +30,7 @@ while read -r victim; do
   ./metrics-write-datapoint.sh "oom-killer,victim=$victim,hostname=$HOSTNAME killed=1"
 done < <( \
   tail --follow=name --retry -n0 $syslog \
-  | sed --unbuffered -n 's/^.* Out of memory: Kill process [1-9][0-9]* (\([^)]*\)) .*/\1/p' \
+  | sed --unbuffered -n "s/^.* earlyoom\[[0-9]*\]: Killing process .\(.*\). with signal .*/\1/p" \
 )
 
 exit 1

sdk/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-sdk"
-version = "0.14.0"
+version = "0.14.1"
 description = "Solana SDK"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

vote-signer/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-vote-signer"
 description = "Solana Vote Signing Service"
-version = "0.14.0"
+version = "0.14.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -17,8 +17,8 @@ jsonrpc-derive = "11.0.0"
 jsonrpc-http-server = "11.0.0"
 serde = "1.0.90"
 serde_json = "1.0.39"
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
-solana-metrics = { path = "../metrics", version = "0.14.0" }
+solana-metrics = { path = "../metrics", version = "0.14.1"  }
 
 [lib]
 name = "solana_vote_signer"

wallet/Cargo.toml

@@ -3,7 +3,7 @@ authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-wallet"
 description = "Blockchain, Rebuilt for Scale"
-version = "0.14.0"
+version = "0.14.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -17,18 +17,18 @@ dirs = "1.0.5"
 log = "0.4.2"
 num-traits = "0.2"
 serde_json = "1.0.39"
-solana-budget-api = { path = "../programs/budget_api", version = "0.14.0" }
+solana-budget-api = { path = "../programs/budget_api", version = "0.14.1"  }
-solana-client = { path = "../client", version = "0.14.0" }
+solana-client = { path = "../client", version = "0.14.1"  }
-solana-drone = { path = "../drone", version = "0.14.0" }
+solana-drone = { path = "../drone", version = "0.14.1"  }
-solana-logger = { path = "../logger", version = "0.14.0" }
+solana-logger = { path = "../logger", version = "0.14.1"  }
-solana-netutil = { path = "../netutil", version = "0.14.0" }
+solana-netutil = { path = "../netutil", version = "0.14.1"  }
-solana-sdk = { path = "../sdk", version = "0.14.0" }
+solana-sdk = { path = "../sdk", version = "0.14.1"  }
-solana-vote-api = { path = "../programs/vote_api", version = "0.14.0" }
+solana-vote-api = { path = "../programs/vote_api", version = "0.14.1"  }
-solana-vote-signer = { path = "../vote-signer", version = "0.14.0" }
+solana-vote-signer = { path = "../vote-signer", version = "0.14.1"  }
 
 [dev-dependencies]
-solana-budget-program = { path = "../programs/budget_program", version = "0.14.0" }
+solana-budget-program = { path = "../programs/budget_program", version = "0.14.1"  }
-solana = { path = "../core", version = "0.14.0" }
+solana = { path = "../core", version = "0.14.1"  }
 
 [features]
 cuda = []