testnet-deploy.sh updates

* Checkout testnet scripts from tip to upstream/0.11

* Add back NO_VALIDATOR_SANITY to testnet sanity

Cargo.lock

@@ -1715,7 +1715,7 @@ dependencies = [
 
 [[package]]
 name = "solana"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "bs58 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1739,19 +1739,19 @@ dependencies = [
  "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-bpfloader 0.11.0",
+ "solana-bpfloader 0.11.1",
- "solana-drone 0.11.0",
+ "solana-drone 0.11.1",
  "solana-jsonrpc-core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-jsonrpc-http-server 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-jsonrpc-macros 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-jsonrpc-pubsub 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-jsonrpc-ws-server 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
- "solana-metrics 0.11.0",
+ "solana-metrics 0.11.1",
- "solana-native-loader 0.11.0",
+ "solana-native-loader 0.11.1",
- "solana-netutil 0.11.0",
+ "solana-netutil 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
- "solana-system-program 0.11.0",
+ "solana-system-program 0.11.1",
  "tokio 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "untrusted 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1759,39 +1759,39 @@ dependencies = [
 
 [[package]]
 name = "solana-bench-streamer"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.11.0",
+ "solana 0.11.1",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
- "solana-netutil 0.11.0",
+ "solana-netutil 0.11.1",
 ]
 
 [[package]]
 name = "solana-bench-tps"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rayon 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.11.0",
+ "solana 0.11.1",
- "solana-drone 0.11.0",
+ "solana-drone 0.11.1",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
- "solana-metrics 0.11.0",
+ "solana-metrics 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-bpf-noop"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "rbpf 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-bpfloader"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1799,27 +1799,27 @@ dependencies = [
  "libc 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
  "solana_rbpf 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "solana-budget-program"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-drone"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1828,43 +1828,43 @@ dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-metrics 0.11.0",
+ "solana-metrics 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
  "tokio 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "solana-erc20"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-fullnode"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.11.0",
+ "solana 0.11.1",
- "solana-drone 0.11.0",
+ "solana-drone 0.11.1",
- "solana-fullnode-config 0.11.0",
+ "solana-fullnode-config 0.11.1",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
- "solana-metrics 0.11.0",
+ "solana-metrics 0.11.1",
- "solana-netutil 0.11.0",
+ "solana-netutil 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-fullnode-config"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "dirs 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1872,19 +1872,19 @@ dependencies = [
  "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-netutil 0.11.0",
+ "solana-netutil 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
  "untrusted 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "solana-genesis"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.11.0",
+ "solana 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
@@ -2025,70 +2025,70 @@ dependencies = [
 
 [[package]]
 name = "solana-keygen"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "dirs 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-ledger-tool"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.11.0",
+ "solana 0.11.1",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
 ]
 
 [[package]]
 name = "solana-logger"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "env_logger 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "solana-lualoader"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "rlua 0.15.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-metrics"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "influx_db_client 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
  "sys-info 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "solana-native-loader"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.45 (registry+https://github.com/rust-lang/crates.io-index)",
  "libloading 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-netutil"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "ipnetwork 0.12.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2097,33 +2097,33 @@ dependencies = [
  "rand 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "socket2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
 ]
 
 [[package]]
 name = "solana-noop"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-replicator"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.11.0",
+ "solana 0.11.1",
- "solana-fullnode-config 0.11.0",
+ "solana-fullnode-config 0.11.1",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-sdk"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "bs58 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2141,50 +2141,50 @@ dependencies = [
 
 [[package]]
 name = "solana-storage-program"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-system-program"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-upload-perf"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-metrics 0.11.0",
+ "solana-metrics 0.11.1",
 ]
 
 [[package]]
 name = "solana-vote-program"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.82 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
- "solana-metrics 0.11.0",
+ "solana-metrics 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-vote-signer"
-version = "0.0.1"
+version = "0.11.1"
 dependencies = [
  "bs58 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2193,13 +2193,13 @@ dependencies = [
  "solana-jsonrpc-core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-jsonrpc-http-server 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-jsonrpc-macros 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana-metrics 0.11.0",
+ "solana-metrics 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]
 name = "solana-wallet"
-version = "0.11.0"
+version = "0.11.1"
 dependencies = [
  "bincode 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "bs58 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2207,10 +2207,10 @@ dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "dirs 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.33 (registry+https://github.com/rust-lang/crates.io-index)",
- "solana 0.11.0",
+ "solana 0.11.1",
- "solana-drone 0.11.0",
+ "solana-drone 0.11.1",
- "solana-logger 0.11.0",
+ "solana-logger 0.11.1",
- "solana-sdk 0.11.0",
+ "solana-sdk 0.11.1",
 ]
 
 [[package]]

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "solana"
 description = "Blockchain, Rebuilt for Scale"
-version = "0.11.0"
+version = "0.11.1"
 documentation = "https://docs.rs/solana"
 homepage = "https://solana.com/"
 readme = "README.md"
@@ -43,19 +43,19 @@ rocksdb = "0.10.1"
 serde = "1.0.82"
 serde_derive = "1.0.82"
 serde_json = "1.0.10"
-solana-bpfloader = { path = "programs/native/bpf_loader", version = "0.11.0" }
+solana-bpfloader = { path = "programs/native/bpf_loader", version = "0.11.1" }
-solana-drone = { path = "drone", version = "0.11.0" }
+solana-drone = { path = "drone", version = "0.11.1" }
 solana-jsonrpc-core = "0.4.0"
 solana-jsonrpc-http-server = "0.4.0"
 solana-jsonrpc-macros = "0.4.0"
 solana-jsonrpc-pubsub = "0.4.0"
 solana-jsonrpc-ws-server = "0.4.0"
-solana-logger = { path = "logger", version = "0.11.0" }
+solana-logger = { path = "logger", version = "0.11.1" }
-solana-metrics = { path = "metrics", version = "0.11.0" }
+solana-metrics = { path = "metrics", version = "0.11.1" }
-solana-native-loader = { path = "programs/native/native_loader", version = "0.11.0" }
+solana-native-loader = { path = "programs/native/native_loader", version = "0.11.1" }
-solana-netutil = { path = "netutil", version = "0.11.0" }
+solana-netutil = { path = "netutil", version = "0.11.1" }
-solana-sdk = { path = "sdk", version = "0.11.0" }
+solana-sdk = { path = "sdk", version = "0.11.1" }
-solana-system-program = { path = "programs/native/system", version = "0.11.0" }
+solana-system-program = { path = "programs/native/system", version = "0.11.1" }
 tokio = "0.1"
 tokio-codec = "0.1"
 untrusted = "0.6.2"

README.md

@@ -26,7 +26,9 @@ Furthermore, and much to our surprise, it can be implemented using a mechanism t
 Architecture
 ===
 
-Before you jump into the code, review the online book [Solana: Blockchain Rebuilt for Scale](https://solana-labs.github.io/solana/).
+Before you jump into the code, review the online book [Solana: Blockchain Rebuilt for Scale](https://solana-labs.github.io/book/).
+
+(The _latest_ development version of the online book is also [available here](https://solana-labs.github.io/book-edge/).)
 
 Developing
 ===

bench-streamer/Cargo.toml

@@ -2,16 +2,16 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-streamer"
-version = "0.11.0"
+version = "0.11.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
 
 [dependencies]
 clap = "2.32.0"
-solana = { path = "..", version = "0.11.0" }
+solana = { path = "..", version = "0.11.1" }
-solana-logger = { path = "../logger", version = "0.11.0" }
+solana-logger = { path = "../logger", version = "0.11.1" }
-solana-netutil = { path = "../netutil", version = "0.11.0" }
+solana-netutil = { path = "../netutil", version = "0.11.1" }
 
 [features]
 cuda = []

bench-tps/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-bench-tps"
-version = "0.11.0"
+version = "0.11.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,11 +11,11 @@ homepage = "https://solana.com/"
 clap = "2.32.0"
 rayon = "1.0.3"
 serde_json = "1.0.10"
-solana = { path = "..", version = "0.11.0" }
+solana = { path = "..", version = "0.11.1" }
-solana-drone = { path = "../drone", version = "0.11.0" }
+solana-drone = { path = "../drone", version = "0.11.1" }
-solana-logger = { path = "../logger", version = "0.11.0" }
+solana-logger = { path = "../logger", version = "0.11.1" }
-solana-metrics = { path = "../metrics", version = "0.11.0" }
+solana-metrics = { path = "../metrics", version = "0.11.1" }
-solana-sdk = { path = "../sdk", version = "0.11.0" }
+solana-sdk = { path = "../sdk", version = "0.11.1" }
 
 [features]
 cuda = []

ci/buildkite-secondary.yml

@@ -6,7 +6,7 @@ steps:
     timeout_in_minutes: 20
     name: "publish docker"
   - command: "ci/publish-crate.sh"
-    timeout_in_minutes: 20
+    timeout_in_minutes: 40
     name: "publish crate"
     branches: "!master"
   - command: "ci/publish-bpf-sdk.sh"

ci/channel-info.sh

@@ -82,10 +82,26 @@ for tag in "${tags[@]}"; do
   fi
 done
 
-echo EDGE_CHANNEL=master
+EDGE_CHANNEL=master
-echo BETA_CHANNEL="${beta:+v$beta}"
+BETA_CHANNEL=${beta:+v$beta}
-echo STABLE_CHANNEL="${stable:+v$stable}"
+STABLE_CHANNEL=${stable:+v$stable}
-echo BETA_CHANNEL_LATEST_TAG="${beta_tag:+v$beta_tag}"
+BETA_CHANNEL_LATEST_TAG=${beta_tag:+v$beta_tag}
-echo STABLE_CHANNEL_LATEST_TAG="${stable_tag:+v$stable_tag}"
+STABLE_CHANNEL_LATEST_TAG=${stable_tag:+v$stable_tag}
+
+
+if [[ $BUILDKITE_BRANCH = "$STABLE_CHANNEL" ]]; then
+  CHANNEL=stable
+elif [[ $BUILDKITE_BRANCH = "$EDGE_CHANNEL" ]]; then
+  CHANNEL=edge
+elif [[ $BUILDKITE_BRANCH = "$BETA_CHANNEL" ]]; then
+  CHANNEL=beta
+fi
+
+echo EDGE_CHANNEL="$EDGE_CHANNEL"
+echo BETA_CHANNEL="$BETA_CHANNEL"
+echo BETA_CHANNEL_LATEST_TAG="$BETA_CHANNEL_LATEST_TAG"
+echo STABLE_CHANNEL="$STABLE_CHANNEL"
+echo STABLE_CHANNEL_LATEST_TAG="$STABLE_CHANNEL_LATEST_TAG"
+echo CHANNEL="$CHANNEL"
 
 exit 0

ci/publish-book.sh

@@ -16,18 +16,29 @@ echo --- create book repo
   git commit -m "${BUILDKITE_COMMIT:-local}"
 )
 
-echo --- publish
+eval "$(ci/channel-info.sh)"
-if [[ $BUILDKITE_BRANCH = master ]]; then
+# Only publish the book from the edge and beta channels for now.
-  cd book/html/
+case $CHANNEL in
-  git remote add origin git@github.com:solana-labs/solana.git
+edge)
-  git fetch origin gh-pages
+  repo=git@github.com:solana-labs/book-edge.git
-  if ! git diff HEAD origin/gh-pages --quiet; then
+  ;;
-    git push -f origin HEAD:gh-pages
+beta)
-  else
+  repo=git@github.com:solana-labs/book.git
-    echo "Content unchanged, publish skipped"
+  ;;
-  fi
+*)
+ echo "--- publish skipped"
+ exit 0
+ ;;
+esac
+
+echo "--- publish $CHANNEL"
+cd book/html/
+git remote add origin $repo
+git fetch origin master
+if ! git diff HEAD origin/master --quiet; then
+  git push -f origin HEAD:master
 else
-  echo "Publish skipped"
+  echo "Content unchanged, publish skipped"
 fi
 
 exit 0

ci/publish-crate.sh

@@ -19,6 +19,7 @@ CRATES=(
   drone
   programs/native/{budget,bpf_loader,lua_loader,native_loader,noop,system,vote}
   .
+  fullnode-config
   fullnode
   genesis
   ledger-tool
@@ -26,22 +27,18 @@ CRATES=(
 )
 
 
-maybePackage="echo Package skipped"
-maybePublish="echo Publish skipped"
-
 # Only package/publish if this is a tagged release
-if [[ -n $TRIGGERED_BUILDKITE_TAG ]]; then
+[[ -n $TRIGGERED_BUILDKITE_TAG ]] || {
-  maybePackage="cargo package"
+  echo TRIGGERED_BUILDKITE_TAG unset, skipped
+  exit 0
+}
 
-  # Only publish if there's no human around
+[[ -n "$CRATES_IO_TOKEN" ]] || {
-  if [[ -n $CI ]]; then
+  echo CRATES_IO_TOKEN undefined
-    maybePublish="cargo publish --token $CRATES_IO_TOKEN"
+  exit 1
-    if [[ -z "$CRATES_IO_TOKEN" ]]; then
+}
-      echo CRATES_IO_TOKEN undefined
+
-      exit 1
+cargoCommand="cargo publish --token $CRATES_IO_TOKEN"
-    fi
-  fi
-fi
 
 for crate in "${CRATES[@]}"; do
   if [[ ! -r $crate/Cargo.toml ]]; then
@@ -49,10 +46,14 @@ for crate in "${CRATES[@]}"; do
     exit 1
   fi
   echo "-- $crate"
-  # TODO: Ensure the published version matches the contents of BUILDKITE_TAG
+  # TODO: Ensure the published version matches the contents of
+  # TRIGGERED_BUILDKITE_TAG
   (
     set -x
-    ci/docker-run.sh rust bash -exc "cd $crate; $maybePackage; $maybePublish"
+    # TODO: the rocksdb package does not build with the stock rust docker image,
+    # so use the solana rust docker image until this is resolved upstream
+    ci/docker-run.sh solanalabs/rust:1.31.0 bash -exc "cd $crate; $cargoCommand"
+    #ci/docker-run.sh rust bash -exc "cd $crate; $cargoCommand"
   )
 done
 

ci/publish-metrics-dashboard.sh

@@ -8,11 +8,11 @@ if [[ -z $BUILDKITE ]]; then
   exit 1
 fi
 
-if [[ -z $CHANNEL ]]; then
+if [[ -z $PUBLISH_CHANNEL ]]; then
-  CHANNEL=$(buildkite-agent meta-data get "channel" --default "")
+  PUBLISH_CHANNEL=$(buildkite-agent meta-data get "channel" --default "")
 fi
 
-if [[ -z $CHANNEL ]]; then
+if [[ -z $PUBLISH_CHANNEL ]]; then
   (
     cat <<EOF
 steps:
@@ -37,7 +37,7 @@ fi
 ci/channel-info.sh
 eval "$(ci/channel-info.sh)"
 
-case $CHANNEL in
+case $PUBLISH_CHANNEL in
 edge)
   CHANNEL_BRANCH=$EDGE_CHANNEL
   ;;
@@ -48,7 +48,7 @@ stable)
   CHANNEL_BRANCH=$STABLE_CHANNEL
   ;;
 *)
-  echo "Error: Invalid CHANNEL=$CHANNEL"
+  echo "Error: Invalid PUBLISH_CHANNEL=$PUBLISH_CHANNEL"
   exit 1
   ;;
 esac
@@ -63,11 +63,11 @@ steps:
       message: "$BUILDKITE_MESSAGE"
       branch: "$CHANNEL_BRANCH"
       env:
-        CHANNEL: "$CHANNEL"
+        PUBLISH_CHANNEL: "$PUBLISH_CHANNEL"
 EOF
   ) | buildkite-agent pipeline upload
   exit 0
 fi
 
 set -x
-exec metrics/publish-metrics-dashboard.sh "$CHANNEL"
+exec metrics/publish-metrics-dashboard.sh "$PUBLISH_CHANNEL"

ci/publish-tarball.sh

@@ -11,14 +11,6 @@ fi
 
 eval "$(ci/channel-info.sh)"
 
-if [[ $BUILDKITE_BRANCH = "$STABLE_CHANNEL" ]]; then
-  CHANNEL=stable
-elif [[ $BUILDKITE_BRANCH = "$EDGE_CHANNEL" ]]; then
-  CHANNEL=edge
-elif [[ $BUILDKITE_BRANCH = "$BETA_CHANNEL" ]]; then
-  CHANNEL=beta
-fi
-
 if [[ -n "$BUILDKITE_TAG" ]]; then
   CHANNEL_OR_TAG=$BUILDKITE_TAG
 elif [[ -n "$TRIGGERED_BUILDKITE_TAG" ]]; then

ci/snap.sh

@@ -19,14 +19,6 @@ fi
 
 eval "$(ci/channel-info.sh)"
 
-if [[ $BUILDKITE_BRANCH = "$STABLE_CHANNEL" ]]; then
-  CHANNEL=stable
-elif [[ $BUILDKITE_BRANCH = "$EDGE_CHANNEL" ]]; then
-  CHANNEL=edge
-elif [[ $BUILDKITE_BRANCH = "$BETA_CHANNEL" ]]; then
-  CHANNEL=beta
-fi
-
 if [[ -z $CHANNEL ]]; then
   echo Unable to determine channel to publish into, exiting.
   exit 1
@@ -61,13 +53,13 @@ if [[ ! -x /usr/bin/multilog ]]; then
   sudo apt-get install -y daemontools
 fi
 
-echo --- build: $CHANNEL channel
+echo "--- build: $CHANNEL channel"
 snapcraft
 
 source ci/upload-ci-artifact.sh
 upload-ci-artifact solana_*.snap
 
 if [[ -z $DO_NOT_PUBLISH_SNAP ]]; then
-  echo --- publish: $CHANNEL channel
+  echo "--- publish: $CHANNEL channel"
-  $DRYRUN snapcraft push solana_*.snap --release $CHANNEL
+  $DRYRUN snapcraft push solana_*.snap --release "$CHANNEL"
 fi

ci/testnet-deploy.sh

@@ -2,6 +2,7 @@
 set -e
 
 cd "$(dirname "$0")"/..
+source ci/upload-ci-artifact.sh
 
 zone=
 bootstrapFullNodeAddress=
@@ -9,12 +10,16 @@ bootstrapFullNodeMachineType=
 clientNodeCount=0
 additionalFullNodeCount=10
 publicNetwork=false
-snapChannel=edge
+skipSetup=false
+skipStart=false
+stopNetwork=false
+externalNode=false
 tarChannelOrTag=edge
 delete=false
 enableGpu=false
+bootDiskType=""
 leaderRotation=true
-useTarReleaseChannel=false
+blockstreamer=false
 
 usage() {
   exitcode=0
@@ -23,29 +28,35 @@ usage() {
     echo "Error: $*"
   fi
   cat <<EOF
-usage: $0 [name] [cloud] [zone] [options...]
+usage: $0 -p network-name -C cloud -z zone1 [-z zone2] ... [-z zoneN] [options...]
 
 Deploys a CD testnet
 
-  name  - name of the network
+  mandatory arguments:
-  cloud - cloud provider to use (gce, ec2)
+  -p [network-name]  - name of the network
-  zone  - cloud provider zone to deploy the network into
+  -C [cloud] - cloud provider to use (gce, ec2)
+  -z [zone]  - cloud provider zone to deploy the network into.  Must specify at least one zone
 
   options:
-   -s edge|beta|stable  - Deploy the specified Snap release channel
-                          (default: $snapChannel)
    -t edge|beta|stable|vX.Y.Z  - Deploy the latest tarball release for the
                                  specified release channel (edge|beta|stable) or release tag
                                  (vX.Y.Z)
                                  (default: $tarChannelOrTag)
    -n [number]          - Number of additional full nodes (default: $additionalFullNodeCount)
    -c [number]          - Number of client bencher nodes (default: $clientNodeCount)
+   -u                   - Include a Blockstreamer (default: $blockstreamer)
    -P                   - Use public network IP addresses (default: $publicNetwork)
    -G                   - Enable GPU, and set count/type of GPUs to use (e.g n1-standard-16 --accelerator count=4,type=nvidia-tesla-k80)
    -g                   - Enable GPU (default: $enableGpu)
    -b                   - Disable leader rotation
    -a [address]         - Set the bootstrap fullnode's external IP address to this GCE address
-   -d                   - Delete the network
+   -d [disk-type]       - Specify a boot disk type (default None) Use pd-ssd to get ssd on GCE.
+   -D                   - Delete the network
+   -r                   - Reuse existing node/ledger configuration from a
+                          previous |start| (ie, don't run ./multinode-demo/setup.sh).
+   -x                   - External node.  Default: false
+   -s                   - Skip start.  Nodes will still be created or configured, but network software will not be started.
+   -S                   - Stop network software without tearing down nodes.
 
    Note: the SOLANA_METRICS_CONFIG environment variable is used to configure
          metrics
@@ -53,19 +64,22 @@ EOF
   exit $exitcode
 }
 
-netName=$1
+zone=()
-cloudProvider=$2
-zone=$3
-[[ -n $netName ]] || usage
-[[ -n $cloudProvider ]] || usage "Cloud provider not specified"
-[[ -n $zone ]] || usage "Zone not specified"
-shift 3
 
-while getopts "h?p:Pn:c:s:t:gG:a:db" opt; do
+while getopts "h?p:Pn:c:t:gG:a:Dbd:rusxz:p:C:S" opt; do
   case $opt in
   h | \?)
     usage
     ;;
+  p)
+    netName=$OPTARG
+    ;;
+  C)
+    cloudProvider=$OPTARG
+    ;;
+  z)
+    zone+=("$OPTARG")
+    ;;
   P)
     publicNetwork=true
     ;;
@@ -75,21 +89,10 @@ while getopts "h?p:Pn:c:s:t:gG:a:db" opt; do
   c)
     clientNodeCount=$OPTARG
     ;;
-  s)
-    case $OPTARG in
-    edge|beta|stable)
-      snapChannel=$OPTARG
-      ;;
-    *)
-      usage "Invalid snap channel: $OPTARG"
-      ;;
-    esac
-    ;;
   t)
     case $OPTARG in
     edge|beta|stable|v*)
       tarChannelOrTag=$OPTARG
-      useTarReleaseChannel=true
       ;;
     *)
       usage "Invalid release channel: $OPTARG"
@@ -110,51 +113,139 @@ while getopts "h?p:Pn:c:s:t:gG:a:db" opt; do
     bootstrapFullNodeAddress=$OPTARG
     ;;
   d)
+    bootDiskType=$OPTARG
+    ;;
+  D)
     delete=true
     ;;
+  r)
+    skipSetup=true
+    ;;
+  s)
+    skipStart=true
+    ;;
+  x)
+    externalNode=true
+    ;;
+  u)
+    blockstreamer=true
+    ;;
+  S)
+    stopNetwork=true
+    ;;
   *)
     usage "Error: unhandled option: $opt"
     ;;
   esac
 done
 
+[[ -n $netName ]] || usage
+[[ -n $cloudProvider ]] || usage "Cloud provider not specified"
+[[ -n ${zone[*]} ]] || usage "At least one zone must be specified"
 
-create_args=(
+shutdown() {
-  -a "$bootstrapFullNodeAddress"
+  exitcode=$?
-  -c "$clientNodeCount"
-  -n "$additionalFullNodeCount"
-  -p "$netName"
-  -z "$zone"
-)
 
-if $enableGpu; then
+  set +e
-  if [[ -z $bootstrapFullNodeMachineType ]]; then
+  if [[ -d net/log ]]; then
-    create_args+=(-g)
+    mv net/log net/log-deploy
-  else
+    for logfile in net/log-deploy/*; do
-    create_args+=(-G "$bootstrapFullNodeMachineType")
+      if [[ -f $logfile ]]; then
+        upload-ci-artifact "$logfile"
+        tail "$logfile"
+      fi
+    done
   fi
-fi
+  exit $exitcode
-
+}
-if ! $leaderRotation; then
+rm -rf net/{log,-deploy}
-  create_args+=(-b)
+trap shutdown EXIT INT
-fi
-
-if $publicNetwork; then
-  create_args+=(-P)
-fi
 
 set -x
 
-echo "--- $cloudProvider.sh delete"
+# Build a string to pass zone opts to $cloudProvider.sh: "-z zone1 -z zone2 ..."
-time net/"$cloudProvider".sh delete -z "$zone" -p "$netName"
+zone_args=()
-if $delete; then
+for val in "${zone[@]}"; do
-  exit 0
+  zone_args+=("-z $val")
+done
+
+if $stopNetwork; then
+  skipSetup=true
 fi
 
-echo "--- $cloudProvider.sh create"
+if $delete; then
-time net/"$cloudProvider".sh create "${create_args[@]}"
+  skipSetup=false
+fi
+
+# Create the network
+if ! $skipSetup; then
+  echo "--- $cloudProvider.sh delete"
+  # shellcheck disable=SC2068
+  time net/"$cloudProvider".sh delete ${zone_args[@]} -p "$netName"
+  if $delete; then
+    exit 0
+  fi
+
+  echo "--- $cloudProvider.sh create"
+  create_args=(
+    -p "$netName"
+    -a "$bootstrapFullNodeAddress"
+    -c "$clientNodeCount"
+    -n "$additionalFullNodeCount"
+  )
+  # shellcheck disable=SC2206
+  create_args+=(${zone_args[@]})
+
+  if $blockstreamer; then
+    create_args+=(-u)
+  fi
+
+  if [[ -n $bootDiskType ]]; then
+    create_args+=(-d "$bootDiskType")
+  fi
+
+  if $enableGpu; then
+    if [[ -z $bootstrapFullNodeMachineType ]]; then
+      create_args+=(-g)
+    else
+      create_args+=(-G "$bootstrapFullNodeMachineType")
+    fi
+  fi
+
+  if ! $leaderRotation; then
+    create_args+=(-b)
+  fi
+
+  if $publicNetwork; then
+    create_args+=(-P)
+  fi
+
+  if $externalNode; then
+    create_args+=(-x)
+  fi
+
+  time net/"$cloudProvider".sh create "${create_args[@]}"
+else
+  echo "--- $cloudProvider.sh config"
+  config_args=(
+    -p "$netName"
+  )
+  # shellcheck disable=SC2206
+  config_args+=(${zone_args[@]})
+  if $publicNetwork; then
+    config_args+=(-P)
+  fi
+
+  time net/"$cloudProvider".sh config "${config_args[@]}"
+fi
 net/init-metrics.sh -e
 
+if $stopNetwork; then
+  echo --- net.sh stop
+  time net/net.sh stop
+  exit 0
+fi
+
 echo --- net.sh start
 maybeRejectExtraNodes=
 if ! $publicNetwork; then
@@ -168,10 +259,38 @@ maybeNoLedgerVerify=
 if [[ -n $NO_LEDGER_VERIFY ]]; then
   maybeNoLedgerVerify="-o noLedgerVerify"
 fi
-# shellcheck disable=SC2086 # Don't want to double quote maybeRejectExtraNodes
+
-if $useTarReleaseChannel; then
+maybeSkipSetup=
-  time net/net.sh start -t "$tarChannelOrTag" $maybeRejectExtraNodes $maybeNoValidatorSanity $maybeNoLedgerVerify
+if $skipSetup; then
-else
+  maybeSkipSetup="-r"
-  time net/net.sh start -s "$snapChannel" $maybeRejectExtraNodes $maybeNoValidatorSanity $maybeNoLedgerVerify
 fi
-exit 0
+
+ok=true
+if ! $skipStart; then
+  (
+    if $skipSetup; then
+      # TODO: Enable rolling updates
+      #op=update
+      op=restart
+    else
+      op=start
+    fi
+
+    maybeUpdateManifestKeypairFile=
+    # shellcheck disable=SC2154 # SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu comes from .buildkite/env/
+    if [[ -n $SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu ]]; then
+      echo "$SOLANA_INSTALL_UPDATE_MANIFEST_KEYPAIR_x86_64_unknown_linux_gnu" > update_manifest_keypair.json
+      maybeUpdateManifestKeypairFile="-i update_manifest_keypair.json"
+    fi
+
+    # shellcheck disable=SC2086 # Don't want to double quote the $maybeXYZ variables
+    time net/net.sh $op -t "$tarChannelOrTag" \
+      $maybeUpdateManifestKeypairFile \
+      $maybeSkipSetup \
+      $maybeRejectExtraNodes \
+      $maybeNoValidatorSanity \
+      $maybeNoLedgerVerify
+  ) || ok=false
+fi
+
+$ok

ci/testnet-manager.sh

@@ -42,18 +42,28 @@ steps:
             value: "testnet-beta"
           - label: "testnet-beta-perf"
             value: "testnet-beta-perf"
+          - label: "testnet-demo"
+            value: "testnet-demo"
       - select: "Operation"
         key: "testnet-operation"
         default: "sanity-or-restart"
         options:
-          - label: "Sanity check.  Restart network on failure"
+          - label: "Create new testnet nodes and then start network software.  If nodes are already created, they will be deleted and then re-created."
-            value: "sanity-or-restart"
+            value: "create-and-start"
-          - label: "Start (or restart) the network"
+          - label: "Create new testnet nodes, but do not start network software.  If nodes are already created, they will be deleted and then re-created."
+            value: "create"
+          - label: "Start network software on already-created testnet nodes.  If software is already running, it will be restarted."
             value: "start"
-          - label: "Stop the network"
+          - label: "Stop network software without deleting testnet nodes"
             value: "stop"
+          - label: "Update the network software.  Restart network software on failure"
+            value: "update-or-restart"
+          - label: "Sanity check.  Restart network software on failure"
+            value: "sanity-or-restart"
           - label: "Sanity check only"
             value: "sanity"
+          - label: "Delete all nodes on a testnet.  Network software will be stopped first if it is running"
+            value: "delete"
   - command: "ci/$(basename "$0")"
     agents:
       - "queue=$BUILDKITE_AGENT_META_DATA_QUEUE"
@@ -62,29 +72,37 @@ EOF
   exit 0
 fi
 
-export SOLANA_METRICS_CONFIG="db=$TESTNET,$SOLANA_METRICS_PARTIAL_CONFIG"
-echo "SOLANA_METRICS_CONFIG: $SOLANA_METRICS_CONFIG"
-
 ci/channel-info.sh
 eval "$(ci/channel-info.sh)"
 
+
+EC2_ZONES=(us-west-1a sa-east-1a ap-northeast-2a eu-central-1a ca-central-1a)
+GCE_ZONES=(us-west1-b asia-east2-a europe-west4-a southamerica-east1-b us-east4-c)
 case $TESTNET in
 testnet-edge|testnet-edge-perf)
   CHANNEL_OR_TAG=edge
   CHANNEL_BRANCH=$EDGE_CHANNEL
+  : "${TESTNET_DB_HOST:=https://clocktower-f1d56615.influxcloud.net:8086}"
   ;;
 testnet-beta|testnet-beta-perf)
   CHANNEL_OR_TAG=beta
   CHANNEL_BRANCH=$BETA_CHANNEL
+  : "${TESTNET_DB_HOST:=https://clocktower-f1d56615.influxcloud.net:8086}"
+  : "${EC2_NODE_COUNT:=10}"
+  : "${GCE_NODE_COUNT:=}"
   ;;
-testnet|testnet-perf)
+testnet)
-  if [[ -n $BETA_CHANNEL_LATEST_TAG ]]; then
+  CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
-    CHANNEL_OR_TAG=$BETA_CHANNEL_LATEST_TAG
+  CHANNEL_BRANCH=$STABLE_CHANNEL
-    CHANNEL_BRANCH=$BETA_CHANNEL
+  : "${TESTNET_DB_HOST:=https://clocktower-f1d56615.influxcloud.net:8086}"
-  else
+  ;;
-    CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
+testnet-perf)
-    CHANNEL_BRANCH=$STABLE_CHANNEL
+  CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
-  fi
+  CHANNEL_BRANCH=$STABLE_CHANNEL
+  ;;
+testnet-demo)
+  CHANNEL_OR_TAG=beta
+  CHANNEL_BRANCH=$BETA_CHANNEL
   ;;
 *)
   echo "Error: Invalid TESTNET=$TESTNET"
@@ -92,9 +110,21 @@ testnet|testnet-perf)
   ;;
 esac
 
-if [[ $BUILDKITE_BRANCH != "$CHANNEL_BRANCH" ]]; then
+if [[ -n $TESTNET_DB_HOST ]]; then
-  (
+  SOLANA_METRICS_PARTIAL_CONFIG="host=$TESTNET_DB_HOST,$SOLANA_METRICS_PARTIAL_CONFIG"
-    cat <<EOF
+fi
+
+export SOLANA_METRICS_CONFIG="db=$TESTNET,$SOLANA_METRICS_PARTIAL_CONFIG"
+echo "SOLANA_METRICS_CONFIG: $SOLANA_METRICS_CONFIG"
+source scripts/configure-metrics.sh
+
+if [[ -n $TESTNET_TAG ]]; then
+  CHANNEL_OR_TAG=$TESTNET_TAG
+else
+
+  if [[ $BUILDKITE_BRANCH != "$CHANNEL_BRANCH" ]]; then
+    (
+      cat <<EOF
 steps:
   - trigger: "$BUILDKITE_PIPELINE_SLUG"
     async: true
@@ -104,78 +134,78 @@ steps:
       env:
         TESTNET: "$TESTNET"
         TESTNET_OP: "$TESTNET_OP"
+        TESTNET_DB_HOST: "$TESTNET_DB_HOST"
+        EC2_NODE_COUNT: "$EC2_NODE_COUNT"
+        GCE_NODE_COUNT: "$GCE_NODE_COUNT"
 EOF
-  ) | buildkite-agent pipeline upload
+    ) | buildkite-agent pipeline upload
-  exit 0
+    exit 0
+  fi
 fi
 
-
 sanity() {
   echo "--- sanity $TESTNET"
   case $TESTNET in
   testnet-edge)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+        ci/testnet-sanity.sh edge-testnet-solana-com ec2 us-west-1a
-      ci/testnet-sanity.sh edge-testnet-solana-com ec2 us-west-1a
     )
     ;;
   testnet-edge-perf)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export REJECT_EXTRA_NODES=1
+      REJECT_EXTRA_NODES=1 \
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
-      ci/testnet-sanity.sh edge-perf-testnet-solana-com ec2 us-west-2b
+        ci/testnet-sanity.sh edge-perf-testnet-solana-com ec2 us-west-2b
     )
     ;;
   testnet-beta)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+
-      export NO_VALIDATOR_SANITY=1
+      ok=true
-      ci/testnet-sanity.sh beta-testnet-solana-com ec2 us-west-1a
+      if [[ -n $EC2_NODE_COUNT ]]; then
+        NO_LEDGER_VERIFY=1 \
+          ci/testnet-sanity.sh beta-testnet-solana-com ec2 "${EC2_ZONES[0]}" || ok=false
+      elif [[ -n $GCE_NODE_COUNT ]]; then
+        NO_LEDGER_VERIFY=1 \
+          ci/testnet-sanity.sh beta-testnet-solana-com gce "${GCE_ZONES[0]}" || ok=false
+      else
+        echo "Error: no EC2 or GCE nodes"
+        ok=false
+      fi
+      $ok
     )
     ;;
   testnet-beta-perf)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export REJECT_EXTRA_NODES=1
+      REJECT_EXTRA_NODES=1 \
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
-      ci/testnet-sanity.sh beta-perf-testnet-solana-com ec2 us-west-2b
+        ci/testnet-sanity.sh beta-perf-testnet-solana-com ec2 us-west-2b
     )
     ;;
   testnet)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
+        ci/testnet-sanity.sh testnet-solana-com ec2 us-west-1a
       #ci/testnet-sanity.sh testnet-solana-com gce us-east1-c
-      ci/testnet-sanity.sh testnet-solana-com ec2 us-west-1a
     )
     ;;
   testnet-perf)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export REJECT_EXTRA_NODES=1
+      REJECT_EXTRA_NODES=1 \
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
+        ci/testnet-sanity.sh perf-testnet-solana-com gce us-west1-b
       #ci/testnet-sanity.sh perf-testnet-solana-com ec2 us-east-1a
-      ci/testnet-sanity.sh perf-testnet-solana-com gce us-west1-b
     )
     ;;
   *)
@@ -185,100 +215,146 @@ sanity() {
   esac
 }
 
+deploy() {
+  declare maybeCreate=$1
+  declare maybeStart=$2
+  declare maybeStop=$3
+  declare maybeDelete=$4
 
-start() {
+  echo "--- deploy \"$maybeCreate\" \"$maybeStart\" \"$maybeStop\" \"$maybeDelete\""
-  declare maybeDelete=$1
+
-  if [[ -z $maybeDelete ]]; then
+  # Create or recreate the nodes
-    echo "--- start $TESTNET"
+  if [[ -z $maybeCreate ]]; then
+    skipCreate=skip
   else
-    echo "--- stop $TESTNET"
+    skipCreate=""
+  fi
+
+  # Start or restart the network software on the nodes
+  if [[ -z $maybeStart ]]; then
+    skipStart=skip
+  else
+    skipStart=""
   fi
 
   case $TESTNET in
   testnet-edge)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      ci/testnet-deploy.sh -p edge-testnet-solana-com -C ec2 -z us-west-1a \
-      export NO_VALIDATOR_SANITY=1
+        -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0ccd4f2239886fa94 \
-      ci/testnet-deploy.sh edge-testnet-solana-com ec2 us-west-1a \
+        ${skipCreate:+-r} \
-        -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0ccd4f2239886fa94 \
+        ${skipStart:+-s} \
-        ${maybeDelete:+-d}
+        ${maybeStop:+-S} \
+        ${maybeDelete:+-D}
     )
     ;;
   testnet-edge-perf)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
-      ci/testnet-deploy.sh edge-perf-testnet-solana-com ec2 us-west-2b \
+      RUST_LOG=solana=warn \
-        -g -t "$CHANNEL_OR_TAG" -c 2 \
+        ci/testnet-deploy.sh -p edge-perf-testnet-solana-com -C ec2 -z us-west-2b \
-        -b \
+          -g -t "$CHANNEL_OR_TAG" -c 2 \
-        ${maybeDelete:+-d}
+          -b \
+          ${skipCreate:+-r} \
+          ${skipStart:+-s} \
+          ${maybeStop:+-S} \
+          ${maybeDelete:+-D}
     )
     ;;
   testnet-beta)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+
-      export NO_VALIDATOR_SANITY=1
+      # Build an array to pass as opts to testnet-deploy.sh: "-z zone1 -z zone2 ..."
-      ci/testnet-deploy.sh beta-testnet-solana-com ec2 us-west-1a \
+      GCE_ZONE_ARGS=()
-        -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0f286cf8a0771ce35 \
+      for val in "${GCE_ZONES[@]}"; do
-        -b \
+        GCE_ZONE_ARGS+=("-z $val")
-        ${maybeDelete:+-d}
+      done
+
+      EC2_ZONE_ARGS=()
+      for val in "${EC2_ZONES[@]}"; do
+        EC2_ZONE_ARGS+=("-z $val")
+      done
+
+      if [[ -n $EC2_NODE_COUNT ]]; then
+        if [[ -n $GCE_NODE_COUNT ]] || [[ -n $skipStart ]]; then
+          maybeSkipStart="skip"
+        fi
+
+        # shellcheck disable=SC2068
+        ci/testnet-deploy.sh -p beta-testnet-solana-com -C ec2 ${EC2_ZONE_ARGS[@]} \
+          -t "$CHANNEL_OR_TAG" -n "$EC2_NODE_COUNT" -c 0 -u -P -a eipalloc-0f286cf8a0771ce35 \
+          ${skipCreate:+-r} \
+          ${maybeSkipStart:+-s} \
+          ${maybeStop:+-S} \
+          ${maybeDelete:+-D}
+      fi
+
+      if [[ -n $GCE_NODE_COUNT ]]; then
+        # shellcheck disable=SC2068
+        ci/testnet-deploy.sh -p beta-testnet-solana-com -C gce ${GCE_ZONE_ARGS[@]} \
+          -t "$CHANNEL_OR_TAG" -n "$GCE_NODE_COUNT" -c 0 -P \
+          ${skipCreate:+-r} \
+          ${skipStart:+-s} \
+          ${maybeStop:+-S} \
+          ${maybeDelete:+-D} \
+          ${EC2_NODE_COUNT:+-x}
+      fi
     )
     ;;
   testnet-beta-perf)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
-      ci/testnet-deploy.sh beta-perf-testnet-solana-com ec2 us-west-2b \
+      RUST_LOG=solana=warn \
-        -g -t "$CHANNEL_OR_TAG" -c 2 \
+        ci/testnet-deploy.sh -p beta-perf-testnet-solana-com -C ec2 -z us-west-2b \
-        -b \
+          -g -t "$CHANNEL_OR_TAG" -c 2 \
-        ${maybeDelete:+-d}
+          -b \
+          ${skipCreate:+-r} \
+          ${skipStart:+-s} \
+          ${maybeStop:+-S} \
+          ${maybeDelete:+-D}
     )
     ;;
   testnet)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      NO_VALIDATOR_SANITY=1 \
-      export NO_VALIDATOR_SANITY=1
+        ci/testnet-deploy.sh -p testnet-solana-com -C ec2 -z us-west-1a \
-      #ci/testnet-deploy.sh testnet-solana-com gce us-east1-c \
+          -t "$CHANNEL_OR_TAG" -n 3 -c 0 -u -P -a eipalloc-0fa502bf95f6f18b2 \
-      #  -s "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a testnet-solana-com  \
+          -b \
-      #  ${maybeDelete:+-d}
+          ${skipCreate:+-r} \
-      ci/testnet-deploy.sh testnet-solana-com ec2 us-west-1a \
+          ${skipStart:+-s} \
-        -t "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a eipalloc-0fa502bf95f6f18b2 \
+          ${maybeStop:+-S} \
-        -b \
+          ${maybeDelete:+-D}
-        ${maybeDelete:+-d}
     )
     ;;
   testnet-perf)
-    # shellcheck disable=2030
-    # shellcheck disable=2031
     (
-      set -ex
+      set -x
-      export NO_LEDGER_VERIFY=1
+      NO_LEDGER_VERIFY=1 \
-      export NO_VALIDATOR_SANITY=1
+      NO_VALIDATOR_SANITY=1 \
-      ci/testnet-deploy.sh perf-testnet-solana-com gce us-west1-b \
+      RUST_LOG=solana=warn \
-        -G "n1-standard-16 --accelerator count=2,type=nvidia-tesla-v100" \
+        ci/testnet-deploy.sh -p perf-testnet-solana-com -C gce -z us-west1-b \
-        -t "$CHANNEL_OR_TAG" -c 2 \
+          -G "n1-standard-16 --accelerator count=2,type=nvidia-tesla-v100" \
-        -b \
+          -t "$CHANNEL_OR_TAG" -c 2 \
-        -d pd-ssd \
+          -b \
-        ${maybeDelete:+-d}
+          -d pd-ssd \
-      #ci/testnet-deploy.sh perf-testnet-solana-com ec2 us-east-1a \
+          ${skipCreate:+-r} \
-      #  -g \
+          ${skipStart:+-s} \
-      #  -t "$CHANNEL_OR_TAG" -c 2 \
+          ${maybeStop:+-S} \
-      #  ${maybeDelete:+-d}
+          ${maybeDelete:+-D}
+    )
+    ;;
+  testnet-demo)
+    (
+      set -x
+      echo "Demo net not yet implemented!"
+      exit 1
     )
     ;;
   *)
@@ -288,29 +364,119 @@ start() {
   esac
 }
 
+ENABLED_LOCKFILE="${HOME}/${TESTNET}.is_enabled"
+CREATED_LOCKFILE="${HOME}/${TESTNET}.is_created"
+
+create-and-start() {
+  rm -f "${CREATED_LOCKFILE}"
+  deploy create start
+  touch "${CREATED_LOCKFILE}"
+}
+create() {
+  rm -f "${CREATED_LOCKFILE}"
+  deploy create
+  touch "${CREATED_LOCKFILE}"
+}
+start() {
+  if [[ -f ${CREATED_LOCKFILE} ]]; then
+    deploy "" start
+  else
+    echo "Unable to start ${TESTNET}.  Are the nodes created?
+    Re-run ci/testnet-manager.sh with \$TESTNET_OP=create or \$TESTNET_OP=create-and-start"
+    exit 1
+  fi
+}
 stop() {
-  start delete
+  deploy "" ""
+}
+delete() {
+  deploy "" "" "" delete
+  rm -f "${CREATED_LOCKFILE}"
+}
+enable_testnet() {
+  touch "${ENABLED_LOCKFILE}"
+  echo "+++ $TESTNET now enabled"
+}
+disable_testnet() {
+  rm -f "${ENABLED_LOCKFILE}"
+  echo "+++ $TESTNET now disabled"
+}
+is_testnet_enabled() {
+  if [[ ! -f ${ENABLED_LOCKFILE} ]]; then
+    echo "--- ${TESTNET} is currently disabled.  Enable ${TESTNET} by running ci/testnet-manager.sh with \$TESTNET_OP=enable, then re-run with current settings."
+    exit 0
+  fi
 }
 
 case $TESTNET_OP in
-sanity)
+enable)
-  sanity
+  enable_testnet
+  ;;
+disable)
+  disable_testnet
+  delete
+  ;;
+create-and-start)
+  is_testnet_enabled
+  create-and-start
+  ;;
+create)
+  is_testnet_enabled
+  create
   ;;
 start)
+  is_testnet_enabled
   start
   ;;
 stop)
+  is_testnet_enabled
   stop
   ;;
+sanity)
+  is_testnet_enabled
+  sanity
+  ;;
+delete)
+  is_testnet_enabled
+  delete
+  ;;
+update-or-restart)
+  is_testnet_enabled
+  if start; then
+    echo Update successful
+  else
+    echo "+++ Update failed, restarting the network"
+    $metricsWriteDatapoint "testnet-manager update-failure=1"
+    create-and-start
+  fi
+  ;;
 sanity-or-restart)
+  is_testnet_enabled
   if sanity; then
     echo Pass
   else
-    echo "Sanity failed, restarting the network"
+    echo "+++ Sanity failed, updating the network"
-    echo "^^^ +++"
+    $metricsWriteDatapoint "testnet-manager sanity-failure=1"
-    start
+
+    # TODO: Restore attempt to restart the cluster before recreating it
+    #       See https://github.com/solana-labs/solana/issues/3774
+    if false; then
+      if start; then
+        echo Update successful
+      else
+        echo "+++ Update failed, restarting the network"
+        $metricsWriteDatapoint "testnet-manager update-failure=1"
+        create-and-start
+      fi
+    else
+      create-and-start
+    fi
   fi
   ;;
+*)
+  echo "Error: Invalid TESTNET_OP=$TESTNET_OP"
+  exit 1
+  ;;
 esac
 
 echo --- fin

drone/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-drone"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana Drone"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,8 +19,8 @@ clap = "2.31"
 log = "0.4.2"
 serde = "1.0.82"
 serde_derive = "1.0.82"
-solana-sdk = { path = "../sdk", version = "0.11.0" }
+solana-sdk = { path = "../sdk", version = "0.11.1" }
-solana-metrics = { path = "../metrics", version = "0.11.0" }
+solana-metrics = { path = "../metrics", version = "0.11.1" }
 tokio = "0.1"
 tokio-codec = "0.1"
 

fullnode-config/Cargo.toml

@@ -2,7 +2,8 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-fullnode-config"
-version = "0.11.0"
+description = "Blockchain, Rebuilt for Scale"
+version = "0.11.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,8 +15,8 @@ log = "0.4.2"
 serde = "1.0.82"
 serde_derive = "1.0.82"
 serde_json = "1.0.10"
-solana-netutil = { path = "../netutil", version = "0.11.0" }
+solana-netutil = { path = "../netutil", version = "0.11.1" }
-solana-sdk = { path = "../sdk", version = "0.11.0" }
+solana-sdk = { path = "../sdk", version = "0.11.1" }
 untrusted = "0.6.2"
 
 [features]

fullnode/Cargo.toml

@@ -2,7 +2,8 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-fullnode"
-version = "0.11.0"
+description = "Blockchain, Rebuilt for Scale"
+version = "0.11.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -11,13 +12,13 @@ homepage = "https://solana.com/"
 clap = "2.32.0"
 log = "0.4.2"
 serde_json = "1.0.10"
-solana = { path = "..", version = "0.11.0" }
+solana = { path = "..", version = "0.11.1" }
-solana-drone = { path = "../drone", version = "0.11.0" }
+solana-drone = { path = "../drone", version = "0.11.1" }
-solana-fullnode-config = { path = "../fullnode-config", version = "0.11.0" }
+solana-fullnode-config = { path = "../fullnode-config", version = "0.11.1" }
-solana-logger = { path = "../logger", version = "0.11.0" }
+solana-logger = { path = "../logger", version = "0.11.1" }
-solana-netutil = { path = "../netutil", version = "0.11.0" }
+solana-netutil = { path = "../netutil", version = "0.11.1" }
-solana-metrics = { path = "../metrics", version = "0.11.0" }
+solana-metrics = { path = "../metrics", version = "0.11.1" }
-solana-sdk = { path = "../sdk", version = "0.11.0" }
+solana-sdk = { path = "../sdk", version = "0.11.1" }
 
 [features]
 chacha = []

genesis/Cargo.toml

@@ -2,7 +2,8 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-genesis"
-version = "0.11.0"
+description = "Blockchain, Rebuilt for Scale"
+version = "0.11.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,8 +11,8 @@ homepage = "https://solana.com/"
 [dependencies]
 clap = "2.32.0"
 serde_json = "1.0.10"
-solana = { path = "..", version = "0.11.0" }
+solana = { path = "..", version = "0.11.1" }
-solana-sdk = { path = "../sdk", version = "0.11.0" }
+solana-sdk = { path = "../sdk", version = "0.11.1" }
 
 [features]
 cuda = []

keygen/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-keygen"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana key generation utility"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,7 +14,7 @@ cuda = []
 [dependencies]
 dirs = "1.0.2"
 clap = "2.31"
-solana-sdk = { path = "../sdk", version = "0.11.0" }
+solana-sdk = { path = "../sdk", version = "0.11.1" }
 
 [[bin]]
 name = "solana-keygen"

ledger-tool/Cargo.toml

@@ -2,7 +2,8 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-ledger-tool"
-version = "0.11.0"
+description = "Blockchain, Rebuilt for Scale"
+version = "0.11.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,8 +11,8 @@ homepage = "https://solana.com/"
 [dependencies]
 clap = "2.32.0"
 serde_json = "1.0.10"
-solana = { path = "..", version = "0.11.0" }
+solana = { path = "..", version = "0.11.1" }
-solana-logger = { path = "../logger", version = "0.11.0" }
+solana-logger = { path = "../logger", version = "0.11.1" }
 
 [features]
 cuda = []

logger/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-logger"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana Logger"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

metrics/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-metrics"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana Metrics"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,7 +14,7 @@ log = "0.4.2"
 reqwest = "0.9.0"
 lazy_static = "1.2.0"
 sys-info = "0.5.6"
-solana-sdk = { path = "../sdk", version = "0.11.0" }
+solana-sdk = { path = "../sdk", version = "0.11.1" }
 
 [dev-dependencies]
 rand = "0.6.1"

net/gce.sh

@@ -154,6 +154,11 @@ case $cloudProvider in
 gce)
   if $enableGpu; then
     # Custom Ubuntu 18.04 LTS image with CUDA 9.2 and CUDA 10.0 installed
+    #
+    # TODO: Unfortunately this image is not public.  When this becomes an issue,
+    # use the stock Ubuntu 18.04 image and programmatically install CUDA after the
+    # instance boots
+    #
     imageName="ubuntu-1804-bionic-v20181029-with-cuda-10-and-cuda-9-2"
   else
     # Upstream Ubuntu 18.04 LTS image
@@ -161,23 +166,45 @@ gce)
   fi
   ;;
 ec2)
-  #
+  if $enableGpu; then
-  # Custom Ubuntu 18.04 LTS image with CUDA 9.2 and CUDA 10.0 installed
+    #
-  #
+    # Custom Ubuntu 18.04 LTS image with CUDA 9.2 and CUDA 10.0 installed
-  case $region in # (region global variable is set by cloud_SetZone)
+    #
-  us-east-1)
+    # TODO: Unfortunately these AMIs are not public.  When this becomes an issue,
-    imageName="ami-0a8bd6fb204473f78"
+    # use the stock Ubuntu 18.04 image and programmatically install CUDA after the
-    ;;
+    # instance boots
-  us-west-1)
+    #
-    imageName="ami-07011f0795513c59d"
+    case $region in
-    ;;
+    us-east-1)
-  us-west-2)
+      imageName="ami-0a8bd6fb204473f78"
-    imageName="ami-0a11ef42b62b82b68"
+      ;;
-    ;;
+    us-west-1)
-  *)
+      imageName="ami-07011f0795513c59d"
-    usage "Unsupported region: $region"
+      ;;
-    ;;
+    us-west-2)
-  esac
+      imageName="ami-0a11ef42b62b82b68"
+      ;;
+    *)
+      usage "Unsupported region: $region"
+      ;;
+    esac
+  else
+    # Select an upstream Ubuntu 18.04 AMI from https://cloud-images.ubuntu.com/locator/ec2/
+    case $region in
+    us-east-1)
+      imageName="ami-0a313d6098716f372"
+      ;;
+    us-west-1)
+      imageName="ami-06397100adf427136"
+      ;;
+    us-west-2)
+      imageName="ami-0dc34f4b016c9ce49"
+      ;;
+    *)
+      usage "Unsupported region: $region"
+      ;;
+    esac
+  fi
   ;;
 *)
   echo "Error: Unknown cloud provider: $cloudProvider"
@@ -444,6 +471,8 @@ touch /.instance-startup-complete
 
 EOF
 
+  cloud_Initialize "$prefix"
+
   cloud_CreateInstances "$prefix" "$prefix-bootstrap-leader" 1 \
     "$imageName" "$bootstrapLeaderMachineType" "$fullNodeBootDiskSizeInGb" \
     "$startupScript" "$bootstrapLeaderAddress" "$bootDiskType"

net/net.sh

@@ -35,6 +35,7 @@ Operate a configured testnet
                                  (ignored if -s or -S is specified)
    -r                          - Reuse existing node/ledger configuration from a
                                  previous |start| (ie, don't run ./mulitnode-demo/setup.sh).
+   -D /path/to/programs        - Deploy custom programs from this location
 
    Note: if RUST_LOG is set in the environment it will be propogated into the
          network nodes.
@@ -59,12 +60,13 @@ sanityExtraArgs=
 cargoFeatures=
 skipSetup=false
 updateNodes=false
+customPrograms=
 
 command=$1
 [[ -n $command ]] || usage
 shift
 
-while getopts "h?S:s:T:t:o:f:r" opt; do
+while getopts "h?S:s:T:t:o:f:r:D:" opt; do
   case $opt in
   h | \?)
     usage
@@ -107,6 +109,9 @@ while getopts "h?S:s:T:t:o:f:r" opt; do
   r)
     skipSetup=true
     ;;
+  D)
+    customPrograms=$OPTARG
+    ;;
   o)
     case $OPTARG in
     noLedgerVerify|noValidatorSanity|rejectExtraNodes)
@@ -146,6 +151,9 @@ build() {
     $MAYBE_DOCKER bash -c "
       set -ex
       scripts/cargo-install-all.sh farf \"$cargoFeatures\"
+      if [[ -n \"$customPrograms\" ]]; then
+        scripts/cargo-install-custom-programs.sh farf $customPrograms
+      fi
     "
   )
   echo "Build took $SECONDS seconds"

net/scripts/ec2-provider.sh

@@ -101,6 +101,33 @@ cloud_FindInstance() {
   __cloud_FindInstances "$name"
 }
 
+#
+# cloud_Initialize [networkName]
+#
+# Perform one-time initialization that may be required for the given testnet.
+#
+# networkName   - unique name of this testnet
+#
+# This function will be called before |cloud_CreateInstances|
+cloud_Initialize() {
+  declare networkName="$1"
+
+  __cloud_SshPrivateKeyCheck
+  (
+    set -x
+    aws ec2 delete-key-pair --region "$region" --key-name "$networkName"
+    aws ec2 import-key-pair --region "$region" --key-name "$networkName" \
+      --public-key-material file://"${sshPrivateKey}".pub
+  )
+
+  (
+    set -x
+    aws ec2 delete-security-group --region "$region" --group-name "$networkName" || true
+    aws ec2 create-security-group --region "$region" --group-name "$networkName" --description "Created automatically by $0"
+    rules=$(cat "$(dirname "${BASH_SOURCE[0]}")"/ec2-security-group-config.json)
+    aws ec2 authorize-security-group-ingress --region "$region" --group-name "$networkName" --cli-input-json "$rules"
+  )
+}
 
 #
 # cloud_CreateInstances [networkName] [namePrefix] [numNodes] [imageName]
@@ -131,21 +158,13 @@ cloud_CreateInstances() {
   declare optionalStartupScript="$7"
   declare optionalAddress="$8"
 
-  __cloud_SshPrivateKeyCheck
-  (
-    set -x
-    aws ec2 delete-key-pair --region "$region" --key-name "$networkName"
-    aws ec2 import-key-pair --region "$region" --key-name "$networkName" \
-      --public-key-material file://"${sshPrivateKey}".pub
-  )
-
   declare -a args
   args=(
     --key-name "$networkName"
     --count "$numNodes"
     --region "$region"
     --placement "AvailabilityZone=$zone"
-    --security-groups testnet
+    --security-groups "$networkName"
     --image-id "$imageName"
     --instance-type "$machineType"
     --tag-specifications "ResourceType=instance,Tags=[{Key=name,Value=$namePrefix}]"

net/scripts/ec2-security-group-config.json

@@ -0,0 +1,137 @@
+{
+    "IpPermissions": [
+        {
+            "PrefixListIds": [],
+            "FromPort": 80,
+            "IpRanges": [
+                {
+                    "CidrIp": "0.0.0.0/0",
+                    "Description": "http for block explorer"
+                }
+            ],
+            "ToPort": 80,
+            "IpProtocol": "tcp",
+            "UserIdGroupPairs": [],
+            "Ipv6Ranges": [
+                {
+                    "CidrIpv6": "::/0",
+                    "Description": "http for block explorer"
+                }
+            ]
+        },
+        {
+            "PrefixListIds": [],
+            "FromPort": 8000,
+            "IpRanges": [
+                {
+                    "Description": "fullnode UDP range",
+                    "CidrIp": "0.0.0.0/0"
+                }
+            ],
+            "ToPort": 10000,
+            "IpProtocol": "udp",
+            "UserIdGroupPairs": [],
+            "Ipv6Ranges": [
+                {
+                    "CidrIpv6": "::/0",
+                    "Description": "fullnode UDP range"
+                }
+            ]
+        },
+        {
+            "PrefixListIds": [],
+            "FromPort": 22,
+            "IpRanges": [
+                {
+                    "CidrIp": "0.0.0.0/0",
+                    "Description": "ssh"
+                }
+            ],
+            "ToPort": 22,
+            "IpProtocol": "tcp",
+            "UserIdGroupPairs": [],
+            "Ipv6Ranges": [
+                {
+                    "CidrIpv6": "::/0",
+                    "Description": "ssh"
+                }
+            ]
+        },
+        {
+            "PrefixListIds": [],
+            "FromPort": 873,
+            "IpRanges": [
+                {
+                    "Description": "rsync",
+                    "CidrIp": "0.0.0.0/0"
+                }
+            ],
+            "ToPort": 873,
+            "IpProtocol": "tcp",
+            "UserIdGroupPairs": [],
+            "Ipv6Ranges": [
+                {
+                    "CidrIpv6": "::/0",
+                    "Description": "rsync"
+                }
+            ]
+        },
+        {
+            "PrefixListIds": [],
+            "FromPort": 3001,
+            "IpRanges": [
+                {
+                    "Description": "blockexplorer API port",
+                    "CidrIp": "0.0.0.0/0"
+                }
+            ],
+            "ToPort": 3001,
+            "IpProtocol": "tcp",
+            "UserIdGroupPairs": [],
+            "Ipv6Ranges": [
+                {
+                    "CidrIpv6": "::/0",
+                    "Description": "blockexplorer API port"
+                }
+            ]
+        },
+        {
+            "PrefixListIds": [],
+            "FromPort": 8000,
+            "IpRanges": [
+                {
+                    "Description": "fullnode TCP range",
+                    "CidrIp": "0.0.0.0/0"
+                }
+            ],
+            "ToPort": 10000,
+            "IpProtocol": "tcp",
+            "UserIdGroupPairs": [],
+            "Ipv6Ranges": [
+                {
+                    "CidrIpv6": "::/0",
+                    "Description": "fullnode TCP range"
+                }
+            ]
+        },
+        {
+            "PrefixListIds": [],
+            "FromPort": 8,
+            "IpRanges": [
+                {
+                    "CidrIp": "0.0.0.0/0",
+                    "Description": "allow ping"
+                }
+            ],
+            "ToPort": -1,
+            "IpProtocol": "icmp",
+            "UserIdGroupPairs": [],
+            "Ipv6Ranges": [
+                {
+                    "CidrIpv6": "::/0",
+                    "Description": "allow ping"
+                }
+            ]
+        }
+    ]
+}

net/scripts/gce-provider.sh

@@ -76,6 +76,21 @@ cloud_FindInstance() {
   __cloud_FindInstances "name=$name"
 }
 
+#
+# cloud_Initialize [networkName]
+#
+# Perform one-time initialization that may be required for the given testnet.
+#
+# networkName   - unique name of this testnet
+#
+# This function will be called before |cloud_CreateInstances|
+cloud_Initialize() {
+  declare networkName="$1"
+  # ec2-provider.sh creates firewall rules programmatically, should do the same
+  # here.
+  echo "TODO: create $networkName firewall rules programmatically instead of assuming the 'testnet' tag exists"
+}
+
 #
 # cloud_CreateInstances [networkName] [namePrefix] [numNodes] [imageName]
 #                       [machineType] [bootDiskSize] [enableGpu]

netutil/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-netutil"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana Network Utilities"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -18,7 +18,7 @@ reqwest = "0.9.0"
 socket2 = "0.3.8"
 
 [dev-dependencies]
-solana-logger = { path = "../logger", version = "0.11.0" }
+solana-logger = { path = "../logger", version = "0.11.1" }
 
 [lib]
 name = "solana_netutil"

programs/bpf/rust/noop/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpf-noop"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana BPF noop program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,4 +9,4 @@ homepage = "https://solana.com/"
 
 [dependencies]
 rbpf = "0.1.0"
-solana-sdk = { path = "../../../../sdk", version = "0.11.0" }
+solana-sdk = { path = "../../../../sdk", version = "0.11.1" }

programs/native/bpf_loader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-bpfloader"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana BPF Loader"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -19,8 +19,8 @@ libc = "0.2.45"
 log = "0.4.2"
 solana_rbpf = "=0.1.5"
 serde = "1.0.82"
-solana-logger = { path = "../../../logger", version = "0.11.0" }
+solana-logger = { path = "../../../logger", version = "0.11.1" }
-solana-sdk = { path = "../../../sdk", version = "0.11.0" }
+solana-sdk = { path = "../../../sdk", version = "0.11.1" }
 
 [lib]
 name = "solana_bpf_loader"

programs/native/budget/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-budget-program"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana budget program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,8 +14,8 @@ chrono = { version = "0.4.0", features = ["serde"] }
 log = "0.4.2"
 serde = "1.0.82"
 serde_derive = "1.0.82"
-solana-logger = { path = "../../../logger", version = "0.11.0" }
+solana-logger = { path = "../../../logger", version = "0.11.1" }
-solana-sdk = { path = "../../../sdk", version = "0.11.0" }
+solana-sdk = { path = "../../../sdk", version = "0.11.1" }
 
 [lib]
 name = "solana_budget_program"

programs/native/erc20/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-erc20"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana reference erc20 program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,8 +13,8 @@ bincode = "1.0.0"
 log = "0.4.2"
 serde = "1.0.82"
 serde_derive = "1.0.82"
-solana-logger = { path = "../../../logger", version = "0.11.0" }
+solana-logger = { path = "../../../logger", version = "0.11.1" }
-solana-sdk = { path = "../../../sdk", version = "0.11.0" }
+solana-sdk = { path = "../../../sdk", version = "0.11.1" }
 
 [lib]
 name = "solana_erc20"

programs/native/lua_loader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-lualoader"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana Lua Loader"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -14,8 +14,8 @@ log = "0.4.2"
 rlua = "0.15.2"
 serde = "1.0.82"
 serde_derive = "1.0.82"
-solana-logger = { path = "../../../logger", version = "0.11.0" }
+solana-logger = { path = "../../../logger", version = "0.11.1" }
-solana-sdk = { path = "../../../sdk", version = "0.11.0" }
+solana-sdk = { path = "../../../sdk", version = "0.11.1" }
 
 [dev-dependencies]
 bincode = "1.0.0"

programs/native/native_loader/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-native-loader"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana Native Loader"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,7 +13,7 @@ bincode = "1.0.0"
 libc = "0.2.45"
 libloading = "0.5.0"
 log = "0.4.2"
-solana-sdk = { path = "../../../sdk", version = "0.11.0" }
+solana-sdk = { path = "../../../sdk", version = "0.11.1" }
 
 [lib]
 name = "solana_native_loader"

programs/native/noop/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-noop"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana noop program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -9,8 +9,8 @@ homepage = "https://solana.com/"
 edition = "2018"
 
 [dependencies]
-solana-sdk = { path = "../../../sdk", version = "0.11.0" }
+solana-sdk = { path = "../../../sdk", version = "0.11.1" }
-solana-logger = { path = "../../../logger", version = "0.11.0" }
+solana-logger = { path = "../../../logger", version = "0.11.1" }
 log = "0.4.2"
 
 [lib]

programs/native/storage/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-storage-program"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana storage program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,8 +13,8 @@ bincode = "1.0.0"
 log = "0.4.2"
 serde = "1.0.27"
 serde_derive = "1.0.27"
-solana-logger = { path = "../../../logger", version = "0.11.0" }
+solana-logger = { path = "../../../logger", version = "0.11.1" }
-solana-sdk = { path = "../../../sdk", version = "0.11.0" }
+solana-sdk = { path = "../../../sdk", version = "0.11.1" }
 
 [lib]
 name = "solana_storage_program"

programs/native/system/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-system-program"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana system program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,7 +12,7 @@ edition = "2018"
 bincode = "1.0.0"
 log = "0.4.2"
 serde = "1.0.27"
-solana-sdk = { path = "../../../sdk", version = "0.11.0" }
+solana-sdk = { path = "../../../sdk", version = "0.11.1" }
 
 [lib]
 name = "solana_system_program"

programs/native/vote/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-vote-program"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana vote program"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -13,9 +13,9 @@ bincode = "1.0.0"
 log = "0.4.2"
 serde = "1.0.82"
 serde_derive = "1.0.82"
-solana-logger = { path = "../../../logger", version = "0.11.0" }
+solana-logger = { path = "../../../logger", version = "0.11.1" }
-solana-metrics = { path = "../../../metrics", version = "0.11.0" }
+solana-metrics = { path = "../../../metrics", version = "0.11.1" }
-solana-sdk = { path = "../../../sdk", version = "0.11.0" }
+solana-sdk = { path = "../../../sdk", version = "0.11.1" }
 
 [lib]
 name = "solana_vote_program"

replicator/Cargo.toml

@@ -2,7 +2,7 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-replicator"
-version = "0.11.0"
+version = "0.11.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -10,10 +10,10 @@ homepage = "https://solana.com/"
 [dependencies]
 clap = "2.32.0"
 serde_json = "1.0.10"
-solana = { path = "..", version = "0.11.0" }
+solana = { path = "..", version = "0.11.1" }
-solana-logger = { path = "../logger", version = "0.11.0" }
+solana-logger = { path = "../logger", version = "0.11.1" }
-solana-fullnode-config = { path = "../fullnode-config", version = "0.11.0" }
+solana-fullnode-config = { path = "../fullnode-config", version = "0.11.1" }
-solana-sdk = { path = "../sdk", version = "0.11.0" }
+solana-sdk = { path = "../sdk", version = "0.11.1" }
 
 [features]
 chacha = ["solana/chacha"]

scripts/cargo-install-custom-programs.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+#
+# This script will install all cargo workspace libraries found in
+# `programDir` as native programs.
+set -e
+
+# Directory to install libraries into
+installDir="$(mkdir -p "$1"; cd "$1"; pwd)"
+
+# Where to find custom programs
+programDir="$2"
+
+(
+  set -x
+  cd "$programDir"
+  cargo build --all --release
+)
+
+for dir in "$programDir"/*; do
+  for program in $programDir/target/release/deps/lib"$(basename "$dir")".{so,dylib,dll}; do
+    if [[ -f $program ]]; then
+      (
+        set -x
+        mkdir -p "$installDir/bin/deps"
+        rm -f "$installDir/bin/deps/$(basename "$program")"
+        cp -v "$program" "$installDir"/bin/deps
+      )
+    fi
+  done
+done

scripts/metrics-write-datapoint.sh

@@ -2,7 +2,6 @@
 #
 # Send a metrics datapoint
 #
-set -e
 
 point=$1
 if [[ -z $point ]]; then
@@ -18,3 +17,4 @@ fi
 
 echo "https://metrics.solana.com:8086/write?db=${INFLUX_DATABASE}&u=${INFLUX_USERNAME}&p=${INFLUX_PASSWORD}" \
   | xargs curl --max-time 5 -XPOST --data-binary "$point"
+exit 0

sdk/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-sdk"
-version = "0.11.0"
+version = "0.11.1"
 description = "Solana SDK"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"

sdk/docker-solana/build.sh

@@ -4,14 +4,6 @@ set -ex
 cd "$(dirname "$0")"
 eval "$(../../ci/channel-info.sh)"
 
-if [[ $BUILDKITE_BRANCH = "$STABLE_CHANNEL" ]]; then
-  CHANNEL=stable
-elif [[ $BUILDKITE_BRANCH = "$EDGE_CHANNEL" ]]; then
-  CHANNEL=edge
-elif [[ $BUILDKITE_BRANCH = "$BETA_CHANNEL" ]]; then
-  CHANNEL=beta
-fi
-
 if [[ -z $CHANNEL ]]; then
   echo Unable to determine channel to publish into, exiting.
   echo "^^^ +++"
@@ -24,7 +16,7 @@ rm -rf usr/
 
 cp -f entrypoint.sh usr/bin/solana-entrypoint.sh
 
-docker build -t solanalabs/solana:$CHANNEL .
+docker build -t solanalabs/solana:"$CHANNEL" .
 
 maybeEcho=
 if [[ -z $CI ]]; then
@@ -38,4 +30,4 @@ else
     fi
   )
 fi
-$maybeEcho docker push solanalabs/solana:$CHANNEL
+$maybeEcho docker push solanalabs/solana:"$CHANNEL"

upload-perf/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-upload-perf"
-version = "0.11.0"
+version = "0.11.1"
 description = "Metrics Upload Utility"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -10,7 +10,7 @@ homepage = "https://solana.com/"
 
 [dependencies]
 serde_json = "1.0.10"
-solana-metrics = { path = "../metrics", version = "0.11.0" }
+solana-metrics = { path = "../metrics", version = "0.11.1" }
 
 [[bin]]
 name = "solana-upload-perf"

vote-signer/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "solana-vote-signer"
-version = "0.0.1"
+version = "0.11.1"
 description = "Solana Vote Signing Service"
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 repository = "https://github.com/solana-labs/solana"
@@ -12,8 +12,8 @@ bs58 = "0.2.0"
 clap = "2.31"
 log = "0.4.2"
 serde_json = "1.0.10"
-solana-sdk = { path = "../sdk", version = "0.11.0" }
+solana-sdk = { path = "../sdk", version = "0.11.1" }
-solana-metrics = { path = "../metrics", version = "0.11.0" }
+solana-metrics = { path = "../metrics", version = "0.11.1" }
 solana-jsonrpc-core = "0.3.0"
 solana-jsonrpc-http-server = "0.3.0"
 solana-jsonrpc-macros = "0.3.0"

wallet/Cargo.toml

@@ -2,7 +2,8 @@
 authors = ["Solana Maintainers <maintainers@solana.com>"]
 edition = "2018"
 name = "solana-wallet"
-version = "0.11.0"
+description = "Blockchain, Rebuilt for Scale"
+version = "0.11.1"
 repository = "https://github.com/solana-labs/solana"
 license = "Apache-2.0"
 homepage = "https://solana.com/"
@@ -14,10 +15,10 @@ clap = "2.32.0"
 chrono = { version = "0.4.0", features = ["serde"] }
 dirs = "1.0.2"
 serde_json = "1.0.10"
-solana = { path = "..", version = "0.11.0" }
+solana = { path = "..", version = "0.11.1" }
-solana-drone = { path = "../drone", version = "0.11.0" }
+solana-drone = { path = "../drone", version = "0.11.1" }
-solana-logger = { path = "../logger", version = "0.11.0" }
+solana-logger = { path = "../logger", version = "0.11.1" }
-solana-sdk = { path = "../sdk", version = "0.11.0" }
+solana-sdk = { path = "../sdk", version = "0.11.1" }
 
 [features]
 cuda = []

wallet/src/wallet.rs

@@ -602,7 +602,7 @@ pub fn process_command(config: &WalletConfig) -> Result<String, Box<dyn error::E
         }
         // Apply time elapsed to contract
         WalletCommand::TimeElapsed(to, pubkey, dt) => {
-            let params = json!(format!("{}", config.id.pubkey()));
+            let params = json!([format!("{}", config.id.pubkey())]);
             let balance = RpcRequest::GetBalance
                 .make_rpc_request(&rpc_client, 1, Some(params))?
                 .as_u64();