s/edge/beta/

Use relocatable BPF ELFs

.buildkite/hooks/post-checkout

@@ -0,0 +1,2 @@
+CI_BUILD_START=$(date +%s)
+export CI_BUILD_START

.buildkite/hooks/post-checkout.sh

@@ -0,0 +1 @@
+post-checkout

.buildkite/hooks/post-command

@@ -0,0 +1,45 @@
+#!/bin/bash -e
+
+#
+# Save target/ for the next CI build on this machine
+#
+if [[ -n $CARGO_TARGET_CACHE_NAME ]]; then
+  (
+    d=$HOME/cargo-target-cache/"$CARGO_TARGET_CACHE_NAME"
+    mkdir -p "$d"
+    set -x
+    rsync -a --delete --link-dest="$PWD" target "$d"
+    du -hs "$d"
+  )
+fi
+
+#
+# Add job_stats data point
+#
+if [[ -z $CI_BUILD_START ]]; then
+  echo Error: CI_BUILD_START empty
+else
+  CI_BUILD_DURATION=$(( $(date +%s) - CI_BUILD_START + 1 ))
+
+  CI_LABEL=${BUILDKITE_LABEL:-build label missing}
+
+  PR=false
+  if [[ $BUILDKITE_BRANCH =~ pull/* ]]; then
+    PR=true
+  fi
+
+  SUCCESS=true
+  if [[ $BUILDKITE_COMMAND_EXIT_STATUS != 0 ]]; then
+    SUCCESS=false
+  fi
+
+  point_tags="pipeline=$BUILDKITE_PIPELINE_SLUG,job=$CI_LABEL,pr=$PR,success=$SUCCESS"
+  point_tags="${point_tags// /\\ }"  # Escape spaces
+
+  point_fields="duration=$CI_BUILD_DURATION"
+  point_fields="${point_fields// /\\ }"  # Escape spaces
+
+  point="job_stats,$point_tags $point_fields"
+
+  scripts/metrics-write-datapoint.sh "$point" || true
+fi

.buildkite/hooks/post-command.sh

@@ -0,0 +1 @@
+post-command

.buildkite/hooks/pre-command

@@ -0,0 +1,27 @@
+#!/bin/bash -e
+
+# Ensure the pattern "+++ ..." never occurs when |set -x| is set, as buildkite
+# interprets this as the start of a log group.
+# Ref: https://buildkite.com/docs/pipelines/managing-log-output
+export PS4="++"
+
+#
+# Restore target/ from the previous CI build on this machine
+#
+[[ -n "$CARGO_TARGET_CACHE_NAME" ]] || (
+  d=$HOME/cargo-target-cache/"$CARGO_TARGET_CACHE_NAME"
+
+  if [[ -d $d ]]; then
+    du -hs "$d"
+    read -r cacheSizeInGB _ < <(du -s --block-size=1000000000 "$d")
+    if [[ $cacheSizeInGB -gt 5 ]]; then
+      echo "$d has gotten too large, removing it"
+      rm -rf "$d"
+    fi
+  fi
+
+  mkdir -p "$d"/target
+  set -x
+  rsync -a --delete --link-dest="$d" "$d"/target .
+)
+

.buildkite/hooks/pre-command.sh

@@ -0,0 +1 @@
+pre-command

.clippy.toml

@@ -0,0 +1 @@
+too-many-arguments-threshold = 9

.codecov.yml

@@ -1,2 +1,5 @@
 ignore:
   - "src/bin" 
+coverage:
+  status:
+    patch: off

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,6 @@
+#### Problem
+
+
+
+#### Proposed Solution
+

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,5 @@
+#### Problem
+
+#### Summary of Changes
+
+Fixes #

.github/RELEASE_TEMPLATE.md

@@ -0,0 +1,28 @@
+# Release v0.X.Y <milestone name>
+
+fun blurb about the name, what's in the release
+
+## Major Features And Improvements
+
+* bulleted
+* list of features and improvements
+
+## Breaking Changes
+
+* bulleted
+* list
+* of
+* protocol changes/breaks
+* API breaks
+* CLI changes
+* etc.
+
+## Bug Fixes and Other Changes
+
+* can be pulled from commit log, or synthesized
+
+## Thanks to our Contributors
+
+This release contains contributions from many people at Solana, as well as:
+
+  pull from commit log

.gitignore

@@ -1,4 +1,19 @@
 Cargo.lock
 /target/
+
 **/*.rs.bk
 .cargo
+
+# node configuration files
+/config/
+/config-private/
+/config-drone/
+/config-validator/
+/config-client/
+/multinode-demo/test/config-client/
+
+# test temp files, ledgers, etc.
+/farf/
+
+# log files
+*.log

CONTRIBUTING.md

@@ -0,0 +1,53 @@
+Solana Coding Guidelines
+===
+
+The goal of these guidelines is to improve developer productivity by allowing developers to
+jump any file in the codebase and not need to adapt to inconsistencies in how the code is
+written. The codebase should appear as if it had been authored by a single developer. If you
+don't agree with a convention, submit a PR patching this document and let's discuss! Once
+the PR is accepted, *all* code should be updated as soon as possible to reflect the new
+conventions.
+
+Rust coding conventions
+---
+
+* All Rust code is formatted using the latest version of `rustfmt`. Once installed, it will be
+  updated automatically when you update the compiler with `rustup`.
+
+* All Rust code is linted with Clippy. If you'd prefer to ignore its advice, do so explicitly:
+
+  ```rust
+  #[cfg_attr(feature = "cargo-clippy", allow(too_many_arguments))]
+  ```
+
+  Note: Clippy defaults can be overridden in the top-level file `.clippy.toml`.
+
+* For variable names, when in doubt, spell it out. The mapping from type names to variable names
+  is to lowercase the type name, putting an underscore before each capital letter. Variable names
+  should *not* be abbreviated unless being used as closure arguments and the brevity improves
+  readability. When a function has multiple instances of the same type, qualify each with a
+  prefix and underscore (i.e. alice_keypair) or a numeric suffix (i.e. tx0).
+
+* For function and method names, use `<verb>_<subject>`. For unit tests, that verb should
+  always be `test` and for benchmarks the verb should always be `bench`. Avoid namespacing
+  function names with some arbitrary word. Avoid abreviating words in function names.
+
+* As they say, "When in Rome, do as the Romans do." A good patch should acknowledge the coding
+  conventions of the code that surrounds it, even in the case where that code has not yet been
+  updated to meet the conventions described here.
+
+
+Terminology
+---
+
+Inventing new terms is allowed, but should only be done when the term is widely used and
+understood. Avoid introducing new 3-letter terms, which can be confused with 3-letter acronyms.
+
+Some terms we currently use regularly in the codebase:
+
+* fullnode: n. A fully participating network node.
+* hash: n. A SHA-256 Hash.
+* keypair: n. A Ed25519 key-pair, containing a public and private key.
+* pubkey: n. The public key of a Ed25519 key-pair.
+* sigverify: v. To verify a Ed25519 digital signature.
+

Cargo.toml

@@ -1,20 +1,34 @@
 [package]
 name = "solana"
-description = "The World's Fastest Blockchain"
-version = "0.6.0-beta.1"
+description = "Blockchain, Rebuilt for Scale"
+version = "0.10.1"
 documentation = "https://docs.rs/solana"
 homepage = "http://solana.com/"
+readme = "README.md"
 repository = "https://github.com/solana-labs/solana"
-authors = [
-    "Anatoly Yakovenko <anatoly@solana.com>",
-    "Greg Fitzgerald <greg@solana.com>",
-    "Stephen Akridge <stephen@solana.com>",
-]
+authors = ["Solana Maintainers <maintainers@solana.com>"]
 license = "Apache-2.0"
 
 [[bin]]
-name = "solana-client-demo"
-path = "src/bin/client-demo.rs"
+name = "solana-upload-perf"
+path = "src/bin/upload-perf.rs"
+
+[[bin]]
+name = "solana-bench-streamer"
+path = "src/bin/bench-streamer.rs"
+
+[[bin]]
+name = "solana-bench-tps"
+path = "src/bin/bench-tps.rs"
+
+[[bin]]
+name = "solana-drone"
+path = "src/bin/drone.rs"
+
+[[bin]]
+required-features = ["chacha"]
+name = "solana-replicator"
+path = "src/bin/replicator.rs"
 
 [[bin]]
 name = "solana-fullnode"
@@ -29,43 +43,101 @@ name = "solana-genesis"
 path = "src/bin/genesis.rs"
 
 [[bin]]
-name = "solana-genesis-demo"
-path = "src/bin/genesis-demo.rs"
+name = "solana-ledger-tool"
+path = "src/bin/ledger-tool.rs"
 
 [[bin]]
-name = "solana-mint"
-path = "src/bin/mint.rs"
+name = "solana-keygen"
+path = "src/bin/keygen.rs"
 
 [[bin]]
-name = "solana-mint-demo"
-path = "src/bin/mint-demo.rs"
+name = "solana-wallet"
+path = "src/bin/wallet.rs"
 
 [badges]
 codecov = { repository = "solana-labs/solana", branch = "master", service = "github" }
 
 [features]
-unstable = []
-ipv6 = []
+bpf_c = []
+chacha = []
 cuda = []
 erasure = []
+ipv6 = []
+test = []
+unstable = []
 
 [dependencies]
+atty = "0.2"
+bincode = "1.0.0"
+bs58 = "0.2.0"
+byteorder = "1.2.1"
+bytes = "0.4"
+chrono = { version = "0.4.0", features = ["serde"] }
+clap = "2.31"
+dirs = "1.0.2"
+elf = "0.0.10"
+env_logger = "0.5.12"
+generic-array = { version = "0.12.0", default-features = false, features = ["serde"] }
+getopts = "0.2"
+hex-literal = "0.1.1"
+influx_db_client = "0.3.6"
+solana-jsonrpc-core = "0.3.0"
+solana-jsonrpc-http-server = "0.3.0"
+solana-jsonrpc-macros = "0.3.0"
+solana-jsonrpc-pubsub = "0.3.0"
+solana-jsonrpc-ws-server = "0.3.0"
+ipnetwork = "0.12.7"
+itertools = "0.7.8"
+libc = "0.2.43"
+libloading = "0.5.0"
+log = "0.4.2"
+matches = "0.1.6"
+nix = "0.11.0"
+pnet_datalink = "0.21.0"
+rand = "0.5.1"
 rayon = "1.0.0"
-sha2 = "0.7.0"
-generic-array = { version = "0.9.0", default-features = false, features = ["serde"] }
+reqwest = "0.9.0"
+ring = "0.13.2"
+sha2 = "0.8.0"
 serde = "1.0.27"
+serde_cbor = "0.9.0"
 serde_derive = "1.0.27"
 serde_json = "1.0.10"
-ring = "0.12.1"
-untrusted = "0.5.1"
-bincode = "1.0.0"
-chrono = { version = "0.4.0", features = ["serde"] }
-log = "^0.4.1"
-env_logger = "^0.4.1"
-matches = "^0.1.6"
-byteorder = "^1.2.1"
-libc = "^0.2.1"
-getopts = "^0.2"
-isatty = "0.1"
-rand = "0.4.2"
-pnet = "^0.21.0"
+socket2 = "0.3.8"
+solana-sdk = { path = "sdk", version = "0.10.1" }
+sys-info = "0.5.6"
+tokio = "0.1"
+tokio-codec = "0.1"
+untrusted = "0.6.2"
+solana-noop = { path = "programs/native/noop", version = "0.10.1" }
+solana-bpfloader = { path = "programs/native/bpf_loader", version = "0.10.1" }
+solana-lualoader = { path = "programs/native/lua_loader", version = "0.10.1" }
+
+[[bench]]
+name = "bank"
+
+[[bench]]
+name = "banking_stage"
+
+[[bench]]
+name = "ledger"
+
+[[bench]]
+name = "signature"
+
+[[bench]]
+name = "sigverify"
+
+[[bench]]
+required-features = ["chacha"]
+name = "chacha"
+
+[workspace]
+members = [
+    ".",
+    "sdk",
+    "programs/native/noop",
+    "programs/native/bpf_loader",
+    "programs/native/lua_loader",
+    "programs/bpf/rust/noop",
+]

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2018 Anatoly Yakovenko, Greg Fitzgerald and Stephen Akridge
+Copyright 2018 Solana Labs, Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

README.md

@@ -1,27 +1,46 @@
 [![Solana crate](https://img.shields.io/crates/v/solana.svg)](https://crates.io/crates/solana)
 [![Solana documentation](https://docs.rs/solana/badge.svg)](https://docs.rs/solana)
-[![Build status](https://badge.buildkite.com/d4c4d7da9154e3a8fb7199325f430ccdb05be5fc1e92777e51.svg?branch=master)](https://buildkite.com/solana-labs/solana)
+[![Build status](https://badge.buildkite.com/d4c4d7da9154e3a8fb7199325f430ccdb05be5fc1e92777e51.svg?branch=master)](https://solana-ci-gate.herokuapp.com/buildkite_public_log?https://buildkite.com/solana-labs/solana/builds/latest/master)
 [![codecov](https://codecov.io/gh/solana-labs/solana/branch/master/graph/badge.svg)](https://codecov.io/gh/solana-labs/solana)
 
+Blockchain, Rebuilt for Scale
+===
+
+Solana™ is a new blockchain architecture built from the ground up for scale. The architecture supports
+up to 710 thousand transactions per second on a gigabit network.
+
 Disclaimer
 ===
 
 All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort.  It is up to the reader to check and validate their accuracy and truthfulness.  Furthermore nothing in this project constitutes a solicitation for investment.
 
-Solana: High Performance Blockchain
-===
-
-Solana™ is a new architecture for a high performance blockchain. It aims to support
-over 700 thousand transactions per second on a gigabit network.
-
 Introduction
 ===
 
-It's possible for a centralized database to process 710,000 transactions per second on a standard gigabit network if the transactions are, on average, no more than 178 bytes. A centralized database can also replicate itself and maintain high availability without significantly compromising that transaction rate using the distributed system technique known as Optimistic Concurrency Control [H.T.Kung, J.T.Robinson (1981)]. At Solana, we're demonstrating that these same theoretical limits apply just as well to blockchain on an adversarial network. The key ingredient? Finding a way to share time when nodes can't trust one-another. Once nodes can trust time, suddenly ~40 years of distributed systems research becomes applicable to blockchain! Furthermore, and much to our surprise, it can implemented using a mechanism that has existed in Bitcoin since day one. The Bitcoin feature is called nLocktime and it can be used to postdate transactions using block height instead of a timestamp. As a Bitcoin client, you'd use block height instead of a timestamp if you don't trust the network. Block height turns out to be an instance of what's being called a Verifiable Delay Function in cryptography circles. It's a cryptographically secure way to say time has passed. In Solana, we use a far more granular verifiable delay function, a SHA 256 hash chain, to checkpoint the ledger and coordinate consensus. With it, we implement Optimistic Concurrency Control and are now well in route towards that theoretical limit of 710,000 transactions per second.
+It's possible for a centralized database to process 710,000 transactions per second on a standard gigabit network if the transactions are, on average, no more than 176 bytes. A centralized database can also replicate itself and maintain high availability without significantly compromising that transaction rate using the distributed system technique known as Optimistic Concurrency Control [\[H.T.Kung, J.T.Robinson (1981)\]](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.65.4735). At Solana, we're demonstrating that these same theoretical limits apply just as well to blockchain on an adversarial network. The key ingredient? Finding a way to share time when nodes can't trust one-another. Once nodes can trust time, suddenly ~40 years of distributed systems research becomes applicable to blockchain!
 
-Running the demo
+> Perhaps the most striking difference between algorithms obtained by our method and ones based upon timeout is that using timeout produces a traditional distributed algorithm in which the processes operate asynchronously, while our method produces a globally synchronous one in which every process does the same thing at (approximately) the same time. Our method seems to contradict the whole purpose of distributed processing, which is to permit different processes to operate independently and perform different functions. However, if a distributed system is really a single system, then the processes must be synchronized in some way. Conceptually, the easiest way to synchronize processes is to get them all to do the same thing at the same time. Therefore, our method is used to implement a kernel that performs the necessary synchronization--for example, making sure that two different processes do not try to modify a file at the same time. Processes might spend only a small fraction of their time executing the synchronizing kernel; the rest of the time, they can operate independently--e.g., accessing different files. This is an approach we have advocated even when fault-tolerance is not required. The method's basic simplicity makes it easier to understand the precise properties of a system, which is crucial if one is to know just how fault-tolerant the system is. [\[L.Lamport (1984)\]](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.71.1078)
+
+Furthermore, and much to our surprise, it can be implemented using a mechanism that has existed in Bitcoin since day one. The Bitcoin feature is called nLocktime and it can be used to postdate transactions using block height instead of a timestamp. As a Bitcoin client, you'd use block height instead of a timestamp if you don't trust the network. Block height turns out to be an instance of what's being called a Verifiable Delay Function in cryptography circles. It's a cryptographically secure way to say time has passed. In Solana, we use a far more granular verifiable delay function, a SHA 256 hash chain, to checkpoint the ledger and coordinate consensus. With it, we implement Optimistic Concurrency Control and are now well en route towards that theoretical limit of 710,000 transactions per second.
+
+
+Testnet Demos
 ===
 
+The Solana repo contains all the scripts you might need to spin up your own
+local testnet. Depending on what you're looking to achieve, you may want to
+run a different variation, as the full-fledged, performance-enhanced
+multinode testnet is considerably more complex to set up than a Rust-only,
+singlenode testnode.  If you are looking to develop high-level features, such
+as experimenting with smart contracts, save yourself some setup headaches and
+stick to the Rust-only singlenode demo.  If you're doing performance optimization
+of the transaction pipeline, consider the enhanced singlenode demo. If you're
+doing consensus work, you'll need at least a Rust-only multinode demo. If you want
+to reproduce our TPS metrics, run the enhanced multinode demo.
+
+For all four variations, you'd need the latest Rust toolchain and the Solana
+source code:
+
 First, install Rust's package manager Cargo.
 
 ```bash
@@ -32,76 +51,216 @@ $ source $HOME/.cargo/env
 Now checkout the code from github:
 
 ```bash
-$ git clone https://github.com/solana-labs/solana.git 
+$ git clone https://github.com/solana-labs/solana.git
 $ cd solana
 ```
 
-The fullnode server is initialized with a ledger from stdin and
-generates new ledger entries on stdout. To create the input ledger, we'll need
-to create *the mint* and use it to generate a *genesis ledger*. It's done in
-two steps because the mint-demo.json file contains private keys that will be
-used later in this demo.
+The demo code is sometimes broken between releases as we add new low-level
+features, so if this is your first time running the demo, you'll improve
+your odds of success if you check out the
+[latest release](https://github.com/solana-labs/solana/releases)
+before proceeding:
 
 ```bash
-$ echo 1000000000 | cargo run --release --bin solana-mint-demo > mint-demo.json
-$ cat mint-demo.json | cargo run --release --bin solana-genesis-demo > genesis.log
+$ git checkout v0.8.0
 ```
 
-Before you start the server, make sure you know the IP address of the machine you
+Configuration Setup
+---
+
+The network is initialized with a genesis ledger and leader/validator configuration files.
+These files can be generated by running the following script.
+
+```bash
+$ ./multinode-demo/setup.sh
+```
+
+Drone
+---
+
+In order for the leader, client and validators to work, we'll need to
+spin up a drone to give out some test tokens.  The drone delivers Milton
+Friedman-style "air drops" (free tokens to requesting clients) to be used in
+test transactions.
+
+Start the drone on the leader node with:
+
+```bash
+$ ./multinode-demo/drone.sh
+```
+
+Singlenode Testnet
+---
+
+Before you start a fullnode, make sure you know the IP address of the machine you
 want to be the leader for the demo, and make sure that udp ports 8000-10000 are
 open on all the machines you want to test with.
 
-Generate a leader configuration file with:
+Now start the server in a separate shell:
 
 ```bash
-cargo run --release --bin solana-fullnode-config > leader.json
+$ ./multinode-demo/leader.sh
 ```
 
-Now start the server:
+Wait a few seconds for the server to initialize. It will print "leader ready..." when it's ready to
+receive transactions. The leader will request some tokens from the drone if it doesn't have any.
+The drone does not need to be running for subsequent leader starts.
+
+Multinode Testnet
+---
+
+To run a multinode testnet, after starting a leader node, spin up some validator nodes in
+separate shells:
 
 ```bash
-$ cat ./multinode-demo/leader.sh
-#!/bin/bash
-export RUST_LOG=solana=info
-sudo sysctl -w net.core.rmem_max=26214400
-cat genesis.log | cargo run --release --bin solana-fullnode -- -l leader.json
-$ ./multinode-demo/leader.sh > leader-txs.log
+$ ./multinode-demo/validator.sh
 ```
 
-Wait a few seconds for the server to initialize. It will print "Ready." when it's safe
-to start sending it transactions.
-
-Now you can start some validators:
+To run a performance-enhanced leader or validator (on Linux),
+[CUDA 9.2](https://developer.nvidia.com/cuda-downloads) must be installed on
+your system:
 
 ```bash
-$ cat ./multinode-demo/validator.sh
-#!/bin/bash
-rsync -v -e ssh $1/mint-demo.json .
-rsync -v -e ssh $1/leader.json .
-rsync -v -e ssh $1/genesis.log .
-export RUST_LOG=solana=info
-sudo sysctl -w net.core.rmem_max=26214400
-cat genesis.log | cargo run --release --bin solana-fullnode -- -l validator.json -v leader.json -b 9000 -d
-$ ./multinode-demo/validator.sh ubuntu@10.0.1.51:~/solana > validator-txs.log #The leader machine
+$ ./fetch-perf-libs.sh
+$ SOLANA_CUDA=1 ./multinode-demo/leader.sh
+$ SOLANA_CUDA=1 ./multinode-demo/validator.sh
 ```
 
 
-Then, in a separate shell, let's execute some transactions. Note we pass in
-the JSON configuration file here, not the genesis ledger.
+Testnet Client Demo
+---
+
+Now that your singlenode or multinode testnet is up and running let's send it
+some transactions!
+
+In a separate shell start the client:
 
 ```bash
-$ cat ./multinode-demo/client.sh
-#!/bin/bash
-export RUST_LOG=solana=info
-rsync -v -e ssh $1/leader.json .
-rsync -v -e ssh $1/mint-demo.json .
-cat mint-demo.json | cargo run --release --bin solana-client-demo -- -l leader.json -c 8100 -n 1
-$ ./multinode-demo/client.sh ubuntu@10.0.1.51:~/solana #The leader machine
+$ ./multinode-demo/client.sh # runs against localhost by default
 ```
 
-Try starting a more validators and reruning the client demo and change the `-n 1` option in `client.sh`!
+What just happened? The client demo spins up several threads to send 500,000 transactions
+to the testnet as quickly as it can. The client then pings the testnet periodically to see
+how many transactions it processed in that time. Take note that the demo intentionally
+floods the network with UDP packets, such that the network will almost certainly drop a
+bunch of them. This ensures the testnet has an opportunity to reach 710k TPS. The client
+demo completes after it has convinced itself the testnet won't process any additional
+transactions. You should see several TPS measurements printed to the screen. In the
+multinode variation, you'll see TPS measurements for each validator node as well.
 
-To enable cuda, downlaod the cuda library (see #Benchmarking) and add `--features=cuda` to the leader and validator scripts (`--release --features=cuda`).
+Public Testnet
+--------------
+In this example the client connects to our public testnet. To run validators on the testnet you would need to open udp ports `8000-10000`.
+
+```bash
+$ ./multinode-demo/client.sh --network $(dig +short testnet.solana.com):8001 --identity config-private/client-id.json --duration 60
+```
+
+You can observe the effects of your client's transactions on our [dashboard](https://metrics.solana.com:3000/d/testnet/testnet-hud?orgId=2&from=now-30m&to=now&refresh=5s&var-testnet=testnet)
+
+
+Linux Snap
+---
+A Linux [Snap](https://snapcraft.io/) is available, which can be used to
+easily get Solana running on supported Linux systems without building anything
+from source.  The `edge` Snap channel is updated daily with the latest
+development from the `master` branch.  To install:
+
+```bash
+$ sudo snap install solana --edge --devmode
+```
+
+(`--devmode` flag is required only for `solana.fullnode-cuda`)
+
+Once installed the usual Solana programs will be available as `solona.*` instead
+of `solana-*`.  For example, `solana.fullnode` instead of `solana-fullnode`.
+
+Update to the latest version at any time with:
+
+```bash
+$ snap info solana
+$ sudo snap refresh solana --devmode
+```
+
+### Daemon support
+The snap supports running a leader, validator or leader+drone node as a system
+daemon.
+
+Run `sudo snap get solana` to view the current daemon configuration.  To view
+daemon logs:
+1. Run `sudo snap logs -n=all solana` to view the daemon initialization log
+2. Runtime logging can be found under `/var/snap/solana/current/leader/`,
+`/var/snap/solana/current/validator/`, or `/var/snap/solana/current/drone/` depending
+on which `mode=` was selected.  Within each log directory the file `current`
+contains the latest log, and the files `*.s` (if present) contain older rotated
+logs.
+
+Disable the daemon at any time by running:
+
+```bash
+$ sudo snap set solana mode=
+```
+
+Runtime configuration files for the daemon can be found in
+`/var/snap/solana/current/config`.
+
+#### Leader daemon
+
+```bash
+$ sudo snap set solana mode=leader
+```
+
+If CUDA is available:
+
+```bash
+$ sudo snap set solana mode=leader enable-cuda=1
+```
+
+`rsync` must be configured and running on the leader.
+
+1. Ensure rsync is installed with `sudo apt-get -y install rsync`
+2. Edit `/etc/rsyncd.conf` to include the following
+```
+[config]
+path = /var/snap/solana/current/config
+hosts allow = *
+read only = true
+```
+3. Run `sudo systemctl enable rsync; sudo systemctl start rsync`
+4. Test by running `rsync -Pzravv rsync://<ip-address-of-leader>/config
+solana-config` from another machine.  **If the leader is running on a cloud
+provider it may be necessary to configure the Firewall rules to permit ingress
+to port tcp:873, tcp:9900 and the port range udp:8000-udp:10000**
+
+
+To run both the Leader and Drone:
+
+```bash
+$ sudo snap set solana mode=leader+drone
+
+```
+
+#### Validator daemon
+
+```bash
+$ sudo snap set solana mode=validator
+
+```
+If CUDA is available:
+
+```bash
+$ sudo snap set solana mode=validator enable-cuda=1
+```
+
+By default the validator will connect to **testnet.solana.com**, override
+the leader IP address by running:
+
+```bash
+$ sudo snap set solana mode=validator leader-address=127.0.0.1 #<-- change IP address
+```
+
+It's assumed that the leader will be running `rsync` configured as described in
+the previous **Leader daemon** section.
 
 Developing
 ===
@@ -123,6 +282,12 @@ If your rustc version is lower than 1.26.1, please update it:
 $ rustup update
 ```
 
+On Linux systems you may need to install libssl-dev, pkg-config, zlib1g-dev, etc.  On Ubuntu:
+
+```bash
+$ sudo apt-get install libssl-dev pkg-config zlib1g-dev
+```
+
 Download the source code:
 
 ```bash
@@ -140,6 +305,7 @@ $ cargo test
 ```
 
 To emulate all the tests that will run on a Pull Request, run:
+
 ```bash
 $ ./ci/run-local.sh
 ```
@@ -148,13 +314,28 @@ Debugging
 ---
 
 There are some useful debug messages in the code, you can enable them on a per-module and per-level
-basis with the normal RUST\_LOG environment variable. Run the fullnode with this syntax:
+basis.  Before running a leader or validator set the normal RUST\_LOG environment variable.
+
+For example, to enable info everywhere and debug only in the solana::banking_stage module:
+
 ```bash
-$ RUST_LOG=solana::streamer=debug,solana::server=info cat genesis.log | ./target/release/solana-fullnode > transactions0.log
+$ export RUST_LOG=info,solana::banking_stage=debug
 ```
-to see the debug and info sections for streamer and server respectively. Generally
-we are using debug for infrequent debug messages, trace for potentially frequent messages and
-info for performance-related logging.
+
+Generally we are using debug for infrequent debug messages, trace for potentially frequent
+messages and info for performance-related logging.
+
+You can also attach to a running process with GDB.  The leader's process is named
+_solana-fullnode_:
+
+```bash
+$ sudo gdb
+attach <PID>
+set logging on
+thread apply all bt
+```
+
+This will dump all the threads stack traces into gdb.txt
 
 Benchmarking
 ---
@@ -171,22 +352,29 @@ Run the benchmarks:
 $ cargo +nightly bench --features="unstable"
 ```
 
-To run the benchmarks on Linux with GPU optimizations enabled:
+Release Process
+---
+The release process for this project is described [here](rfcs/rfc-005-branches-tags-and-channels.md).
 
-```bash
-$ wget https://solana-build-artifacts.s3.amazonaws.com/v0.6.0/libcuda_verify_ed25519.a
-$ cargo +nightly bench --features="unstable,cuda"
-```
 
 Code coverage
 ---
 
-To generate code coverage statistics, run kcov via Docker:
+To generate code coverage statistics, install cargo-cov. Note: the tool currently only works
+in Rust nightly.
 
 ```bash
-$ ./ci/coverage.sh
+$ cargo +nightly install cargo-cov
 ```
-The coverage report will be written to `./target/cov/index.html`
+
+Run cargo-cov and generate a report:
+
+```bash
+$ cargo +nightly cov test
+$ cargo +nightly cov report --open
+```
+
+The coverage report will be written to `./target/cov/report/index.html`
 
 
 Why coverage? While most see coverage as a code quality metric, we see it primarily as a developer

RELEASE.md

@@ -0,0 +1,32 @@
+# Solana Release process
+
+## Introduction
+
+Solana uses a channel-oriented, date-based branching process described [here](https://github.com/solana-labs/solana/blob/master/rfcs/rfc-005-branches-tags-and-channels.md).
+
+## Release Steps
+
+### Changing channels
+
+When cutting a new channel branch these pre-steps are required:
+
+1. Pick your branch point for release on master.
+2. Create the branch.  The name should be "v" + the first 2 "version" fields from Cargo.toml.  For example, a Cargo.toml with version = "0.9.0" implies the next branch name is "v0.9".
+3. Update Cargo.toml to the next semantic version (e.g. 0.9.0 -> 0.10.0) by running `./scripts/increment-cargo-version.sh`.
+4. Push your new branch to solana.git
+5. Land your Cargo.toml change as a master PR.
+
+At this point, ci/channel-info.sh should show your freshly cut release branch as "BETA_CHANNEL" and the previous release branch as "STABLE_CHANNEL".
+
+### Updating channels (i.e. "making a release")
+
+We use [github's Releases UI](https://github.com/solana-labs/solana/releases) for tagging a release.
+
+1. Go [there ;)](https://github.com/solana-labs/solana/releases).
+2. Click "Draft new release".
+3. If the first major release on the branch (e.g. v0.8.0), paste in [this template](https://raw.githubusercontent.com/solana-labs/solana/master/.github/RELEASE_TEMPLATE.md) and fill it in.
+4. Test the release by generating a tag using semver's rules.  First try at a release should be <branchname>.X-rc.0.
+5. Verify release automation:
+   1. [Crates.io](https://crates.io/crates/solana) should have an updated Solana version.
+   2. ...
+6. After testnet deployment, verify that testnets are running correct software.  http://metrics.solana.com should show testnet running on a hash from your newly created branch.

_config.yml

@@ -1 +0,0 @@
-theme: jekyll-theme-slate

benches/bank.rs

@@ -0,0 +1,57 @@
+#![feature(test)]
+extern crate bincode;
+extern crate rayon;
+extern crate solana;
+extern crate test;
+
+use solana::bank::*;
+use solana::hash::hash;
+use solana::mint::Mint;
+use solana::signature::{Keypair, KeypairUtil};
+use solana::system_transaction::SystemTransaction;
+use solana::transaction::Transaction;
+use test::Bencher;
+
+#[bench]
+fn bench_process_transaction(bencher: &mut Bencher) {
+    let mint = Mint::new(100_000_000);
+    let bank = Bank::new(&mint);
+
+    // Create transactions between unrelated parties.
+    let transactions: Vec<_> = (0..4096)
+        .into_iter()
+        .map(|_| {
+            // Seed the 'from' account.
+            let rando0 = Keypair::new();
+            let tx = Transaction::system_move(
+                &mint.keypair(),
+                rando0.pubkey(),
+                10_000,
+                bank.last_id(),
+                0,
+            );
+            assert_eq!(bank.process_transaction(&tx), Ok(()));
+
+            // Seed the 'to' account and a cell for its signature.
+            let rando1 = Keypair::new();
+            let tx = Transaction::system_move(&rando0, rando1.pubkey(), 1, bank.last_id(), 0);
+            assert_eq!(bank.process_transaction(&tx), Ok(()));
+
+            // Finally, return the transaction to the benchmark.
+            tx
+        }).collect();
+
+    let mut id = bank.last_id();
+
+    for _ in 0..(MAX_ENTRY_IDS - 1) {
+        bank.register_entry_id(&id);
+        id = hash(&id.as_ref())
+    }
+
+    bencher.iter(|| {
+        // Since benchmarker runs this multiple times, we need to clear the signatures.
+        bank.clear_signatures();
+        let results = bank.process_transactions(&transactions);
+        assert!(results.iter().all(Result::is_ok));
+    })
+}

benches/banking_stage.rs

@@ -0,0 +1,228 @@
+#![feature(test)]
+extern crate bincode;
+extern crate rand;
+extern crate rayon;
+extern crate solana;
+extern crate solana_sdk;
+extern crate test;
+
+use rand::{thread_rng, Rng};
+use rayon::prelude::*;
+use solana::bank::{Bank, MAX_ENTRY_IDS};
+use solana::banking_stage::{BankingStage, NUM_THREADS};
+use solana::entry::Entry;
+use solana::hash::hash;
+use solana::mint::Mint;
+use solana::packet::to_packets_chunked;
+use solana::signature::{KeypairUtil, Signature};
+use solana::system_transaction::SystemTransaction;
+use solana::transaction::Transaction;
+use solana_sdk::pubkey::Pubkey;
+use std::iter;
+use std::sync::mpsc::{channel, Receiver};
+use std::sync::Arc;
+use std::time::Duration;
+use test::Bencher;
+
+fn check_txs(receiver: &Receiver<Vec<Entry>>, ref_tx_count: usize) {
+    let mut total = 0;
+    loop {
+        let entries = receiver.recv_timeout(Duration::new(1, 0));
+        if let Ok(entries) = entries {
+            for entry in &entries {
+                total += entry.transactions.len();
+            }
+        } else {
+            break;
+        }
+        if total >= ref_tx_count {
+            break;
+        }
+    }
+    assert_eq!(total, ref_tx_count);
+}
+
+#[bench]
+fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) {
+    let txes = 1000 * NUM_THREADS;
+    let mint_total = 1_000_000_000_000;
+    let mint = Mint::new(mint_total);
+
+    let (verified_sender, verified_receiver) = channel();
+    let bank = Arc::new(Bank::new(&mint));
+    let dummy = Transaction::system_move(
+        &mint.keypair(),
+        mint.keypair().pubkey(),
+        1,
+        mint.last_id(),
+        0,
+    );
+    let transactions: Vec<_> = (0..txes)
+        .into_par_iter()
+        .map(|_| {
+            let mut new = dummy.clone();
+            let from: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+            let to: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+            let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+            new.account_keys[0] = Pubkey::new(&from[0..32]);
+            new.account_keys[1] = Pubkey::new(&to[0..32]);
+            new.signature = Signature::new(&sig[0..64]);
+            new
+        }).collect();
+    // fund all the accounts
+    transactions.iter().for_each(|tx| {
+        let fund = Transaction::system_move(
+            &mint.keypair(),
+            tx.account_keys[0],
+            mint_total / txes as i64,
+            mint.last_id(),
+            0,
+        );
+        assert!(bank.process_transaction(&fund).is_ok());
+    });
+    //sanity check, make sure all the transactions can execute sequentially
+    transactions.iter().for_each(|tx| {
+        let res = bank.process_transaction(&tx);
+        assert!(res.is_ok(), "sanity test transactions");
+    });
+    bank.clear_signatures();
+    //sanity check, make sure all the transactions can execute in parallel
+    let res = bank.process_transactions(&transactions);
+    for r in res {
+        assert!(r.is_ok(), "sanity parallel execution");
+    }
+    bank.clear_signatures();
+    let verified: Vec<_> = to_packets_chunked(&transactions.clone(), 192)
+        .into_iter()
+        .map(|x| {
+            let len = x.read().unwrap().packets.len();
+            (x, iter::repeat(1).take(len).collect())
+        }).collect();
+    let (_stage, signal_receiver) = BankingStage::new(
+        &bank,
+        verified_receiver,
+        Default::default(),
+        &mint.last_id(),
+        0,
+        None,
+    );
+
+    let mut id = mint.last_id();
+    for _ in 0..MAX_ENTRY_IDS {
+        id = hash(&id.as_ref());
+        bank.register_entry_id(&id);
+    }
+
+    bencher.iter(move || {
+        // make sure the tx last id is still registered
+        if bank.count_valid_ids(&[mint.last_id()]).len() == 0 {
+            bank.register_entry_id(&mint.last_id());
+        }
+        for v in verified.chunks(verified.len() / NUM_THREADS) {
+            verified_sender.send(v.to_vec()).unwrap();
+        }
+        check_txs(&signal_receiver, txes);
+        bank.clear_signatures();
+    });
+}
+
+#[bench]
+fn bench_banking_stage_multi_programs(bencher: &mut Bencher) {
+    let progs = 5;
+    let txes = 1000 * NUM_THREADS;
+    let mint_total = 1_000_000_000_000;
+    let mint = Mint::new(mint_total);
+
+    let (verified_sender, verified_receiver) = channel();
+    let bank = Arc::new(Bank::new(&mint));
+    let dummy = Transaction::system_move(
+        &mint.keypair(),
+        mint.keypair().pubkey(),
+        1,
+        mint.last_id(),
+        0,
+    );
+    let transactions: Vec<_> = (0..txes)
+        .into_par_iter()
+        .map(|_| {
+            let mut new = dummy.clone();
+            let from: Vec<u8> = (0..32).map(|_| thread_rng().gen()).collect();
+            let sig: Vec<u8> = (0..64).map(|_| thread_rng().gen()).collect();
+            let to: Vec<u8> = (0..32).map(|_| thread_rng().gen()).collect();
+            new.account_keys[0] = Pubkey::new(&from[0..32]);
+            new.account_keys[1] = Pubkey::new(&to[0..32]);
+            let prog = new.instructions[0].clone();
+            for i in 1..progs {
+                //generate programs that spend to random keys
+                let to: Vec<u8> = (0..32).map(|_| thread_rng().gen()).collect();
+                let to_key = Pubkey::new(&to[0..32]);
+                new.account_keys.push(to_key);
+                assert_eq!(new.account_keys.len(), i + 2);
+                new.instructions.push(prog.clone());
+                assert_eq!(new.instructions.len(), i + 1);
+                new.instructions[i].accounts[1] = 1 + i as u8;
+                assert_eq!(new.key(i, 1), Some(&to_key));
+                assert_eq!(
+                    new.account_keys[new.instructions[i].accounts[1] as usize],
+                    to_key
+                );
+            }
+            assert_eq!(new.instructions.len(), progs);
+            new.signature = Signature::new(&sig[0..64]);
+            new
+        }).collect();
+    transactions.iter().for_each(|tx| {
+        let fund = Transaction::system_move(
+            &mint.keypair(),
+            tx.account_keys[0],
+            mint_total / txes as i64,
+            mint.last_id(),
+            0,
+        );
+        assert!(bank.process_transaction(&fund).is_ok());
+    });
+    //sanity check, make sure all the transactions can execute sequentially
+    transactions.iter().for_each(|tx| {
+        let res = bank.process_transaction(&tx);
+        assert!(res.is_ok(), "sanity test transactions");
+    });
+    bank.clear_signatures();
+    //sanity check, make sure all the transactions can execute in parallel
+    let res = bank.process_transactions(&transactions);
+    for r in res {
+        assert!(r.is_ok(), "sanity parallel execution");
+    }
+    bank.clear_signatures();
+    let verified: Vec<_> = to_packets_chunked(&transactions.clone(), 96)
+        .into_iter()
+        .map(|x| {
+            let len = x.read().unwrap().packets.len();
+            (x, iter::repeat(1).take(len).collect())
+        }).collect();
+    let (_stage, signal_receiver) = BankingStage::new(
+        &bank,
+        verified_receiver,
+        Default::default(),
+        &mint.last_id(),
+        0,
+        None,
+    );
+
+    let mut id = mint.last_id();
+    for _ in 0..MAX_ENTRY_IDS {
+        id = hash(&id.as_ref());
+        bank.register_entry_id(&id);
+    }
+
+    bencher.iter(move || {
+        // make sure the transactions are still valid
+        if bank.count_valid_ids(&[mint.last_id()]).len() == 0 {
+            bank.register_entry_id(&mint.last_id());
+        }
+        for v in verified.chunks(verified.len() / NUM_THREADS) {
+            verified_sender.send(v.to_vec()).unwrap();
+        }
+        check_txs(&signal_receiver, txes);
+        bank.clear_signatures();
+    });
+}

benches/chacha.rs

@@ -0,0 +1,29 @@
+#![feature(test)]
+
+extern crate solana;
+extern crate test;
+
+use solana::chacha::chacha_cbc_encrypt_files;
+use std::fs::remove_file;
+use std::fs::File;
+use std::io::Write;
+use std::path::Path;
+use test::Bencher;
+
+#[bench]
+fn bench_chacha_encrypt(bench: &mut Bencher) {
+    let in_path = Path::new("bench_chacha_encrypt_file_input.txt");
+    let out_path = Path::new("bench_chacha_encrypt_file_output.txt.enc");
+    {
+        let mut in_file = File::create(in_path).unwrap();
+        for _ in 0..1024 {
+            in_file.write("123456foobar".as_bytes()).unwrap();
+        }
+    }
+    bench.iter(move || {
+        chacha_cbc_encrypt_files(in_path, out_path, "thetestkey".to_string()).unwrap();
+    });
+
+    remove_file(in_path).unwrap();
+    remove_file(out_path).unwrap();
+}

benches/ledger.rs

@@ -0,0 +1,25 @@
+#![feature(test)]
+extern crate solana;
+extern crate test;
+
+use solana::hash::{hash, Hash};
+use solana::ledger::{next_entries, reconstruct_entries_from_blobs, Block};
+use solana::signature::{Keypair, KeypairUtil};
+use solana::system_transaction::SystemTransaction;
+use solana::transaction::Transaction;
+use test::Bencher;
+
+#[bench]
+fn bench_block_to_blobs_to_block(bencher: &mut Bencher) {
+    let zero = Hash::default();
+    let one = hash(&zero.as_ref());
+    let keypair = Keypair::new();
+    let tx0 = Transaction::system_move(&keypair, keypair.pubkey(), 1, one, 0);
+    let transactions = vec![tx0; 10];
+    let entries = next_entries(&zero, 1, transactions);
+
+    bencher.iter(|| {
+        let blobs = entries.to_blobs();
+        assert_eq!(reconstruct_entries_from_blobs(blobs).unwrap(), entries);
+    });
+}

benches/signature.rs

@@ -0,0 +1,12 @@
+#![feature(test)]
+extern crate solana;
+extern crate test;
+
+use solana::signature::GenKeys;
+use test::Bencher;
+
+#[bench]
+fn bench_gen_keys(b: &mut Bencher) {
+    let mut rnd = GenKeys::new([0u8; 32]);
+    b.iter(|| rnd.gen_n_keypairs(1000));
+}

benches/sigverify.rs

@@ -0,0 +1,23 @@
+#![feature(test)]
+extern crate bincode;
+extern crate rayon;
+extern crate solana;
+extern crate test;
+
+use solana::packet::to_packets;
+use solana::sigverify;
+use solana::system_transaction::test_tx;
+use test::Bencher;
+
+#[bench]
+fn bench_sigverify(bencher: &mut Bencher) {
+    let tx = test_tx();
+
+    // generate packet vector
+    let batches = to_packets(&vec![tx; 128]);
+
+    // verify packets
+    bencher.iter(|| {
+        let _ans = sigverify::ed25519_verify(&batches);
+    })
+}

build.rs

@@ -1,15 +1,62 @@
 use std::env;
+use std::fs;
+use std::process::Command;
 
 fn main() {
-    println!("cargo:rustc-link-search=native=.");
-    if !env::var("CARGO_FEATURE_CUDA").is_err() {
-        println!("cargo:rustc-link-lib=static=cuda_verify_ed25519");
+    println!("cargo:rerun-if-changed=build.rs");
+
+    // Ensure target/perf-libs/ exists.  It's been observed that
+    // a cargo:rerun-if-changed= directive with a non-existent
+    // directory triggers a rebuild on every |cargo build| invocation
+    fs::create_dir("target/perf-libs").unwrap_or_else(|err| {
+        if err.kind() != std::io::ErrorKind::AlreadyExists {
+            panic!("Unable to create target/perf-libs: {:?}", err);
+        }
+    });
+
+    let bpf_c = !env::var("CARGO_FEATURE_BPF_C").is_err();
+    let chacha = !env::var("CARGO_FEATURE_CHACHA").is_err();
+    let cuda = !env::var("CARGO_FEATURE_CUDA").is_err();
+    let erasure = !env::var("CARGO_FEATURE_ERASURE").is_err();
+
+    if bpf_c {
+        let out_dir = "OUT_DIR=../../../target/".to_string()
+            + &env::var("PROFILE").unwrap()
+            + &"/bpf".to_string();
+
+        println!("cargo:rerun-if-changed=programs/bpf/c/sdk/bpf.mk");
+        println!("cargo:rerun-if-changed=programs/bpf/c/sdk/inc/solana_sdk.h");
+        println!("cargo:rerun-if-changed=programs/bpf/c/makefile");
+        println!("cargo:rerun-if-changed=programs/bpf/c/src/move_funds.c");
+        println!("cargo:rerun-if-changed=programs/bpf/c/src/noop.c");
+        println!("cargo:warning=(not a warning) Compiling C-based BPF programs");
+        let status = Command::new("make")
+            .current_dir("programs/bpf/c")
+            .arg("all")
+            .arg(&out_dir)
+            .status()
+            .expect("Failed to build C-based BPF programs");
+        assert!(status.success());
+    }
+    if chacha || cuda || erasure {
+        println!("cargo:rerun-if-changed=target/perf-libs");
+        println!("cargo:rustc-link-search=native=target/perf-libs");
+    }
+    if chacha {
+        println!("cargo:rerun-if-changed=target/perf-libs/libcpu-crypt.a");
+    }
+    if cuda {
+        println!("cargo:rerun-if-changed=target/perf-libs/libcuda-crypt.a");
+        println!("cargo:rustc-link-lib=static=cuda-crypt");
         println!("cargo:rustc-link-search=native=/usr/local/cuda/lib64");
         println!("cargo:rustc-link-lib=dylib=cudart");
         println!("cargo:rustc-link-lib=dylib=cuda");
         println!("cargo:rustc-link-lib=dylib=cudadevrt");
     }
-    if !env::var("CARGO_FEATURE_ERASURE").is_err() {
+    if erasure {
+        println!("cargo:rerun-if-changed=target/perf-libs/libgf_complete.so");
+        println!("cargo:rerun-if-changed=target/perf-libs/libJerasure.so");
         println!("cargo:rustc-link-lib=dylib=Jerasure");
+        println!("cargo:rustc-link-lib=dylib=gf_complete");
     }
 }

ci/.gitignore

@@ -1,2 +1,3 @@
 /node_modules/
 /package-lock.json
+/snapcraft.credentials

ci/README.md

@@ -0,0 +1,89 @@
+
+Our CI infrastructure is built around [BuildKite](https://buildkite.com) with some
+additional GitHub integration provided by https://github.com/mvines/ci-gate
+
+## Agent Queues
+
+We define two [Agent Queues](https://buildkite.com/docs/agent/v3/queues):
+`queue=default` and `queue=cuda`.  The `default` queue should be favored and
+runs on lower-cost CPU instances.  The `cuda` queue is only necessary for
+running **tests** that depend on GPU (via CUDA) access -- CUDA builds may still
+be run on the `default` queue, and the [buildkite artifact
+system](https://buildkite.com/docs/builds/artifacts) used to transfer build
+products over to a GPU instance for testing.
+
+## Buildkite Agent Management
+
+### Buildkite GCP Setup
+
+CI runs on Google Cloud Platform via two Compute Engine Instance groups:
+`ci-default` and `ci-cuda`.  Autoscaling is currently disabled and the number of
+VM Instances in each group is manually adjusted.
+
+#### Updating a CI Disk Image
+
+Each Instance group has its own disk image, `ci-default-vX` and
+`ci-cuda-vY`, where *X* and *Y* are incremented each time the image is changed.
+
+The process to update a disk image is as follows (TODO: make this less manual):
+
+1. Create a new VM Instance using the disk image to modify.
+2. Once the VM boots, ssh to it and modify the disk as desired.
+3. Stop the VM Instance running the modified disk.  Remember the name of the VM disk
+4. From another machine, `gcloud auth login`, then create a new Disk Image based
+off the modified VM Instance:
+```
+ $ gcloud compute images create ci-default-$(date +%Y%m%d%H%M) --source-disk xxx --source-disk-zone us-east1-b --family ci-default
+
+```
+or
+```
+  $ gcloud compute images create ci-cuda-$(date +%Y%m%d%H%M) --source-disk xxx --source-disk-zone us-east1-b --family ci-cuda
+```
+5. Delete the new VM instance.
+6. Go to the Instance templates tab, find the existing template named
+`ci-default-vX` or `ci-cuda-vY` and select it.  Use the "Copy" button to create
+a new Instance template called `ci-default-vX+1` or `ci-cuda-vY+1` with the
+newly created Disk image.
+7. Go to the Instance Groups tag and find the applicable group, `ci-default` or
+`ci-cuda`.  Edit the Instance Group in two steps: (a) Set the number of
+instances to 0 and wait for them all to terminate, (b) Update the Instance
+template and restore the number of instances to the original value.
+8. Clean up the previous version by deleting it from Instance Templates and
+Images. 
+
+
+## Reference
+
+### Buildkite AWS CloudFormation Setup
+
+**AWS CloudFormation is currently inactive, although it may be restored in the
+future**
+
+AWS CloudFormation can be used to scale machines up and down based on the
+current CI load.  If no machine is currently running it can take up to 60
+seconds to spin up a new instance, please remain calm during this time.
+
+#### AMI
+We use a custom AWS AMI built via https://github.com/solana-labs/elastic-ci-stack-for-aws/tree/solana/cuda.
+
+Use the following process to update this AMI as dependencies change:
+```bash
+$ export AWS_ACCESS_KEY_ID=my_access_key
+$ export AWS_SECRET_ACCESS_KEY=my_secret_access_key
+$ git clone https://github.com/solana-labs/elastic-ci-stack-for-aws.git -b solana/cuda
+$ cd elastic-ci-stack-for-aws/
+$ make build
+$ make build-ami
+```
+
+Watch for the *"amazon-ebs: AMI:"* log message to extract the name of the new
+AMI.  For example:
+```
+amazon-ebs: AMI: ami-07118545e8b4ce6dc
+```
+The new AMI should also now be visible in your EC2 Dashboard.  Go to the desired
+AWS CloudFormation stack, update the **ImageId** field to the new AMI id, and
+*apply* the stack changes.
+
+

ci/audit.sh

@@ -0,0 +1,32 @@
+#!/bin/bash -e
+#
+# Audits project dependencies for security vulnerabilities
+#
+
+cd "$(dirname "$0")/.."
+
+export RUST_BACKTRACE=1
+rustc --version
+cargo --version
+
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+maybe_cargo_install() {
+  for cmd in "$@"; do
+    set +e
+    cargo "$cmd" --help > /dev/null 2>&1
+    declare exitcode=$?
+    set -e
+    if [[ $exitcode -eq 101 ]]; then
+      _ cargo install cargo-"$cmd"
+    fi
+  done
+}
+
+maybe_cargo_install audit tree
+
+_ cargo tree
+_ cargo audit

ci/buildkite-secondary.yml

@@ -0,0 +1,16 @@
+steps:
+  - command: "ci/snap.sh"
+    timeout_in_minutes: 40
+    name: "snap [public]"
+  - command: "ci/docker-solana/build.sh"
+    timeout_in_minutes: 20
+    name: "docker-solana"
+  - command: "ci/publish-crate.sh"
+    timeout_in_minutes: 20
+    name: "publish crate [public]"
+  - command: "ci/publish-bpf-sdk.sh"
+    timeout_in_minutes: 5
+    name: "publish bpf sdk"
+  - command: "ci/publish-solana-tar.sh"
+    timeout_in_minutes: 15
+    name: "publish solana release tar"

ci/buildkite.yml

@@ -1,16 +1,47 @@
 steps:
-  - command: "ci/coverage.sh"
-    name: "coverage [public]"
-  - command: "ci/docker-run.sh rust ci/test-stable.sh"
+  - command: "ci/docker-run.sh solanalabs/rust:1.30.0 ci/test-stable.sh"
     name: "stable [public]"
-  - command: "ci/docker-run.sh rustlang/rust:nightly ci/test-nightly.sh || true"
-    name: "nightly - FAILURES IGNORED [public]"
-  - command: "ci/docker-run.sh rust ci/test-ignored.sh"
-    name: "ignored [public]"
-  - command: "ci/test-cuda.sh"
-    name: "cuda"
-  - command: "ci/shellcheck.sh || true"
+    env:
+      CARGO_TARGET_CACHE_NAME: "stable"
+    timeout_in_minutes: 30
+  - command: "ci/docker-run.sh solanalabs/rust-nightly:2018-10-04 ci/test-bench.sh"
+    name: "bench [public]"
+    env:
+      CARGO_TARGET_CACHE_NAME: "nightly"
+    timeout_in_minutes: 30
+  - command: "ci/shellcheck.sh"
     name: "shellcheck [public]"
+    timeout_in_minutes: 20
+  - command: "ci/docker-run.sh solanalabs/rust-nightly:2018-10-04 ci/test-nightly.sh"
+    name: "nightly [public]"
+    env:
+      CARGO_TARGET_CACHE_NAME: "nightly"
+    timeout_in_minutes: 30
+  - command: "ci/test-stable-perf.sh"
+    name: "stable-perf [public]"
+    env:
+      CARGO_TARGET_CACHE_NAME: "stable-perf"
+    timeout_in_minutes: 20
+    agents:
+      - "queue=cuda"
+  # TODO: Fix and re-enable test-large-network.sh
+  # - command: "ci/test-large-network.sh || true"
+  #   name: "large-network [public] [ignored]"
+  #   env:
+  #     CARGO_TARGET_CACHE_NAME: "stable"
+  #   timeout_in_minutes: 20
+  #   agents:
+  #     - "queue=large"
+  - command: "ci/pr-snap.sh"
+    timeout_in_minutes: 20
+    name: "snap [public]"
   - wait
-  - command: "ci/publish.sh"
-    name: "publish release artifacts"
+  - trigger: "solana-secondary"
+    branches: "!pull/*"
+    async: true
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        TRIGGERED_BUILDKITE_TAG: "${BUILDKITE_TAG}"

ci/channel-info.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# Computes the current branch names of the edge, beta and stable
+# channels, as well as the latest tagged release for beta and stable.
+#
+# stdout of this script may be eval-ed
+#
+
+here="$(dirname "$0")"
+
+# shellcheck source=ci/semver_bash/semver.sh
+source "$here"/semver_bash/semver.sh
+
+remote=https://github.com/solana-labs/solana.git
+
+# Fetch all vX.Y.Z tags
+#
+# NOTE: pre-release tags are explicitly ignored
+#
+# shellcheck disable=SC2207
+tags=( \
+  $(git ls-remote --tags $remote \
+    | cut -c52- \
+    | grep '^v[[:digit:]][[:digit:]]*\.[[:digit:]][[:digit:]]*.[[:digit:]][[:digit:]]*$' \
+    | cut -c2- \
+  ) \
+)
+
+# Fetch all the vX.Y branches
+#
+# shellcheck disable=SC2207
+heads=( \
+  $(git ls-remote --heads $remote \
+    | cut -c53- \
+    | grep '^v[[:digit:]][[:digit:]]*\.[[:digit:]][[:digit:]]*$' \
+    | cut -c2- \
+  ) \
+)
+
+# Figure the beta channel by looking for the largest vX.Y branch
+beta=
+for head in "${heads[@]}"; do
+  if [[ -n $beta ]]; then
+    if semverLT "$head.0" "$beta.0"; then
+      continue
+    fi
+  fi
+  beta=$head
+done
+
+# Figure the stable channel by looking for the second largest vX.Y branch
+stable=
+for head in "${heads[@]}"; do
+  if [[ $head = "$beta" ]]; then
+    continue
+  fi
+  if [[ -n $stable ]]; then
+    if semverLT "$head.0" "$stable.0"; then
+      continue
+    fi
+  fi
+  stable=$head
+done
+
+for tag in "${tags[@]}"; do
+  if [[ -n $beta && $tag = $beta* ]]; then
+    if [[ -n $beta_tag ]]; then
+      if semverLT "$tag" "$beta_tag"; then
+        continue
+      fi
+    fi
+    beta_tag=$tag
+  fi
+
+  if [[ -n $stable && $tag = $stable* ]]; then
+    if [[ -n $stable_tag ]]; then
+      if semverLT "$tag" "$stable_tag"; then
+        continue
+      fi
+    fi
+    stable_tag=$tag
+  fi
+done
+
+echo EDGE_CHANNEL=master
+echo BETA_CHANNEL="${beta:+v$beta}"
+echo STABLE_CHANNEL="${stable:+v$stable}"
+echo BETA_CHANNEL_LATEST_TAG="${beta_tag:+v$beta_tag}"
+echo STABLE_CHANNEL_LATEST_TAG="${stable_tag:+v$stable_tag}"
+
+exit 0

ci/coverage.sh

@@ -1,21 +0,0 @@
-#!/bin/bash -e
-
-cd "$(dirname "$0")/.."
-
-ci/docker-run.sh evilmachines/rust-cargo-kcov \
-  bash -exc "\
-    export RUST_BACKTRACE=1; \
-    cargo build --verbose; \
-    cargo kcov --lib --verbose; \
-  "
-
-echo Coverage report:
-ls -l target/cov/index.html
-
-if [[ -z "$CODECOV_TOKEN" ]]; then
-  echo CODECOV_TOKEN undefined
-else
-  bash <(curl -s https://codecov.io/bash)
-fi
-
-exit 0

ci/crate-version.sh

@@ -0,0 +1,16 @@
+#!/bin/bash -e
+#
+# Outputs the current crate version
+#
+
+cd "$(dirname "$0")"/..
+
+while read -r name equals value _; do
+  if [[ $name = version && $equals = = ]]; then
+    echo "${value//\"/}"
+    exit 0
+  fi
+done < <(cat Cargo.toml)
+
+echo Unable to locate version in Cargo.toml 1>&2
+exit 1

ci/docker-run.sh

@@ -1,41 +1,85 @@
 #!/bin/bash -e
 
 usage() {
-  echo "Usage: $0 [docker image name] [command]"
+  echo "Usage: $0 [--nopull] [docker image name] [command]"
   echo
   echo Runs command in the specified docker image with
-  echo a CI-appropriate environment
+  echo a CI-appropriate environment.
+  echo
+  echo "--nopull   Skip the dockerhub image update"
+  echo "--shell    Skip command and enter an interactive shell"
   echo
 }
 
 cd "$(dirname "$0")/.."
 
+INTERACTIVE=false
+if [[ $1 = --shell ]]; then
+  INTERACTIVE=true
+  shift
+fi
+
+NOPULL=false
+if [[ $1 = --nopull ]]; then
+  NOPULL=true
+  shift
+fi
+
 IMAGE="$1"
 if [[ -z "$IMAGE" ]]; then
   echo Error: image not defined
   exit 1
 fi
 
-docker pull "$IMAGE"
+$NOPULL || docker pull "$IMAGE"
 shift
 
-ARGS=(--workdir /solana --volume "$PWD:/solana" --rm)
+ARGS=(
+  --workdir /solana
+  --volume "$PWD:/solana"
+  --rm
+)
 
-ARGS+=(--env "CARGO_HOME=/solana/.cargo")
+if [[ -n $CI ]]; then
+  # Share the real ~/.cargo between docker containers in CI for speed
+  ARGS+=(--volume "$HOME:/home")
+else
+  # Avoid sharing ~/.cargo when building locally to avoid a mixed macOS/Linux
+  # ~/.cargo
+  ARGS+=(--volume "$PWD:/home")
+fi
+ARGS+=(--env "CARGO_HOME=/home/.cargo")
 
 # kcov tries to set the personality of the binary which docker
 # doesn't allow by default.
 ARGS+=(--security-opt "seccomp=unconfined")
 
 # Ensure files are created with the current host uid/gid
-ARGS+=(--user "$(id -u):$(id -g)")
+if [[ -z "$SOLANA_DOCKER_RUN_NOSETUID" ]]; then
+  ARGS+=(--user "$(id -u):$(id -g)")
+fi
 
 # Environment variables to propagate into the container
 ARGS+=(
+  --env BUILDKITE
+  --env BUILDKITE_AGENT_ACCESS_TOKEN
+  --env BUILDKITE_BRANCH
+  --env BUILDKITE_JOB_ID
   --env BUILDKITE_TAG
   --env CODECOV_TOKEN
   --env CRATES_IO_TOKEN
+  --env SNAPCRAFT_CREDENTIALS_KEY
 )
 
+if $INTERACTIVE; then
+  if [[ -n $1 ]]; then
+    echo
+    echo "Note: '$*' ignored due to --shell argument"
+    echo
+  fi
+  set -x
+  exec docker run --interactive --tty "${ARGS[@]}" "$IMAGE" bash
+fi
+
 set -x
-docker run "${ARGS[@]}" "$IMAGE" "$@"
+exec docker run "${ARGS[@]}" "$IMAGE" "$@"

ci/docker-rust-nightly/Dockerfile

@@ -0,0 +1,9 @@
+FROM solanalabs/rust
+ARG date
+
+RUN set -x && \
+    rustup install nightly-$date && \
+    rustup default nightly-$date && \
+    rustc --version && \
+    cargo --version && \
+    cargo +nightly-$date install cargo-cov

ci/docker-rust-nightly/README.md

@@ -0,0 +1,36 @@
+Docker image containing rust nightly and some preinstalled crates used in CI.
+
+This image may be manually updated by running `CI=true ./build.sh` if you are a member
+of the [Solana Labs](https://hub.docker.com/u/solanalabs/) Docker Hub
+organization, but it is also automatically updated periodically by
+[this automation](https://buildkite.com/solana-labs/solana-ci-docker-rust-nightly).
+
+## Moving to a newer nightly
+
+We pin the version of nightly (see the `ARG nightly=xyz` line in `Dockerfile`)
+to avoid the build breaking at unexpected times, as occasionally nightly will
+introduce breaking changes.
+
+To update the pinned version:
+1. Run `ci/docker-rust-nightly/build.sh` to rebuild the nightly image locally,
+   or potentially `ci/docker-rust-nightly/build.sh YYYY-MM-DD` if there's a
+   specific YYYY-MM-DD that is desired (default is today's build).
+1. Run `SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-nightly.sh`
+   to confirm the new nightly image builds.  Fix any issues as needed
+1. Run `docker login` to enable pushing images to Docker Hub, if you're authorized.
+1. Run `CI=true ci/docker-rust-nightly/build.sh YYYY-MM-DD` to push the new nightly image to dockerhub.com.
+1. Modify the `solanalabs/rust-nightly:YYYY-MM-DD` reference in `ci/buildkite.yml` from the previous to
+   new *YYYY-MM-DD* value, send a PR with this change and any codebase adjustments needed.
+
+## Troubleshooting
+
+### Resource is denied
+
+When running `CI=true ci/docker-rust-nightly/build.sh`, you see:
+
+```
+denied: requested access to the resource is denied
+```
+
+Run `docker login` to enable pushing images to Docker Hub. Contact @mvines or @garious
+to get write access.

ci/docker-rust-nightly/build.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -ex
+
+cd "$(dirname "$0")"
+
+nightlyDate=${1:-$(date +%Y-%m-%d)}
+docker build -t solanalabs/rust-nightly:"$nightlyDate" --build-arg date="$nightlyDate" .
+
+maybeEcho=
+if [[ -z $CI ]]; then
+  echo "Not CI, skipping |docker push|"
+  maybeEcho="echo"
+fi
+$maybeEcho docker push solanalabs/rust-nightly:"$nightlyDate"

ci/docker-rust/Dockerfile

@@ -0,0 +1,26 @@
+# Note: when the rust version is changed also modify
+# ci/buildkite.yml to pick up the new image tag
+FROM rust:1.30.0
+
+RUN set -x && \
+    apt update && \
+    apt-get install apt-transport-https && \
+    echo deb https://apt.buildkite.com/buildkite-agent stable main > /etc/apt/sources.list.d/buildkite-agent.list && \
+    echo deb http://apt.llvm.org/stretch/ llvm-toolchain-stretch-7 main > /etc/apt/sources.list.d/llvm.list && \
+    apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    apt update && \
+    apt install -y \
+      buildkite-agent \
+      cmake \
+      lcov \
+      libclang-common-7-dev \
+      llvm-7 \
+      rsync \
+      sudo \
+      && \
+    rustup component add rustfmt-preview && \
+    rustup component add clippy-preview && \
+    rm -rf /var/lib/apt/lists/* && \
+    rustc --version && \
+    cargo --version

ci/docker-rust/README.md

@@ -0,0 +1,6 @@
+Docker image containing rust and some preinstalled packages used in CI.
+
+This image may be manually updated by running `./build.sh` if you are a member
+of the [Solana Labs](https://hub.docker.com/u/solanalabs/) Docker Hub
+organization, but it is also automatically updated periodically by
+[this automation](https://buildkite.com/solana-labs/solana-ci-docker-rust).

ci/docker-rust/build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -ex
+
+cd "$(dirname "$0")"
+
+docker build -t solanalabs/rust .
+
+read -r rustc version _ < <(docker run solanalabs/rust rustc --version)
+[[ $rustc = rustc ]]
+docker tag solanalabs/rust:latest solanalabs/rust:"$version"
+
+docker push solanalabs/rust

ci/docker-snapcraft/Dockerfile

@@ -0,0 +1,7 @@
+FROM snapcraft/xenial-amd64
+
+# Update snapcraft to latest version
+RUN apt-get update -qq \
+ && apt-get install -y snapcraft daemontools \
+ && rm -rf /var/lib/apt/lists/* \
+ && snapcraft --version

ci/docker-snapcraft/build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -ex
+
+cd "$(dirname "$0")"
+
+docker build -t solanalabs/snapcraft .
+docker push solanalabs/snapcraft

ci/docker-solana/.gitignore

@@ -0,0 +1 @@
+cargo-install/

ci/docker-solana/Dockerfile

@@ -0,0 +1,13 @@
+FROM debian:stretch
+
+# JSON RPC port
+EXPOSE 8899/tcp
+
+# Install libssl
+RUN apt update && \
+    apt-get install -y libssl-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY usr/bin /usr/bin/
+ENTRYPOINT [ "/usr/bin/solana-entrypoint.sh" ]
+CMD [""]

ci/docker-solana/README.md

@@ -0,0 +1,17 @@
+## Minimal Solana Docker image
+This image is automatically updated by CI
+
+https://hub.docker.com/r/solanalabs/solana/
+
+### Usage:
+Run the latest beta image:
+```bash
+$ docker run --rm -p 8899:8899 solanalabs/solana:beta
+```
+
+Run the latest edge image:
+```bash
+$ docker run --rm -p 8899:8899 solanalabs/solana:edge
+```
+
+Port *8899* is the JSON RPC port, which is used by clients to communicate with the network.

ci/docker-solana/build.sh

@@ -0,0 +1,39 @@
+#!/bin/bash -ex
+
+cd "$(dirname "$0")"
+eval "$(../channel-info.sh)"
+
+if [[ $BUILDKITE_BRANCH = "$STABLE_CHANNEL" ]]; then
+  CHANNEL=stable
+elif [[ $BUILDKITE_BRANCH = "$EDGE_CHANNEL" ]]; then
+  CHANNEL=edge
+elif [[ $BUILDKITE_BRANCH = "$BETA_CHANNEL" ]]; then
+  CHANNEL=beta
+fi
+
+if [[ -z $CHANNEL ]]; then
+  echo Unable to determine channel to publish into, exiting.
+  exit 0
+fi
+
+rm -rf usr/
+../docker-run.sh solanalabs/rust:1.30.0 \
+  cargo install --path . --root ci/docker-solana/usr
+cp -f entrypoint.sh usr/bin/solana-entrypoint.sh
+../../scripts/install-native-programs.sh usr/bin/
+
+docker build -t solanalabs/solana:$CHANNEL .
+
+maybeEcho=
+if [[ -z $CI ]]; then
+  echo "Not CI, skipping |docker push|"
+  maybeEcho="echo"
+else
+  (
+    set +x
+    if [[ -n $DOCKER_PASSWORD && -n $DOCKER_USERNAME ]]; then
+      echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
+    fi
+  )
+fi
+$maybeEcho docker push solanalabs/solana:$CHANNEL

ci/docker-solana/entrypoint.sh

@@ -0,0 +1,23 @@
+#!/bin/bash -ex
+
+export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info
+export RUST_BACKTRACE=1
+
+solana-keygen -o /config/leader-keypair.json
+solana-keygen -o /config/drone-keypair.json
+
+solana-genesis --tokens=1000000000 --ledger /ledger < /config/drone-keypair.json
+solana-fullnode-config --keypair=/config/leader-keypair.json -l > /config/leader-config.json
+
+solana-drone --keypair /config/drone-keypair.json --network 127.0.0.1:8001 &
+drone=$!
+solana-fullnode --identity /config/leader-config.json --ledger /ledger/ &
+fullnode=$!
+
+abort() {
+  kill "$drone" "$fullnode"
+}
+
+trap abort SIGINT SIGTERM
+wait "$fullnode"
+kill "$drone" "$fullnode"

ci/hoover.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Regular maintenance performed on a buildkite agent to control disk usage
+#
+
+
+echo --- Delete all exited containers first
+(
+  set -x
+  exited=$(docker ps -aq --no-trunc --filter "status=exited")
+  if [[ -n "$exited" ]]; then
+    # shellcheck disable=SC2086 # Don't want to double quote "$exited"
+    docker rm $exited
+  fi
+)
+
+echo --- Delete untagged images
+(
+  set -x
+  untagged=$(docker images | grep '<none>'| awk '{ print $3 }')
+  if [[ -n "$untagged" ]]; then
+    # shellcheck disable=SC2086 # Don't want to double quote "$untagged"
+    docker rmi $untagged
+  fi
+)
+
+echo --- Delete all dangling images
+(
+  set -x
+  dangling=$(docker images --filter 'dangling=true' -q --no-trunc | sort | uniq)
+  if [[ -n "$dangling" ]]; then
+    # shellcheck disable=SC2086 # Don't want to double quote "$dangling"
+    docker rmi $dangling
+  fi
+)
+
+echo --- Remove unused docker networks
+(
+  set -x
+  docker network prune -f
+)
+
+echo "--- Delete /tmp files older than 1 day owned by $(id -un)"
+(
+  set -x
+  find /tmp -maxdepth 1 -user "$(id -un)" -mtime +1 -print0 | xargs -0 rm -rf
+)
+
+echo --- Deleting stale buildkite agent build directories
+if [[ ! -d ../../../../builds/$BUILDKITE_AGENT_NAME ]]; then
+  # We might not be where we think we are, do nothing
+  echo Warning: Skipping flush of stale agent build directories
+  echo "  PWD=$PWD"
+else
+  # NOTE: this will be horribly broken if we ever decide to run multiple
+  #       agents on the same machine.
+  (
+    for keepDir in "$BUILDKITE_PIPELINE_SLUG" \
+                   "$BUILDKITE_ORGANIZATION_SLUG" \
+                   "$BUILDKITE_AGENT_NAME"; do
+      cd .. || exit 1
+      for dir in *; do
+        if [[ -d $dir && $dir != "$keepDir" ]]; then
+          echo "Removing $dir"
+          rm -rf "${dir:?}"/
+        fi
+      done
+    done
+  )
+fi
+
+echo --- System Status
+(
+  set -x
+  docker images
+  docker ps
+  docker network ls
+  df -h
+)
+
+exit 0

ci/is-pr.sh

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+#
+# The standard BUILDKITE_PULL_REQUEST environment variable is always "false" due
+# to how solana-ci-gate is used to trigger PR builds rather than using the
+# standard Buildkite PR trigger.
+#
+
+[[ $BUILDKITE_BRANCH =~ pull/* ]]

ci/localnet-sanity.sh

@@ -0,0 +1,93 @@
+#!/bin/bash -e
+#
+# Perform a quick sanity test on a leader, drone, validator and client running
+# locally on the same machine
+#
+
+cd "$(dirname "$0")"/..
+source ci/upload_ci_artifact.sh
+source scripts/configure-metrics.sh
+
+multinode-demo/setup.sh
+
+backgroundCommands="drone leader validator validator-x"
+pids=()
+
+for cmd in $backgroundCommands; do
+  echo "--- Start $cmd"
+  rm -f log-"$cmd".txt
+  multinode-demo/"$cmd".sh > log-"$cmd".txt 2>&1 &
+  declare pid=$!
+  pids+=("$pid")
+  echo "pid: $pid"
+done
+
+killBackgroundCommands() {
+  set +e
+  for pid in "${pids[@]}"; do
+    if kill "$pid"; then
+      wait "$pid"
+    else
+      echo -e "^^^ +++\\nWarning: unable to kill $pid"
+    fi
+  done
+  set -e
+  pids=()
+}
+
+shutdown() {
+  exitcode=$?
+  killBackgroundCommands
+
+  set +e
+
+  echo "--- Upload artifacts"
+  for cmd in $backgroundCommands; do
+    declare logfile=log-$cmd.txt
+    upload_ci_artifact "$logfile"
+    tail "$logfile"
+  done
+
+  exit $exitcode
+}
+
+trap shutdown EXIT INT
+
+set -e
+
+flag_error() {
+  echo Failed
+  echo "^^^ +++"
+  exit 1
+}
+
+echo "--- Wallet sanity"
+(
+  set -x
+  scripts/wallet-sanity.sh
+) || flag_error
+
+echo "--- Node count"
+(
+  source multinode-demo/common.sh
+  set -x
+  client_id=/tmp/client-id.json-$$
+  $solana_keygen -o $client_id
+  $solana_bench_tps --identity $client_id --num-nodes 3 --reject-extra-nodes --converge-only
+  rm -rf $client_id
+) || flag_error
+
+killBackgroundCommands
+
+echo "--- Ledger verification"
+(
+  source multinode-demo/common.sh
+  set -x
+  cp -R "$SOLANA_CONFIG_DIR"/ledger /tmp/ledger-$$
+  $solana_ledger_tool --ledger /tmp/ledger-$$ verify
+  rm -rf /tmp/ledger-$$
+) || flag_error
+
+echo +++
+echo Ok
+exit 0

ci/pr-snap.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+#
+# Only run snap.sh for pull requests that modify files under /snap
+#
+
+cd "$(dirname "$0")"
+
+if ./is-pr.sh; then
+  affected_files="$(buildkite-agent meta-data get affected_files)"
+  echo "Affected files in this PR: $affected_files"
+  if [[ ! ":$affected_files:" =~ :snap/ ]]; then
+    echo "Skipping snap build as no files under /snap were modified"
+    exit 0
+  fi
+  exec ./snap.sh
+else
+  echo "Skipping snap build as this is not a pull request"
+fi

ci/publish-bpf-sdk.sh

@@ -0,0 +1,36 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+version=$(./ci/crate-version.sh)
+
+echo --- Creating tarball
+(
+  set -x
+  rm -rf bpf-sdk/
+  mkdir bpf-sdk/
+  (
+    echo "$version"
+    git rev-parse HEAD
+  ) > bpf-sdk/version.txt
+
+  cp -ra programs/bpf/c/sdk/* bpf-sdk/
+
+  tar jvcf bpf-sdk.tar.bz2 bpf-sdk/
+)
+
+
+echo --- AWS S3 Store
+
+set -x
+if [[ ! -r s3cmd-2.0.1/s3cmd ]]; then
+  rm -rf s3cmd-2.0.1.tar.gz s3cmd-2.0.1
+  wget https://github.com/s3tools/s3cmd/releases/download/v2.0.1/s3cmd-2.0.1.tar.gz
+  tar zxf s3cmd-2.0.1.tar.gz
+fi
+
+python ./s3cmd-2.0.1/s3cmd --acl-public put bpf-sdk.tar.bz2 \
+  s3://solana-sdk/"$version"/bpf-sdk.tar.bz2
+
+exit 0
+

ci/publish-crate.sh

@@ -0,0 +1,29 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+if [[ -z "$BUILDKITE_TAG" && -z "$TRIGGERED_BUILDKITE_TAG" ]]; then
+  # Skip publish if this is not a tagged release
+  exit 0
+fi
+
+if [[ -z "$CRATES_IO_TOKEN" ]]; then
+  echo CRATES_IO_TOKEN undefined
+  exit 1
+fi
+
+maybePublish="echo Publish skipped"
+if [[ -n $CI ]]; then
+  maybePublish="cargo publish --token $CRATES_IO_TOKEN"
+fi
+
+# shellcheck disable=2044 # Disable 'For loops over find output are fragile...'
+for Cargo_toml in {sdk,programs/native/{bpf_loader,lua_loader,noop},.}/Cargo.toml; do
+  # TODO: Ensure the published version matches the contents of BUILDKITE_TAG
+  (
+    set -x
+    ci/docker-run.sh rust bash -exc "cd $(dirname "$Cargo_toml"); cargo package; $maybePublish"
+  )
+done
+
+exit 0

ci/publish-solana-tar.sh

@@ -0,0 +1,66 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+DRYRUN=
+if [[ -z $BUILDKITE_BRANCH ]]; then
+  DRYRUN="echo"
+fi
+
+eval "$(ci/channel-info.sh)"
+
+if [[ $BUILDKITE_BRANCH = "$STABLE_CHANNEL" ]]; then
+  CHANNEL=stable
+elif [[ $BUILDKITE_BRANCH = "$EDGE_CHANNEL" ]]; then
+  CHANNEL=edge
+elif [[ $BUILDKITE_BRANCH = "$BETA_CHANNEL" ]]; then
+  CHANNEL=beta
+fi
+
+if [[ -z $CHANNEL ]]; then
+  echo Unable to determine channel to publish into, exiting.
+  exit 0
+fi
+
+if [[ -n "$BUILDKITE_TAG" ]]; then
+  CHANNEL_OR_TAG=$BUILDKITE_TAG
+elif [[ -n "$TRIGGERED_BUILDKITE_TAG" ]]; then
+  CHANNEL_OR_TAG=$TRIGGERED_BUILDKITE_TAG
+else
+  CHANNEL_OR_TAG=$CHANNEL
+fi
+
+echo --- Creating tarball
+if [[ -z $DRYRUN ]]; then
+(
+  set -x
+  rm -rf solana-release/
+  mkdir solana-release/
+  (
+    echo "$CHANNEL"
+    git rev-parse HEAD
+  ) > solana-release/version.txt
+
+  ./fetch-perf-libs.sh
+  cargo install --features=cuda --root solana-release
+  ./scripts/install-native-programs.sh solana-release
+
+  tar jvcf solana-release.tar.bz2 solana-release/
+)
+fi
+
+
+echo --- AWS S3 Store
+
+set -x
+if [[ ! -r s3cmd-2.0.1/s3cmd ]]; then
+  rm -rf s3cmd-2.0.1.tar.gz s3cmd-2.0.1
+  $DRYRUN wget https://github.com/s3tools/s3cmd/releases/download/v2.0.1/s3cmd-2.0.1.tar.gz
+  $DRYRUN tar zxf s3cmd-2.0.1.tar.gz
+fi
+
+$DRYRUN python ./s3cmd-2.0.1/s3cmd --acl-public put solana-release.tar.bz2 \
+  s3://solana-release/"$CHANNEL_OR_TAG"/solana-release.tar.bz2
+
+exit 0
+

ci/publish.sh

@@ -1,19 +0,0 @@
-#!/bin/bash -e
-
-cd "$(dirname "$0")/.."
-
-if [[ -z "$BUILDKITE_TAG" ]]; then
-  # Skip publish if this is not a tagged release
-  exit 0
-fi
-
-if [[ -z "$CRATES_IO_TOKEN" ]]; then
-  echo CRATES_IO_TOKEN undefined
-  exit 1
-fi
-
-# TODO: Ensure the published version matches the contents of BUILDKITE_TAG
-ci/docker-run.sh rust \
-  bash -exc "cargo package; cargo publish --token $CRATES_IO_TOKEN"
-
-exit 0

ci/run-local.sh

@@ -16,4 +16,4 @@ if [[ ! -x $BKRUN ]]; then
 fi
 
 set -x
-./ci/node_modules/.bin/bkrun ci/buildkite.yml
+exec ./ci/node_modules/.bin/bkrun ci/buildkite.yml

ci/semver_bash/LICENSE

@@ -0,0 +1,26 @@
+Copyright (c) 2013, Ray Bejjani
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met: 
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer. 
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution. 
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation are those
+of the authors and should not be interpreted as representing official policies, 
+either expressed or implied, of the FreeBSD Project.

ci/semver_bash/README.md

@@ -0,0 +1,31 @@
+semver_bash is a bash parser for semantic versioning
+====================================================
+
+[Semantic Versioning](http://semver.org/) is a set of guidelines that help keep
+version and version management sane. This is a bash based parser to help manage
+a project's versions. Use it from a Makefile or any scripts you use in your
+project.
+
+Usage
+-----
+semver_bash can be used from the command line as:  
+
+    $ ./semver.sh "3.2.1" "3.2.1-alpha"  
+    3.2.1 -> M: 3 m:2 p:1 s:  
+    3.2.1-alpha -> M: 3 m:2 p:1 s:-alpha  
+    3.2.1 == 3.2.1-alpha -> 1.  
+    3.2.1 < 3.2.1-alpha -> 1.  
+    3.2.1 > 3.2.1-alpha -> 0.
+
+
+Alternatively, you can source it from within a script:
+
+    . ./semver.sh  
+    
+    local MAJOR=0  
+    local MINOR=0  
+    local PATCH=0  
+    local SPECIAL=""
+    
+    semverParseInto "1.2.3" MAJOR MINOR PATCH SPECIAL  
+    semverParseInto "3.2.1" MAJOR MINOR PATCH SPECIAL  

ci/semver_bash/semver.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env sh
+
+function semverParseInto() {
+    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
+    #MAJOR
+    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
+    #MINOR
+    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
+    #MINOR
+    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
+    #SPECIAL
+    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
+}
+
+function semverEQ() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -ne $MAJOR_B ]; then
+        return 1
+    fi
+
+    if [ $MINOR_A -ne $MINOR_B ]; then
+        return 1
+    fi
+
+    if [ $PATCH_A -ne $PATCH_B ]; then
+        return 1
+    fi
+
+    if [[ "_$SPECIAL_A" != "_$SPECIAL_B" ]]; then
+        return 1
+    fi
+
+
+    return 0
+
+}
+
+function semverLT() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -lt $MAJOR_B ]; then
+        return 0
+    fi
+
+    if [[ $MAJOR_A -le $MAJOR_B  && $MINOR_A -lt $MINOR_B ]]; then
+        return 0
+    fi
+    
+    if [[ $MAJOR_A -le $MAJOR_B  && $MINOR_A -le $MINOR_B && $PATCH_A -lt $PATCH_B ]]; then
+        return 0
+    fi
+
+    if [[ "_$SPECIAL_A"  == "_" ]] && [[ "_$SPECIAL_B"  == "_" ]] ; then
+        return 1
+    fi
+    if [[ "_$SPECIAL_A"  == "_" ]] && [[ "_$SPECIAL_B"  != "_" ]] ; then
+        return 1
+    fi
+    if [[ "_$SPECIAL_A"  != "_" ]] && [[ "_$SPECIAL_B"  == "_" ]] ; then
+        return 0
+    fi
+
+    if [[ "_$SPECIAL_A" < "_$SPECIAL_B" ]]; then
+        return 0
+    fi
+
+    return 1
+
+}
+
+function semverGT() {
+    semverEQ $1 $2
+    local EQ=$?
+
+    semverLT $1 $2
+    local LT=$?
+
+    if [ $EQ -ne 0 ] && [ $LT -ne 0 ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+if [ "___semver.sh" == "___`basename $0`" ]; then
+
+MAJOR=0
+MINOR=0
+PATCH=0
+SPECIAL=""
+
+semverParseInto $1 MAJOR MINOR PATCH SPECIAL
+echo "$1 -> M: $MAJOR m:$MINOR p:$PATCH s:$SPECIAL"
+
+semverParseInto $2 MAJOR MINOR PATCH SPECIAL
+echo "$2 -> M: $MAJOR m:$MINOR p:$PATCH s:$SPECIAL"
+
+semverEQ $1 $2
+echo "$1 == $2 -> $?."
+
+semverLT $1 $2
+echo "$1 < $2 -> $?."
+
+semverGT $1 $2
+echo "$1 > $2 -> $?."
+
+fi

ci/semver_bash/semver_test.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+
+. ./semver.sh
+
+semverTest() {
+local A=R1.3.2
+local B=R2.3.2
+local C=R1.4.2
+local D=R1.3.3
+local E=R1.3.2a
+local F=R1.3.2b
+local G=R1.2.3
+
+local MAJOR=0
+local MINOR=0
+local PATCH=0
+local SPECIAL=""
+
+semverParseInto $A MAJOR MINOR PATCH SPECIAL
+echo "$A -> M:$MAJOR m:$MINOR p:$PATCH s:$SPECIAL. Expect M:1 m:3 p:2 s:"
+semverParseInto $E MAJOR MINOR PATCH SPECIAL
+echo "$E -> M:$MAJOR m:$MINOR p:$PATCH s:$SPECIAL. Expect M:1 m:3 p:2 s:a"
+
+echo "Equality comparisions"
+semverEQ $A $A
+echo "$A == $A -> $?. Expect 0."
+
+semverLT $A $A
+echo "$A < $A -> $?. Expect 1."
+
+semverGT $A $A
+echo "$A > $A -> $?. Expect 1."
+
+
+echo "Major number comparisions"
+semverEQ $A $B
+echo "$A == $B -> $?. Expect 1."
+
+semverLT $A $B
+echo "$A < $B -> $?. Expect 0."
+
+semverGT $A $B
+echo "$A > $B -> $?. Expect 1."
+
+semverEQ $B $A
+echo "$B == $A -> $?. Expect 1."
+
+semverLT $B $A
+echo "$B < $A -> $?. Expect 1."
+
+semverGT $B $A
+echo "$B > $A -> $?. Expect 0."
+
+
+echo "Minor number comparisions"
+semverEQ $A $C
+echo "$A == $C -> $?. Expect 1."
+
+semverLT $A $C
+echo "$A < $C -> $?. Expect 0."
+
+semverGT $A $C
+echo "$A > $C -> $?. Expect 1."
+
+semverEQ $C $A
+echo "$C == $A -> $?. Expect 1."
+
+semverLT $C $A
+echo "$C < $A -> $?. Expect 1."
+
+semverGT $C $A
+echo "$C > $A -> $?. Expect 0."
+
+echo "patch number comparisions"
+semverEQ $A $D
+echo "$A == $D -> $?. Expect 1."
+
+semverLT $A $D
+echo "$A < $D -> $?. Expect 0."
+
+semverGT $A $D
+echo "$A > $D -> $?. Expect 1."
+
+semverEQ $D $A
+echo "$D == $A -> $?. Expect 1."
+
+semverLT $D $A
+echo "$D < $A -> $?. Expect 1."
+
+semverGT $D $A
+echo "$D > $A -> $?. Expect 0."
+
+echo "special section vs no special comparisions"
+semverEQ $A $E
+echo "$A == $E -> $?. Expect 1."
+
+semverLT $A $E
+echo "$A < $E -> $?. Expect 1."
+
+semverGT $A $E
+echo "$A > $E -> $?. Expect 0."
+
+semverEQ $E $A
+echo "$E == $A -> $?. Expect 1."
+
+semverLT $E $A
+echo "$E < $A -> $?. Expect 0."
+
+semverGT $E $A
+echo "$E > $A -> $?. Expect 1."
+
+echo "special section vs special comparisions"
+semverEQ $E $F
+echo "$E == $F -> $?. Expect 1."
+
+semverLT $E $F
+echo "$E < $F -> $?. Expect 0."
+
+semverGT $E $F
+echo "$E > $F -> $?. Expect 1."
+
+semverEQ $F $E
+echo "$F == $E -> $?. Expect 1."
+
+semverLT $F $E
+echo "$F < $E -> $?. Expect 1."
+
+semverGT $F $E
+echo "$F > $E -> $?. Expect 0."
+
+echo "Minor and patch number comparisons"
+semverEQ $A $G
+echo "$A == $G -> $?. Expect 1."
+
+semverLT $A $G
+echo "$A < $G -> $?. Expect 1."
+
+semverGT $A $G
+echo "$A > $G -> $?. Expect 0."
+
+semverEQ $G $A
+echo "$G == $A -> $?. Expect 1."
+
+semverLT $G $A
+echo "$G < $A -> $?. Expect 0."
+
+semverGT $G $A
+echo "$G > $A -> $?. Expect 1."
+}
+
+semverTest

ci/shellcheck.sh

@@ -5,7 +5,13 @@
 cd "$(dirname "$0")/.."
 
 set -x
-find . -name "*.sh" -not -regex ".*/.cargo/.*" -not -regex ".*/node_modules/.*" -print0 \
+find . -name "*.sh" \
+    -not -regex ".*/ci/semver_bash/.*" \
+    -not -regex ".*/.cargo/.*" \
+    -not -regex ".*/node_modules/.*" \
+    -not -regex ".*/target/.*" \
+    -print0 \
   | xargs -0 \
       ci/docker-run.sh koalaman/shellcheck --color=always --external-sources --shell=bash
+
 exit 0

ci/snap.sh

@@ -0,0 +1,70 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+if ! ci/version-check.sh stable; then
+  # This job doesn't run within a container, try once to upgrade tooling on a
+  # version check failure
+  rustup install stable
+  ci/version-check.sh stable
+fi
+
+DRYRUN=
+if [[ -z $BUILDKITE_BRANCH ]] || ./ci/is-pr.sh; then
+  DRYRUN="echo"
+fi
+
+eval "$(ci/channel-info.sh)"
+
+if [[ $BUILDKITE_BRANCH = "$STABLE_CHANNEL" ]]; then
+  CHANNEL=stable
+elif [[ $BUILDKITE_BRANCH = "$EDGE_CHANNEL" ]]; then
+  CHANNEL=edge
+elif [[ $BUILDKITE_BRANCH = "$BETA_CHANNEL" ]]; then
+  CHANNEL=beta
+fi
+
+if [[ -z $CHANNEL ]]; then
+  echo Unable to determine channel to publish into, exiting.
+  exit 0
+fi
+
+if [[ -z $DRYRUN ]]; then
+  [[ -n $SNAPCRAFT_CREDENTIALS_KEY ]] || {
+    echo SNAPCRAFT_CREDENTIALS_KEY not defined
+    exit 1;
+  }
+  (
+    openssl aes-256-cbc -d \
+      -in ci/snapcraft.credentials.enc \
+      -out ci/snapcraft.credentials \
+      -k "$SNAPCRAFT_CREDENTIALS_KEY"
+
+    snapcraft login --with ci/snapcraft.credentials
+  ) || {
+    rm -f ci/snapcraft.credentials;
+    exit 1
+  }
+fi
+
+set -x
+
+echo --- checking for multilog
+if [[ ! -x /usr/bin/multilog ]]; then
+  if [[ -z $CI ]]; then
+    echo "multilog not found, install with: sudo apt-get install -y daemontools"
+    exit 1
+  fi
+  sudo apt-get install -y daemontools
+fi
+
+echo --- build: $CHANNEL channel
+snapcraft
+
+source ci/upload_ci_artifact.sh
+upload_ci_artifact solana_*.snap
+
+if [[ -z $DO_NOT_PUBLISH_SNAP ]]; then
+  echo --- publish: $CHANNEL channel
+  $DRYRUN snapcraft push solana_*.snap --release $CHANNEL
+fi

ci/solana-testnet.yml

@@ -0,0 +1,18 @@
+steps:
+  - command: "ci/snap.sh"
+    label: "create snap"
+
+  - wait
+
+  - command: "ci/testnet-automation.sh"
+    label: "run testnet"
+    agents:
+      - "queue=testnet-deploy"
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: "ci/testnet-automation-cleanup.sh"
+    label: "delete testnet"
+    agents:
+      - "queue=testnet-deploy"

ci/test-bench.sh

@@ -0,0 +1,33 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+# shellcheck disable=SC1091
+source ci/upload_ci_artifact.sh
+
+eval "$(ci/channel-info.sh)"
+
+ci/version-check.sh nightly
+export RUST_BACKTRACE=1
+
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+set -o pipefail
+
+UPLOAD_METRICS=""
+TARGET_BRANCH=$BUILDKITE_BRANCH
+if [[ -z $BUILDKITE_BRANCH ]] || ./ci/is-pr.sh; then
+  TARGET_BRANCH=$EDGE_CHANNEL
+else
+  UPLOAD_METRICS="upload"
+fi
+
+BENCH_FILE=bench_output.log
+BENCH_ARTIFACT=current_bench_results.log
+_ cargo bench --features=unstable --verbose -- -Z unstable-options --format=json | tee "$BENCH_FILE"
+_ cargo run --release --bin solana-upload-perf -- "$BENCH_FILE" "$TARGET_BRANCH" "$UPLOAD_METRICS" >"$BENCH_ARTIFACT"
+
+upload_ci_artifact "$BENCH_ARTIFACT"

ci/test-cuda.sh

@@ -1,22 +0,0 @@
-#!/bin/bash -e
-
-cd "$(dirname "$0")/.."
-
-LIB=libcuda_verify_ed25519.a
-if [[ ! -r $LIB ]]; then
-  if [[ -z "${libcuda_verify_ed25519_URL:-}" ]]; then
-    echo "$0 skipped.  Unable to locate $LIB"
-    exit 0
-  fi
-
-  export LD_LIBRARY_PATH=/usr/local/cuda/lib64
-  export PATH=$PATH:/usr/local/cuda/bin
-  curl -X GET -o $LIB "$libcuda_verify_ed25519_URL"
-fi
-
-# shellcheck disable=SC1090    # <-- shellcheck can't follow ~
-source ~/.cargo/env
-export RUST_BACKTRACE=1
-cargo test --features=cuda
-
-exit 0

ci/test-ignored.sh

@@ -1,9 +0,0 @@
-#!/bin/bash -e
-
-cd "$(dirname "$0")/.."
-
-rustc --version
-cargo --version
-
-export RUST_BACKTRACE=1
-cargo test -- --ignored

ci/test-large-network.sh

@@ -0,0 +1,45 @@
+#!/bin/bash -e
+
+here=$(dirname "$0")
+cd "$here"/..
+
+if ! ci/version-check.sh stable; then
+  # This job doesn't run within a container, try once to upgrade tooling on a
+  # version check failure
+  rustup install stable
+  ci/version-check.sh stable
+fi
+export RUST_BACKTRACE=1
+
+./fetch-perf-libs.sh
+export LD_LIBRARY_PATH=$PWD/target/perf-libs:$LD_LIBRARY_PATH
+
+export RUST_LOG=multinode=info
+
+if [[ $(ulimit -n) -lt 65000 ]]; then
+  echo 'Error: nofiles too small, run "ulimit -n 65000" to continue'
+  exit 1
+fi
+
+if [[ $(sysctl -n net.core.rmem_default) -lt 1610612736 ]]; then
+  echo 'Error: rmem_default too small, run "sudo sysctl -w net.core.rmem_default=1610612736" to continue'
+  exit 1
+fi
+
+if [[ $(sysctl -n net.core.rmem_max) -lt 1610612736 ]]; then
+  echo 'Error: rmem_max too small, run "sudo sysctl -w net.core.rmem_max=1610612736" to continue'
+  exit 1
+fi
+
+if [[ $(sysctl -n net.core.wmem_default) -lt 1610612736 ]]; then
+  echo 'Error: rmem_default too small, run "sudo sysctl -w net.core.wmem_default=1610612736" to continue'
+  exit 1
+fi
+
+if [[ $(sysctl -n net.core.wmem_max) -lt 1610612736 ]]; then
+  echo 'Error: rmem_max too small, run "sudo sysctl -w net.core.wmem_max=1610612736" to continue'
+  exit 1
+fi
+
+set -x
+exec cargo test --release --features=erasure test_multi_node_dynamic_network -- --ignored

ci/test-nightly.sh

@@ -1,14 +1,60 @@
 #!/bin/bash -e
 
 cd "$(dirname "$0")/.."
+source ci/upload_ci_artifact.sh
 
-rustc --version
-cargo --version
-
+ci/version-check.sh nightly
 export RUST_BACKTRACE=1
-rustup component add rustfmt-preview
-cargo build --verbose --features unstable
-cargo test --verbose --features unstable
-cargo bench --verbose --features unstable
 
-exit 0
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+# Uncomment this to run nightly test suit
+# _ cargo test --verbose --features=unstable
+
+maybe_cargo_install() {
+  for cmd in "$@"; do
+    set +e
+    cargo "$cmd" --help > /dev/null 2>&1
+    declare exitcode=$?
+    set -e
+    if [[ $exitcode -eq 101 ]]; then
+      _ cargo install cargo-"$cmd"
+    fi
+  done
+}
+
+maybe_cargo_install cov
+
+# Generate coverage data and report via unit-test suite.
+_ cargo cov clean
+_ cargo cov test --lib
+_ cargo cov report
+
+# Generate a coverage report with grcov via lcov.
+if [[ ! -f ./grcov ]]; then
+  uname=$(uname | tr '[:upper:]' '[:lower:]')
+  uname_m=$(uname -m | tr '[:upper:]' '[:lower:]')
+  name=grcov-${uname}-${uname_m}.tar.bz2
+  _ wget "https://github.com/mozilla/grcov/releases/download/v0.2.3/${name}"
+  _ tar -xjf "${name}"
+fi
+_ ./grcov . -t lcov > lcov.info
+_ genhtml -o target/cov/report-lcov --show-details --highlight --ignore-errors source --legend lcov.info
+
+# Upload to tarballs to buildkite.
+_ cd target/cov && tar -cjf cov-report.tar.bz2 report/* && cd -
+_ upload_ci_artifact "target/cov/cov-report.tar.bz2"
+
+_ cd target/cov && tar -cjf lcov-report.tar.bz2 report-lcov/* && cd -
+_ upload_ci_artifact "target/cov/lcov-report.tar.bz2"
+
+if [[ -z "$CODECOV_TOKEN" ]]; then
+  echo CODECOV_TOKEN undefined
+else
+  true
+  # TODO: Why doesn't codecov grok our lcov files?
+  #bash <(curl -s https://codecov.io/bash) -X gcov
+fi

ci/test-stable-perf.sh

@@ -0,0 +1,39 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+if ! ci/version-check.sh stable; then
+  # This job doesn't run within a container, try once to upgrade tooling on a
+  # version check failure
+  rustup install stable
+  ci/version-check.sh stable
+fi
+export RUST_BACKTRACE=1
+export RUSTFLAGS="-D warnings"
+
+./fetch-perf-libs.sh
+export LD_LIBRARY_PATH=$PWD/target/perf-libs:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+export PATH=$PATH:/usr/local/cuda/bin
+
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+FEATURES=cuda,erasure,chacha
+_ cargo test --verbose --features="$FEATURES" --lib
+
+# Run integration tests serially
+for test in tests/*.rs; do
+  test=${test##*/} # basename x
+  test=${test%.rs} # basename x .rs
+  _ cargo test --verbose --jobs=1 --features="$FEATURES" --test="$test"
+done
+
+echo --- ci/localnet-sanity.sh
+(
+  set -x
+  # Assume |cargo build| has populated target/debug/ successfully.
+  export PATH=$PWD/target/debug:$PATH
+  USE_INSTALL=1 ci/localnet-sanity.sh
+)

ci/test-stable.sh

@@ -2,13 +2,43 @@
 
 cd "$(dirname "$0")/.."
 
-rustc --version
-cargo --version
-
+ci/version-check.sh stable
 export RUST_BACKTRACE=1
-rustup component add rustfmt-preview
-cargo fmt -- --write-mode=diff
-cargo build --verbose
-cargo test --verbose
+export RUSTFLAGS="-D warnings"
 
-exit 0
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+_ cargo fmt -- --check
+_ cargo build --verbose
+_ cargo test --verbose --lib
+_ cargo clippy -- --deny=warnings
+
+# Run integration tests serially
+for test in tests/*.rs; do
+  test=${test##*/} # basename x
+  test=${test%.rs} # basename x .rs
+  _ cargo test --verbose --jobs=1 --test="$test"
+done
+
+# Run native program's tests
+for program in programs/native/*; do
+  echo --- "$program"
+  ( 
+    set -x
+    cd "$program"
+    cargo test --verbose
+  )
+done
+
+echo --- ci/localnet-sanity.sh
+(
+  set -x
+  # Assume |cargo build| has populated target/debug/ successfully.
+  export PATH=$PWD/target/debug:$PATH
+  USE_INSTALL=1 ci/localnet-sanity.sh
+)
+
+_ ci/audit.sh

ci/testnet-automation-cleanup.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+echo --- find testnet configuration
+net/gce.sh config -p testnet-automation
+
+echo --- delete testnet
+net/gce.sh delete -p testnet-automation

ci/testnet-automation-json-parser.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+import sys, json
+
+data=json.load(sys.stdin)
+print[\
+   ([result['series'][0]['columns'][1].encode(), result['series'][0]['values'][0][1]]) \
+   for result in data['results']]

ci/testnet-automation.sh

@@ -0,0 +1,80 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+echo --- downloading snap from build artifacts
+buildkite-agent artifact download "solana_*.snap" .
+
+# shellcheck disable=SC1091
+source ci/upload_ci_artifact.sh
+
+[[ -n $ITERATION_WAIT ]] || ITERATION_WAIT=300
+[[ -n $NUMBER_OF_NODES ]] || NUMBER_OF_NODES="10 25 50 100"
+[[ -n $LEADER_CPU_MACHINE_TYPE ]] ||
+  LEADER_CPU_MACHINE_TYPE="n1-standard-16 --accelerator count=2,type=nvidia-tesla-v100"
+[[ -n $CLIENT_COUNT ]] || CLIENT_COUNT=2
+[[ -n $TESTNET_TAG ]] || TESTNET_TAG=testnet-automation
+[[ -n $TESTNET_ZONE ]] || TESTNET_ZONE=us-west1-b
+
+launchTestnet() {
+  declare nodeCount=$1
+  echo --- setup "$nodeCount" node test
+  net/gce.sh create \
+    -n "$nodeCount" -c "$CLIENT_COUNT" \
+    -G "$LEADER_CPU_MACHINE_TYPE" \
+    -p "$TESTNET_TAG" -z "$TESTNET_ZONE"
+
+  echo --- configure database
+  net/init-metrics.sh -e
+
+  echo --- start "$nodeCount" node test
+  net/net.sh start -o noValidatorSanity -S solana_*.snap
+
+  echo --- wait "$ITERATION_WAIT" seconds to complete test
+  sleep "$ITERATION_WAIT"
+
+  declare q_mean_tps='
+    SELECT round(mean("sum_count")) AS "mean_tps" FROM (
+      SELECT sum("count") AS "sum_count"
+        FROM "testnet-automation"."autogen"."counter-banking_stage-process_transactions"
+        WHERE time > now() - 300s GROUP BY time(1s)
+    )'
+
+  declare q_max_tps='
+    SELECT round(max("sum_count")) AS "max_tps" FROM (
+      SELECT sum("count") AS "sum_count"
+        FROM "testnet-automation"."autogen"."counter-banking_stage-process_transactions"
+        WHERE time > now() - 300s GROUP BY time(1s)
+    )'
+
+  declare q_mean_finality='
+    SELECT round(mean("duration_ms")) as "mean_finality"
+      FROM "testnet-automation"."autogen"."leader-finality"
+      WHERE time > now() - 300s'
+
+  declare q_max_finality='
+    SELECT round(max("duration_ms")) as "max_finality"
+      FROM "testnet-automation"."autogen"."leader-finality"
+      WHERE time > now() - 300s'
+
+  declare q_99th_finality='
+    SELECT round(percentile("duration_ms", 99)) as "99th_finality"
+      FROM "testnet-automation"."autogen"."leader-finality"
+      WHERE time > now() - 300s'
+
+  curl -G "https://metrics.solana.com:8086/query?u=${INFLUX_USERNAME}&p=${INFLUX_PASSWORD}" \
+    --data-urlencode "db=$INFLUX_DATABASE" \
+    --data-urlencode "q=$q_mean_tps;$q_max_tps;$q_mean_finality;$q_max_finality;$q_99th_finality" |
+    python ci/testnet-automation-json-parser.py >>TPS"$nodeCount".log
+
+  upload_ci_artifact TPS"$nodeCount".log
+}
+
+# This is needed, because buildkite doesn't let us define an array of numbers.
+# The array is defined as a space separated string of numbers
+# shellcheck disable=SC2206
+nodes_count_array=($NUMBER_OF_NODES)
+
+for n in "${nodes_count_array[@]}"; do
+  launchTestnet "$n"
+done

ci/testnet-deploy.sh

@@ -0,0 +1,165 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")"/..
+
+zone=
+leaderAddress=
+leaderMachineType=
+clientNodeCount=0
+validatorNodeCount=10
+publicNetwork=false
+snapChannel=edge
+releaseChannel=edge
+delete=false
+enableGpu=false
+useReleaseChannel=false
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [name] [cloud] [zone] [options...]
+
+Deploys a CD testnet
+
+  name  - name of the network
+  cloud - cloud provider to use (gce, ec2)
+  zone  - cloud provider zone to deploy the network into
+
+  options:
+   -s edge|beta|stable  - Deploy the specified Snap release channel
+                          (default: $snapChannel)
+   -t edge|beta|stable  - Deploy the specified prebuilt tar from channel
+                          (default: $releaseChannel)
+   -n [number]          - Number of validator nodes (default: $validatorNodeCount)
+   -c [number]          - Number of client nodes (default: $clientNodeCount)
+   -P                   - Use public network IP addresses (default: $publicNetwork)
+   -G                   - Enable GPU, and set count/type of GPUs to use (e.g n1-standard-16 --accelerator count=4,type=nvidia-tesla-k80)
+   -g                   - Enable GPU (default: $enableGpu)
+   -a [address]         - Set the leader node's external IP address to this GCE address
+   -d                   - Delete the network
+
+   Note: the SOLANA_METRICS_CONFIG environment variable is used to configure
+         metrics
+EOF
+  exit $exitcode
+}
+
+netName=$1
+cloudProvider=$2
+zone=$3
+[[ -n $netName ]] || usage
+[[ -n $cloudProvider ]] || usage "Cloud provider not specified"
+[[ -n $zone ]] || usage "Zone not specified"
+shift 3
+
+while getopts "h?p:Pn:c:s:t:gG:a:d" opt; do
+  case $opt in
+  h | \?)
+    usage
+    ;;
+  P)
+    publicNetwork=true
+    ;;
+  n)
+    validatorNodeCount=$OPTARG
+    ;;
+  c)
+    clientNodeCount=$OPTARG
+    ;;
+  s)
+    case $OPTARG in
+    edge|beta|stable)
+      snapChannel=$OPTARG
+      ;;
+    *)
+      usage "Invalid snap channel: $OPTARG"
+      ;;
+    esac
+    ;;
+  t)
+    case $OPTARG in
+    edge|beta|stable)
+      releaseChannel=$OPTARG
+      useReleaseChannel=true
+      ;;
+    *)
+      usage "Invalid release channel: $OPTARG"
+      ;;
+    esac
+    ;;
+  g)
+    enableGpu=true
+    ;;
+  G)
+    enableGpu=true
+    leaderMachineType=$OPTARG
+    ;;
+  a)
+    leaderAddress=$OPTARG
+    ;;
+  d)
+    delete=true
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+
+
+create_args=(
+  -a "$leaderAddress"
+  -c "$clientNodeCount"
+  -n "$validatorNodeCount"
+  -p "$netName"
+  -z "$zone"
+)
+
+if $enableGpu; then
+  if [[ -z $leaderMachineType ]]; then
+    create_args+=(-g)
+  else
+    create_args+=(-G "$leaderMachineType")
+  fi
+fi
+
+if $publicNetwork; then
+  create_args+=(-P)
+fi
+
+set -x
+
+echo "--- $cloudProvider.sh delete"
+time net/"$cloudProvider".sh delete -z "$zone" -p "$netName"
+if $delete; then
+  exit 0
+fi
+
+echo "--- $cloudProvider.sh create"
+time net/"$cloudProvider".sh create "${create_args[@]}"
+net/init-metrics.sh -e
+
+echo --- net.sh start
+maybeRejectExtraNodes=
+if ! $publicNetwork; then
+  maybeRejectExtraNodes="-o rejectExtraNodes"
+fi
+maybeNoValidatorSanity=
+if [[ -n $NO_VALIDATOR_SANITY ]]; then
+  maybeNoValidatorSanity="-o noValidatorSanity"
+fi
+maybeNoLedgerVerify=
+if [[ -n $NO_LEDGER_VERIFY ]]; then
+  maybeNoLedgerVerify="-o noLedgerVerify"
+fi
+# shellcheck disable=SC2086 # Don't want to double quote maybeRejectExtraNodes
+if ! $useReleaseChannel; then
+  time net/net.sh start -s "$snapChannel" $maybeRejectExtraNodes $maybeNoValidatorSanity $maybeNoLedgerVerify
+else
+  time net/net.sh start -t "$releaseChannel" $maybeRejectExtraNodes $maybeNoValidatorSanity $maybeNoLedgerVerify
+fi
+exit 0

ci/testnet-manager.sh

@@ -0,0 +1,350 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")"/..
+
+if [[ -z $BUILDKITE ]]; then
+  echo BUILDKITE not defined
+  exit 1
+fi
+
+if [[ -z $SOLANA_METRICS_PARTIAL_CONFIG ]]; then
+  echo SOLANA_METRICS_PARTIAL_CONFIG not defined
+  exit 1
+fi
+
+if [[ -z $TESTNET ]]; then
+  TESTNET=$(buildkite-agent meta-data get "testnet" --default "")
+fi
+
+if [[ -z $TESTNET_OP ]]; then
+  TESTNET_OP=$(buildkite-agent meta-data get "testnet-operation" --default "")
+fi
+
+if [[ -z $TESTNET || -z $TESTNET_OP ]]; then
+  (
+    cat <<EOF
+steps:
+  - block: "Manage Testnet"
+    fields:
+      - select: "Network"
+        key: "testnet"
+        options:
+          - label: "testnet"
+            value: "testnet"
+          - label: "testnet-perf"
+            value: "testnet-perf"
+          - label: "testnet-master"
+            value: "testnet-master"
+          - label: "testnet-master-perf"
+            value: "testnet-master-perf"
+          - label: "testnet-edge"
+            value: "testnet-edge"
+          - label: "testnet-edge-perf"
+            value: "testnet-edge-perf"
+          - label: "testnet-beta"
+            value: "testnet-beta"
+          - label: "testnet-beta-perf"
+            value: "testnet-beta-perf"
+      - select: "Operation"
+        key: "testnet-operation"
+        default: "sanity-or-restart"
+        options:
+          - label: "Sanity check.  Restart network on failure"
+            value: "sanity-or-restart"
+          - label: "Start (or restart) the network"
+            value: "start"
+          - label: "Stop the network"
+            value: "stop"
+          - label: "Sanity check only"
+            value: "sanity"
+  - command: "ci/$(basename "$0")"
+    agents:
+      - "queue=$BUILDKITE_AGENT_META_DATA_QUEUE"
+EOF
+  ) | buildkite-agent pipeline upload
+  exit 0
+fi
+
+export SOLANA_METRICS_CONFIG="db=$TESTNET,$SOLANA_METRICS_PARTIAL_CONFIG"
+echo "SOLANA_METRICS_CONFIG: $SOLANA_METRICS_CONFIG"
+
+ci/channel-info.sh
+eval "$(ci/channel-info.sh)"
+
+case $TESTNET in
+testnet-edge|testnet-edge-perf|testnet-master|testnet-master-perf)
+  CHANNEL_OR_TAG=edge
+  CHANNEL_BRANCH=$EDGE_CHANNEL
+  ;;
+testnet-beta|testnet-beta-perf)
+  CHANNEL_OR_TAG=beta
+  CHANNEL_BRANCH=$BETA_CHANNEL
+  ;;
+testnet|testnet-perf)
+  if [[ -n $BETA_CHANNEL_LATEST_TAG ]]; then
+    CHANNEL_OR_TAG=$BETA_CHANNEL_LATEST_TAG
+  else
+    CHANNEL_OR_TAG=$STABLE_CHANNEL_LATEST_TAG
+  fi
+  CHANNEL_BRANCH=$BETA_CHANNEL
+  ;;
+*)
+  echo "Error: Invalid TESTNET=$TESTNET"
+  exit 1
+  ;;
+esac
+
+if [[ $BUILDKITE_BRANCH != "$CHANNEL_BRANCH" ]]; then
+  (
+    cat <<EOF
+steps:
+  - trigger: "$BUILDKITE_PIPELINE_SLUG"
+    async: true
+    build:
+      message: "$BUILDKITE_MESSAGE"
+      branch: "$CHANNEL_BRANCH"
+      env:
+        TESTNET: "$TESTNET"
+        TESTNET_OP: "$TESTNET_OP"
+EOF
+  ) | buildkite-agent pipeline upload
+  exit 0
+fi
+
+
+sanity() {
+  echo "--- sanity $TESTNET"
+  case $TESTNET in
+  testnet-edge)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-sanity.sh edge-testnet-solana-com ec2 us-east-1b
+    )
+    ;;
+  testnet-edge-perf)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export REJECT_EXTRA_NODES=1
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-sanity.sh edge-perf-testnet-solana-com ec2 us-east-1a
+    )
+    ;;
+  testnet-beta)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-sanity.sh beta-testnet-solana-com ec2 us-west-1a
+    )
+    ;;
+  testnet-beta-perf)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export REJECT_EXTRA_NODES=1
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-sanity.sh beta-perf-testnet-solana-com ec2 us-west-2b
+    )
+    ;;
+  testnet-master)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-sanity.sh master-testnet-solana-com gce us-west1-b
+    )
+    ;;
+  testnet-master-perf)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export REJECT_EXTRA_NODES=1
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-sanity.sh master-perf-testnet-solana-com gce us-west1-b
+    )
+    ;;
+  testnet)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-sanity.sh testnet-solana-com gce us-east1-c
+    )
+    ;;
+  testnet-perf)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export REJECT_EXTRA_NODES=1
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-sanity.sh perf-testnet-solana-com gce us-west1-b
+    )
+    ;;
+  *)
+    echo "Error: Invalid TESTNET=$TESTNET"
+    exit 1
+    ;;
+  esac
+}
+
+
+start() {
+  declare maybeDelete=$1
+  if [[ -z $maybeDelete ]]; then
+    echo "--- start $TESTNET"
+  else
+    echo "--- stop $TESTNET"
+  fi
+
+  case $TESTNET in
+  testnet-edge)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-deploy.sh edge-testnet-solana-com ec2 us-east-1b \
+        -s "$CHANNEL_OR_TAG" -n 3 -c 0 -P \
+        ${maybeDelete:+-d}
+    )
+    ;;
+  testnet-edge-perf)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-deploy.sh edge-perf-testnet-solana-com ec2 us-east-1a \
+        -g -t "$CHANNEL_OR_TAG" -c 2 \
+        ${maybeDelete:+-d}
+    )
+    ;;
+  testnet-beta)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-deploy.sh beta-testnet-solana-com ec2 us-west-1a \
+        -s "$CHANNEL_OR_TAG" -n 3 -c 0 -P \
+        ${maybeDelete:+-d}
+    )
+    ;;
+  testnet-beta-perf)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-deploy.sh beta-perf-testnet-solana-com ec2 us-west-2b \
+        -g -t "$CHANNEL_OR_TAG" -c 2 \
+        ${maybeDelete:+-d}
+    )
+    ;;
+  testnet-master)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-deploy.sh master-testnet-solana-com gce us-west1-b \
+        -s "$CHANNEL_OR_TAG" -n 3 -c 0 -P -a master-testnet-solana-com \
+        ${maybeDelete:+-d}
+    )
+    ;;
+  testnet-master-perf)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-deploy.sh master-perf-testnet-solana-com gce us-west1-b \
+        -G "n1-standard-16 --accelerator count=2,type=nvidia-tesla-v100" \
+        -t "$CHANNEL_OR_TAG" -c 2 \
+        ${maybeDelete:+-d}
+    )
+    ;;
+  testnet)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-deploy.sh testnet-solana-com gce us-east1-c \
+        -s "$CHANNEL_OR_TAG" -n 3 -g -c 0 -P -a testnet-solana-com  \
+        ${maybeDelete:+-d}
+    )
+    ;;
+  testnet-perf)
+    # shellcheck disable=2030
+    # shellcheck disable=2031
+    (
+      set -ex
+      export NO_LEDGER_VERIFY=1
+      export NO_VALIDATOR_SANITY=1
+      ci/testnet-deploy.sh perf-testnet-solana-com gce us-west1-b \
+        -G "n1-standard-16 --accelerator count=2,type=nvidia-tesla-v100" \
+        -t "$CHANNEL_OR_TAG" -c 2 \
+        ${maybeDelete:+-d}
+    )
+    ;;
+  *)
+    echo "Error: Invalid TESTNET=$TESTNET"
+    exit 1
+    ;;
+  esac
+}
+
+stop() {
+  start delete
+}
+
+case $TESTNET_OP in
+sanity)
+  sanity
+  ;;
+start)
+  start
+  ;;
+stop)
+  stop
+  ;;
+sanity-or-restart)
+  if sanity; then
+    echo Pass
+  else
+    echo "Sanity failed, restarting the network"
+    echo "^^^ +++"
+    start
+  fi
+  ;;
+esac
+
+echo --- fin
+exit 0

ci/testnet-sanity.sh

@@ -0,0 +1,43 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [name] [cloud] [zone]
+
+Sanity check a CD testnet
+
+  name  - name of the network
+  cloud - cloud provider to use (gce, ec2)
+  zone  - cloud provider zone of the network
+
+  Note: the SOLANA_METRICS_CONFIG environment variable is used to configure
+        metrics
+EOF
+  exit $exitcode
+}
+
+netName=$1
+cloudProvider=$2
+zone=$3
+[[ -n $netName ]] || usage ""
+[[ -n $cloudProvider ]] || usage "Cloud provider not specified"
+[[ -n $zone ]] || usage "Zone not specified"
+
+set -x
+echo "--- $cloudProvider.sh config"
+net/"$cloudProvider".sh config -p "$netName" -z "$zone"
+net/init-metrics.sh -e
+echo --- net.sh sanity
+net/net.sh sanity \
+  ${NO_LEDGER_VERIFY:+-o noLedgerVerify} \
+  ${NO_VALIDATOR_SANITY:+-o noValidatorSanity} \
+  ${REJECT_EXTRA_NODES:+-o rejectExtraNodes} \
+
+exit 0

ci/upload_ci_artifact.sh

@@ -0,0 +1,18 @@
+# |source| me
+
+upload_ci_artifact() {
+  echo "--- artifact: $1"
+  if [[ -r "$1" ]]; then
+    ls -l "$1"
+    if ${BUILDKITE:-false}; then
+      (
+        set -x
+        buildkite-agent artifact upload "$1"
+      )
+    fi
+  else
+    echo ^^^ +++
+    echo "$1 not found"
+  fi
+}
+

ci/version-check.sh

@@ -0,0 +1,35 @@
+#!/bin/bash -e
+
+require() {
+  declare expectedProgram="$1"
+  declare expectedVersion="$2"
+
+  read -r program version _ < <($expectedProgram -V)
+
+  declare ok=true
+  [[ $program = "$expectedProgram" ]] || ok=false
+  [[ $version =~ $expectedVersion ]] || ok=false
+
+  echo "Found $program $version"
+  if ! $ok; then
+    echo Error: expected "$expectedProgram $expectedVersion"
+    exit 1
+  fi
+}
+
+case ${1:-stable} in
+nightly)
+  require rustc 1.31.[0-9]+-nightly
+  require cargo 1.31.[0-9]+-nightly
+  ;;
+stable)
+  require rustc 1.30.[0-9]+
+  require cargo 1.30.[0-9]+
+  ;;
+*)
+  echo Error: unknown argument: "$1"
+  exit 1
+  ;;
+esac
+
+exit 0

doc/historian.md

@@ -1,65 +0,0 @@
-The Historian
-===
-
-Create a *Historian* and send it *events* to generate an *event log*, where each *entry*
-is tagged with the historian's latest *hash*. Then ensure the order of events was not tampered
-with by verifying each entry's hash can be generated from the hash in the previous entry:
-
-![historian](https://user-images.githubusercontent.com/55449/36950845-459bdb58-1fb9-11e8-850e-894586f3729b.png)
-
-```rust
-extern crate solana;
-
-use solana::historian::Historian;
-use solana::ledger::{Block, Entry, Hash};
-use solana::event::{generate_keypair, get_pubkey, sign_claim_data, Event};
-use std::thread::sleep;
-use std::time::Duration;
-use std::sync::mpsc::SendError;
-
-fn create_ledger(hist: &Historian<Hash>) -> Result<(), SendError<Event<Hash>>> {
-    sleep(Duration::from_millis(15));
-    let tokens = 42;
-    let keypair = generate_keypair();
-    let event0 = Event::new_claim(get_pubkey(&keypair), tokens, sign_claim_data(&tokens, &keypair));
-    hist.sender.send(event0)?;
-    sleep(Duration::from_millis(10));
-    Ok(())
-}
-
-fn main() {
-    let seed = Hash::default();
-    let hist = Historian::new(&seed, Some(10));
-    create_ledger(&hist).expect("send error");
-    drop(hist.sender);
-    let entries: Vec<Entry<Hash>> = hist.receiver.iter().collect();
-    for entry in &entries {
-        println!("{:?}", entry);
-    }
-    // Proof-of-History: Verify the historian learned about the events
-    // in the same order they appear in the vector.
-    assert!(entries[..].verify(&seed));
-}
-```
-
-Running the program should produce a ledger similar to:
-
-```rust
-Entry { num_hashes: 0, id: [0, ...], event: Tick }
-Entry { num_hashes: 3, id: [67, ...], event: Transaction { tokens: 42 } }
-Entry { num_hashes: 3, id: [123, ...], event: Tick }
-```
-
-Proof-of-History
----
-
-Take note of the last line:
-
-```rust
-assert!(entries[..].verify(&seed));
-```
-
-[It's a proof!](https://en.wikipedia.org/wiki/Curry–Howard_correspondence) For each entry returned by the
-historian, we can verify that `id` is the result of applying a sha256 hash to the previous `id`
-exactly `num_hashes` times, and then hashing then event data on top of that. Because the event data is
-included in the hash, the events cannot be reordered without regenerating all the hashes.

doc/historian.msc

@@ -1,18 +0,0 @@
-msc {
-  client,historian,recorder;
-
-  recorder=>historian [ label = "e0 = Entry{id: h0, n: 0, event: Tick}" ] ;
-  recorder=>recorder [ label = "h1 = hash(h0)" ] ;
-  recorder=>recorder [ label = "h2 = hash(h1)" ] ;
-  client=>historian [ label = "Transaction(d0)" ] ;
-  historian=>recorder [ label = "Transaction(d0)" ] ;
-  recorder=>recorder [ label = "h3 = hash(h2 + d0)" ] ;
-  recorder=>historian [ label = "e1 = Entry{id: hash(h3), n: 3, event: Transaction(d0)}" ] ;
-  recorder=>recorder [ label = "h4 = hash(h3)" ] ;
-  recorder=>recorder [ label = "h5 = hash(h4)" ] ;
-  recorder=>recorder [ label = "h6 = hash(h5)" ] ;
-  recorder=>historian [ label = "e2 = Entry{id: h6, n: 3, event: Tick}" ] ;
-  client=>historian [ label = "collect()" ] ;
-  historian=>client [ label = "entries = [e0, e1, e2]" ] ;
-  client=>client [ label = "entries.verify(h0)" ] ;
-}

doc/json-rpc.md

@@ -0,0 +1,339 @@
+Solana JSON RPC API
+===
+
+Solana nodes accept HTTP requests using the [JSON-RPC 2.0](https://www.jsonrpc.org/specification) specification.
+
+To interact with a Solana node inside a JavaScript application, use the [solana-web3.js](https://github.com/solana-labs/solana-web3.js) library, which gives a convenient interface for the RPC methods.
+
+RPC HTTP Endpoint
+---
+
+**Default port:** 8899
+eg. http://localhost:8899, http://192.168.1.88:8899
+
+RPC PubSub WebSocket Endpoint
+---
+
+**Default port:** 8900
+eg. ws://localhost:8900, http://192.168.1.88:8900
+
+
+Methods
+---
+
+* [confirmTransaction](#confirmtransaction)
+* [getBalance](#getbalance)
+* [getAccountInfo](#getaccountinfo)
+* [getLastId](#getlastid)
+* [getSignatureStatus](#getsignaturestatus)
+* [getTransactionCount](#gettransactioncount)
+* [requestAirdrop](#requestairdrop)
+* [sendTransaction](#sendtransaction)
+* [startSubscriptionChannel](#startsubscriptionchannel)
+
+* [Subscription Websocket](#subscription-websocket)
+  * [accountSubscribe](#accountsubscribe)
+  * [accountUnsubscribe](#accountunsubscribe)
+  * [signatureSubscribe](#signaturesubscribe)
+  * [signatureUnsubscribe](#signatureunsubscribe)
+
+Request Formatting
+---
+
+To make a JSON-RPC request, send an HTTP POST request with a `Content-Type: application/json` header. The JSON request data should contain 4 fields:
+
+* `jsonrpc`, set to `"2.0"`
+* `id`, a unique client-generated identifying integer
+* `method`, a string containing the method to be invoked
+* `params`, a JSON array of ordered parameter values
+
+Example using curl:
+```bash
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getBalance", "params":["83astBRguLMdt2h5U1Tpdq5tjFoJ6noeGwaY3mDLVcri"]}' 192.168.1.88:8899
+```
+
+The response output will be a JSON object with the following fields:
+
+* `jsonrpc`, matching the request specification
+* `id`, matching the request identifier
+* `result`, requested data or success confirmation
+
+Requests can be sent in batches by sending an array of JSON-RPC request objects as the data for a single POST.
+
+Definitions
+---
+
+* Hash: A SHA-256 hash of a chunk of data.
+* Pubkey: The public key of a Ed25519 key-pair.
+* Signature: An Ed25519 signature of a chunk of data.
+* Transaction: A Solana instruction signed by a client key-pair.
+
+JSON RPC API Reference
+---
+
+### confirmTransaction
+Returns a transaction receipt
+
+##### Parameters:
+* `string` - Signature of Transaction to confirm, as base-58 encoded string
+
+##### Results:
+* `boolean` - Transaction status, true if Transaction is confirmed
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"confirmTransaction", "params":["5VERv8NMvzbJMEkV8xnrLkEaWRtSz9CosKDYjCJjBRnbJLgp8uirBgmQpjKhoR4tjF3ZpRzrFmBV6UjKdiSZkQUW"]}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":true,"id":1}
+```
+
+---
+
+### getBalance
+Returns the balance of the account of provided Pubkey
+
+##### Parameters:
+* `string` - Pubkey of account to query, as base-58 encoded string
+
+##### Results:
+* `integer` - quantity, as a signed 64-bit integer
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getBalance", "params":["83astBRguLMdt2h5U1Tpdq5tjFoJ6noeGwaY3mDLVcri"]}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":0,"id":1}
+```
+
+---
+
+### getAccountInfo
+Returns all information associated with the account of provided Pubkey
+
+##### Parameters:
+* `string` - Pubkey of account to query, as base-58 encoded string
+
+##### Results:
+The result field will be a JSON object with the following sub fields:
+
+* `tokens`, number of tokens assigned to this account, as a signed 64-bit integer
+* `program_id`, array of 32 bytes representing the program this account has been assigned to
+* `userdata`, array of bytes representing any userdata associated with the account
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getAccountInfo", "params":["FVxxngPx368XvMCoeskdd6U8cZJFsfa1BEtGWqyAxRj4"]}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":{"program_id":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"tokens":1,"userdata":[3,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,20,0,0,0,0,0,0,0,50,48,53,48,45,48,49,45,48,49,84,48,48,58,48,48,58,48,48,90,252,10,7,28,246,140,88,177,98,82,10,227,89,81,18,30,194,101,199,16,11,73,133,20,246,62,114,39,20,113,189,32,50,0,0,0,0,0,0,0,247,15,36,102,167,83,225,42,133,127,82,34,36,224,207,130,109,230,224,188,163,33,213,13,5,117,211,251,65,159,197,51,0,0,0,0,0,0]},"id":1}
+```
+
+---
+
+### getLastId
+Returns the last entry ID from the ledger
+
+##### Parameters:
+None
+
+##### Results:
+* `string` - the ID of last entry, a Hash as base-58 encoded string
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getLastId"}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":"GH7ome3EiwEr7tu9JuTh2dpYWBJK3z69Xm1ZE3MEE6JC","id":1}
+```
+
+---
+
+### getSignatureStatus
+Returns the status of a given signature.  This method is similar to
+[confirmTransaction](#confirmtransaction) but provides more resolution for error
+events.
+
+##### Parameters:
+* `string` - Signature of Transaction to confirm, as base-58 encoded string
+
+##### Results:
+* `string` - Transaction status:
+    * `Confirmed` - Transaction was successful
+    * `SignatureNotFound` - Unknown transaction
+    * `ProgramRuntimeError` - An error occurred in the program that processed this Transaction
+    * `AccountInUse` - Another Transaction had a write lock one of the Accounts specified in this Transaction.  The Transaction may succeed if retried
+    * `GenericFailure` - Some other error occurred.  **Note**: In the future new Transaction statuses may be added to this list.  It's safe to assume that all new statuses will be more specific error conditions that previously presented as `GenericFailure`
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getSignatureStatus", "params":["5VERv8NMvzbJMEkV8xnrLkEaWRtSz9CosKDYjCJjBRnbJLgp8uirBgmQpjKhoR4tjF3ZpRzrFmBV6UjKdiSZkQUW"]}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":"SignatureNotFound","id":1}
+```
+
+---
+### getTransactionCount
+Returns the current Transaction count from the ledger
+
+##### Parameters:
+None
+
+##### Results:
+* `integer` - count, as unsigned 64-bit integer
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getTransactionCount"}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":268,"id":1}
+```
+
+---
+
+### requestAirdrop
+Requests an airdrop of tokens to a Pubkey
+
+##### Parameters:
+* `string` - Pubkey of account to receive tokens, as base-58 encoded string
+* `integer` - token quantity, as a signed 64-bit integer
+
+##### Results:
+* `string` - Transaction Signature of airdrop, as base-58 encoded string
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"requestAirdrop", "params":["83astBRguLMdt2h5U1Tpdq5tjFoJ6noeGwaY3mDLVcri", 50]}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":"5VERv8NMvzbJMEkV8xnrLkEaWRtSz9CosKDYjCJjBRnbJLgp8uirBgmQpjKhoR4tjF3ZpRzrFmBV6UjKdiSZkQUW","id":1}
+```
+
+---
+
+### sendTransaction
+Creates new transaction
+
+##### Parameters:
+* `array` - array of octets containing a fully-signed Transaction
+
+##### Results:
+* `string` - Transaction Signature, as base-58 encoded string
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"sendTransaction", "params":[[61, 98, 55, 49, 15, 187, 41, 215, 176, 49, 234, 229, 228, 77, 129, 221, 239, 88, 145, 227, 81, 158, 223, 123, 14, 229, 235, 247, 191, 115, 199, 71, 121, 17, 32, 67, 63, 209, 239, 160, 161, 2, 94, 105, 48, 159, 235, 235, 93, 98, 172, 97, 63, 197, 160, 164, 192, 20, 92, 111, 57, 145, 251, 6, 40, 240, 124, 194, 149, 155, 16, 138, 31, 113, 119, 101, 212, 128, 103, 78, 191, 80, 182, 234, 216, 21, 121, 243, 35, 100, 122, 68, 47, 57, 13, 39, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 40, 240, 124, 194, 149, 155, 16, 138, 31, 113, 119, 101, 212, 128, 103, 78, 191, 80, 182, 234, 216, 21, 121, 243, 35, 100, 122, 68, 47, 57, 11, 12, 106, 49, 74, 226, 201, 16, 161, 192, 28, 84, 124, 97, 190, 201, 171, 186, 6, 18, 70, 142, 89, 185, 176, 154, 115, 61, 26, 163, 77, 1, 88, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":"2EBVM6cB8vAAD93Ktr6Vd8p67XPbQzCJX47MpReuiCXJAtcjaxpvWpcg9Ege1Nr5Tk3a2GFrByT7WPBjdsTycY9b","id":1}
+```
+
+---
+
+### Subscription Websocket
+After connect to the RPC PubSub websocket at `ws://<ADDRESS>/`:
+- Submit subscription requests to the websocket using the methods below
+- Multiple subscriptions may be active at once
+
+---
+
+### accountSubscribe
+Subscribe to an account to receive notifications when the userdata for a given account public key changes
+
+##### Parameters:
+* `string` - account Pubkey, as base-58 encoded string
+
+##### Results:
+* `integer` - Subscription id (needed to unsubscribe)
+
+##### Example:
+```bash
+// Request
+{"jsonrpc":"2.0", "id":1, "method":"accountSubscribe", "params":["CM78CPUeXjn8o3yroDHxUtKsZZgoy4GPkPPXfouKNH12"]}
+
+// Result
+{"jsonrpc": "2.0","result": 0,"id": 1}
+```
+
+##### Notification Format:
+```bash
+{"jsonrpc": "2.0","method": "accountNotification", "params": {"result": {"program_id":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"tokens":1,"userdata":[3,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,20,0,0,0,0,0,0,0,50,48,53,48,45,48,49,45,48,49,84,48,48,58,48,48,58,48,48,90,252,10,7,28,246,140,88,177,98,82,10,227,89,81,18,30,194,101,199,16,11,73,133,20,246,62,114,39,20,113,189,32,50,0,0,0,0,0,0,0,247,15,36,102,167,83,225,42,133,127,82,34,36,224,207,130,109,230,224,188,163,33,213,13,5,117,211,251,65,159,197,51,0,0,0,0,0,0]},"subscription":0}}
+```
+
+---
+
+### accountUnsubscribe
+Unsubscribe from account userdata change notifications
+
+##### Parameters:
+* `integer` - id of account Subscription to cancel
+
+##### Results:
+* `bool` - unsubscribe success message
+
+##### Example:
+```bash
+// Request
+{"jsonrpc":"2.0", "id":1, "method":"accountUnsubscribe", "params":[0]}
+
+// Result
+{"jsonrpc": "2.0","result": true,"id": 1}
+```
+
+---
+
+### signatureSubscribe
+Subscribe to a transaction signature to receive notification when the transaction is confirmed
+On `signatureNotification`, the subscription is automatically cancelled
+
+##### Parameters:
+* `string` - Transaction Signature, as base-58 encoded string
+
+##### Results:
+* `integer` - subscription id (needed to unsubscribe)
+
+##### Example:
+```bash
+// Request
+{"jsonrpc":"2.0", "id":1, "method":"signatureSubscribe", "params":["2EBVM6cB8vAAD93Ktr6Vd8p67XPbQzCJX47MpReuiCXJAtcjaxpvWpcg9Ege1Nr5Tk3a2GFrByT7WPBjdsTycY9b"]}
+
+// Result
+{"jsonrpc": "2.0","result": 0,"id": 1}
+```
+
+##### Notification Format:
+```bash
+{"jsonrpc": "2.0","method": "signatureNotification", "params": {"result": "Confirmed","subscription":0}}
+```
+
+---
+
+### signatureUnsubscribe
+Unsubscribe from account userdata change notifications
+
+##### Parameters:
+* `integer` - id of account subscription to cancel
+
+##### Results:
+* `bool` - unsubscribe success message
+
+##### Example:
+```bash
+// Request
+{"jsonrpc":"2.0", "id":1, "method":"signatureUnsubscribe", "params":[0]}
+
+// Result
+{"jsonrpc": "2.0","result": true,"id": 1}
+```

doc/testnet.md

@@ -0,0 +1,52 @@
+# TestNet debugging info
+
+Currently we have three testnets:
+* `testnet` - public beta channel testnet accessible via testnet.solana.com. Runs 24/7
+* `testnet-perf` - private beta channel testnet with clients trying to flood the network
+with transactions until failure.  Runs 24/7
+* `testnet-msater` - public edge channel testnet accessible via master.testnet.solana.com. Runs 24/7
+* `testnet-master-perf` - private edge channel testnet with clients trying to flood the network
+with transactions until failure.  Runs on weekday mornings for a couple hours
+
+## Deploy process
+
+They are deployed with the `ci/testnet-manager.sh` script through a list of [scheduled
+buildkite jobs](https://buildkite.com/solana-labs/testnet-management/settings/schedules).
+Each testnet can be manually manipulated from buildkite as well.  The `-perf`
+testnets use a release tarball while the non`-perf` builds use the snap build
+(we've observed that the snap build runs slower than a tarball but this has yet
+to be root caused).
+
+## Where are the testnet logs?
+
+Attach to the testnet first by running one of:
+```bash
+$ net/gce.sh config testnet-solana-com
+$ net/gce.sh config master-testnet-solana-com
+$ net/gce.sh config perf-testnet-solana-com
+```
+
+Then run:
+```bash
+$ net/ssh.sh
+```
+for log location details
+
+## How do I reset the testnet?
+Manually trigger the [testnet-management](https://buildkite.com/solana-labs/testnet-management) pipeline
+and when prompted select the desired testnet
+
+## How can I scale the tx generation rate?
+
+Increase the TX rate by increasing the number of cores on the client machine which is running
+`bench-tps` or run multiple clients. Decrease by lowering cores or using the rayon env
+variable `RAYON_NUM_THREADS=<xx>`
+
+## How can I test a change on the testnet?
+
+Currently, a merged PR is the only way to test a change on the testnet.  But you
+can run your own testnet using the scripts in the `net/` directory.
+
+## Adjusting the number of clients or validators on the testnet
+Edit `ci/testnet-manager.sh`
+

fetch-perf-libs.sh

@@ -0,0 +1,39 @@
+#!/bin/bash -e
+
+if [[ $(uname) != Linux ]]; then
+  echo Performance libraries are only available for Linux
+  exit 1
+fi
+
+if [[ $(uname -m) != x86_64 ]]; then
+  echo Performance libraries are only available for x86_64 architecture
+  exit 1
+fi
+
+mkdir -p target/perf-libs
+(
+  cd target/perf-libs
+  (
+    set -x
+    curl https://solana-perf.s3.amazonaws.com/v0.10.3/x86_64-unknown-linux-gnu/solana-perf.tgz | tar zxvf -
+  )
+
+  if [[ -r /usr/local/cuda/version.txt && -r cuda-version.txt ]]; then
+    if ! diff /usr/local/cuda/version.txt cuda-version.txt > /dev/null; then
+        echo ==============================================
+        echo Warning: possible CUDA version mismatch
+        echo
+        echo "Expected version: $(cat cuda-version.txt)"
+        echo "Detected version: $(cat /usr/local/cuda/version.txt)"
+        echo ==============================================
+    fi
+  else
+    echo ==============================================
+    echo Warning: unable to validate CUDA version
+    echo ==============================================
+  fi
+
+  echo "Downloaded solana-perf version: $(cat solana-perf-HEAD.txt)"
+)
+
+exit 0

multinode-demo/client.sh

@@ -1,16 +1,25 @@
 #!/bin/bash -e
 
-if [[ -z "$1" ]]; then
-  echo "usage: $0 [network path to solana repo on leader machine]"
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+usage() {
+  if [[ -n $1 ]]; then
+    echo "$*"
+    echo
+  fi
+  echo "usage: $0 [extra args]"
+  echo
+  echo " Run bench-tps "
+  echo
+  echo "   extra args: additional arguments are pass along to solana-bench-tps"
+  echo
   exit 1
+}
+
+if [[ -z $1 ]]; then # default behavior
+  $solana_bench_tps --identity config-private/client-id.json --network 127.0.0.1:8001 --duration 90
+else
+  $solana_bench_tps "$@"
 fi
-
-LEADER="$1"
-
-set -x
-export RUST_LOG=solana=info
-rsync -v -e ssh "$LEADER/leader.json" .
-rsync -v -e ssh "$LEADER/mint-demo.json" .
-
-cargo run --release --bin solana-client-demo -- \
-  -l leader.json < mint-demo.json 2>&1 | tee client.log

multinode-demo/common.sh

@@ -0,0 +1,202 @@
+# |source| this file
+#
+# Common utilities shared by other scripts in this directory
+#
+# The following directive disable complaints about unused variables in this
+# file:
+# shellcheck disable=2034
+#
+
+rsync=rsync
+leader_logger="tee leader.log"
+validator_logger="tee validator.log"
+drone_logger="tee drone.log"
+
+if [[ $(uname) != Linux ]]; then
+  # Protect against unsupported configurations to prevent non-obvious errors
+  # later. Arguably these should be fatal errors but for now prefer tolerance.
+  if [[ -n $USE_SNAP ]]; then
+    echo "Warning: Snap is not supported on $(uname)"
+    USE_SNAP=
+  fi
+  if [[ -n $SOLANA_CUDA ]]; then
+    echo "Warning: CUDA is not supported on $(uname)"
+    SOLANA_CUDA=
+  fi
+fi
+
+if [[ -d $SNAP ]]; then # Running inside a Linux Snap?
+  solana_program() {
+    declare program="$1"
+    printf "%s/command-%s.wrapper" "$SNAP" "$program"
+  }
+  rsync="$SNAP"/bin/rsync
+  multilog="$SNAP/bin/multilog t s16777215 n200"
+  leader_logger="$multilog $SNAP_DATA/leader"
+  validator_logger="$multilog t $SNAP_DATA/validator"
+  drone_logger="$multilog $SNAP_DATA/drone"
+  # Create log directories manually to prevent multilog from creating them as
+  # 0700
+  mkdir -p "$SNAP_DATA"/{drone,leader,validator}
+
+elif [[ -n $USE_SNAP ]]; then # Use the Linux Snap binaries
+  solana_program() {
+    declare program="$1"
+    printf "solana.%s" "$program"
+  }
+elif [[ -n $USE_INSTALL ]]; then # Assume |cargo install| was run
+  solana_program() {
+    declare program="$1"
+    printf "solana-%s" "$program"
+  }
+  # CUDA was/wasn't selected at build time, can't affect CUDA state here
+  unset SOLANA_CUDA
+else
+  solana_program() {
+    declare program="$1"
+    declare features=""
+    if [[ "$program" =~ ^(.*)-cuda$ ]]; then
+      program=${BASH_REMATCH[1]}
+      features="--features=cuda"
+    fi
+    if [[ -z $DEBUG ]]; then
+      maybe_release=--release
+    fi
+    printf "cargo run $maybe_release --bin solana-%s %s -- " "$program" "$features"
+  }
+  if [[ -n $SOLANA_CUDA ]]; then
+    # shellcheck disable=2154 # 'here' is referenced but not assigned
+    if [[ -z $here ]]; then
+      echo "|here| is not defined"
+      exit 1
+    fi
+
+    # Locate perf libs downloaded by |./fetch-perf-libs.sh|
+    LD_LIBRARY_PATH=$(cd "$here" && dirname "$PWD"/target/perf-libs):$LD_LIBRARY_PATH
+    export LD_LIBRARY_PATH
+  fi
+fi
+
+solana_bench_tps=$(solana_program bench-tps)
+solana_wallet=$(solana_program wallet)
+solana_drone=$(solana_program drone)
+solana_fullnode=$(solana_program fullnode)
+solana_fullnode_config=$(solana_program fullnode-config)
+solana_fullnode_cuda=$(solana_program fullnode-cuda)
+solana_genesis=$(solana_program genesis)
+solana_keygen=$(solana_program keygen)
+solana_ledger_tool=$(solana_program ledger-tool)
+
+export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info
+export RUST_BACKTRACE=1
+
+# shellcheck source=scripts/configure-metrics.sh
+source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh
+
+tune_networking() {
+  # Skip in CI
+  [[ -z $CI ]] || return 0
+
+  # Reference: https://medium.com/@CameronSparr/increase-os-udp-buffers-to-improve-performance-51d167bb1360
+  if [[ $(uname) = Linux ]]; then
+    (
+      set -x +e
+      # test the existence of the sysctls before trying to set them
+      # go ahead and return true and don't exit if these calls fail
+      sysctl net.core.rmem_max 2>/dev/null 1>/dev/null &&
+          sudo sysctl -w net.core.rmem_max=1610612736 1>/dev/null 2>/dev/null
+
+      sysctl net.core.rmem_default 2>/dev/null 1>/dev/null &&
+          sudo sysctl -w net.core.rmem_default=1610612736 1>/dev/null 2>/dev/null
+
+      sysctl net.core.wmem_max 2>/dev/null 1>/dev/null &&
+          sudo sysctl -w net.core.wmem_max=1610612736 1>/dev/null 2>/dev/null
+
+      sysctl net.core.wmem_default 2>/dev/null 1>/dev/null &&
+          sudo sysctl -w net.core.wmem_default=1610612736 1>/dev/null 2>/dev/null
+    ) || true
+  fi
+
+  if [[ $(uname) = Darwin ]]; then
+    (
+      if [[ $(sysctl net.inet.udp.maxdgram | cut -d\  -f2) != 65535 ]]; then
+        echo "Adjusting maxdgram to allow for large UDP packets, see BLOB_SIZE in src/packet.rs:"
+        set -x
+        sudo sysctl net.inet.udp.maxdgram=65535
+      fi
+    )
+
+  fi
+}
+
+
+SOLANA_CONFIG_DIR=${SNAP_DATA:-$PWD}/config
+SOLANA_CONFIG_PRIVATE_DIR=${SNAP_DATA:-$PWD}/config-private
+SOLANA_CONFIG_VALIDATOR_DIR=${SNAP_DATA:-$PWD}/config-validator
+SOLANA_CONFIG_CLIENT_DIR=${SNAP_USER_DATA:-$PWD}/config-client
+
+rsync_url() { # adds the 'rsync://` prefix to URLs that need it
+  declare url="$1"
+
+  if [[ $url =~ ^.*:.*$ ]]; then
+    # assume remote-shell transport when colon is present, use $url unmodified
+    echo "$url"
+    return 0
+  fi
+
+  if [[ -d $url ]]; then
+    # assume local directory if $url is a valid directory, use $url unmodified
+    echo "$url"
+    return 0
+  fi
+
+  # Default to rsync:// URL
+  echo "rsync://$url"
+}
+
+# called from drone, validator, client
+find_leader() {
+  declare leader leader_address
+  declare shift=0
+
+  if [[ -d $SNAP ]]; then
+    if [[ -n $1 ]]; then
+      usage "Error: unexpected parameter: $1"
+    fi
+
+    # Select leader from the Snap configuration
+    leader_ip=$(snapctl get leader-ip)
+    if [[ -z $leader_ip ]]; then
+      leader=testnet.solana.com
+      leader_ip=$(dig +short "${leader%:*}" | head -n1)
+      if [[ -z $leader_ip ]]; then
+          usage "Error: unable to resolve IP address for $leader"
+      fi
+    fi
+    leader=$leader_ip
+    leader_address=$leader_ip:8001
+  else
+    if [[ -z $1 ]]; then
+      leader=${here}/..        # Default to local tree for rsync
+      leader_address=127.0.0.1:8001 # Default to local leader
+    elif [[ -z $2 ]]; then
+      leader=$1
+
+      declare leader_ip
+      leader_ip=$(dig +short "${leader%:*}" | head -n1)
+
+      if [[ -z $leader_ip ]]; then
+          usage "Error: unable to resolve IP address for $leader"
+      fi
+
+      leader_address=$leader_ip:8001
+      shift=1
+    else
+      leader=$1
+      leader_address=$2
+      shift=2
+    fi
+  fi
+
+  echo "$leader" "$leader_address" "$shift"
+}

multinode-demo/drone.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Starts an instance of solana-drone
+#
+here=$(dirname "$0")
+
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+usage() {
+  if [[ -n $1 ]]; then
+    echo "$*"
+    echo
+  fi
+  echo "usage: $0 [network entry point]"
+  echo
+  echo " Run an airdrop drone for the specified network"
+  echo
+  exit 1
+}
+
+read -r _ leader_address shift < <(find_leader "${@:1:1}")
+shift "$shift"
+
+[[ -f "$SOLANA_CONFIG_PRIVATE_DIR"/mint.json ]] || {
+  echo "$SOLANA_CONFIG_PRIVATE_DIR/mint.json not found, create it by running:"
+  echo
+  echo "  ${here}/setup.sh -t leader"
+  exit 1
+}
+
+set -ex
+
+trap 'kill "$pid" && wait "$pid"' INT TERM
+$solana_drone \
+  --keypair "$SOLANA_CONFIG_PRIVATE_DIR"/mint.json \
+  --network "$leader_address" \
+  > >($drone_logger) 2>&1 &
+pid=$!
+wait "$pid"

multinode-demo/leader.sh

@@ -1,4 +1,41 @@
 #!/bin/bash
-export RUST_LOG=solana=info
-sudo sysctl -w net.core.rmem_max=26214400
-cat genesis.log | cargo run --bin solana-fullnode -- -l leader.json
+#
+# Starts a leader node
+#
+
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+# shellcheck source=scripts/oom-score-adj.sh
+source "$here"/../scripts/oom-score-adj.sh
+
+if [[ -d "$SNAP" ]]; then
+  # Exit if mode is not yet configured
+  # (typically the case after the Snap is first installed)
+  [[ -n "$(snapctl get mode)" ]] || exit 0
+fi
+
+[[ -f "$SOLANA_CONFIG_DIR"/leader.json ]] || {
+  echo "$SOLANA_CONFIG_DIR/leader.json not found, create it by running:"
+  echo
+  echo "  ${here}/setup.sh"
+  exit 1
+}
+
+if [[ -n "$SOLANA_CUDA" ]]; then
+  program="$solana_fullnode_cuda"
+else
+  program="$solana_fullnode"
+fi
+
+tune_networking
+
+trap 'kill "$pid" && wait "$pid"' INT TERM
+$program \
+  --identity "$SOLANA_CONFIG_DIR"/leader.json \
+  --ledger "$SOLANA_CONFIG_DIR"/ledger \
+  > >($leader_logger) 2>&1 &
+pid=$!
+oom_score_adj "$pid" 1000
+wait "$pid"

multinode-demo/setup.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+#
+# Creates a fullnode configuration
+#
+
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+usage () {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [-n num_tokens] [-l] [-p] [-t node_type]
+
+Creates a fullnode configuration
+
+ -n num_tokens  - Number of tokens to create
+ -l             - Detect network address from local machine configuration, which
+                  may be a private IP address unaccessible on the Intenet (default)
+ -p             - Detect public address using public Internet servers
+ -t node_type   - Create configuration files only for this kind of node.  Valid
+                  options are validator or leader.  Creates configuration files
+                  for both by default
+
+EOF
+  exit $exitcode
+}
+
+ip_address_arg=-l
+num_tokens=1000000000
+node_type_leader=true
+node_type_validator=true
+node_type_client=true
+while getopts "h?n:lpt:" opt; do
+  case $opt in
+  h|\?)
+    usage
+    exit 0
+    ;;
+  l)
+    ip_address_arg=-l
+    ;;
+  p)
+    ip_address_arg=-p
+    ;;
+  n)
+    num_tokens="$OPTARG"
+    ;;
+  t)
+    node_type="$OPTARG"
+    case $OPTARG in
+    leader)
+      node_type_leader=true
+      node_type_validator=false
+      node_type_client=false
+      ;;
+    validator)
+      node_type_leader=false
+      node_type_validator=true
+      node_type_client=false
+      ;;
+    client)
+      node_type_leader=false
+      node_type_validator=false
+      node_type_client=true
+      ;;
+    *)
+      usage "Error: unknown node type: $node_type"
+      ;;
+    esac
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+
+
+set -e
+
+for i in "$SOLANA_CONFIG_DIR" "$SOLANA_CONFIG_VALIDATOR_DIR" "$SOLANA_CONFIG_PRIVATE_DIR"; do
+  echo "Cleaning $i"
+  rm -rvf "$i"
+  mkdir -p "$i"
+done
+
+if $node_type_client; then
+  client_id_path="$SOLANA_CONFIG_PRIVATE_DIR"/client-id.json
+  $solana_keygen -o "$client_id_path"
+  ls -lhR "$SOLANA_CONFIG_PRIVATE_DIR"/
+fi
+
+if $node_type_leader; then
+  leader_address_args=("$ip_address_arg")
+  leader_id_path="$SOLANA_CONFIG_PRIVATE_DIR"/leader-id.json
+  mint_path="$SOLANA_CONFIG_PRIVATE_DIR"/mint.json
+
+  $solana_keygen -o "$leader_id_path"
+
+  echo "Creating $mint_path with $num_tokens tokens"
+  $solana_keygen -o "$mint_path"
+
+  echo "Creating $SOLANA_CONFIG_DIR/ledger"
+  $solana_genesis --tokens="$num_tokens" --ledger "$SOLANA_CONFIG_DIR"/ledger < "$mint_path"
+
+  echo "Creating $SOLANA_CONFIG_DIR/leader.json"
+  $solana_fullnode_config --keypair="$leader_id_path" "${leader_address_args[@]}" > "$SOLANA_CONFIG_DIR"/leader.json
+
+  ls -lhR "$SOLANA_CONFIG_DIR"/
+  ls -lhR "$SOLANA_CONFIG_PRIVATE_DIR"/
+fi
+
+
+if $node_type_validator; then
+  validator_address_args=("$ip_address_arg" -b 9000)
+  validator_id_path="$SOLANA_CONFIG_PRIVATE_DIR"/validator-id.json
+
+  $solana_keygen -o "$validator_id_path"
+
+  echo "Creating $SOLANA_CONFIG_VALIDATOR_DIR/validator.json"
+  $solana_fullnode_config --keypair="$validator_id_path" "${validator_address_args[@]}" > "$SOLANA_CONFIG_VALIDATOR_DIR"/validator.json
+
+  ls -lhR "$SOLANA_CONFIG_VALIDATOR_DIR"/
+fi

multinode-demo/validator-x.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+# Start a dynamically-configured validator node
+#
+
+here=$(dirname "$0")
+
+exec "$here"/validator.sh -x "$@"

multinode-demo/validator.sh

@@ -1,23 +1,100 @@
-#!/bin/bash -e
+#!/bin/bash
+#
+# Start a validator node
+#
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
 
-if [[ -z "$1" ]]; then
-  echo "usage: $0 [network path to solana repo on leader machine]"
-  exit 1
+# shellcheck source=scripts/oom-score-adj.sh
+source "$here"/../scripts/oom-score-adj.sh
+
+if [[ -d "$SNAP" ]]; then
+  # Exit if mode is not yet configured
+  # (typically the case after the Snap is first installed)
+  [[ -n "$(snapctl get mode)" ]] || exit 0
 fi
 
-LEADER="$1"
+usage() {
+  if [[ -n $1 ]]; then
+    echo "$*"
+    echo
+  fi
+  echo "usage: $0 [-x] [rsync network path to leader] [network entry point]"
+  echo
+  echo " Start a validator on the specified network"
+  echo
+  echo "   -x: runs a new, dynamically-configured validator"
+  echo
+  exit 1
+}
 
-set -x
+if [[ $1 = -h ]]; then
+  usage
+fi
 
-rsync -v -e ssh "$LEADER/mint-demo.json" .
-rsync -v -e ssh "$LEADER/leader.json" .
-rsync -v -e ssh "$LEADER/genesis.log" .
-rsync -v -e ssh "$LEADER/libcuda_verify_ed25519.a" .
+if [[ $1 == -x ]]; then
+  self_setup=1
+  shift
+else
+  self_setup=0
+fi
 
-export RUST_LOG=solana=info
+if [[ -n $3 ]]; then
+  usage
+fi
 
-sudo sysctl -w net.core.rmem_max=26214400
+read -r leader leader_address shift < <(find_leader "${@:1:2}")
+shift "$shift"
 
-cat genesis.log | \
-  cargo run --release --features=cuda --bin solana-fullnode -- \
-    -l validator.json -s validator.json -v leader.json -b 9000 -d 2>&1 | tee validator-tee.log
+if [[ -n $SOLANA_CUDA ]]; then
+  program=$solana_fullnode_cuda
+else
+  program=$solana_fullnode
+fi
+
+if ((!self_setup)); then
+  [[ -f $SOLANA_CONFIG_VALIDATOR_DIR/validator.json ]] || {
+    echo "$SOLANA_CONFIG_VALIDATOR_DIR/validator.json not found, create it by running:"
+    echo
+    echo "  ${here}/setup.sh"
+    exit 1
+  }
+  validator_json_path=$SOLANA_CONFIG_VALIDATOR_DIR/validator.json
+  SOLANA_LEADER_CONFIG_DIR=$SOLANA_CONFIG_VALIDATOR_DIR/leader-config
+else
+  mkdir -p "$SOLANA_CONFIG_PRIVATE_DIR"
+  validator_id_path=$SOLANA_CONFIG_PRIVATE_DIR/validator-id-x$$.json
+  $solana_keygen -o "$validator_id_path"
+
+  mkdir -p "$SOLANA_CONFIG_VALIDATOR_DIR"
+  validator_json_path=$SOLANA_CONFIG_VALIDATOR_DIR/validator-x$$.json
+
+  port=9000
+  (((port += ($$ % 1000)) && (port == 9000) && port++))
+
+  $solana_fullnode_config --keypair="$validator_id_path" -l -b "$port" > "$validator_json_path"
+
+  SOLANA_LEADER_CONFIG_DIR=$SOLANA_CONFIG_VALIDATOR_DIR/leader-config-x$$
+fi
+
+rsync_leader_url=$(rsync_url "$leader")
+
+tune_networking
+
+set -ex
+$rsync -vPr "$rsync_leader_url"/config/ "$SOLANA_LEADER_CONFIG_DIR"
+[[ -d $SOLANA_LEADER_CONFIG_DIR/ledger ]] || {
+  echo "Unable to retrieve ledger from $rsync_leader_url"
+  exit 1
+}
+
+trap 'kill "$pid" && wait "$pid"' INT TERM
+$program \
+  --identity "$validator_json_path" \
+  --network "$leader_address" \
+  --ledger "$SOLANA_LEADER_CONFIG_DIR"/ledger \
+  > >($validator_logger) 2>&1 &
+pid=$!
+oom_score_adj "$pid" 1000
+wait "$pid"

net/.gitignore

@@ -0,0 +1,2 @@
+/config/
+/log/

net/README.md

@@ -0,0 +1,100 @@
+
+# Network Management
+This directory contains scripts useful for working with a test network.  It's
+intended to be both dev and CD friendly.
+
+### User Account Prerequisites
+
+GCP and AWS are supported.
+
+#### GCP
+First authenticate with
+```bash
+$ gcloud auth login
+```
+
+#### AWS
+Obtain your credentials from the AWS IAM Console and configure the AWS CLI with
+```bash
+$ aws configure
+```
+More information on AWS CLI configuration can be found [here](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-quick-configuration)
+
+### Metrics configuration
+Ensure that `$(whoami)` is the name of an InfluxDB user account with enough
+access to create a new InfluxDB database.  Ask mvines@ for help if needed.
+
+## Quick Start
+
+NOTE: This example uses GCP.  If you are using AWS, replace `./gce.sh` with
+`./ec2.sh` in the commands.
+
+```bash
+$ cd net/
+$ ./gce.sh create -n 5 -c 1  #<-- Create a GCE testnet with 5 validators, 1 client (billing starts here)
+$ ./init-metrics.sh $(whoami)   #<-- Configure a metrics database for the testnet
+$ ./net.sh start             #<-- Deploy the network from the local workspace
+$ ./ssh.sh                   #<-- Details on how to ssh into any testnet node
+$ ./gce.sh delete            #<-- Dispose of the network (billing stops here)
+```
+
+## Tips
+
+### Running the network over public IP addresses
+By default private IP addresses are used with all instances in the same
+availability zone to avoid GCE network engress charges.  However to run the
+network over public IP addresses:
+```bash
+$ ./gce.sh create -P ...
+```
+or
+```bash
+$ ./ec2.sh create -P ...
+```
+
+### Deploying a Snap-based network
+To deploy the latest pre-built `edge` channel Snap (ie, latest from the `master`
+branch), once the testnet has been created run:
+
+```bash
+$ ./net.sh start -s edge
+```
+
+### Enabling CUDA
+First ensure the network instances are created with GPU enabled:
+```bash
+$ ./gce.sh create -g ...
+```
+or
+```bash
+$ ./ec2.sh create -g ...
+```
+
+If deploying a Snap-based network nothing further is required, as GPU presence
+is detected at runtime and the CUDA build is auto selected.
+
+If deploying a locally-built network, first run `./fetch-perf-libs.sh` then
+ensure the `cuda` feature is specified at network start:
+```bash
+$ ./net.sh start -f "cuda,erasure"
+```
+
+### How to interact with a CD testnet deployed by ci/testnet-deploy.sh
+
+**AWS-Specific Extra Setup**: Follow the steps in `scripts/add-solana-user-authorized_keys.sh`,
+then redeploy the testnet before continuing in this section.
+
+Taking **master-testnet-solana-com** as an example, configure your workspace for
+the testnet using:
+```bash
+$ ./gce.sh config -p master-testnet-solana-com
+```
+or
+```bash
+$ ./ec2.sh config -p master-testnet-solana-com
+```
+
+Then run the following for details on how to ssh into any testnet node
+```bash
+$ ./ssh.sh
+```

net/common.sh

@@ -0,0 +1,58 @@
+# |source| this file
+#
+# Common utilities shared by other scripts in this directory
+#
+# The following directive disable complaints about unused variables in this
+# file:
+# shellcheck disable=2034
+#
+
+netDir=$(
+  cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+  echo "$PWD"
+)
+netConfigDir="$netDir"/config
+netLogDir="$netDir"/log
+mkdir -p "$netConfigDir" "$netLogDir"
+
+# shellcheck source=scripts/configure-metrics.sh
+source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh
+
+configFile="$netConfigDir/config"
+
+entrypointIp=
+publicNetwork=
+leaderIp=
+netBasename=
+sshPrivateKey=
+clientIpList=()
+sshOptions=()
+validatorIpList=()
+
+buildSshOptions() {
+  sshOptions=(
+    -o "BatchMode=yes"
+    -o "StrictHostKeyChecking=no"
+    -o "UserKnownHostsFile=/dev/null"
+    -o "User=solana"
+    -o "IdentityFile=$sshPrivateKey"
+    -o "LogLevel=ERROR"
+    -F /dev/null
+  )
+}
+
+loadConfigFile() {
+  [[ -r $configFile ]] || usage "Config file unreadable: $configFile"
+
+  # shellcheck source=/dev/null
+  source "$configFile"
+  [[ -n "$entrypointIp" ]] || usage "Config file invalid, entrypointIp unspecified: $configFile"
+  [[ -n "$publicNetwork" ]] || usage "Config file invalid, publicNetwork unspecified: $configFile"
+  [[ -n "$leaderIp" ]] || usage "Config file invalid, leaderIp unspecified: $configFile"
+  [[ -n "$netBasename" ]] || usage "Config file invalid, netBasename unspecified: $configFile"
+  [[ -n $sshPrivateKey ]] || usage "Config file invalid, sshPrivateKey unspecified: $configFile"
+  [[ ${#validatorIpList[@]} -gt 0 ]] || usage "Config file invalid, validatorIpList unspecified: $configFile"
+
+  buildSshOptions
+  configureMetrics
+}

net/ec2.sh

@@ -0,0 +1 @@
+gce.sh

net/gce.sh

@@ -0,0 +1,430 @@
+#!/bin/bash -e
+
+here=$(dirname "$0")
+# shellcheck source=net/common.sh
+source "$here"/common.sh
+
+cloudProvider=$(basename "$0" .sh)
+bootDiskType=""
+case $cloudProvider in
+gce)
+  # shellcheck source=net/scripts/gce-provider.sh
+  source "$here"/scripts/gce-provider.sh
+
+  cpuImageName="ubuntu-1804-bionic-v20181029 --image-project ubuntu-os-cloud"
+
+  # TODO: GPU image is still 16.04-based pending resolution of
+  #       https://github.com/solana-labs/solana/issues/1702
+  gpuImageName="ubuntu-16-04-cuda-9-2-new"
+  imageName=$cpuImageName
+
+  cpuLeaderMachineType=n1-standard-16
+  gpuLeaderMachineType="$cpuLeaderMachineType --accelerator count=4,type=nvidia-tesla-k80"
+  leaderMachineType=$cpuLeaderMachineType
+  validatorMachineType=n1-standard-16
+  clientMachineType=n1-standard-16
+  ;;
+ec2)
+  # shellcheck source=net/scripts/ec2-provider.sh
+  source "$here"/scripts/ec2-provider.sh
+
+  # Deep Learning AMI (Ubuntu 16.04-based)
+  cpuImageName="ami-0466e26ccc0e752c1"
+  gpuImageName="$cpuImageName"
+  imageName=$cpuImageName
+
+  cpuLeaderMachineType=m4.4xlarge
+  gpuLeaderMachineType=p2.xlarge
+  leaderMachineType=$cpuLeaderMachineType
+  validatorMachineType=m4.2xlarge
+  clientMachineType=m4.2xlarge
+  ;;
+*)
+  echo "Error: Unknown cloud provider: $cloudProvider"
+  ;;
+esac
+
+
+prefix=testnet-dev-${USER//[^A-Za-z0-9]/}
+validatorNodeCount=5
+clientNodeCount=1
+leaderBootDiskSizeInGb=1000
+validatorBootDiskSizeInGb=$leaderBootDiskSizeInGb
+clientBootDiskSizeInGb=75
+
+publicNetwork=false
+enableGpu=false
+leaderAddress=
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [create|config|delete] [common options] [command-specific options]
+
+Manage testnet instances
+
+ create - create a new testnet (implies 'config')
+ config - configure the testnet and write a config file describing it
+ delete - delete the testnet
+
+ common options:
+   -p [prefix]      - Optional common prefix for instance names to avoid
+                      collisions (default: $prefix)
+
+ create-specific options:
+   -n [number]      - Number of validator nodes (default: $validatorNodeCount)
+   -c [number]      - Number of client nodes (default: $clientNodeCount)
+   -P               - Use public network IP addresses (default: $publicNetwork)
+   -z [zone]        - Zone for the nodes (default: $zone)
+   -g               - Enable GPU (default: $enableGpu)
+   -G               - Enable GPU, and set count/type of GPUs to use (e.g $cpuLeaderMachineType --accelerator count=4,type=nvidia-tesla-k80)
+   -a [address]     - Set the leader node's external IP address to this value.
+                      For GCE, [address] is the "name" of the desired External
+                      IP Address.
+                      For EC2, [address] is the "allocation ID" of the desired
+                      Elastic IP.
+   -d [disk-type]   - Specify a boot disk type (default None) Use pd-ssd to get ssd on GCE.
+
+ config-specific options:
+   none
+
+ delete-specific options:
+   none
+
+EOF
+  exit $exitcode
+}
+
+
+command=$1
+[[ -n $command ]] || usage
+shift
+[[ $command = create || $command = config || $command = delete ]] || usage "Invalid command: $command"
+
+while getopts "h?p:Pn:c:z:gG:a:d:" opt; do
+  case $opt in
+  h | \?)
+    usage
+    ;;
+  p)
+    [[ ${OPTARG//[^A-Za-z0-9-]/} == "$OPTARG" ]] || usage "Invalid prefix: \"$OPTARG\", alphanumeric only"
+    prefix=$OPTARG
+    ;;
+  P)
+    publicNetwork=true
+    ;;
+  n)
+    validatorNodeCount=$OPTARG
+    ;;
+  c)
+    clientNodeCount=$OPTARG
+    ;;
+  z)
+    cloud_SetZone "$OPTARG"
+    ;;
+  g)
+    enableGpu=true
+    leaderMachineType=$gpuLeaderMachineType
+    imageName=$gpuImageName
+    ;;
+  G)
+    enableGpu=true
+    leaderMachineType="$OPTARG"
+    imageName=$gpuImageName
+    ;;
+  a)
+    leaderAddress=$OPTARG
+    ;;
+  d)
+    bootDiskType=$OPTARG
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+shift $((OPTIND - 1))
+
+[[ -z $1 ]] || usage "Unexpected argument: $1"
+sshPrivateKey="$netConfigDir/id_$prefix"
+
+
+# cloud_ForEachInstance [cmd] [extra args to cmd]
+#
+# Execute a command for each element in the `instances` array
+#
+#   cmd   - The command to execute on each instance
+#           The command will receive arguments followed by any
+#           additionl arguments supplied to cloud_ForEachInstance:
+#               name     - name of the instance
+#               publicIp - The public IP address of this instance
+#               privateIp - The priate IP address of this instance
+#               count    - Monotonically increasing count for each
+#                          invocation of cmd, starting at 1
+#               ...      - Extra args to cmd..
+#
+#
+cloud_ForEachInstance() {
+  declare cmd="$1"
+  shift
+  [[ -n $cmd ]] || { echo cloud_ForEachInstance: cmd not specified; exit 1; }
+
+  declare count=1
+  for info in "${instances[@]}"; do
+    declare name publicIp privateIp
+    IFS=: read -r name publicIp privateIp < <(echo "$info")
+
+    eval "$cmd" "$name" "$publicIp" "$privateIp" "$count" "$@"
+    count=$((count + 1))
+  done
+}
+
+prepareInstancesAndWriteConfigFile() {
+  $metricsWriteDatapoint "testnet-deploy net-config-begin=1"
+
+  cat >> "$configFile" <<EOF
+# autogenerated at $(date)
+netBasename=$prefix
+publicNetwork=$publicNetwork
+sshPrivateKey=$sshPrivateKey
+EOF
+
+  buildSshOptions
+
+  recordInstanceIp() {
+    declare name="$1"
+    declare publicIp="$2"
+    declare privateIp="$3"
+
+    declare arrayName="$5"
+
+    echo "$arrayName+=($publicIp)  # $name" >> "$configFile"
+    if [[ $arrayName = "leaderIp" ]]; then
+      if $publicNetwork; then
+        echo "entrypointIp=$publicIp" >> "$configFile"
+      else
+        echo "entrypointIp=$privateIp" >> "$configFile"
+      fi
+    fi
+  }
+
+  waitForStartupComplete() {
+    declare name="$1"
+    declare publicIp="$2"
+
+    echo "Waiting for $name to finish booting..."
+    (
+      set -x
+      for i in $(seq 1 60); do
+        if timeout 20s ssh "${sshOptions[@]}" "$publicIp" "test -f /.instance-startup-complete"; then
+          exit 0
+        fi
+        sleep 2
+        echo "Retry $i..."
+      done
+      echo "$name failed to boot."
+      exit 1
+    )
+    echo "$name has booted."
+  }
+
+  echo "Looking for leader instance..."
+  cloud_FindInstance "$prefix-leader"
+  [[ ${#instances[@]} -eq 1 ]] || {
+    echo "Unable to find leader"
+    exit 1
+  }
+
+  (
+    declare leaderName
+    declare leaderIp
+    IFS=: read -r leaderName leaderIp _ < <(echo "${instances[0]}")
+
+    # Try to ping the machine first.
+    timeout 60s bash -c "set -o pipefail; until ping -c 3 $leaderIp | tr - _; do echo .; done"
+
+    if [[ ! -r $sshPrivateKey ]]; then
+      echo "Fetching $sshPrivateKey from $leaderName"
+
+      # Try to scp in a couple times, sshd may not yet be up even though the
+      # machine can be pinged...
+      set -x -o pipefail
+      for i in $(seq 1 30); do
+        if cloud_FetchFile "$leaderName" "$leaderIp" /solana-id_ecdsa "$sshPrivateKey"; then
+          break
+        fi
+
+        sleep 1
+        echo "Retry $i..."
+      done
+
+      chmod 400 "$sshPrivateKey"
+      ls -l "$sshPrivateKey"
+    fi
+  )
+
+  echo "leaderIp=()" >> "$configFile"
+  cloud_ForEachInstance recordInstanceIp leaderIp
+  cloud_ForEachInstance waitForStartupComplete
+
+  echo "Looking for validator instances..."
+  cloud_FindInstances "$prefix-validator"
+  [[ ${#instances[@]} -gt 0 ]] || {
+    echo "Unable to find validators"
+    exit 1
+  }
+  echo "validatorIpList=()" >> "$configFile"
+  cloud_ForEachInstance recordInstanceIp validatorIpList
+  cloud_ForEachInstance waitForStartupComplete
+
+  echo "clientIpList=()" >> "$configFile"
+  echo "Looking for client instances..."
+  cloud_FindInstances "$prefix-client"
+  [[ ${#instances[@]} -eq 0 ]] || {
+    cloud_ForEachInstance recordInstanceIp clientIpList
+    cloud_ForEachInstance waitForStartupComplete
+  }
+
+  echo "Wrote $configFile"
+  $metricsWriteDatapoint "testnet-deploy net-config-complete=1"
+}
+
+delete() {
+  $metricsWriteDatapoint "testnet-deploy net-delete-begin=1"
+
+  # Delete the leader node first to prevent unusual metrics on the dashboard
+  # during shutdown.
+  # TODO: It would be better to fully cut-off metrics reporting before any
+  # instances are deleted.
+  for filter in "$prefix-leader" "$prefix-"; do
+    echo "Searching for instances: $filter"
+    cloud_FindInstances "$filter"
+
+    if [[ ${#instances[@]} -eq 0 ]]; then
+      echo "No instances found matching '$filter'"
+    else
+      cloud_DeleteInstances true
+    fi
+  done
+  rm -f "$configFile"
+
+  $metricsWriteDatapoint "testnet-deploy net-delete-complete=1"
+
+}
+
+case $command in
+delete)
+  delete
+  ;;
+
+create)
+  [[ -n $validatorNodeCount ]] || usage "Need number of nodes"
+  if [[ $validatorNodeCount -le 0 ]]; then
+    usage "One or more validator nodes is required"
+  fi
+
+  delete
+
+  $metricsWriteDatapoint "testnet-deploy net-create-begin=1"
+
+  rm -rf "$sshPrivateKey"{,.pub}
+
+  # Note: using rsa because |aws ec2 import-key-pair| seems to fail for ecdsa
+  ssh-keygen -t rsa -N '' -f "$sshPrivateKey"
+
+  printNetworkInfo() {
+    cat <<EOF
+========================================================================================
+
+Network composition:
+  Leader = $leaderMachineType (GPU=$enableGpu)
+  Validators = $validatorNodeCount x $validatorMachineType
+  Client(s) = $clientNodeCount x $clientMachineType
+
+========================================================================================
+
+EOF
+  }
+  printNetworkInfo
+
+  declare startupScript="$netConfigDir"/instance-startup-script.sh
+  cat > "$startupScript" <<EOF
+#!/bin/bash -ex
+# autogenerated at $(date)
+
+cat > /etc/motd <<EOM
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+  This instance has not been fully configured.
+
+  See startup script log messages in /var/log/syslog for status:
+    $ sudo cat /var/log/syslog | egrep \\(startup-script\\|cloud-init\)
+
+  To block until setup is complete, run:
+    $ until [[ -f /.instance-startup-complete ]]; do sleep 1; done
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+EOM
+
+# Place the generated private key at /solana-id_ecdsa so it's retrievable by anybody
+# who is able to log into this machine
+cat > /solana-id_ecdsa <<EOK
+$(cat "$sshPrivateKey")
+EOK
+cat > /solana-id_ecdsa.pub <<EOK
+$(cat "$sshPrivateKey.pub")
+EOK
+chmod 444 /solana-id_ecdsa
+
+USER=\$(id -un)
+
+$(
+  cd "$here"/scripts/
+  cat \
+    disable-background-upgrades.sh \
+    create-solana-user.sh \
+    add-solana-user-authorized_keys.sh \
+    install-earlyoom.sh \
+    install-libssl-compatability.sh \
+    install-rsync.sh \
+    network-config.sh \
+
+)
+
+cat > /etc/motd <<EOM
+$(printNetworkInfo)
+EOM
+
+touch /.instance-startup-complete
+
+EOF
+
+  cloud_CreateInstances "$prefix" "$prefix-leader" 1 \
+    "$imageName" "$leaderMachineType" "$leaderBootDiskSizeInGb" \
+    "$startupScript" "$leaderAddress" "$bootDiskType"
+
+  cloud_CreateInstances "$prefix" "$prefix-validator" "$validatorNodeCount" \
+    "$imageName" "$validatorMachineType" "$validatorBootDiskSizeInGb" \
+    "$startupScript" "" "$bootDiskType"
+
+  if [[ $clientNodeCount -gt 0 ]]; then
+    cloud_CreateInstances "$prefix" "$prefix-client" "$clientNodeCount" \
+      "$imageName" "$clientMachineType" "$clientBootDiskSizeInGb" \
+      "$startupScript" "" "$bootDiskType"
+  fi
+
+  $metricsWriteDatapoint "testnet-deploy net-create-complete=1"
+
+  prepareInstancesAndWriteConfigFile
+  ;;
+
+config)
+  prepareInstancesAndWriteConfigFile
+  ;;
+*)
+  usage "Unknown command: $command"
+esac

net/init-metrics.sh

@@ -0,0 +1,80 @@
+#!/bin/bash -e
+
+here=$(dirname "$0")
+# shellcheck source=net/common.sh
+source "$here"/common.sh
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [-e] [-d] [username]
+
+Creates a testnet dev metrics database
+
+  username        InfluxDB user with access to create a new database
+  -d              Delete the database instead of creating it
+  -e              Assume database already exists and SOLANA_METRICS_CONFIG is
+                  defined in the environment already
+
+EOF
+  exit $exitcode
+}
+
+loadConfigFile
+
+useEnv=false
+delete=false
+while getopts "hde" opt; do
+  case $opt in
+  h|\?)
+    usage
+    exit 0
+    ;;
+  d)
+    delete=true
+    ;;
+  e)
+    useEnv=true
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+shift $((OPTIND - 1))
+
+if $useEnv; then
+  [[ -n $SOLANA_METRICS_CONFIG ]] ||
+    usage "Error: SOLANA_METRICS_CONFIG is not defined in the environment"
+else
+  username=$1
+  [[ -n "$username" ]] || usage "username not specified"
+
+  read -rs -p "InfluxDB password for $username: " password
+  [[ -n $password ]] || { echo "Password not specified"; exit 1; }
+  echo
+
+  query() {
+    echo "$*"
+    curl -XPOST \
+      "https://metrics.solana.com:8086/query?u=${username}&p=${password}" \
+      --data-urlencode "q=$*"
+  }
+
+  query "DROP DATABASE \"$netBasename\""
+  ! $delete || exit 0
+  query "CREATE DATABASE \"$netBasename\""
+  query "ALTER RETENTION POLICY autogen ON \"$netBasename\" DURATION 7d"
+  query "GRANT READ ON \"$netBasename\" TO \"ro\""
+  query "GRANT WRITE ON \"$netBasename\" TO \"scratch_writer\""
+
+  SOLANA_METRICS_CONFIG="db=$netBasename,u=scratch_writer,p=topsecret"
+fi
+
+echo "export SOLANA_METRICS_CONFIG=\"$SOLANA_METRICS_CONFIG\"" >> "$configFile"
+
+exit 0

net/net.sh

@@ -0,0 +1,393 @@
+#!/bin/bash -e
+
+here=$(dirname "$0")
+SOLANA_ROOT="$(cd "$here"/..; pwd)"
+
+# shellcheck source=net/common.sh
+source "$here"/common.sh
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [start|stop|restart|sanity] [command-specific options]
+
+Operate a configured testnet
+
+ start    - Start the network
+ sanity   - Sanity check the network
+ stop     - Stop the network
+ restart  - Shortcut for stop then start
+
+ start-specific options:
+   -S [snapFilename]           - Deploy the specified Snap file
+   -s edge|beta|stable         - Deploy the latest Snap on the specified Snap release channel
+   -t edge|beta|stable|vX.Y.Z  - Deploy the latest tarball release for the
+                                 specified release channel (edge|beta|stable) or release tag
+                                 (vX.Y.Z)
+   -f [cargoFeatures]          - List of |cargo --feaures=| to activate
+                                 (ignored if -s or -S is specified)
+
+   Note: if RUST_LOG is set in the environment it will be propogated into the
+         network nodes.
+
+ sanity/start-specific options:
+   -o noLedgerVerify    - Skip ledger verification
+   -o noValidatorSanity - Skip validator sanity
+   -o rejectExtraNodes  - Require the exact number of nodes
+
+ stop-specific options:
+   none
+
+EOF
+  exit $exitcode
+}
+
+snapChannel=
+releaseChannel=
+snapFilename=
+deployMethod=local
+sanityExtraArgs=
+cargoFeatures=
+
+command=$1
+[[ -n $command ]] || usage
+shift
+
+while getopts "h?S:s:t:o:f:" opt; do
+  case $opt in
+  h | \?)
+    usage
+    ;;
+  S)
+    snapFilename=$OPTARG
+    [[ -f $snapFilename ]] || usage "Snap not readable: $snapFilename"
+    deployMethod=snap
+    ;;
+  s)
+    case $OPTARG in
+    edge|beta|stable)
+      snapChannel=$OPTARG
+      deployMethod=snap
+      ;;
+    *)
+      usage "Invalid snap channel: $OPTARG"
+      ;;
+    esac
+    ;;
+  t)
+    case $OPTARG in
+    edge|beta|stable|v*)
+      releaseChannel=$OPTARG
+      deployMethod=tar
+      ;;
+    *)
+      usage "Invalid release channel: $OPTARG"
+      ;;
+    esac
+    ;;
+  f)
+    cargoFeatures=$OPTARG
+    ;;
+  o)
+    case $OPTARG in
+    noLedgerVerify|noValidatorSanity|rejectExtraNodes)
+      sanityExtraArgs="$sanityExtraArgs -o $OPTARG"
+      ;;
+    *)
+      echo "Error: unknown option: $OPTARG"
+      exit 1
+      ;;
+    esac
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+
+loadConfigFile
+expectedNodeCount=$((${#validatorIpList[@]} + 1))
+
+build() {
+  declare MAYBE_DOCKER=
+  if [[ $(uname) != Linux ]]; then
+    MAYBE_DOCKER="ci/docker-run.sh solanalabs/rust"
+  fi
+  SECONDS=0
+  (
+    cd "$SOLANA_ROOT"
+    echo "--- Build started at $(date)"
+
+    set -x
+    rm -rf farf
+    $MAYBE_DOCKER cargo install --features="$cargoFeatures" --root farf
+    ./scripts/install-native-programs.sh farf/
+  )
+  echo "Build took $SECONDS seconds"
+}
+
+startCommon() {
+  declare ipAddress=$1
+  test -d "$SOLANA_ROOT"
+  ssh "${sshOptions[@]}" "$ipAddress" "mkdir -p ~/solana ~/.cargo/bin"
+  rsync -vPrc -e "ssh ${sshOptions[*]}" \
+    "$SOLANA_ROOT"/{fetch-perf-libs.sh,scripts,net,multinode-demo} \
+    "$ipAddress":~/solana/
+}
+
+startLeader() {
+  declare ipAddress=$1
+  declare logFile="$2"
+  echo "--- Starting leader: $leaderIp"
+  echo "start log: $logFile"
+
+  # Deploy local binaries to leader.  Validators and clients later fetch the
+  # binaries from the leader.
+  (
+    set -x
+    startCommon "$ipAddress" || exit 1
+    case $deployMethod in
+    snap)
+      rsync -vPrc -e "ssh ${sshOptions[*]}" "$snapFilename" "$ipAddress:~/solana/solana.snap"
+      ;;
+    tar)
+      rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/solana-release/bin/* "$ipAddress:~/.cargo/bin/"
+      ;;
+    local)
+      rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/farf/bin/* "$ipAddress:~/.cargo/bin/"
+      ;;
+    *)
+      usage "Internal error: invalid deployMethod: $deployMethod"
+      ;;
+    esac
+
+    ssh "${sshOptions[@]}" -n "$ipAddress" \
+      "./solana/net/remote/remote-node.sh $deployMethod leader $publicNetwork $entrypointIp $expectedNodeCount \"$RUST_LOG\""
+  ) >> "$logFile" 2>&1 || {
+    cat "$logFile"
+    echo "^^^ +++"
+    exit 1
+  }
+}
+
+startValidator() {
+  declare ipAddress=$1
+  declare logFile="$netLogDir/validator-$ipAddress.log"
+
+  echo "--- Starting validator: $ipAddress"
+  echo "start log: $logFile"
+  (
+    set -x
+    startCommon "$ipAddress"
+    ssh "${sshOptions[@]}" -n "$ipAddress" \
+      "./solana/net/remote/remote-node.sh $deployMethod validator $publicNetwork $entrypointIp $expectedNodeCount \"$RUST_LOG\""
+  ) >> "$logFile" 2>&1 &
+  declare pid=$!
+  ln -sfT "validator-$ipAddress.log" "$netLogDir/validator-$pid.log"
+  pids+=("$pid")
+}
+
+startClient() {
+  declare ipAddress=$1
+  declare logFile="$2"
+  echo "--- Starting client: $ipAddress"
+  echo "start log: $logFile"
+  (
+    set -x
+    startCommon "$ipAddress"
+    ssh "${sshOptions[@]}" -f "$ipAddress" \
+      "./solana/net/remote/remote-client.sh $deployMethod $entrypointIp \"$RUST_LOG\""
+  ) >> "$logFile" 2>&1 || {
+    cat "$logFile"
+    echo "^^^ +++"
+    exit 1
+  }
+}
+
+sanity() {
+  declare expectedNodeCount=$((${#validatorIpList[@]} + 1))
+  declare ok=true
+
+  echo "--- Sanity"
+  $metricsWriteDatapoint "testnet-deploy net-sanity-begin=1"
+
+  declare host=$leaderIp # TODO: maybe use ${validatorIpList[0]} ?
+  (
+    set -x
+    # shellcheck disable=SC2029 # remote-client.sh args are expanded on client side intentionally
+    ssh "${sshOptions[@]}" "$host" \
+      "./solana/net/remote/remote-sanity.sh $sanityExtraArgs"
+  ) || ok=false
+
+  $metricsWriteDatapoint "testnet-deploy net-sanity-complete=1"
+  $ok || exit 1
+}
+
+start() {
+  case $deployMethod in
+  snap)
+    if [[ -n $snapChannel ]]; then
+      rm -f "$SOLANA_ROOT"/solana_*.snap
+      if [[ $(uname) != Linux ]]; then
+        (
+          set -x
+          SOLANA_DOCKER_RUN_NOSETUID=1 "$SOLANA_ROOT"/ci/docker-run.sh ubuntu:18.04 bash -c "
+            set -ex;
+            apt-get -qq update;
+            apt-get -qq -y install snapd;
+            until snap download --channel=$snapChannel solana; do
+              sleep 1;
+            done
+          "
+        )
+      else
+        (
+          cd "$SOLANA_ROOT"
+          until snap download --channel="$snapChannel" solana; do
+            sleep 1
+          done
+        )
+      fi
+      snapFilename="$(echo "$SOLANA_ROOT"/solana_*.snap)"
+      [[ -r $snapFilename ]] || {
+        echo "Error: Snap not readable: $snapFilename"
+        exit 1
+      }
+    fi
+    ;;
+  tar)
+    if [[ -n $releaseChannel ]]; then
+      rm -f "$SOLANA_ROOT"/solana-release.tar.bz2
+      cd "$SOLANA_ROOT"
+
+      set -x
+      curl -o solana-release.tar.bz2 http://solana-release.s3.amazonaws.com/"$releaseChannel"/solana-release.tar.bz2
+      tar jxvf solana-release.tar.bz2
+    fi
+    ;;
+  local)
+    build
+    ;;
+  *)
+    usage "Internal error: invalid deployMethod: $deployMethod"
+    ;;
+  esac
+
+  echo "Deployment started at $(date)"
+  $metricsWriteDatapoint "testnet-deploy net-start-begin=1"
+
+  SECONDS=0
+  declare leaderDeployTime=
+  startLeader "$leaderIp" "$netLogDir/leader-$leaderIp.log"
+  leaderDeployTime=$SECONDS
+  $metricsWriteDatapoint "testnet-deploy net-leader-started=1"
+
+  SECONDS=0
+  pids=()
+  loopCount=0
+  for ipAddress in "${validatorIpList[@]}"; do
+    startValidator "$ipAddress"
+
+    # Staggering validator startup time. If too many validators
+    # bootup simultaneously, leader node gets more rsync requests
+    # from the validators than it can handle.
+    ((loopCount++ % 2 == 0)) && sleep 2
+  done
+
+  for pid in "${pids[@]}"; do
+    declare ok=true
+    wait "$pid" || ok=false
+    if ! $ok; then
+      cat "$netLogDir/validator-$pid.log"
+      echo ^^^ +++
+      exit 1
+    fi
+  done
+
+  $metricsWriteDatapoint "testnet-deploy net-validators-started=1"
+  validatorDeployTime=$SECONDS
+
+  sanity
+
+  SECONDS=0
+  for ipAddress in "${clientIpList[@]}"; do
+    startClient "$ipAddress" "$netLogDir/client-$ipAddress.log"
+  done
+  clientDeployTime=$SECONDS
+  $metricsWriteDatapoint "testnet-deploy net-start-complete=1"
+
+  if [[ $deployMethod = "snap" ]]; then
+    declare networkVersion=unknown
+    IFS=\  read -r _ networkVersion _ < <(
+      ssh "${sshOptions[@]}" "$leaderIp" \
+        "snap info solana | grep \"^installed:\""
+    )
+    networkVersion=${networkVersion/0+git./}
+    $metricsWriteDatapoint "testnet-deploy version=\"$networkVersion\""
+  fi
+
+  echo
+  echo "+++ Deployment Successful"
+  echo "Leader deployment took $leaderDeployTime seconds"
+  echo "Validator deployment (${#validatorIpList[@]} instances) took $validatorDeployTime seconds"
+  echo "Client deployment (${#clientIpList[@]} instances) took $clientDeployTime seconds"
+  echo "Network start logs in $netLogDir:"
+  ls -l "$netLogDir"
+}
+
+
+stopNode() {
+  local ipAddress=$1
+  echo "--- Stopping node: $ipAddress"
+  (
+    set -x
+    ssh "${sshOptions[@]}" "$ipAddress" "
+      set -x
+      if snap list solana; then
+        sudo snap set solana mode=
+        sudo snap remove solana
+      fi
+      ! tmux list-sessions || tmux kill-session
+      for pattern in solana- remote- oom-monitor net-stats; do
+        pkill -9 \$pattern
+      done
+    "
+  ) || true
+}
+
+stop() {
+  SECONDS=0
+  $metricsWriteDatapoint "testnet-deploy net-stop-begin=1"
+
+  stopNode "$leaderIp"
+
+  for ipAddress in "${validatorIpList[@]}" "${clientIpList[@]}"; do
+    stopNode "$ipAddress"
+  done
+
+  $metricsWriteDatapoint "testnet-deploy net-stop-complete=1"
+  echo "Stopping nodes took $SECONDS seconds"
+}
+
+case $command in
+restart)
+  stop
+  start
+  ;;
+start)
+  start
+  ;;
+sanity)
+  sanity
+  ;;
+stop)
+  stop
+  ;;
+*)
+  echo "Internal error: Unknown command: $command"
+  exit 1
+esac

net/remote/README.md

@@ -0,0 +1 @@
+Scripts that run on the remote testnet nodes

net/remote/remote-client.sh

@@ -0,0 +1,80 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")"/../..
+
+echo "$(date) | $0 $*" > client.log
+
+deployMethod="$1"
+entrypointIp="$2"
+RUST_LOG="$3"
+export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info
+
+missing() {
+  echo "Error: $1 not specified"
+  exit 1
+}
+
+[[ -n $deployMethod ]] || missing deployMethod
+[[ -n $entrypointIp ]] || missing entrypointIp
+
+source net/common.sh
+loadConfigFile
+
+threadCount=$(nproc)
+if [[ $threadCount -gt 4 ]]; then
+  threadCount=4
+fi
+
+case $deployMethod in
+snap)
+  net/scripts/rsync-retry.sh -vPrc "$entrypointIp:~/solana/solana.snap" .
+  sudo snap install solana.snap --devmode --dangerous
+
+  solana_bench_tps=/snap/bin/solana.bench-tps
+  solana_keygen=/snap/bin/solana.keygen
+  ;;
+local|tar)
+  PATH="$HOME"/.cargo/bin:"$PATH"
+  export USE_INSTALL=1
+  export SOLANA_DEFAULT_METRICS_RATE=1
+
+  net/scripts/rsync-retry.sh -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/
+  solana_bench_tps=solana-bench-tps
+  solana_keygen=solana-keygen
+  ;;
+*)
+  echo "Unknown deployment method: $deployMethod"
+  exit 1
+esac
+
+scripts/oom-monitor.sh > oom-monitor.log 2>&1 &
+scripts/net-stats.sh  > net-stats.log 2>&1 &
+
+! tmux list-sessions || tmux kill-session
+
+clientCommand="\
+  $solana_bench_tps \
+    --network $entrypointIp:8001 \
+    --identity client.json \
+    --duration 7500 \
+    --sustained \
+    --threads $threadCount \
+"
+
+keygenCommand="$solana_keygen -o client.json"
+tmux new -s solana-bench-tps -d "
+  [[ -r client.json ]] || {
+    echo '$ $keygenCommand'  | tee -a client.log
+    $keygenCommand >> client.log 2>&1
+  }
+
+  while true; do
+    echo === Client start: \$(date) | tee -a client.log
+    $metricsWriteDatapoint 'testnet-deploy client-begin=1'
+    echo '$ $clientCommand' | tee -a client.log
+    $clientCommand >> client.log 2>&1
+    $metricsWriteDatapoint 'testnet-deploy client-complete=1'
+  done
+"
+sleep 1
+tmux capture-pane -t solana-bench-tps -p -S -100