Permit testnets without a GPU

- Also use rayon to parallelize to_blobs() to maximize CPU usage

.buildkite/hooks/post-checkout

@@ -0,0 +1,2 @@
+CI_BUILD_START=$(date +%s)
+export CI_BUILD_START

.buildkite/hooks/post-checkout.sh

@@ -0,0 +1 @@
+post-checkout

.buildkite/hooks/post-command

@@ -0,0 +1,45 @@
+#!/bin/bash -e
+
+#
+# Save target/ for the next CI build on this machine
+#
+if [[ -n $CARGO_TARGET_CACHE_NAME ]]; then
+  (
+    d=$HOME/cargo-target-cache/"$CARGO_TARGET_CACHE_NAME"
+    mkdir -p "$d"
+    set -x
+    rsync -a --delete --link-dest="$PWD" target "$d"
+    du -hs "$d"
+  )
+fi
+
+#
+# Add job_stats data point
+#
+if [[ -z $CI_BUILD_START ]]; then
+  echo Error: CI_BUILD_START empty
+else
+  CI_BUILD_DURATION=$(( $(date +%s) - CI_BUILD_START + 1 ))
+
+  CI_LABEL=${BUILDKITE_LABEL:-build label missing}
+
+  PR=false
+  if [[ $BUILDKITE_BRANCH =~ pull/* ]]; then
+    PR=true
+  fi
+
+  SUCCESS=true
+  if [[ $BUILDKITE_COMMAND_EXIT_STATUS != 0 ]]; then
+    SUCCESS=false
+  fi
+
+  point_tags="pipeline=$BUILDKITE_PIPELINE_SLUG,job=$CI_LABEL,pr=$PR,success=$SUCCESS"
+  point_tags="${point_tags// /\\ }"  # Escape spaces
+
+  point_fields="duration=$CI_BUILD_DURATION"
+  point_fields="${point_fields// /\\ }"  # Escape spaces
+
+  point="job_stats,$point_tags $point_fields"
+
+  scripts/metrics-write-datapoint.sh "$point" || true
+fi

.buildkite/hooks/post-command.sh

@@ -0,0 +1 @@
+post-command

.buildkite/hooks/pre-command

@@ -0,0 +1,27 @@
+#!/bin/bash -e
+
+# Ensure the pattern "+++ ..." never occurs when |set -x| is set, as buildkite
+# interprets this as the start of a log group.
+# Ref: https://buildkite.com/docs/pipelines/managing-log-output
+export PS4="++"
+
+#
+# Restore target/ from the previous CI build on this machine
+#
+[[ -n "$CARGO_TARGET_CACHE_NAME" ]] || (
+  d=$HOME/cargo-target-cache/"$CARGO_TARGET_CACHE_NAME"
+
+  if [[ -d $d ]]; then
+    du -hs "$d"
+    read -r cacheSizeInGB _ < <(du -s --block-size=1000000000 "$d")
+    if [[ $cacheSizeInGB -gt 5 ]]; then
+      echo "$d has gotten too large, removing it"
+      rm -rf "$d"
+    fi
+  fi
+
+  mkdir -p "$d"/target
+  set -x
+  rsync -a --delete --link-dest="$d" "$d"/target .
+)
+

.buildkite/hooks/pre-command.sh

@@ -0,0 +1 @@
+pre-command

.clippy.toml

@@ -0,0 +1 @@
+too-many-arguments-threshold = 9

.codecov.yml

@@ -1,2 +1,5 @@
 ignore:
   - "src/bin" 
+coverage:
+  status:
+    patch: off

.gitignore

@@ -1,4 +1,16 @@
-
-/target/
-**/*.rs.bk
 Cargo.lock
+/target/
+
+**/*.rs.bk
+.cargo
+
+# node configuration files
+/config/
+/config-private/
+/config-drone/
+/config-validator/
+/config-client/
+/multinode-demo/test/config-client/
+
+# test temp files, ledgers, etc.
+/farf/

.travis.yml

@@ -1,22 +0,0 @@
-language: rust
-required: sudo
-services:
-- docker
-matrix:
-  allow_failures:
-  - rust: nightly
-  include:
-  - rust: stable
-  - rust: nightly
-    env:
-    - FEATURES='asm,unstable'
-before_script: |
-  export PATH="$PATH:$HOME/.cargo/bin"
-  rustup component add rustfmt-preview
-script:
-- cargo fmt -- --write-mode=diff
-- cargo build --verbose --features "$FEATURES"
-- cargo test --verbose --features "$FEATURES"
-after_success: |
-  docker run -it --rm --security-opt seccomp=unconfined --volume "$PWD:/volume" elmtai/docker-rust-kcov
-  bash <(curl -s https://codecov.io/bash) -s target/cov

CONTRIBUTING.md

@@ -0,0 +1,53 @@
+Solana Coding Guidelines
+===
+
+The goal of these guidelines is to improve developer productivity by allowing developers to
+jump any file in the codebase and not need to adapt to inconsistencies in how the code is
+written. The codebase should appear as if it had been authored by a single developer. If you
+don't agree with a convention, submit a PR patching this document and let's discuss! Once
+the PR is accepted, *all* code should be updated as soon as possible to reflect the new
+conventions.
+
+Rust coding conventions
+---
+
+* All Rust code is formatted using the latest version of `rustfmt`. Once installed, it will be
+  updated automatically when you update the compiler with `rustup`.
+
+* All Rust code is linted with Clippy. If you'd prefer to ignore its advice, do so explicitly:
+
+  ```rust
+  #[cfg_attr(feature = "cargo-clippy", allow(too_many_arguments))]
+  ```
+
+  Note: Clippy defaults can be overridden in the top-level file `.clippy.toml`.
+
+* For variable names, when in doubt, spell it out. The mapping from type names to variable names
+  is to lowercase the type name, putting an underscore before each capital letter. Variable names
+  should *not* be abbreviated unless being used as closure arguments and the brevity improves
+  readability. When a function has multiple instances of the same type, qualify each with a
+  prefix and underscore (i.e. alice_keypair) or a numeric suffix (i.e. tx0).
+
+* For function and method names, use `<verb>_<subject>`. For unit tests, that verb should
+  always be `test` and for benchmarks the verb should always be `bench`. Avoid namespacing
+  function names with some arbitrary word. Avoid abreviating words in function names.
+
+* As they say, "When in Rome, do as the Romans do." A good patch should acknowledge the coding
+  conventions of the code that surrounds it, even in the case where that code has not yet been
+  updated to meet the conventions described here.
+
+
+Terminology
+---
+
+Inventing new terms is allowed, but should only be done when the term is widely used and
+understood. Avoid introducing new 3-letter terms, which can be confused with 3-letter acronyms.
+
+Some terms we currently use regularly in the codebase:
+
+* fullnode: n. A fully participating network node.
+* hash: n. A SHA-256 Hash.
+* keypair: n. A Ed25519 key-pair, containing a public and private key.
+* pubkey: n. The public key of a Ed25519 key-pair.
+* sigverify: v. To verify a Ed25519 digital signature.
+

Cargo.toml

@@ -1,42 +1,114 @@
 [package]
-name = "silk"
-description = "A silky smooth implementation of the Loom architecture"
-version = "0.3.0"
-documentation = "https://docs.rs/silk"
-homepage = "http://loomprotocol.com/"
-repository = "https://github.com/loomprotocol/silk"
+name = "solana"
+description = "Blockchain, Rebuilt for Scale"
+version = "0.8.0"
+documentation = "https://docs.rs/solana"
+homepage = "http://solana.com/"
+readme = "README.md"
+repository = "https://github.com/solana-labs/solana"
 authors = [
-    "Anatoly Yakovenko <aeyakovenko@gmail.com>",
-    "Greg Fitzgerald <garious@gmail.com>",
+    "Anatoly Yakovenko <anatoly@solana.com>",
+    "Greg Fitzgerald <greg@solana.com>",
+    "Stephen Akridge <stephen@solana.com>",
+    "Michael Vines <mvines@solana.com>",
+    "Rob Walker <rob@solana.com>",
+    "Pankaj Garg <pankaj@solana.com>",
+    "Tyera Eulberg <tyera@solana.com>",
 ]
 license = "Apache-2.0"
 
 [[bin]]
-name = "silk-demo"
-path = "src/bin/demo.rs"
+name = "solana-bench-tps"
+path = "src/bin/bench-tps.rs"
 
 [[bin]]
-name = "silk-client-demo"
-path = "src/bin/client-demo.rs"
+name = "solana-bench-streamer"
+path = "src/bin/bench-streamer.rs"
 
 [[bin]]
-name = "silk-testnode"
-path = "src/bin/testnode.rs"
+name = "solana-drone"
+path = "src/bin/drone.rs"
+
+[[bin]]
+name = "solana-fullnode"
+path = "src/bin/fullnode.rs"
+
+[[bin]]
+name = "solana-fullnode-config"
+path = "src/bin/fullnode-config.rs"
+
+[[bin]]
+name = "solana-genesis"
+path = "src/bin/genesis.rs"
+
+[[bin]]
+name = "solana-ledger-tool"
+path = "src/bin/ledger-tool.rs"
+
+[[bin]]
+name = "solana-keygen"
+path = "src/bin/keygen.rs"
+
+[[bin]]
+name = "solana-wallet"
+path = "src/bin/wallet.rs"
 
 [badges]
-codecov = { repository = "loomprotocol/silk", branch = "master", service = "github" }
+codecov = { repository = "solana-labs/solana", branch = "master", service = "github" }
 
 [features]
 unstable = []
-asm = ["sha2-asm"]
+ipv6 = []
+cuda = []
+erasure = []
+test = []
 
 [dependencies]
+atty = "0.2"
+bincode = "1.0.0"
+bs58 = "0.2.0"
+byteorder = "1.2.1"
+bytes = "0.4"
+chrono = { version = "0.4.0", features = ["serde"] }
+clap = "2.31"
+dirs = "1.0.2"
+env_logger = "0.5.12"
+generic-array = { version = "0.12.0", default-features = false, features = ["serde"] }
+getopts = "0.2"
+influx_db_client = "0.3.4"
+jsonrpc-core = { git = "https://github.com/paritytech/jsonrpc", rev = "4b6060b" }
+jsonrpc-http-server = { git = "https://github.com/paritytech/jsonrpc", rev = "4b6060b" }
+jsonrpc-macros = { git = "https://github.com/paritytech/jsonrpc", rev = "4b6060b" }
+itertools = "0.7.8"
+log = "0.4.2"
+matches = "0.1.6"
+nix = "0.11.0"
+pnet_datalink = "0.21.0"
+rand = "0.5.1"
 rayon = "1.0.0"
+reqwest = "0.8.6"
+ring = "0.13.2"
 sha2 = "0.7.0"
-sha2-asm = {version="0.3", optional=true}
-generic-array = { version = "0.9.0", default-features = false, features = ["serde"] }
 serde = "1.0.27"
 serde_derive = "1.0.27"
-ring = "0.12.1"
-untrusted = "0.5.1"
-bincode = "1.0.0"
+serde_json = "1.0.10"
+socket2 = "0.3.8"
+sys-info = "0.5.6"
+tokio = "0.1"
+tokio-codec = "0.1"
+untrusted = "0.6.2"
+
+[[bench]]
+name = "bank"
+
+[[bench]]
+name = "banking_stage"
+
+[[bench]]
+name = "ledger"
+
+[[bench]]
+name = "signature"
+
+[[bench]]
+name = "sigverify"

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2018 Anatoly Yakovenko <anatoly@loomprotocol.com> and Greg Fitzgerald <garious@gmail.com>
+Copyright 2018 Solana Labs, Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

README.md

@@ -1,87 +1,269 @@
-[![Silk crate](https://img.shields.io/crates/v/silk.svg)](https://crates.io/crates/silk)
-[![Silk documentation](https://docs.rs/silk/badge.svg)](https://docs.rs/silk)
-[![Build Status](https://travis-ci.org/loomprotocol/silk.svg?branch=master)](https://travis-ci.org/loomprotocol/silk)
-[![codecov](https://codecov.io/gh/loomprotocol/silk/branch/master/graph/badge.svg)](https://codecov.io/gh/loomprotocol/silk)
+[![Solana crate](https://img.shields.io/crates/v/solana.svg)](https://crates.io/crates/solana)
+[![Solana documentation](https://docs.rs/solana/badge.svg)](https://docs.rs/solana)
+[![Build status](https://badge.buildkite.com/d4c4d7da9154e3a8fb7199325f430ccdb05be5fc1e92777e51.svg?branch=master)](https://solana-ci-gate.herokuapp.com/buildkite_public_log?https://buildkite.com/solana-labs/solana/builds/latest/master)
+[![codecov](https://codecov.io/gh/solana-labs/solana/branch/master/graph/badge.svg)](https://codecov.io/gh/solana-labs/solana)
 
-# Silk, a silky smooth implementation of the Loom specification
+Blockchain, Rebuilt for Scale
+===
 
-Loom is a new achitecture for a high performance blockchain. Its whitepaper boasts a theoretical
-throughput of 710k transactions per second on a 1 gbps network. The specification is implemented
-in two git repositories. Reserach is performed in the loom repository. That work drives the
-Loom specification forward. This repository, on the other hand, aims to implement the specification
-as-is.  We care a great deal about quality, clarity and short learning curve. We avoid the use
-of `unsafe` Rust and write tests for *everything*.  Optimizations are only added when
-corresponding benchmarks are also added that demonstrate real performance boosts. We expect the
-feature set here will always be a ways behind the loom repo, but that this is an implementation
-you can take to the bank, literally.
+Solana™ is a new blockchain architecture built from the ground up for scale. The architecture supports
+up to 710 thousand transactions per second on a gigabit network.
 
-# Usage
+Disclaimer
+===
 
-Add the latest [silk package](https://crates.io/crates/silk) to the `[dependencies]` section
-of your Cargo.toml.
+All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort.  It is up to the reader to check and validate their accuracy and truthfulness.  Furthermore nothing in this project constitutes a solicitation for investment.
 
-Create a *Historian* and send it *events* to generate an *event log*, where each log *entry*
-is tagged with the historian's latest *hash*. Then ensure the order of events was not tampered
-with by verifying each entry's hash can be generated from the hash in the previous entry:
+Introduction
+===
 
-![historian](https://user-images.githubusercontent.com/55449/36633440-f76f7bb8-1952-11e8-8328-387861d3d464.png)
+It's possible for a centralized database to process 710,000 transactions per second on a standard gigabit network if the transactions are, on average, no more than 176 bytes. A centralized database can also replicate itself and maintain high availability without significantly compromising that transaction rate using the distributed system technique known as Optimistic Concurrency Control [\[H.T.Kung, J.T.Robinson (1981)\]](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.65.4735). At Solana, we're demonstrating that these same theoretical limits apply just as well to blockchain on an adversarial network. The key ingredient? Finding a way to share time when nodes can't trust one-another. Once nodes can trust time, suddenly ~40 years of distributed systems research becomes applicable to blockchain!
 
-```rust
-extern crate silk;
+> Perhaps the most striking difference between algorithms obtained by our method and ones based upon timeout is that using timeout produces a traditional distributed algorithm in which the processes operate asynchronously, while our method produces a globally synchronous one in which every process does the same thing at (approximately) the same time. Our method seems to contradict the whole purpose of distributed processing, which is to permit different processes to operate independently and perform different functions. However, if a distributed system is really a single system, then the processes must be synchronized in some way. Conceptually, the easiest way to synchronize processes is to get them all to do the same thing at the same time. Therefore, our method is used to implement a kernel that performs the necessary synchronization--for example, making sure that two different processes do not try to modify a file at the same time. Processes might spend only a small fraction of their time executing the synchronizing kernel; the rest of the time, they can operate independently--e.g., accessing different files. This is an approach we have advocated even when fault-tolerance is not required. The method's basic simplicity makes it easier to understand the precise properties of a system, which is crucial if one is to know just how fault-tolerant the system is. [\[L.Lamport (1984)\]](http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.71.1078)
 
-use silk::historian::Historian;
-use silk::log::{verify_slice, Entry, Event, Sha256Hash};
-use std::thread::sleep;
-use std::time::Duration;
-use std::sync::mpsc::SendError;
+Furthermore, and much to our surprise, it can be implemented using a mechanism that has existed in Bitcoin since day one. The Bitcoin feature is called nLocktime and it can be used to postdate transactions using block height instead of a timestamp. As a Bitcoin client, you'd use block height instead of a timestamp if you don't trust the network. Block height turns out to be an instance of what's being called a Verifiable Delay Function in cryptography circles. It's a cryptographically secure way to say time has passed. In Solana, we use a far more granular verifiable delay function, a SHA 256 hash chain, to checkpoint the ledger and coordinate consensus. With it, we implement Optimistic Concurrency Control and are now well in route towards that theoretical limit of 710,000 transactions per second.
 
-fn create_log(hist: &Historian) -> Result<(), SendError<Event>> {
-    sleep(Duration::from_millis(15));
-    let data = Sha256Hash::default();
-    hist.sender.send(Event::Discovery { data })?;
-    sleep(Duration::from_millis(10));
-    Ok(())
-}
 
-fn main() {
-    let seed = Sha256Hash::default();
-    let hist = Historian::new(&seed, Some(10));
-    create_log(&hist).expect("send error");
-    drop(hist.sender);
-    let entries: Vec<Entry> = hist.receiver.iter().collect();
-    for entry in &entries {
-        println!("{:?}", entry);
-    }
+Testnet Demos
+===
 
-    // Proof-of-History: Verify the historian learned about the events
-    // in the same order they appear in the vector.
-    assert!(verify_slice(&entries, &seed));
-}
+The Solana repo contains all the scripts you might need to spin up your own
+local testnet. Depending on what you're looking to achieve, you may want to
+run a different variation, as the full-fledged, performance-enhanced
+multinode testnet is considerably more complex to set up than a Rust-only,
+singlenode testnode.  If you are looking to develop high-level features, such
+as experimenting with smart contracts, save yourself some setup headaches and
+stick to the Rust-only singlenode demo.  If you're doing performance optimization
+of the transaction pipeline, consider the enhanced singlenode demo. If you're
+doing consensus work, you'll need at least a Rust-only multinode demo. If you want
+to reproduce our TPS metrics, run the enhanced multinode demo.
+
+For all four variations, you'd need the latest Rust toolchain and the Solana
+source code:
+
+First, install Rust's package manager Cargo.
+
+```bash
+$ curl https://sh.rustup.rs -sSf | sh
+$ source $HOME/.cargo/env
 ```
 
-Running the program should produce a log similar to:
+Now checkout the code from github:
 
-```rust
-Entry { num_hashes: 0, end_hash: [0, ...], event: Tick }
-Entry { num_hashes: 2, end_hash: [67, ...], event: Discovery { data: [37, ...] } }
-Entry { num_hashes: 3, end_hash: [123, ...], event: Tick }
+```bash
+$ git clone https://github.com/solana-labs/solana.git
+$ cd solana
 ```
 
-Proof-of-History
+The demo code is sometimes broken between releases as we add new low-level
+features, so if this is your first time running the demo, you'll improve
+your odds of success if you check out the
+[latest release](https://github.com/solana-labs/solana/releases)
+before proceeding:
+
+```bash
+$ git checkout v0.8.0
+```
+
+Configuration Setup
 ---
 
-Take note of the last line:
+The network is initialized with a genesis ledger and leader/validator configuration files.
+These files can be generated by running the following script.
 
-```rust
-assert!(verify_slice(&entries, &seed));
+```bash
+$ ./multinode-demo/setup.sh
 ```
 
-[It's a proof!](https://en.wikipedia.org/wiki/Curry–Howard_correspondence) For each entry returned by the
-historian, we can verify that `end_hash` is the result of applying a sha256 hash to the previous `end_hash`
-exactly `num_hashes` times, and then hashing then event data on top of that. Because the event data is
-included in the hash, the events cannot be reordered without regenerating all the hashes.
+Drone
+---
 
-# Developing
+In order for the leader, client and validators to work, we'll need to
+spin up a drone to give out some test tokens.  The drone delivers Milton
+Friedman-style "air drops" (free tokens to requesting clients) to be used in
+test transactions.
+
+Start the drone on the leader node with:
+
+```bash
+$ ./multinode-demo/drone.sh
+```
+
+Singlenode Testnet
+---
+
+Before you start a fullnode, make sure you know the IP address of the machine you
+want to be the leader for the demo, and make sure that udp ports 8000-10000 are
+open on all the machines you want to test with.
+
+Now start the server in a separate shell:
+
+```bash
+$ ./multinode-demo/leader.sh
+```
+
+Wait a few seconds for the server to initialize. It will print "leader ready..." when it's ready to
+receive transactions. The leader will request some tokens from the drone if it doesn't have any.
+The drone does not need to be running for subsequent leader starts.
+
+Multinode Testnet
+---
+
+To run a multinode testnet, after starting a leader node, spin up some validator nodes in
+separate shells:
+
+```bash
+$ ./multinode-demo/validator.sh
+```
+
+To run a performance-enhanced leader or validator (on Linux),
+[CUDA 9.2](https://developer.nvidia.com/cuda-downloads) must be installed on
+your system:
+
+```bash
+$ ./fetch-perf-libs.sh
+$ SOLANA_CUDA=1 ./multinode-demo/leader.sh
+$ SOLANA_CUDA=1 ./multinode-demo/validator.sh
+```
+
+
+Testnet Client Demo
+---
+
+Now that your singlenode or multinode testnet is up and running let's send it
+some transactions!
+
+In a separate shell start the client:
+
+```bash
+$ ./multinode-demo/client.sh # runs against localhost by default
+```
+
+What just happened? The client demo spins up several threads to send 500,000 transactions
+to the testnet as quickly as it can. The client then pings the testnet periodically to see
+how many transactions it processed in that time. Take note that the demo intentionally
+floods the network with UDP packets, such that the network will almost certainly drop a
+bunch of them. This ensures the testnet has an opportunity to reach 710k TPS. The client
+demo completes after it has convinced itself the testnet won't process any additional
+transactions. You should see several TPS measurements printed to the screen. In the
+multinode variation, you'll see TPS measurements for each validator node as well.
+
+Public Testnet
+--------------
+In this example the client connects to our public testnet. To run validators on the testnet you would need to open udp ports `8000-10000`.
+
+```bash
+$ ./multinode-demo/client.sh --network $(dig +short testnet.solana.com):8001 --identity config-private/client-id.json --duration 60
+```
+
+You can observe the effects of your client's transactions on our [dashboard](https://metrics.solana.com:3000/d/testnet/testnet-hud?orgId=2&from=now-30m&to=now&refresh=5s&var-testnet=testnet)
+
+
+Linux Snap
+---
+A Linux [Snap](https://snapcraft.io/) is available, which can be used to
+easily get Solana running on supported Linux systems without building anything
+from source.  The `edge` Snap channel is updated daily with the latest
+development from the `master` branch.  To install:
+
+```bash
+$ sudo snap install solana --edge --devmode
+```
+
+(`--devmode` flag is required only for `solana.fullnode-cuda`)
+
+Once installed the usual Solana programs will be available as `solona.*` instead
+of `solana-*`.  For example, `solana.fullnode` instead of `solana-fullnode`.
+
+Update to the latest version at any time with:
+
+```bash
+$ snap info solana
+$ sudo snap refresh solana --devmode
+```
+
+### Daemon support
+The snap supports running a leader, validator or leader+drone node as a system
+daemon.
+
+Run `sudo snap get solana` to view the current daemon configuration.  To view
+daemon logs:
+1. Run `sudo snap logs -n=all solana` to view the daemon initialization log
+2. Runtime logging can be found under `/var/snap/solana/current/leader/`,
+`/var/snap/solana/current/validator/`, or `/var/snap/solana/current/drone/` depending
+on which `mode=` was selected.  Within each log directory the file `current`
+contains the latest log, and the files `*.s` (if present) contain older rotated
+logs.
+
+Disable the daemon at any time by running:
+
+```bash
+$ sudo snap set solana mode=
+```
+
+Runtime configuration files for the daemon can be found in
+`/var/snap/solana/current/config`.
+
+#### Leader daemon
+
+```bash
+$ sudo snap set solana mode=leader
+```
+
+If CUDA is available:
+
+```bash
+$ sudo snap set solana mode=leader enable-cuda=1
+```
+
+`rsync` must be configured and running on the leader.
+
+1. Ensure rsync is installed with `sudo apt-get -y install rsync`
+2. Edit `/etc/rsyncd.conf` to include the following
+```
+[config]
+path = /var/snap/solana/current/config
+hosts allow = *
+read only = true
+```
+3. Run `sudo systemctl enable rsync; sudo systemctl start rsync`
+4. Test by running `rsync -Pzravv rsync://<ip-address-of-leader>/config
+solana-config` from another machine.  **If the leader is running on a cloud
+provider it may be necessary to configure the Firewall rules to permit ingress
+to port tcp:873, tcp:9900 and the port range udp:8000-udp:10000**
+
+
+To run both the Leader and Drone:
+
+```bash
+$ sudo snap set solana mode=leader+drone
+
+```
+
+#### Validator daemon
+
+```bash
+$ sudo snap set solana mode=validator
+
+```
+If CUDA is available:
+
+```bash
+$ sudo snap set solana mode=validator enable-cuda=1
+```
+
+By default the validator will connect to **testnet.solana.com**, override
+the leader IP address by running:
+
+```bash
+$ sudo snap set solana mode=validator leader-address=127.0.0.1 #<-- change IP address
+```
+
+It's assumed that the leader will be running `rsync` configured as described in
+the previous **Leader daemon** section.
+
+Developing
+===
 
 Building
 ---
@@ -94,11 +276,23 @@ $ source $HOME/.cargo/env
 $ rustup component add rustfmt-preview
 ```
 
+If your rustc version is lower than 1.26.1, please update it:
+
+```bash
+$ rustup update
+```
+
+On Linux systems you may need to install libssl-dev, pkg-config, zlib1g-dev, etc.  On Ubuntu:
+
+```bash
+$ sudo apt-get install libssl-dev pkg-config zlib1g-dev
+```
+
 Download the source code:
 
 ```bash
-$ git clone https://github.com/loomprotocol/silk.git
-$ cd silk
+$ git clone https://github.com/solana-labs/solana.git
+$ cd solana
 ```
 
 Testing
@@ -107,9 +301,42 @@ Testing
 Run the test suite:
 
 ```bash
-cargo test
+$ cargo test
 ```
 
+To emulate all the tests that will run on a Pull Request, run:
+
+```bash
+$ ./ci/run-local.sh
+```
+
+Debugging
+---
+
+There are some useful debug messages in the code, you can enable them on a per-module and per-level
+basis.  Before running a leader or validator set the normal RUST\_LOG environment variable.
+
+For example, to enable info everywhere and debug only in the solana::banking_stage module:
+
+```bash
+$ export RUST_LOG=info,solana::banking_stage=debug
+```
+
+Generally we are using debug for infrequent debug messages, trace for potentially frequent
+messages and info for performance-related logging.
+
+You can also attach to a running process with GDB.  The leader's process is named
+_solana-fullnode_:
+
+```bash
+$ sudo gdb
+attach <PID>
+set logging on
+thread apply all bt
+```
+
+This will dump all the threads stack traces into gdb.txt
+
 Benchmarking
 ---
 
@@ -122,5 +349,42 @@ $ rustup install nightly
 Run the benchmarks:
 
 ```bash
-$ cargo +nightly bench --features="asm,unstable"
+$ cargo +nightly bench --features="unstable"
 ```
+
+Release Process
+---
+The release process for this project is described [here](rfcs/rfc-005-branches-tags-and-channels.md).
+
+
+Code coverage
+---
+
+To generate code coverage statistics, install cargo-cov. Note: the tool currently only works
+in Rust nightly.
+
+```bash
+$ cargo +nightly install cargo-cov
+```
+
+Run cargo-cov and generate a report:
+
+```bash
+$ cargo +nightly cov test
+$ cargo +nightly cov report --open
+```
+
+The coverage report will be written to `./target/cov/report/index.html`
+
+
+Why coverage? While most see coverage as a code quality metric, we see it primarily as a developer
+productivity metric. When a developer makes a change to the codebase, presumably it's a *solution* to
+some problem.  Our unit-test suite is how we encode the set of *problems* the codebase solves. Running
+the test suite should indicate that your change didn't *infringe* on anyone else's solutions. Adding a
+test *protects* your solution from future changes. Say you don't understand why a line of code exists,
+try deleting it and running the unit-tests. The nearest test failure should tell you what problem
+was solved by that code. If no test fails, go ahead and submit a Pull Request that asks, "what
+problem is solved by this code?" On the other hand, if a test does fail and you can think of a
+better way to solve the same problem, a Pull Request with your solution would most certainly be
+welcome! Likewise, if rewriting a test can better communicate what code it's protecting, please
+send us that patch!

benches/bank.rs

@@ -0,0 +1,48 @@
+#![feature(test)]
+extern crate bincode;
+extern crate rayon;
+extern crate solana;
+extern crate test;
+
+use bincode::serialize;
+use rayon::prelude::*;
+use solana::bank::*;
+use solana::hash::hash;
+use solana::mint::Mint;
+use solana::signature::{Keypair, KeypairUtil};
+use solana::transaction::Transaction;
+use test::Bencher;
+
+#[bench]
+fn bench_process_transaction(bencher: &mut Bencher) {
+    let mint = Mint::new(100_000_000);
+    let bank = Bank::new(&mint);
+
+    // Create transactions between unrelated parties.
+    let transactions: Vec<_> = (0..4096)
+        .into_par_iter()
+        .map(|i| {
+            // Seed the 'from' account.
+            let rando0 = Keypair::new();
+            let tx = Transaction::new(&mint.keypair(), rando0.pubkey(), 10_000, mint.last_id());
+            assert!(bank.process_transaction(&tx).is_ok());
+
+            // Seed the 'to' account and a cell for its signature.
+            let last_id = hash(&serialize(&i).unwrap()); // Unique hash
+            bank.register_entry_id(&last_id);
+
+            let rando1 = Keypair::new();
+            let tx = Transaction::new(&rando0, rando1.pubkey(), 1, last_id);
+            assert!(bank.process_transaction(&tx).is_ok());
+
+            // Finally, return the transaction to the benchmark.
+            tx
+        }).collect();
+
+    bencher.iter(|| {
+        // Since benchmarker runs this multiple times, we need to clear the signatures.
+        bank.clear_signatures();
+        let results = bank.process_transactions(transactions.clone());
+        assert!(results.iter().all(Result::is_ok));
+    })
+}

benches/banking_stage.rs

@@ -0,0 +1,206 @@
+#![feature(test)]
+extern crate bincode;
+extern crate rayon;
+extern crate solana;
+extern crate test;
+
+use rayon::prelude::*;
+use solana::bank::Bank;
+use solana::banking_stage::BankingStage;
+use solana::mint::Mint;
+use solana::packet::{to_packets_chunked, PacketRecycler};
+use solana::record_stage::Signal;
+use solana::signature::{Keypair, KeypairUtil};
+use solana::transaction::Transaction;
+use std::iter;
+use std::sync::mpsc::{channel, Receiver};
+use std::sync::Arc;
+use test::Bencher;
+
+// use self::test::Bencher;
+// use bank::{Bank, MAX_ENTRY_IDS};
+// use bincode::serialize;
+// use hash::hash;
+// use mint::Mint;
+// use rayon::prelude::*;
+// use signature::{Keypair, KeypairUtil};
+// use std::collections::HashSet;
+// use std::time::Instant;
+// use transaction::Transaction;
+//
+// fn bench_process_transactions(_bencher: &mut Bencher) {
+//     let mint = Mint::new(100_000_000);
+//     let bank = Bank::new(&mint);
+//     // Create transactions between unrelated parties.
+//     let txs = 100_000;
+//     let last_ids: Mutex<HashSet<Hash>> = Mutex::new(HashSet::new());
+//     let transactions: Vec<_> = (0..txs)
+//         .into_par_iter()
+//         .map(|i| {
+//             // Seed the 'to' account and a cell for its signature.
+//             let dummy_id = i % (MAX_ENTRY_IDS as i32);
+//             let last_id = hash(&serialize(&dummy_id).unwrap()); // Semi-unique hash
+//             {
+//                 let mut last_ids = last_ids.lock().unwrap();
+//                 if !last_ids.contains(&last_id) {
+//                     last_ids.insert(last_id);
+//                     bank.register_entry_id(&last_id);
+//                 }
+//             }
+//
+//             // Seed the 'from' account.
+//             let rando0 = Keypair::new();
+//             let tx = Transaction::new(&mint.keypair(), rando0.pubkey(), 1_000, last_id);
+//             bank.process_transaction(&tx).unwrap();
+//
+//             let rando1 = Keypair::new();
+//             let tx = Transaction::new(&rando0, rando1.pubkey(), 2, last_id);
+//             bank.process_transaction(&tx).unwrap();
+//
+//             // Finally, return a transaction that's unique
+//             Transaction::new(&rando0, rando1.pubkey(), 1, last_id)
+//         })
+//         .collect();
+//
+//     let banking_stage = EventProcessor::new(bank, &mint.last_id(), None);
+//
+//     let now = Instant::now();
+//     assert!(banking_stage.process_transactions(transactions).is_ok());
+//     let duration = now.elapsed();
+//     let sec = duration.as_secs() as f64 + duration.subsec_nanos() as f64 / 1_000_000_000.0;
+//     let tps = txs as f64 / sec;
+//
+//     // Ensure that all transactions were successfully logged.
+//     drop(banking_stage.historian_input);
+//     let entries: Vec<Entry> = banking_stage.output.lock().unwrap().iter().collect();
+//     assert_eq!(entries.len(), 1);
+//     assert_eq!(entries[0].transactions.len(), txs as usize);
+//
+//     println!("{} tps", tps);
+// }
+
+fn check_txs(receiver: &Receiver<Signal>, ref_tx_count: usize) {
+    let mut total = 0;
+    loop {
+        let signal = receiver.recv().unwrap();
+        if let Signal::Transactions(transactions) = signal {
+            total += transactions.len();
+            if total >= ref_tx_count {
+                break;
+            }
+        } else {
+            assert!(false);
+        }
+    }
+    assert_eq!(total, ref_tx_count);
+}
+
+#[bench]
+fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) {
+    let tx = 10_000_usize;
+    let mint_total = 1_000_000_000_000;
+    let mint = Mint::new(mint_total);
+    let num_dst_accounts = 8 * 1024;
+    let num_src_accounts = 8 * 1024;
+
+    let srckeys: Vec<_> = (0..num_src_accounts).map(|_| Keypair::new()).collect();
+    let dstkeys: Vec<_> = (0..num_dst_accounts)
+        .map(|_| Keypair::new().pubkey())
+        .collect();
+
+    let transactions: Vec<_> = (0..tx)
+        .map(|i| {
+            Transaction::new(
+                &srckeys[i % num_src_accounts],
+                dstkeys[i % num_dst_accounts],
+                i as i64,
+                mint.last_id(),
+            )
+        }).collect();
+
+    let (verified_sender, verified_receiver) = channel();
+    let (signal_sender, signal_receiver) = channel();
+    let packet_recycler = PacketRecycler::default();
+
+    let setup_transactions: Vec<_> = (0..num_src_accounts)
+        .map(|i| {
+            Transaction::new(
+                &mint.keypair(),
+                srckeys[i].pubkey(),
+                mint_total / num_src_accounts as i64,
+                mint.last_id(),
+            )
+        }).collect();
+
+    bencher.iter(move || {
+        let bank = Arc::new(Bank::new(&mint));
+
+        let verified_setup: Vec<_> =
+            to_packets_chunked(&packet_recycler, &setup_transactions.clone(), tx)
+                .into_iter()
+                .map(|x| {
+                    let len = (*x).read().unwrap().packets.len();
+                    (x, iter::repeat(1).take(len).collect())
+                }).collect();
+
+        verified_sender.send(verified_setup).unwrap();
+        BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler)
+            .unwrap();
+
+        check_txs(&signal_receiver, num_src_accounts);
+
+        let verified: Vec<_> = to_packets_chunked(&packet_recycler, &transactions.clone(), 192)
+            .into_iter()
+            .map(|x| {
+                let len = (*x).read().unwrap().packets.len();
+                (x, iter::repeat(1).take(len).collect())
+            }).collect();
+
+        verified_sender.send(verified).unwrap();
+        BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler)
+            .unwrap();
+
+        check_txs(&signal_receiver, tx);
+    });
+}
+
+#[bench]
+fn bench_banking_stage_single_from(bencher: &mut Bencher) {
+    let tx = 10_000_usize;
+    let mint = Mint::new(1_000_000_000_000);
+    let mut pubkeys = Vec::new();
+    let num_keys = 8;
+    for _ in 0..num_keys {
+        pubkeys.push(Keypair::new().pubkey());
+    }
+
+    let transactions: Vec<_> = (0..tx)
+        .into_par_iter()
+        .map(|i| {
+            Transaction::new(
+                &mint.keypair(),
+                pubkeys[i % num_keys],
+                i as i64,
+                mint.last_id(),
+            )
+        }).collect();
+
+    let (verified_sender, verified_receiver) = channel();
+    let (signal_sender, signal_receiver) = channel();
+    let packet_recycler = PacketRecycler::default();
+
+    bencher.iter(move || {
+        let bank = Arc::new(Bank::new(&mint));
+        let verified: Vec<_> = to_packets_chunked(&packet_recycler, &transactions.clone(), tx)
+            .into_iter()
+            .map(|x| {
+                let len = (*x).read().unwrap().packets.len();
+                (x, iter::repeat(1).take(len).collect())
+            }).collect();
+        verified_sender.send(verified).unwrap();
+        BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler)
+            .unwrap();
+
+        check_txs(&signal_receiver, tx);
+    });
+}

benches/ledger.rs

@@ -0,0 +1,26 @@
+#![feature(test)]
+extern crate solana;
+extern crate test;
+
+use solana::hash::{hash, Hash};
+use solana::ledger::{next_entries, reconstruct_entries_from_blobs, Block};
+use solana::packet::BlobRecycler;
+use solana::signature::{Keypair, KeypairUtil};
+use solana::transaction::Transaction;
+use test::Bencher;
+
+#[bench]
+fn bench_block_to_blobs_to_block(bencher: &mut Bencher) {
+    let zero = Hash::default();
+    let one = hash(&zero.as_ref());
+    let keypair = Keypair::new();
+    let tx0 = Transaction::new(&keypair, keypair.pubkey(), 1, one);
+    let transactions = vec![tx0; 10];
+    let entries = next_entries(&zero, 1, transactions);
+
+    let blob_recycler = BlobRecycler::default();
+    bencher.iter(|| {
+        let blobs = entries.to_blobs(&blob_recycler);
+        assert_eq!(reconstruct_entries_from_blobs(blobs).unwrap(), entries);
+    });
+}

benches/signature.rs

@@ -0,0 +1,12 @@
+#![feature(test)]
+extern crate solana;
+extern crate test;
+
+use solana::signature::GenKeys;
+use test::Bencher;
+
+#[bench]
+fn bench_gen_keys(b: &mut Bencher) {
+    let mut rnd = GenKeys::new([0u8; 32]);
+    b.iter(|| rnd.gen_n_keypairs(1000));
+}

benches/sigverify.rs

@@ -0,0 +1,24 @@
+#![feature(test)]
+extern crate bincode;
+extern crate rayon;
+extern crate solana;
+extern crate test;
+
+use solana::packet::{to_packets, PacketRecycler};
+use solana::sigverify;
+use solana::transaction::test_tx;
+use test::Bencher;
+
+#[bench]
+fn bench_sigverify(bencher: &mut Bencher) {
+    let tx = test_tx();
+
+    // generate packet vector
+    let packet_recycler = PacketRecycler::default();
+    let batches = to_packets(&packet_recycler, &vec![tx; 128]);
+
+    // verify packets
+    bencher.iter(|| {
+        let _ans = sigverify::ed25519_verify(&batches);
+    })
+}

build.rs

@@ -0,0 +1,34 @@
+use std::env;
+use std::fs;
+
+fn main() {
+    println!("cargo:rerun-if-changed=target/perf-libs");
+    println!("cargo:rerun-if-changed=build.rs");
+
+    // Ensure target/perf-libs/ exists.  It's been observed that
+    // a cargo:rerun-if-changed= directive with a non-existent
+    // directory triggers a rebuild on every |cargo build| invocation
+    fs::create_dir("target/perf-libs").unwrap_or_else(|err| {
+        if err.kind() != std::io::ErrorKind::AlreadyExists {
+            panic!("Unable to create target/perf-libs: {:?}", err);
+        }
+    });
+
+    let cuda = !env::var("CARGO_FEATURE_CUDA").is_err();
+    let erasure = !env::var("CARGO_FEATURE_ERASURE").is_err();
+
+    if cuda || erasure {
+        println!("cargo:rustc-link-search=native=target/perf-libs");
+    }
+    if cuda {
+        println!("cargo:rustc-link-lib=static=cuda_verify_ed25519");
+        println!("cargo:rustc-link-search=native=/usr/local/cuda/lib64");
+        println!("cargo:rustc-link-lib=dylib=cudart");
+        println!("cargo:rustc-link-lib=dylib=cuda");
+        println!("cargo:rustc-link-lib=dylib=cudadevrt");
+    }
+    if erasure {
+        println!("cargo:rustc-link-lib=dylib=Jerasure");
+        println!("cargo:rustc-link-lib=dylib=gf_complete");
+    }
+}

ci/.gitignore

@@ -0,0 +1,3 @@
+/node_modules/
+/package-lock.json
+/snapcraft.credentials

ci/README.md

@@ -0,0 +1,89 @@
+
+Our CI infrastructure is built around [BuildKite](https://buildkite.com) with some
+additional GitHub integration provided by https://github.com/mvines/ci-gate
+
+## Agent Queues
+
+We define two [Agent Queues](https://buildkite.com/docs/agent/v3/queues):
+`queue=default` and `queue=cuda`.  The `default` queue should be favored and
+runs on lower-cost CPU instances.  The `cuda` queue is only necessary for
+running **tests** that depend on GPU (via CUDA) access -- CUDA builds may still
+be run on the `default` queue, and the [buildkite artifact
+system](https://buildkite.com/docs/builds/artifacts) used to transfer build
+products over to a GPU instance for testing.
+
+## Buildkite Agent Management
+
+### Buildkite GCP Setup
+
+CI runs on Google Cloud Platform via two Compute Engine Instance groups:
+`ci-default` and `ci-cuda`.  Autoscaling is currently disabled and the number of
+VM Instances in each group is manually adjusted.
+
+#### Updating a CI Disk Image
+
+Each Instance group has its own disk image, `ci-default-vX` and
+`ci-cuda-vY`, where *X* and *Y* are incremented each time the image is changed.
+
+The process to update a disk image is as follows (TODO: make this less manual):
+
+1. Create a new VM Instance using the disk image to modify.
+2. Once the VM boots, ssh to it and modify the disk as desired.
+3. Stop the VM Instance running the modified disk.  Remember the name of the VM disk
+4. From another machine, `gcloud auth login`, then create a new Disk Image based
+off the modified VM Instance:
+```
+ $ gcloud compute images create ci-default-$(date +%Y%m%d%H%M) --source-disk xxx --source-disk-zone us-east1-b --family ci-default
+
+```
+or
+```
+  $ gcloud compute images create ci-cuda-$(date +%Y%m%d%H%M) --source-disk xxx --source-disk-zone us-east1-b --family ci-cuda
+```
+5. Delete the new VM instance.
+6. Go to the Instance templates tab, find the existing template named
+`ci-default-vX` or `ci-cuda-vY` and select it.  Use the "Copy" button to create
+a new Instance template called `ci-default-vX+1` or `ci-cuda-vY+1` with the
+newly created Disk image.
+7. Go to the Instance Groups tag and find the applicable group, `ci-default` or
+`ci-cuda`.  Edit the Instance Group in two steps: (a) Set the number of
+instances to 0 and wait for them all to terminate, (b) Update the Instance
+template and restore the number of instances to the original value.
+8. Clean up the previous version by deleting it from Instance Templates and
+Images. 
+
+
+## Reference
+
+### Buildkite AWS CloudFormation Setup
+
+**AWS CloudFormation is currently inactive, although it may be restored in the
+future**
+
+AWS CloudFormation can be used to scale machines up and down based on the
+current CI load.  If no machine is currently running it can take up to 60
+seconds to spin up a new instance, please remain calm during this time.
+
+#### AMI
+We use a custom AWS AMI built via https://github.com/solana-labs/elastic-ci-stack-for-aws/tree/solana/cuda.
+
+Use the following process to update this AMI as dependencies change:
+```bash
+$ export AWS_ACCESS_KEY_ID=my_access_key
+$ export AWS_SECRET_ACCESS_KEY=my_secret_access_key
+$ git clone https://github.com/solana-labs/elastic-ci-stack-for-aws.git -b solana/cuda
+$ cd elastic-ci-stack-for-aws/
+$ make build
+$ make build-ami
+```
+
+Watch for the *"amazon-ebs: AMI:"* log message to extract the name of the new
+AMI.  For example:
+```
+amazon-ebs: AMI: ami-07118545e8b4ce6dc
+```
+The new AMI should also now be visible in your EC2 Dashboard.  Go to the desired
+AWS CloudFormation stack, update the **ImageId** field to the new AMI id, and
+*apply* the stack changes.
+
+

ci/audit.sh

@@ -0,0 +1,32 @@
+#!/bin/bash -e
+#
+# Audits project dependencies for security vulnerabilities
+#
+
+cd "$(dirname "$0")/.."
+
+export RUST_BACKTRACE=1
+rustc --version
+cargo --version
+
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+maybe_cargo_install() {
+  for cmd in "$@"; do
+    set +e
+    cargo "$cmd" --help > /dev/null 2>&1
+    declare exitcode=$?
+    set -e
+    if [[ $exitcode -eq 101 ]]; then
+      _ cargo install cargo-"$cmd"
+    fi
+  done
+}
+
+maybe_cargo_install audit tree
+
+_ cargo tree
+_ cargo audit || true

ci/buildkite-snap.yml

@@ -0,0 +1,4 @@
+steps:
+  - command: "ci/snap.sh"
+    timeout_in_minutes: 40
+    name: "snap [public]"

ci/buildkite.yml

@@ -0,0 +1,42 @@
+steps:
+  - command: "ci/docker-run.sh solanalabs/rust:1.29.0 ci/test-stable.sh"
+    name: "stable [public]"
+    env:
+      CARGO_TARGET_CACHE_NAME: "stable"
+    timeout_in_minutes: 30
+    #  - command: "ci/docker-run.sh solanalabs/rust-nightly ci/test-bench.sh"
+    #    name: "bench [public]"
+    #    env:
+    #      CARGO_TARGET_CACHE_NAME: "nightly"
+    #    timeout_in_minutes: 30
+  - command: "ci/shellcheck.sh"
+    name: "shellcheck [public]"
+    timeout_in_minutes: 20
+  - command: "ci/docker-run.sh solanalabs/rust-nightly:2018-09-03 ci/test-nightly.sh || true"
+    name: "nightly [public]"
+    env:
+      CARGO_TARGET_CACHE_NAME: "nightly"
+    timeout_in_minutes: 30
+  - command: "ci/test-stable-perf.sh"
+    name: "stable-perf [public]"
+    env:
+      CARGO_TARGET_CACHE_NAME: "stable-perf"
+    timeout_in_minutes: 20
+    agents:
+      - "queue=cuda"
+  - command: "ci/pr-snap.sh"
+    timeout_in_minutes: 20
+    name: "snap [public]"
+  - wait
+  - command: "ci/publish-crate.sh"
+    timeout_in_minutes: 20
+    name: "publish crate [public]"
+  - trigger: "solana-snap"
+    branches: "!pull/*"
+    async: true
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        TRIGGERED_BUILDKITE_TAG: "${BUILDKITE_TAG}"

ci/channel-info.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# Computes the current branch names of the edge, beta and stable
+# channels, as well as the latest tagged release for beta and stable.
+#
+# stdout of this script may be eval-ed
+#
+
+here="$(dirname "$0")"
+
+# shellcheck source=ci/semver_bash/semver.sh
+source "$here"/semver_bash/semver.sh
+
+remote=https://github.com/solana-labs/solana.git
+
+# Fetch all vX.Y.Z tags
+#
+# NOTE: pre-release tags are explicitly ignored
+#
+# shellcheck disable=SC2207
+tags=( \
+  $(git ls-remote --tags $remote \
+    | cut -c52- \
+    | grep '^v[[:digit:]][[:digit:]]*\.[[:digit:]][[:digit:]]*.[[:digit:]][[:digit:]]*$' \
+    | cut -c2- \
+  ) \
+)
+
+# Fetch all the vX.Y branches
+#
+# shellcheck disable=SC2207
+heads=( \
+  $(git ls-remote --heads $remote \
+    | cut -c53- \
+    | grep '^v[[:digit:]][[:digit:]]*\.[[:digit:]][[:digit:]]*$' \
+    | cut -c2- \
+  ) \
+)
+
+# Figure the beta channel by looking for the largest vX.Y branch
+beta=
+for head in "${heads[@]}"; do
+  if [[ -n $beta ]]; then
+    if semverLT "$head.0" "$beta.0"; then
+      continue
+    fi
+  fi
+  beta=$head
+done
+
+# Figure the stable channel by looking for the second largest vX.Y branch
+stable=
+for head in "${heads[@]}"; do
+  if [[ $head = "$beta" ]]; then
+    continue
+  fi
+  if [[ -n $stable ]]; then
+    if semverLT "$head.0" "$stable.0"; then
+      continue
+    fi
+  fi
+  stable=$head
+done
+
+for tag in "${tags[@]}"; do
+  if [[ -n $beta && $tag = $beta* ]]; then
+    if [[ -n $beta_tag ]]; then
+      if semverLT "$tag" "$beta_tag"; then
+        continue
+      fi
+    fi
+    beta_tag=$tag
+  fi
+
+  if [[ -n $stable && $tag = $stable* ]]; then
+    if [[ -n $stable_tag ]]; then
+      if semverLT "$tag" "$stable_tag"; then
+        continue
+      fi
+    fi
+    stable_tag=$tag
+  fi
+done
+
+echo EDGE_CHANNEL=master
+echo BETA_CHANNEL="${beta:+v$beta}"
+echo STABLE_CHANNEL="${stable:+v$stable}"
+echo BETA_CHANNEL_LATEST_TAG="${beta_tag:+v$beta_tag}"
+echo STABLE_CHANNEL_LATEST_TAG="${stable_tag:+v$stable_tag}"
+
+exit 0

ci/docker-run.sh

@@ -0,0 +1,68 @@
+#!/bin/bash -e
+
+usage() {
+  echo "Usage: $0 [--nopull] [docker image name] [command]"
+  echo
+  echo Runs command in the specified docker image with
+  echo a CI-appropriate environment.
+  echo
+  echo "--nopull   Skip the dockerhub image update"
+  echo
+}
+
+cd "$(dirname "$0")/.."
+
+NOPULL=false
+if [[ $1 = --nopull ]]; then
+  NOPULL=true
+  shift
+fi
+
+IMAGE="$1"
+if [[ -z "$IMAGE" ]]; then
+  echo Error: image not defined
+  exit 1
+fi
+
+$NOPULL || docker pull "$IMAGE"
+shift
+
+ARGS=(
+  --workdir /solana
+  --volume "$PWD:/solana"
+  --rm
+)
+
+if [[ -n $CI ]]; then
+  # Share the real ~/.cargo between docker containers in CI for speed
+  ARGS+=(--volume "$HOME:/home")
+else
+  # Avoid sharing ~/.cargo when building locally to avoid a mixed macOS/Linux
+  # ~/.cargo
+  ARGS+=(--volume "$PWD:/home")
+fi
+ARGS+=(--env "CARGO_HOME=/home/.cargo")
+
+# kcov tries to set the personality of the binary which docker
+# doesn't allow by default.
+ARGS+=(--security-opt "seccomp=unconfined")
+
+# Ensure files are created with the current host uid/gid
+if [[ -z "$SOLANA_DOCKER_RUN_NOSETUID" ]]; then
+  ARGS+=(--user "$(id -u):$(id -g)")
+fi
+
+# Environment variables to propagate into the container
+ARGS+=(
+  --env BUILDKITE
+  --env BUILDKITE_AGENT_ACCESS_TOKEN
+  --env BUILDKITE_BRANCH
+  --env BUILDKITE_JOB_ID
+  --env BUILDKITE_TAG
+  --env CODECOV_TOKEN
+  --env CRATES_IO_TOKEN
+  --env SNAPCRAFT_CREDENTIALS_KEY
+)
+
+set -x
+exec docker run "${ARGS[@]}" "$IMAGE" "$@"

ci/docker-rust-nightly/Dockerfile

@@ -0,0 +1,10 @@
+FROM solanalabs/rust
+ARG date
+
+RUN set -x && \
+    rustup install nightly-$date && \
+    rustup default nightly-$date && \
+    rustup component add clippy-preview --toolchain=nightly-$date && \
+    rustc --version && \
+    cargo --version && \
+    cargo +nightly-$date install cargo-cov

ci/docker-rust-nightly/README.md

@@ -0,0 +1,36 @@
+Docker image containing rust nightly and some preinstalled crates used in CI.
+
+This image may be manually updated by running `CI=true ./build.sh` if you are a member
+of the [Solana Labs](https://hub.docker.com/u/solanalabs/) Docker Hub
+organization, but it is also automatically updated periodically by
+[this automation](https://buildkite.com/solana-labs/solana-ci-docker-rust-nightly).
+
+## Moving to a newer nightly
+
+We pin the version of nightly (see the `ARG nightly=xyz` line in `Dockerfile`)
+to avoid the build breaking at unexpected times, as occasionally nightly will
+introduce breaking changes.
+
+To update the pinned version:
+1. Run `ci/docker-rust-nightly/build.sh` to rebuild the nightly image locally,
+   or potentially `ci/docker-rust-nightly/build.sh YYYY-MM-DD` if there's a
+   specific YYYY-MM-DD that is desired (default is today's build).
+1. Run `SOLANA_DOCKER_RUN_NOSETUID=1 ci/docker-run.sh --nopull solanalabs/rust-nightly:YYYY-MM-DD ci/test-nightly.sh`
+   to confirm the new nightly image builds.  Fix any issues as needed
+1. Run `docker login` to enable pushing images to Docker Hub, if you're authorized.
+1. Run `CI=true ci/docker-rust-nightly/build.sh YYYY-MM-DD` to push the new nightly image to dockerhub.com.
+1. Modify the `solanalabs/rust-nightly:YYYY-MM-DD` reference in `ci/buildkite.yml` from the previous to
+   new *YYYY-MM-DD* value, send a PR with this change and any codebase adjustments needed.
+
+## Troubleshooting
+
+### Resource is denied
+
+When running `CI=true ci/docker-rust-nightly/build.sh`, you see:
+
+```
+denied: requested access to the resource is denied
+```
+
+Run `docker login` to enable pushing images to Docker Hub. Contact @mvines or @garious
+to get write access.

ci/docker-rust-nightly/build.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -ex
+
+cd "$(dirname "$0")"
+
+nightlyDate=${1:-$(date +%Y-%m-%d)}
+docker build -t solanalabs/rust-nightly:"$nightlyDate" --build-arg date="$nightlyDate" .
+
+maybeEcho=
+if [[ -z $CI ]]; then
+  echo "Not CI, skipping |docker push|"
+  maybeEcho="echo"
+fi
+$maybeEcho docker push solanalabs/rust-nightly:"$nightlyDate"

ci/docker-rust/Dockerfile

@@ -0,0 +1,23 @@
+# Note: when the rust version (1.28) is changed also modify
+# ci/buildkite.yml to pick up the new image tag
+FROM rust:1.29.0
+
+RUN set -x && \
+    apt update && \
+    apt-get install apt-transport-https && \
+    echo deb https://apt.buildkite.com/buildkite-agent stable main > /etc/apt/sources.list.d/buildkite-agent.list && \
+    echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main > /etc/apt/sources.list.d/llvm.list && \
+    apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 32A37959C2FA5C3C99EFBC32A79206696452D198 && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    apt update && \
+    apt install -y \
+      buildkite-agent \
+      cmake \
+      llvm-6.0 \
+      rsync \
+      sudo \
+      && \
+    rustup component add rustfmt-preview && \
+    rm -rf /var/lib/apt/lists/* && \
+    rustc --version && \
+    cargo --version

ci/docker-rust/README.md

@@ -0,0 +1,6 @@
+Docker image containing rust and some preinstalled packages used in CI.
+
+This image may be manually updated by running `./build.sh` if you are a member
+of the [Solana Labs](https://hub.docker.com/u/solanalabs/) Docker Hub
+organization, but it is also automatically updated periodically by
+[this automation](https://buildkite.com/solana-labs/solana-ci-docker-rust).

ci/docker-rust/build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -ex
+
+cd "$(dirname "$0")"
+
+docker build -t solanalabs/rust .
+
+read -r rustc version _ < <(docker run solanalabs/rust rustc --version)
+[[ $rustc = rustc ]]
+docker tag solanalabs/rust:latest solanalabs/rust:"$version"
+
+docker push solanalabs/rust

ci/docker-snapcraft/Dockerfile

@@ -0,0 +1,7 @@
+FROM snapcraft/xenial-amd64
+
+# Update snapcraft to latest version
+RUN apt-get update -qq \
+ && apt-get install -y snapcraft daemontools \
+ && rm -rf /var/lib/apt/lists/* \
+ && snapcraft --version

ci/docker-snapcraft/build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -ex
+
+cd "$(dirname "$0")"
+
+docker build -t solanalabs/snapcraft .
+docker push solanalabs/snapcraft

ci/hoover.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Regular maintenance performed on a buildkite agent to control disk usage
+#
+
+
+echo --- Delete all exited containers first
+(
+  set -x
+  exited=$(docker ps -aq --no-trunc --filter "status=exited")
+  if [[ -n "$exited" ]]; then
+    # shellcheck disable=SC2086 # Don't want to double quote "$exited"
+    docker rm $exited
+  fi
+)
+
+echo --- Delete untagged images
+(
+  set -x
+  untagged=$(docker images | grep '<none>'| awk '{ print $3 }')
+  if [[ -n "$untagged" ]]; then
+    # shellcheck disable=SC2086 # Don't want to double quote "$untagged"
+    docker rmi $untagged
+  fi
+)
+
+echo --- Delete all dangling images
+(
+  set -x
+  dangling=$(docker images --filter 'dangling=true' -q --no-trunc | sort | uniq)
+  if [[ -n "$dangling" ]]; then
+    # shellcheck disable=SC2086 # Don't want to double quote "$dangling"
+    docker rmi $dangling
+  fi
+)
+
+echo --- Remove unused docker networks
+(
+  set -x
+  docker network prune -f
+)
+
+echo "--- Delete /tmp files older than 1 day owned by $(id -un)"
+(
+  set -x
+  find /tmp -maxdepth 1 -user "$(id -un)" -mtime +1 -print0 | xargs -0 rm -rf
+)
+
+echo --- Deleting stale buildkite agent build directories
+if [[ ! -d ../../../../builds/$BUILDKITE_AGENT_NAME ]]; then
+  # We might not be where we think we are, do nothing
+  echo Warning: Skipping flush of stale agent build directories
+  echo "  PWD=$PWD"
+else
+  # NOTE: this will be horribly broken if we ever decide to run multiple
+  #       agents on the same machine.
+  (
+    for keepDir in "$BUILDKITE_PIPELINE_SLUG" \
+                   "$BUILDKITE_ORGANIZATION_SLUG" \
+                   "$BUILDKITE_AGENT_NAME"; do
+      cd .. || exit 1
+      for dir in *; do
+        if [[ -d $dir && $dir != "$keepDir" ]]; then
+          echo "Removing $dir"
+          rm -rf "${dir:?}"/
+        fi
+      done
+    done
+  )
+fi
+
+echo --- System Status
+(
+  set -x
+  docker images
+  docker ps
+  docker network ls
+  df -h
+)
+
+exit 0

ci/is-pr.sh

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+#
+# The standard BUILDKITE_PULL_REQUEST environment variable is always "false" due
+# to how solana-ci-gate is used to trigger PR builds rather than using the
+# standard Buildkite PR trigger.
+#
+
+[[ $BUILDKITE_BRANCH =~ pull/* ]]

ci/localnet-sanity.sh

@@ -0,0 +1,93 @@
+#!/bin/bash -e
+#
+# Perform a quick sanity test on a leader, drone, validator and client running
+# locally on the same machine
+#
+
+cd "$(dirname "$0")"/..
+source ci/upload_ci_artifact.sh
+source scripts/configure-metrics.sh
+
+multinode-demo/setup.sh
+
+backgroundCommands="drone leader validator validator-x"
+pids=()
+
+for cmd in $backgroundCommands; do
+  echo "--- Start $cmd"
+  rm -f log-"$cmd".txt
+  multinode-demo/"$cmd".sh > log-"$cmd".txt 2>&1 &
+  declare pid=$!
+  pids+=("$pid")
+  echo "pid: $pid"
+done
+
+killBackgroundCommands() {
+  set +e
+  for pid in "${pids[@]}"; do
+    if kill "$pid"; then
+      wait "$pid"
+    else
+      echo -e "^^^ +++\\nWarning: unable to kill $pid"
+    fi
+  done
+  set -e
+  pids=()
+}
+
+shutdown() {
+  exitcode=$?
+  killBackgroundCommands
+
+  set +e
+
+  echo "--- Upload artifacts"
+  for cmd in $backgroundCommands; do
+    declare logfile=log-$cmd.txt
+    upload_ci_artifact "$logfile"
+    tail "$logfile"
+  done
+
+  exit $exitcode
+}
+
+trap shutdown EXIT INT
+
+set -e
+
+flag_error() {
+  echo Failed
+  echo "^^^ +++"
+  exit 1
+}
+
+echo "--- Wallet sanity"
+(
+  set -x
+  scripts/wallet-sanity.sh
+) || flag_error
+
+echo "--- Node count"
+(
+  source multinode-demo/common.sh
+  set -x
+  client_id=/tmp/client-id.json-$$
+  $solana_keygen -o $client_id
+  $solana_bench_tps --identity $client_id --num-nodes 3 --converge-only
+  rm -rf $client_id
+) || flag_error
+
+killBackgroundCommands
+
+echo "--- Ledger verification"
+(
+  source multinode-demo/common.sh
+  set -x
+  cp -R "$SOLANA_CONFIG_DIR"/ledger /tmp/ledger-$$
+  $solana_ledger_tool --ledger /tmp/ledger-$$ verify
+  rm -rf /tmp/ledger-$$
+) || flag_error
+
+echo +++
+echo Ok
+exit 0

ci/pr-snap.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+#
+# Only run snap.sh for pull requests that modify files under /snap
+#
+
+cd "$(dirname "$0")"
+
+if ./is-pr.sh; then
+  affected_files="$(buildkite-agent meta-data get affected_files)"
+  echo "Affected files in this PR: $affected_files"
+  if [[ ! ":$affected_files:" =~ :snap/ ]]; then
+    echo "Skipping snap build as no files under /snap were modified"
+    exit 0
+  fi
+  exec ./snap.sh
+else
+  echo "Skipping snap build as this is not a pull request"
+fi

ci/publish-crate.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+if [[ -z "$BUILDKITE_TAG" ]]; then
+  # Skip publish if this is not a tagged release
+  exit 0
+fi
+
+if [[ -z "$CRATES_IO_TOKEN" ]]; then
+  echo CRATES_IO_TOKEN undefined
+  exit 1
+fi
+
+# TODO: Ensure the published version matches the contents of BUILDKITE_TAG
+ci/docker-run.sh rust \
+  bash -exc "cargo package; cargo publish --token $CRATES_IO_TOKEN"
+
+exit 0

ci/run-local.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+#
+# Run the entire buildkite CI pipeline locally for pre-testing before sending a
+# Github pull request
+#
+
+cd "$(dirname "$0")/.."
+BKRUN=ci/node_modules/.bin/bkrun
+
+if [[ ! -x $BKRUN ]]; then
+  (
+    set -x
+    cd ci/
+    npm install bkrun
+  )
+fi
+
+set -x
+exec ./ci/node_modules/.bin/bkrun ci/buildkite.yml

ci/semver_bash/LICENSE

@@ -0,0 +1,26 @@
+Copyright (c) 2013, Ray Bejjani
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met: 
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer. 
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution. 
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation are those
+of the authors and should not be interpreted as representing official policies, 
+either expressed or implied, of the FreeBSD Project.

ci/semver_bash/README.md

@@ -0,0 +1,31 @@
+semver_bash is a bash parser for semantic versioning
+====================================================
+
+[Semantic Versioning](http://semver.org/) is a set of guidelines that help keep
+version and version management sane. This is a bash based parser to help manage
+a project's versions. Use it from a Makefile or any scripts you use in your
+project.
+
+Usage
+-----
+semver_bash can be used from the command line as:  
+
+    $ ./semver.sh "3.2.1" "3.2.1-alpha"  
+    3.2.1 -> M: 3 m:2 p:1 s:  
+    3.2.1-alpha -> M: 3 m:2 p:1 s:-alpha  
+    3.2.1 == 3.2.1-alpha -> 1.  
+    3.2.1 < 3.2.1-alpha -> 1.  
+    3.2.1 > 3.2.1-alpha -> 0.
+
+
+Alternatively, you can source it from within a script:
+
+    . ./semver.sh  
+    
+    local MAJOR=0  
+    local MINOR=0  
+    local PATCH=0  
+    local SPECIAL=""
+    
+    semverParseInto "1.2.3" MAJOR MINOR PATCH SPECIAL  
+    semverParseInto "3.2.1" MAJOR MINOR PATCH SPECIAL  

ci/semver_bash/semver.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env sh
+
+function semverParseInto() {
+    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
+    #MAJOR
+    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
+    #MINOR
+    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
+    #MINOR
+    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
+    #SPECIAL
+    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
+}
+
+function semverEQ() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -ne $MAJOR_B ]; then
+        return 1
+    fi
+
+    if [ $MINOR_A -ne $MINOR_B ]; then
+        return 1
+    fi
+
+    if [ $PATCH_A -ne $PATCH_B ]; then
+        return 1
+    fi
+
+    if [[ "_$SPECIAL_A" != "_$SPECIAL_B" ]]; then
+        return 1
+    fi
+
+
+    return 0
+
+}
+
+function semverLT() {
+    local MAJOR_A=0
+    local MINOR_A=0
+    local PATCH_A=0
+    local SPECIAL_A=0
+
+    local MAJOR_B=0
+    local MINOR_B=0
+    local PATCH_B=0
+    local SPECIAL_B=0
+
+    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
+    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
+
+    if [ $MAJOR_A -lt $MAJOR_B ]; then
+        return 0
+    fi
+
+    if [[ $MAJOR_A -le $MAJOR_B  && $MINOR_A -lt $MINOR_B ]]; then
+        return 0
+    fi
+    
+    if [[ $MAJOR_A -le $MAJOR_B  && $MINOR_A -le $MINOR_B && $PATCH_A -lt $PATCH_B ]]; then
+        return 0
+    fi
+
+    if [[ "_$SPECIAL_A"  == "_" ]] && [[ "_$SPECIAL_B"  == "_" ]] ; then
+        return 1
+    fi
+    if [[ "_$SPECIAL_A"  == "_" ]] && [[ "_$SPECIAL_B"  != "_" ]] ; then
+        return 1
+    fi
+    if [[ "_$SPECIAL_A"  != "_" ]] && [[ "_$SPECIAL_B"  == "_" ]] ; then
+        return 0
+    fi
+
+    if [[ "_$SPECIAL_A" < "_$SPECIAL_B" ]]; then
+        return 0
+    fi
+
+    return 1
+
+}
+
+function semverGT() {
+    semverEQ $1 $2
+    local EQ=$?
+
+    semverLT $1 $2
+    local LT=$?
+
+    if [ $EQ -ne 0 ] && [ $LT -ne 0 ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+if [ "___semver.sh" == "___`basename $0`" ]; then
+
+MAJOR=0
+MINOR=0
+PATCH=0
+SPECIAL=""
+
+semverParseInto $1 MAJOR MINOR PATCH SPECIAL
+echo "$1 -> M: $MAJOR m:$MINOR p:$PATCH s:$SPECIAL"
+
+semverParseInto $2 MAJOR MINOR PATCH SPECIAL
+echo "$2 -> M: $MAJOR m:$MINOR p:$PATCH s:$SPECIAL"
+
+semverEQ $1 $2
+echo "$1 == $2 -> $?."
+
+semverLT $1 $2
+echo "$1 < $2 -> $?."
+
+semverGT $1 $2
+echo "$1 > $2 -> $?."
+
+fi

ci/semver_bash/semver_test.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+
+. ./semver.sh
+
+semverTest() {
+local A=R1.3.2
+local B=R2.3.2
+local C=R1.4.2
+local D=R1.3.3
+local E=R1.3.2a
+local F=R1.3.2b
+local G=R1.2.3
+
+local MAJOR=0
+local MINOR=0
+local PATCH=0
+local SPECIAL=""
+
+semverParseInto $A MAJOR MINOR PATCH SPECIAL
+echo "$A -> M:$MAJOR m:$MINOR p:$PATCH s:$SPECIAL. Expect M:1 m:3 p:2 s:"
+semverParseInto $E MAJOR MINOR PATCH SPECIAL
+echo "$E -> M:$MAJOR m:$MINOR p:$PATCH s:$SPECIAL. Expect M:1 m:3 p:2 s:a"
+
+echo "Equality comparisions"
+semverEQ $A $A
+echo "$A == $A -> $?. Expect 0."
+
+semverLT $A $A
+echo "$A < $A -> $?. Expect 1."
+
+semverGT $A $A
+echo "$A > $A -> $?. Expect 1."
+
+
+echo "Major number comparisions"
+semverEQ $A $B
+echo "$A == $B -> $?. Expect 1."
+
+semverLT $A $B
+echo "$A < $B -> $?. Expect 0."
+
+semverGT $A $B
+echo "$A > $B -> $?. Expect 1."
+
+semverEQ $B $A
+echo "$B == $A -> $?. Expect 1."
+
+semverLT $B $A
+echo "$B < $A -> $?. Expect 1."
+
+semverGT $B $A
+echo "$B > $A -> $?. Expect 0."
+
+
+echo "Minor number comparisions"
+semverEQ $A $C
+echo "$A == $C -> $?. Expect 1."
+
+semverLT $A $C
+echo "$A < $C -> $?. Expect 0."
+
+semverGT $A $C
+echo "$A > $C -> $?. Expect 1."
+
+semverEQ $C $A
+echo "$C == $A -> $?. Expect 1."
+
+semverLT $C $A
+echo "$C < $A -> $?. Expect 1."
+
+semverGT $C $A
+echo "$C > $A -> $?. Expect 0."
+
+echo "patch number comparisions"
+semverEQ $A $D
+echo "$A == $D -> $?. Expect 1."
+
+semverLT $A $D
+echo "$A < $D -> $?. Expect 0."
+
+semverGT $A $D
+echo "$A > $D -> $?. Expect 1."
+
+semverEQ $D $A
+echo "$D == $A -> $?. Expect 1."
+
+semverLT $D $A
+echo "$D < $A -> $?. Expect 1."
+
+semverGT $D $A
+echo "$D > $A -> $?. Expect 0."
+
+echo "special section vs no special comparisions"
+semverEQ $A $E
+echo "$A == $E -> $?. Expect 1."
+
+semverLT $A $E
+echo "$A < $E -> $?. Expect 1."
+
+semverGT $A $E
+echo "$A > $E -> $?. Expect 0."
+
+semverEQ $E $A
+echo "$E == $A -> $?. Expect 1."
+
+semverLT $E $A
+echo "$E < $A -> $?. Expect 0."
+
+semverGT $E $A
+echo "$E > $A -> $?. Expect 1."
+
+echo "special section vs special comparisions"
+semverEQ $E $F
+echo "$E == $F -> $?. Expect 1."
+
+semverLT $E $F
+echo "$E < $F -> $?. Expect 0."
+
+semverGT $E $F
+echo "$E > $F -> $?. Expect 1."
+
+semverEQ $F $E
+echo "$F == $E -> $?. Expect 1."
+
+semverLT $F $E
+echo "$F < $E -> $?. Expect 1."
+
+semverGT $F $E
+echo "$F > $E -> $?. Expect 0."
+
+echo "Minor and patch number comparisons"
+semverEQ $A $G
+echo "$A == $G -> $?. Expect 1."
+
+semverLT $A $G
+echo "$A < $G -> $?. Expect 1."
+
+semverGT $A $G
+echo "$A > $G -> $?. Expect 0."
+
+semverEQ $G $A
+echo "$G == $A -> $?. Expect 1."
+
+semverLT $G $A
+echo "$G < $A -> $?. Expect 0."
+
+semverGT $G $A
+echo "$G > $A -> $?. Expect 1."
+}
+
+semverTest

ci/shellcheck.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -e
+#
+# Reference: https://github.com/koalaman/shellcheck/wiki/Directive
+
+cd "$(dirname "$0")/.."
+
+set -x
+find . -name "*.sh" \
+    -not -regex ".*/ci/semver_bash/.*" \
+    -not -regex ".*/.cargo/.*" \
+    -not -regex ".*/node_modules/.*" \
+    -not -regex ".*/target/.*" \
+    -print0 \
+  | xargs -0 \
+      ci/docker-run.sh koalaman/shellcheck --color=always --external-sources --shell=bash
+
+exit 0

ci/snap.sh

@@ -0,0 +1,61 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+DRYRUN=
+if [[ -z $BUILDKITE_BRANCH ]] || ./ci/is-pr.sh; then
+  DRYRUN="echo"
+fi
+
+eval "$(ci/channel-info.sh)"
+
+if [[ $BUILDKITE_BRANCH = "$STABLE_CHANNEL" ]]; then
+  SNAP_CHANNEL=stable
+elif [[ $BUILDKITE_BRANCH = "$EDGE_CHANNEL" ]]; then
+  SNAP_CHANNEL=edge
+elif [[ $BUILDKITE_BRANCH = "$BETA_CHANNEL" ]]; then
+  SNAP_CHANNEL=beta
+fi
+
+if [[ -z $SNAP_CHANNEL ]]; then
+  echo Unable to determine channel to publish into, exiting.
+  exit 0
+fi
+
+if [[ -z $DRYRUN ]]; then
+  [[ -n $SNAPCRAFT_CREDENTIALS_KEY ]] || {
+    echo SNAPCRAFT_CREDENTIALS_KEY not defined
+    exit 1;
+  }
+  (
+    openssl aes-256-cbc -d \
+      -in ci/snapcraft.credentials.enc \
+      -out ci/snapcraft.credentials \
+      -k "$SNAPCRAFT_CREDENTIALS_KEY"
+
+    snapcraft login --with ci/snapcraft.credentials
+  ) || {
+    rm -f ci/snapcraft.credentials;
+    exit 1
+  }
+fi
+
+set -x
+
+echo --- checking for multilog
+if [[ ! -x /usr/bin/multilog ]]; then
+  if [[ -z $CI ]]; then
+    echo "multilog not found, install with: sudo apt-get install -y daemontools"
+    exit 1
+  fi
+  sudo apt-get install -y daemontools
+fi
+
+echo --- build: $SNAP_CHANNEL channel
+snapcraft
+
+source ci/upload_ci_artifact.sh
+upload_ci_artifact solana_*.snap
+
+echo --- publish: $SNAP_CHANNEL channel
+$DRYRUN snapcraft push solana_*.snap --release $SNAP_CHANNEL

ci/test-bench.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+ci/version-check.sh nightly
+export RUST_BACKTRACE=1
+
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+_ cargo bench --features=unstable --verbose

ci/test-large-network.sh

@@ -0,0 +1,45 @@
+#!/bin/bash -e
+
+here=$(dirname "$0")
+cd "$here"/..
+
+if ! ci/version-check.sh stable; then
+  # This job doesn't run within a container, try once to upgrade tooling on a
+  # version check failure
+  rustup install stable
+  ci/version-check.sh stable
+fi
+export RUST_BACKTRACE=1
+
+./fetch-perf-libs.sh
+export LD_LIBRARY_PATH=$PWD/target/perf-libs:$LD_LIBRARY_PATH
+
+export RUST_LOG=multinode=info
+
+if [[ $(ulimit -n) -lt 65000 ]]; then
+  echo 'Error: nofiles too small, run "ulimit -n 65000" to continue'
+  exit 1
+fi
+
+if [[ $(sysctl -n net.core.rmem_default) -lt 1610612736 ]]; then
+  echo 'Error: rmem_default too small, run "sudo sysctl -w net.core.rmem_default=1610612736" to continue'
+  exit 1
+fi
+
+if [[ $(sysctl -n net.core.rmem_max) -lt 1610612736 ]]; then
+  echo 'Error: rmem_max too small, run "sudo sysctl -w net.core.rmem_max=1610612736" to continue'
+  exit 1
+fi
+
+if [[ $(sysctl -n net.core.wmem_default) -lt 1610612736 ]]; then
+  echo 'Error: rmem_default too small, run "sudo sysctl -w net.core.wmem_default=1610612736" to continue'
+  exit 1
+fi
+
+if [[ $(sysctl -n net.core.wmem_max) -lt 1610612736 ]]; then
+  echo 'Error: rmem_max too small, run "sudo sysctl -w net.core.wmem_max=1610612736" to continue'
+  exit 1
+fi
+
+set -x
+exec cargo test --release --features=erasure test_multi_node_dynamic_network -- --ignored

ci/test-nightly.sh

@@ -0,0 +1,33 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+ci/version-check.sh nightly
+export RUST_BACKTRACE=1
+
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+_ cargo build --verbose --features unstable
+_ cargo test --verbose --features=unstable
+
+# TODO: Re-enable warnings-as-errors after clippy offers a way to not warn on unscoped lint names.
+#_ cargo clippy -- --deny=warnings
+_ cargo clippy
+
+exit 0
+
+# Coverage disabled (see issue #433)
+_ cargo cov test
+_ cargo cov report
+
+echo --- Coverage report:
+ls -l target/cov/report/index.html
+
+if [[ -z "$CODECOV_TOKEN" ]]; then
+  echo CODECOV_TOKEN undefined
+else
+  bash <(curl -s https://codecov.io/bash) -x 'llvm-cov-6.0 gcov'
+fi

ci/test-stable-perf.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+if ! ci/version-check.sh stable; then
+  # This job doesn't run within a container, try once to upgrade tooling on a
+  # version check failure
+  rustup install stable
+  ci/version-check.sh stable
+fi
+export RUST_BACKTRACE=1
+
+./fetch-perf-libs.sh
+export LD_LIBRARY_PATH=$PWD/target/perf-libs:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+export PATH=$PATH:/usr/local/cuda/bin
+
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+_ cargo test --features=cuda,erasure
+
+echo --- ci/localnet-sanity.sh
+(
+  set -x
+  # Assume |cargo build| has populated target/debug/ successfully.
+  export PATH=$PWD/target/debug:$PATH
+  USE_INSTALL=1 ci/localnet-sanity.sh
+)

ci/test-stable.sh

@@ -0,0 +1,25 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+ci/version-check.sh stable
+export RUST_BACKTRACE=1
+
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+_ cargo fmt -- --check
+_ cargo build --verbose
+_ cargo test --verbose
+
+echo --- ci/localnet-sanity.sh
+(
+  set -x
+  # Assume |cargo build| has populated target/debug/ successfully.
+  export PATH=$PWD/target/debug:$PATH
+  USE_INSTALL=1 ci/localnet-sanity.sh
+)
+
+_ ci/audit.sh || true

ci/testnet-deploy.sh

@@ -0,0 +1,121 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")"/..
+
+zone=
+leaderAddress=
+clientNodeCount=0
+validatorNodeCount=10
+publicNetwork=false
+snapChannel=edge
+delete=false
+enableGpu=false
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [name] [zone] [options...]
+
+Deploys a CD testnet
+
+  name  - name of the network
+  zone   - GCE to deploy the network into
+
+  options:
+   -s edge|beta|stable  - Deploy the specified Snap release channel
+                          (default: $snapChannel)
+   -n [number]          - Number of validator nodes (default: $validatorNodeCount)
+   -c [number]          - Number of client nodes (default: $clientNodeCount)
+   -P                   - Use public network IP addresses (default: $publicNetwork)
+   -g                   - Enable GPU (default: $enableGpu)
+   -a [address]         - Set the leader node's external IP address to this GCE address
+   -d                   - Delete the network
+
+   Note: the SOLANA_METRICS_CONFIG environment variable is used to configure
+         metrics
+EOF
+  exit $exitcode
+}
+
+netName=$1
+zone=$2
+[[ -n $netName ]] || usage
+[[ -n $zone ]] || usage "Zone not specified"
+shift 2
+
+while getopts "h?p:Pn:c:s:ga:d" opt; do
+  case $opt in
+  h | \?)
+    usage
+    ;;
+  P)
+    publicNetwork=true
+    ;;
+  n)
+    validatorNodeCount=$OPTARG
+    ;;
+  c)
+    clientNodeCount=$OPTARG
+    ;;
+  s)
+    case $OPTARG in
+    edge|beta|stable)
+      snapChannel=$OPTARG
+      ;;
+    *)
+      usage "Invalid snap channel: $OPTARG"
+      ;;
+    esac
+    ;;
+  g)
+    enableGpu=true
+    ;;
+  a)
+    leaderAddress=$OPTARG
+    ;;
+  d)
+    delete=true
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+
+
+gce_create_args=(
+  -a "$leaderAddress"
+  -c "$clientNodeCount"
+  -n "$validatorNodeCount"
+  -p "$netName"
+  -z "$zone"
+)
+
+if $enableGpu; then
+  gce_create_args+=(-g)
+fi
+
+if $publicNetwork; then
+  gce_create_args+=(-P)
+fi
+
+set -x
+
+echo --- gce.sh delete
+time net/gce.sh delete -p "$netName"
+if $delete; then
+  exit 0
+fi
+
+echo --- gce.sh create
+time net/gce.sh create "${gce_create_args[@]}"
+net/init-metrics.sh -e
+
+echo --- net.sh start
+time net/net.sh start -s "$snapChannel"
+
+exit 0

ci/testnet-sanity.sh

@@ -0,0 +1,36 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [name]
+
+Sanity check a CD testnet
+
+  name  - name of the network
+
+  Note: the SOLANA_METRICS_CONFIG environment variable is used to configure
+        metrics
+EOF
+  exit $exitcode
+}
+
+netName=$1
+[[ -n $netName ]] || usage ""
+
+set -x
+echo --- gce.sh config
+net/gce.sh config -p "$netName"
+net/init-metrics.sh -e
+echo --- net.sh sanity
+net/net.sh sanity \
+  ${NO_LEDGER_VERIFY:+-o noLedgerVerify} \
+  ${NO_VALIDATOR_SANITY:+-o noValidatorSanity} \
+
+exit 0

ci/upload_ci_artifact.sh

@@ -0,0 +1,18 @@
+# |source| me
+
+upload_ci_artifact() {
+  echo "--- artifact: $1"
+  if [[ -r "$1" ]]; then
+    ls -l "$1"
+    if ${BUILDKITE:-false}; then
+      (
+        set -x
+        buildkite-agent artifact upload "$1"
+      )
+    fi
+  else
+    echo ^^^ +++
+    echo "$1 not found"
+  fi
+}
+

ci/version-check.sh

@@ -0,0 +1,35 @@
+#!/bin/bash -e
+
+require() {
+  declare expectedProgram="$1"
+  declare expectedVersion="$2"
+
+  read -r program version _ < <($expectedProgram -V)
+
+  declare ok=true
+  [[ $program = "$expectedProgram" ]] || ok=false
+  [[ $version =~ $expectedVersion ]] || ok=false
+
+  echo "Found $program $version"
+  if ! $ok; then
+    echo Error: expected "$expectedProgram $expectedVersion"
+    exit 1
+  fi
+}
+
+case ${1:-stable} in
+nightly)
+  require rustc 1.30.[0-9]+-nightly
+  require cargo 1.29.[0-9]+-nightly
+  ;;
+stable)
+  require rustc 1.29.[0-9]+
+  require cargo 1.29.[0-9]+
+  ;;
+*)
+  echo Error: unknown argument: "$1"
+  exit 1
+  ;;
+esac
+
+exit 0

diagrams/historian.msc

@@ -1,18 +0,0 @@
-msc {
-  client,historian,logger;
-
-  logger=>historian [ label = "e0 = Entry{hash: h0, n: 0, event: Tick}" ] ;
-  logger=>logger [ label = "h1 = hash(h0)" ] ;
-  logger=>logger [ label = "h2 = hash(h1)" ] ;
-  client=>historian [ label = "Discovery(d0)" ] ;
-  historian=>logger [ label = "Discovery(d0)" ] ;
-  logger=>logger [ label = "h3 = hash(h2 + d0)" ] ;
-  logger=>historian [ label = "e1 = Entry{hash: hash(h3), n: 2, event: Discovery(d0)}" ] ;
-  logger=>logger [ label = "h4 = hash(h3)" ] ;
-  logger=>logger [ label = "h5 = hash(h4)" ] ;
-  logger=>logger [ label = "h6 = hash(h5)" ] ;
-  logger=>historian [ label = "e2 = Entry{hash: h6, n: 3, event: Tick}" ] ;
-  client=>historian [ label = "collect()" ] ;
-  historian=>client [ label = "entries = [e0, e1, e2]" ] ;
-  client=>client [ label = "verify_slice(entries, h0)" ] ;
-}

doc/consensus.msc

@@ -0,0 +1,15 @@
+msc {
+  client,leader,verifier_a,verifier_b,verifier_c;
+
+  client=>leader [ label = "SUBMIT" ] ;
+  leader=>client [ label = "CONFIRMED" ] ;
+  leader=>verifier_a [ label = "CONFIRMED" ] ;
+  leader=>verifier_b [ label = "CONFIRMED" ] ;
+  leader=>verifier_c [ label = "CONFIRMED" ] ;
+  verifier_a=>leader [ label = "VERIFIED" ] ;
+  verifier_b=>leader [ label = "VERIFIED" ] ;
+  leader=>client [ label = "FINALIZED" ] ;
+  leader=>verifier_a [ label = "FINALIZED" ] ;
+  leader=>verifier_b [ label = "FINALIZED" ] ;
+  leader=>verifier_c [ label = "FINALIZED" ] ;
+}

doc/json-rpc.md

@@ -0,0 +1,178 @@
+Solana JSON RPC API
+===
+
+Solana nodes accept HTTP requests using the [JSON-RPC 2.0](https://www.jsonrpc.org/specification) specification.
+
+To interact with a Solana node inside a JavaScript application, use the [solana-web3.js](https://github.com/solana-labs/solana-web3.js) library, which gives a convenient interface for the RPC methods.
+
+RPC Endpoint
+---
+
+**Default port:** 8899  
+eg. http://localhost:8899, http://192.168.1.88:8899
+
+Methods
+---
+
+* [confirmTransaction](#confirmtransaction)
+* [getAddress](#getaddress)
+* [getBalance](#getbalance)
+* [getLastId](#getlastid)
+* [getTransactionCount](#gettransactioncount)
+* [requestAirdrop](#requestairdrop)
+* [sendTransaction](#sendtransaction)
+
+Request Formatting
+---
+
+To make a JSON-RPC request, send an HTTP POST request with a `Content-Type: application/json` header. The JSON request data should contain 4 fields:
+
+* `jsonrpc`, set to `"2.0"`
+* `id`, a unique client-generated identifying integer
+* `method`, a string containing the method to be invoked
+* `params`, a JSON array of ordered parameter values
+
+Example using curl:
+```bash
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getBalance", "params":["83astBRguLMdt2h5U1Tpdq5tjFoJ6noeGwaY3mDLVcri"]}' 192.168.1.88:8899
+```
+
+The response output will be a JSON object with the following fields:
+
+* `jsonrpc`, matching the request specification
+* `id`, matching the request identifier
+* `result`, requested data or success confirmation
+
+Requests can be sent in batches by sending an array of JSON-RPC request objects as the data for a single POST.
+
+Definitions
+---
+
+* Hash: A SHA-256 hash of a chunk of data.
+* Pubkey: The public key of a Ed25519 key-pair.
+* Signature: An Ed25519 signature of a chunk of data.
+* Transaction: A Solana instruction signed by a client key-pair.
+
+JSON RPC API Reference
+---
+
+### confirmTransaction
+Returns a transaction receipt
+
+##### Parameters:
+* `string` - Signature of Transaction to confirm, as base-58 encoded string
+
+##### Results:
+* `boolean` - Transaction status, true if Transaction is confirmed
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"confirmTransaction", "params":["5VERv8NMvzbJMEkV8xnrLkEaWRtSz9CosKDYjCJjBRnbJLgp8uirBgmQpjKhoR4tjF3ZpRzrFmBV6UjKdiSZkQUW"]}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":true,"id":1}
+```
+
+---
+
+### getBalance
+Returns the balance of the account of provided Pubkey
+
+##### Parameters:
+* `string` - Pubkey of account to query, as base-58 encoded string
+
+##### Results:
+* `integer` - quantity, as a signed 64-bit integer
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0", "id":1, "method":"getBalance", "params":["83astBRguLMdt2h5U1Tpdq5tjFoJ6noeGwaY3mDLVcri"]}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":0,"id":1}
+```
+
+---
+
+### getLastId
+Returns the last entry ID from the ledger
+
+##### Parameters:
+None
+
+##### Results:
+* `string` - the ID of last entry, a Hash as base-58 encoded string
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getLastId"}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":"GH7ome3EiwEr7tu9JuTh2dpYWBJK3z69Xm1ZE3MEE6JC","id":1}
+```
+
+---
+
+### getTransactionCount
+Returns the current Transaction count from the ledger
+
+##### Parameters:
+None
+
+##### Results:
+* `integer` - count, as unsigned 64-bit integer
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"getTransactionCount"}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":268,"id":1}
+```
+
+---
+
+### requestAirdrop
+Requests an airdrop of tokens to a Pubkey
+
+##### Parameters:
+* `string` - Pubkey of account to receive tokens, as base-58 encoded string
+* `integer` - token quantity, as a signed 64-bit integer
+
+##### Results:
+* `string` - Transaction Signature of airdrop, as base-58 encoded string
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"requestAirdrop", "params":["83astBRguLMdt2h5U1Tpdq5tjFoJ6noeGwaY3mDLVcri", 50]}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":"5VERv8NMvzbJMEkV8xnrLkEaWRtSz9CosKDYjCJjBRnbJLgp8uirBgmQpjKhoR4tjF3ZpRzrFmBV6UjKdiSZkQUW","id":1}
+```
+
+---
+
+### sendTransaction
+Creates new transaction
+
+##### Parameters:
+* `array` - array of octets containing a fully-signed Transaction
+
+##### Results:
+* `string` - Transaction Signature, as base-58 encoded string
+
+##### Example:
+```bash
+// Request
+curl -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","id":1, "method":"sendTransaction", "params":[[61, 98, 55, 49, 15, 187, 41, 215, 176, 49, 234, 229, 228, 77, 129, 221, 239, 88, 145, 227, 81, 158, 223, 123, 14, 229, 235, 247, 191, 115, 199, 71, 121, 17, 32, 67, 63, 209, 239, 160, 161, 2, 94, 105, 48, 159, 235, 235, 93, 98, 172, 97, 63, 197, 160, 164, 192, 20, 92, 111, 57, 145, 251, 6, 40, 240, 124, 194, 149, 155, 16, 138, 31, 113, 119, 101, 212, 128, 103, 78, 191, 80, 182, 234, 216, 21, 121, 243, 35, 100, 122, 68, 47, 57, 13, 39, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 40, 240, 124, 194, 149, 155, 16, 138, 31, 113, 119, 101, 212, 128, 103, 78, 191, 80, 182, 234, 216, 21, 121, 243, 35, 100, 122, 68, 47, 57, 11, 12, 106, 49, 74, 226, 201, 16, 161, 192, 28, 84, 124, 97, 190, 201, 171, 186, 6, 18, 70, 142, 89, 185, 176, 154, 115, 61, 26, 163, 77, 1, 88, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}' http://localhost:8899
+
+// Result
+{"jsonrpc":"2.0","result":"2EBVM6cB8vAAD93Ktr6Vd8p67XPbQzCJX47MpReuiCXJAtcjaxpvWpcg9Ege1Nr5Tk3a2GFrByT7WPBjdsTycY9b","id":1}
+```
+
+---

doc/testnet.md

@@ -0,0 +1,44 @@
+# TestNet debugging info
+
+Currently we have two testnets, 'perf' and 'master', both on the master branch of the solana repo. Deploys happen
+at the top of every hour with the latest code. 'perf' has more cores for the client machine to flood the network
+with transactions until failure.
+
+## Deploy process
+
+They are deployed with the `ci/testnet-deploy.sh` script. There is a scheduled buildkite job which runs to do the deploy,
+look at `testnet-deploy` to see the agent which ran it and the logs. There is also a manual job to do the deploy manually..
+Validators are selected based on their machine name and everyone gets the binaries installed from snap.
+
+## Where are the testnet logs?
+
+For the client they are put in `/tmp/solana`; for validators and leaders they are in `/var/snap/solana/current/`.
+You can also see the backtrace of the client by ssh'ing into the client node and doing:
+
+```bash
+$ sudo -u testnet-deploy
+$ tmux attach -t solana
+```
+
+## How do I reset the testnet?
+
+Through buildkite.
+
+## How can I scale the tx generation rate?
+
+Increase the TX rate by increasing the number of cores on the client machine which is running
+`bench-tps` or run multiple clients. Decrease by lowering cores or using the rayon env
+variable `RAYON_NUM_THREADS=<xx>`
+
+## How can I test a change on the testnet?
+
+Currently, a merged PR is the only way to test a change on the testnet.
+
+## Adjusting the number of clients or validators on the testnet
+
+1. Go to the [GCP Instance Group](https://console.cloud.google.com/compute/instanceGroups/list?project=principal-lane-200702) tab
+2. Find the client or validator instance group you'd like to adjust
+3. Edit it (pencil icon), change the "Number of instances", then click "Save" button
+4. Refresh until the change to number of instances has been executed
+5. Click the "New Build" button on the [testnet-deploy](https://buildkite.com/solana-labs/testnet-deploy/)
+   buildkite job to initiate a redeploy of the network with the updated instance count.

fetch-perf-libs.sh

@@ -0,0 +1,39 @@
+#!/bin/bash -e
+
+if [[ $(uname) != Linux ]]; then
+  echo Performance libraries are only available for Linux
+  exit 1
+fi
+
+if [[ $(uname -m) != x86_64 ]]; then
+  echo Performance libraries are only available for x86_64 architecture
+  exit 1
+fi
+
+mkdir -p target/perf-libs
+(
+  cd target/perf-libs
+  (
+    set -x
+    curl https://solana-perf.s3.amazonaws.com/v0.8.0/x86_64-unknown-linux-gnu/solana-perf.tgz | tar zxvf -
+  )
+
+  if [[ -r /usr/local/cuda/version.txt && -r cuda-version.txt ]]; then
+    if ! diff /usr/local/cuda/version.txt cuda-version.txt > /dev/null; then
+        echo ==============================================
+        echo Warning: possible CUDA version mismatch
+        echo
+        echo "Expected version: $(cat cuda-version.txt)"
+        echo "Detected version: $(cat /usr/local/cuda/version.txt)"
+        echo ==============================================
+    fi
+  else
+    echo ==============================================
+    echo Warning: unable to validate CUDA version
+    echo ==============================================
+  fi
+
+  echo "Downloaded solana-perf version: $(cat solana-perf-HEAD.txt)"
+)
+
+exit 0

multinode-demo/client.sh

@@ -0,0 +1,25 @@
+#!/bin/bash -e
+
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+usage() {
+  if [[ -n $1 ]]; then
+    echo "$*"
+    echo
+  fi
+  echo "usage: $0 [extra args]"
+  echo
+  echo " Run bench-tps "
+  echo
+  echo "   extra args: additional arguments are pass along to solana-bench-tps"
+  echo
+  exit 1
+}
+
+if [[ -z $1 ]]; then # default behavior
+  $solana_bench_tps --identity config-private/client-id.json --network 127.0.0.1:8001 --duration 90
+else
+  $solana_bench_tps "$@"
+fi

multinode-demo/common.sh

@@ -0,0 +1,202 @@
+# |source| this file
+#
+# Common utilities shared by other scripts in this directory
+#
+# The following directive disable complaints about unused variables in this
+# file:
+# shellcheck disable=2034
+#
+
+rsync=rsync
+leader_logger="cat"
+validator_logger="cat"
+drone_logger="cat"
+
+if [[ $(uname) != Linux ]]; then
+  # Protect against unsupported configurations to prevent non-obvious errors
+  # later. Arguably these should be fatal errors but for now prefer tolerance.
+  if [[ -n $USE_SNAP ]]; then
+    echo "Warning: Snap is not supported on $(uname)"
+    USE_SNAP=
+  fi
+  if [[ -n $SOLANA_CUDA ]]; then
+    echo "Warning: CUDA is not supported on $(uname)"
+    SOLANA_CUDA=
+  fi
+fi
+
+if [[ -d $SNAP ]]; then # Running inside a Linux Snap?
+  solana_program() {
+    declare program="$1"
+    if [[ "$program" = wallet || "$program" = bench-tps ]]; then
+      # TODO: Merge wallet.sh/client.sh functionality into
+      #       solana-wallet/solana-demo-client proper and remove this special case
+      printf "%s/bin/solana-%s" "$SNAP" "$program"
+    else
+      printf "%s/command-%s.wrapper" "$SNAP" "$program"
+    fi
+  }
+  rsync="$SNAP"/bin/rsync
+  multilog="$SNAP/bin/multilog t s16777215 n200"
+  leader_logger="$multilog $SNAP_DATA/leader"
+  validator_logger="$multilog t $SNAP_DATA/validator"
+  drone_logger="$multilog $SNAP_DATA/drone"
+  # Create log directories manually to prevent multilog from creating them as
+  # 0700
+  mkdir -p "$SNAP_DATA"/{drone,leader,validator}
+
+elif [[ -n $USE_SNAP ]]; then # Use the Linux Snap binaries
+  solana_program() {
+    declare program="$1"
+    printf "solana.%s" "$program"
+  }
+elif [[ -n $USE_INSTALL ]]; then # Assume |cargo install| was run
+  solana_program() {
+    declare program="$1"
+    printf "solana-%s" "$program"
+  }
+  # CUDA was/wasn't selected at build time, can't affect CUDA state here
+  unset SOLANA_CUDA
+else
+  solana_program() {
+    declare program="$1"
+    declare features=""
+    if [[ "$program" =~ ^(.*)-cuda$ ]]; then
+      program=${BASH_REMATCH[1]}
+      features="--features=cuda"
+    fi
+    if [[ -z $DEBUG ]]; then
+      maybe_release=--release
+    fi
+    printf "cargo run $maybe_release --bin solana-%s %s -- " "$program" "$features"
+  }
+  if [[ -n $SOLANA_CUDA ]]; then
+    # shellcheck disable=2154 # 'here' is referenced but not assigned
+    if [[ -z $here ]]; then
+      echo "|here| is not defined"
+      exit 1
+    fi
+
+    # Locate perf libs downloaded by |./fetch-perf-libs.sh|
+    LD_LIBRARY_PATH=$(cd "$here" && dirname "$PWD"/target/perf-libs):$LD_LIBRARY_PATH
+    export LD_LIBRARY_PATH
+  fi
+fi
+
+solana_bench_tps=$(solana_program bench-tps)
+solana_wallet=$(solana_program wallet)
+solana_drone=$(solana_program drone)
+solana_fullnode=$(solana_program fullnode)
+solana_fullnode_config=$(solana_program fullnode-config)
+solana_fullnode_cuda=$(solana_program fullnode-cuda)
+solana_genesis=$(solana_program genesis)
+solana_keygen=$(solana_program keygen)
+solana_ledger_tool=$(solana_program ledger-tool)
+
+export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info
+export RUST_BACKTRACE=1
+
+# shellcheck source=scripts/configure-metrics.sh
+source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh
+
+tune_networking() {
+  # Skip in CI
+  [[ -z $CI ]] || return 0
+
+  # Reference: https://medium.com/@CameronSparr/increase-os-udp-buffers-to-improve-performance-51d167bb1360
+  if [[ $(uname) = Linux ]]; then
+    (
+      set -x +e
+      # test the existence of the sysctls before trying to set them
+      # go ahead and return true and don't exit if these calls fail
+      sysctl net.core.rmem_max 2>/dev/null 1>/dev/null &&
+          sudo sysctl -w net.core.rmem_max=67108864 1>/dev/null 2>/dev/null
+
+      sysctl net.core.rmem_default 2>/dev/null 1>/dev/null &&
+          sudo sysctl -w net.core.rmem_default=26214400 1>/dev/null 2>/dev/null
+    ) || true
+  fi
+
+  if [[ $(uname) = Darwin ]]; then
+    (
+      if [[ $(sysctl net.inet.udp.maxdgram | cut -d\  -f2) != 65535 ]]; then
+        echo "Adjusting maxdgram to allow for large UDP packets, see BLOB_SIZE in src/packet.rs:"
+        set -x
+        sudo sysctl net.inet.udp.maxdgram=65535
+      fi
+    )
+
+  fi
+}
+
+
+SOLANA_CONFIG_DIR=${SNAP_DATA:-$PWD}/config
+SOLANA_CONFIG_PRIVATE_DIR=${SNAP_DATA:-$PWD}/config-private
+SOLANA_CONFIG_VALIDATOR_DIR=${SNAP_DATA:-$PWD}/config-validator
+SOLANA_CONFIG_CLIENT_DIR=${SNAP_USER_DATA:-$PWD}/config-client
+
+rsync_url() { # adds the 'rsync://` prefix to URLs that need it
+  declare url="$1"
+
+  if [[ $url =~ ^.*:.*$ ]]; then
+    # assume remote-shell transport when colon is present, use $url unmodified
+    echo "$url"
+    return 0
+  fi
+
+  if [[ -d $url ]]; then
+    # assume local directory if $url is a valid directory, use $url unmodified
+    echo "$url"
+    return 0
+  fi
+
+  # Default to rsync:// URL
+  echo "rsync://$url"
+}
+
+# called from drone, validator, client
+find_leader() {
+  declare leader leader_address
+  declare shift=0
+
+  if [[ -d $SNAP ]]; then
+    if [[ -n $1 ]]; then
+      usage "Error: unexpected parameter: $1"
+    fi
+
+    # Select leader from the Snap configuration
+    leader_ip=$(snapctl get leader-ip)
+    if [[ -z $leader_ip ]]; then
+      leader=testnet.solana.com
+      leader_ip=$(dig +short "${leader%:*}" | head -n1)
+      if [[ -z $leader_ip ]]; then
+          usage "Error: unable to resolve IP address for $leader"
+      fi
+    fi
+    leader=$leader_ip
+    leader_address=$leader_ip:8001
+  else
+    if [[ -z $1 ]]; then
+      leader=${here}/..        # Default to local tree for rsync
+      leader_address=127.0.0.1:8001 # Default to local leader
+    elif [[ -z $2 ]]; then
+      leader=$1
+
+      declare leader_ip
+      leader_ip=$(dig +short "${leader%:*}" | head -n1)
+
+      if [[ -z $leader_ip ]]; then
+          usage "Error: unable to resolve IP address for $leader"
+      fi
+
+      leader_address=$leader_ip:8001
+      shift=1
+    else
+      leader=$1
+      leader_address=$2
+      shift=2
+    fi
+  fi
+
+  echo "$leader" "$leader_address" "$shift"
+}

multinode-demo/drone.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Starts an instance of solana-drone
+#
+here=$(dirname "$0")
+
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+usage() {
+  if [[ -n $1 ]]; then
+    echo "$*"
+    echo
+  fi
+  echo "usage: $0 [network entry point]"
+  echo
+  echo " Run an airdrop drone for the specified network"
+  echo
+  exit 1
+}
+
+read -r _ leader_address shift < <(find_leader "${@:1:1}")
+shift "$shift"
+
+[[ -f "$SOLANA_CONFIG_PRIVATE_DIR"/mint.json ]] || {
+  echo "$SOLANA_CONFIG_PRIVATE_DIR/mint.json not found, create it by running:"
+  echo
+  echo "  ${here}/setup.sh -t leader"
+  exit 1
+}
+
+set -ex
+
+trap 'kill "$pid" && wait "$pid"' INT TERM
+$solana_drone \
+  --keypair "$SOLANA_CONFIG_PRIVATE_DIR"/mint.json \
+  --network "$leader_address" \
+  > >($drone_logger) 2>&1 &
+pid=$!
+wait "$pid"

multinode-demo/leader.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Starts a leader node
+#
+
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+# shellcheck source=scripts/oom-score-adj.sh
+source "$here"/../scripts/oom-score-adj.sh
+
+if [[ -d "$SNAP" ]]; then
+  # Exit if mode is not yet configured
+  # (typically the case after the Snap is first installed)
+  [[ -n "$(snapctl get mode)" ]] || exit 0
+fi
+
+[[ -f "$SOLANA_CONFIG_DIR"/leader.json ]] || {
+  echo "$SOLANA_CONFIG_DIR/leader.json not found, create it by running:"
+  echo
+  echo "  ${here}/setup.sh"
+  exit 1
+}
+
+if [[ -n "$SOLANA_CUDA" ]]; then
+  program="$solana_fullnode_cuda"
+else
+  program="$solana_fullnode"
+fi
+
+tune_networking
+
+trap 'kill "$pid" && wait "$pid"' INT TERM
+$program \
+  --identity "$SOLANA_CONFIG_DIR"/leader.json \
+  --ledger "$SOLANA_CONFIG_DIR"/ledger \
+  > >($leader_logger) 2>&1 &
+pid=$!
+oom_score_adj "$pid" 1000
+wait "$pid"

multinode-demo/setup.sh

@@ -0,0 +1,128 @@
+#!/bin/bash
+#
+# Creates a fullnode configuration
+#
+
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+usage () {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [-n num_tokens] [-l] [-p] [-t node_type]
+
+Creates a fullnode configuration
+
+ -n num_tokens  - Number of tokens to create
+ -l             - Detect network address from local machine configuration, which
+                  may be a private IP address unaccessible on the Intenet (default)
+ -p             - Detect public address using public Internet servers
+ -t node_type   - Create configuration files only for this kind of node.  Valid
+                  options are validator or leader.  Creates configuration files
+                  for both by default
+
+EOF
+  exit $exitcode
+}
+
+ip_address_arg=-l
+num_tokens=1000000000
+node_type_leader=true
+node_type_validator=true
+node_type_client=true
+while getopts "h?n:lpt:" opt; do
+  case $opt in
+  h|\?)
+    usage
+    exit 0
+    ;;
+  l)
+    ip_address_arg=-l
+    ;;
+  p)
+    ip_address_arg=-p
+    ;;
+  n)
+    num_tokens="$OPTARG"
+    ;;
+  t)
+    node_type="$OPTARG"
+    case $OPTARG in
+    leader)
+      node_type_leader=true
+      node_type_validator=false
+      node_type_client=false
+      ;;
+    validator)
+      node_type_leader=false
+      node_type_validator=true
+      node_type_client=false
+      ;;
+    client)
+      node_type_leader=false
+      node_type_validator=false
+      node_type_client=true
+      ;;
+    *)
+      usage "Error: unknown node type: $node_type"
+      ;;
+    esac
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+
+
+set -e
+
+for i in "$SOLANA_CONFIG_DIR" "$SOLANA_CONFIG_VALIDATOR_DIR" "$SOLANA_CONFIG_PRIVATE_DIR"; do
+  echo "Cleaning $i"
+  rm -rvf "$i"
+  mkdir -p "$i"
+done
+
+if $node_type_client; then
+  client_id_path="$SOLANA_CONFIG_PRIVATE_DIR"/client-id.json
+  $solana_keygen -o "$client_id_path"
+  ls -lhR "$SOLANA_CONFIG_PRIVATE_DIR"/
+fi
+
+if $node_type_leader; then
+  leader_address_args=("$ip_address_arg")
+  leader_id_path="$SOLANA_CONFIG_PRIVATE_DIR"/leader-id.json
+  mint_path="$SOLANA_CONFIG_PRIVATE_DIR"/mint.json
+
+  $solana_keygen -o "$leader_id_path"
+
+  echo "Creating $mint_path with $num_tokens tokens"
+  $solana_keygen -o "$mint_path"
+
+  echo "Creating $SOLANA_CONFIG_DIR/ledger"
+  $solana_genesis --tokens="$num_tokens" --ledger "$SOLANA_CONFIG_DIR"/ledger < "$mint_path"
+
+  echo "Creating $SOLANA_CONFIG_DIR/leader.json"
+  $solana_fullnode_config --keypair="$leader_id_path" "${leader_address_args[@]}" > "$SOLANA_CONFIG_DIR"/leader.json
+
+  ls -lhR "$SOLANA_CONFIG_DIR"/
+  ls -lhR "$SOLANA_CONFIG_PRIVATE_DIR"/
+fi
+
+
+if $node_type_validator; then
+  validator_address_args=("$ip_address_arg" -b 9000)
+  validator_id_path="$SOLANA_CONFIG_PRIVATE_DIR"/validator-id.json
+
+  $solana_keygen -o "$validator_id_path"
+
+  echo "Creating $SOLANA_CONFIG_VALIDATOR_DIR/validator.json"
+  $solana_fullnode_config --keypair="$validator_id_path" "${validator_address_args[@]}" > "$SOLANA_CONFIG_VALIDATOR_DIR"/validator.json
+
+  ls -lhR "$SOLANA_CONFIG_VALIDATOR_DIR"/
+fi

multinode-demo/validator-x.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+# Start a dynamically-configured validator node
+#
+
+here=$(dirname "$0")
+
+exec "$here"/validator.sh -x "$@"

multinode-demo/validator.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+#
+# Start a validator node
+#
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+# shellcheck source=scripts/oom-score-adj.sh
+source "$here"/../scripts/oom-score-adj.sh
+
+if [[ -d "$SNAP" ]]; then
+  # Exit if mode is not yet configured
+  # (typically the case after the Snap is first installed)
+  [[ -n "$(snapctl get mode)" ]] || exit 0
+fi
+
+usage() {
+  if [[ -n $1 ]]; then
+    echo "$*"
+    echo
+  fi
+  echo "usage: $0 [-x] [rsync network path to leader] [network entry point]"
+  echo
+  echo " Start a validator on the specified network"
+  echo
+  echo "   -x: runs a new, dynamically-configured validator"
+  echo
+  exit 1
+}
+
+if [[ $1 = -h ]]; then
+  usage
+fi
+
+if [[ $1 == -x ]]; then
+  self_setup=1
+  shift
+else
+  self_setup=0
+fi
+
+if [[ -n $3 ]]; then
+  usage
+fi
+
+read -r leader leader_address shift < <(find_leader "${@:1:2}")
+shift "$shift"
+
+if [[ -n $SOLANA_CUDA ]]; then
+  program=$solana_fullnode_cuda
+else
+  program=$solana_fullnode
+fi
+
+if ((!self_setup)); then
+  [[ -f $SOLANA_CONFIG_VALIDATOR_DIR/validator.json ]] || {
+    echo "$SOLANA_CONFIG_VALIDATOR_DIR/validator.json not found, create it by running:"
+    echo
+    echo "  ${here}/setup.sh"
+    exit 1
+  }
+  validator_json_path=$SOLANA_CONFIG_VALIDATOR_DIR/validator.json
+  SOLANA_LEADER_CONFIG_DIR=$SOLANA_CONFIG_VALIDATOR_DIR/leader-config
+else
+  mkdir -p "$SOLANA_CONFIG_PRIVATE_DIR"
+  validator_id_path=$SOLANA_CONFIG_PRIVATE_DIR/validator-id-x$$.json
+  $solana_keygen -o "$validator_id_path"
+
+  mkdir -p "$SOLANA_CONFIG_VALIDATOR_DIR"
+  validator_json_path=$SOLANA_CONFIG_VALIDATOR_DIR/validator-x$$.json
+
+  port=9000
+  (((port += ($$ % 1000)) && (port == 9000) && port++))
+
+  $solana_fullnode_config --keypair="$validator_id_path" -l -b "$port" > "$validator_json_path"
+
+  SOLANA_LEADER_CONFIG_DIR=$SOLANA_CONFIG_VALIDATOR_DIR/leader-config-x$$
+fi
+
+rsync_leader_url=$(rsync_url "$leader")
+
+tune_networking
+
+set -ex
+$rsync -vPr "$rsync_leader_url"/config/ "$SOLANA_LEADER_CONFIG_DIR"
+[[ -d $SOLANA_LEADER_CONFIG_DIR/ledger ]] || {
+  echo "Unable to retrieve ledger from $rsync_leader_url"
+  exit 1
+}
+
+trap 'kill "$pid" && wait "$pid"' INT TERM
+$program \
+  --identity "$validator_json_path" \
+  --network "$leader_address" \
+  --ledger "$SOLANA_LEADER_CONFIG_DIR"/ledger \
+  > >($validator_logger) 2>&1 &
+pid=$!
+oom_score_adj "$pid" 1000
+wait "$pid"

multinode-demo/wallet.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Runs solana-wallet against the specified network
+#
+# usage: $0 <rsync network path to solana repo on leader machine>"
+#
+
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+# shellcheck source=scripts/oom-score-adj.sh
+source "$here"/../scripts/oom-score-adj.sh
+
+# if $1 isn't host:path, something.com, or a valid local path
+if [[ ${1%:} != "$1" || "$1" =~ [^.]\.[^.] || -d $1 ]]; then
+  leader=$1 # interpret
+  shift
+else
+  if [[ -d "$SNAP" ]]; then
+    leader=testnet.solana.com # Default to testnet when running as a Snap
+  else
+    leader=$here/.. # Default to local solana repo
+  fi
+fi
+
+if [[ "$1" = "reset" ]]; then
+  echo Wallet resetting
+  rm -rf "$SOLANA_CONFIG_CLIENT_DIR"
+  exit 0
+fi
+
+rsync_leader_url=$(rsync_url "$leader")
+
+set -e
+mkdir -p "$SOLANA_CONFIG_CLIENT_DIR"
+if [[ ! -r "$SOLANA_CONFIG_CLIENT_DIR"/leader.json ]]; then
+  echo "Fetching leader configuration from $rsync_leader_url"
+  $rsync -Pz "$rsync_leader_url"/config/leader.json "$SOLANA_CONFIG_CLIENT_DIR"/
+fi
+
+client_id_path="$SOLANA_CONFIG_CLIENT_DIR"/id.json
+if [[ ! -r $client_id_path ]]; then
+  echo "Generating client identity: $client_id_path"
+  $solana_keygen -o "$client_id_path"
+fi
+
+# shellcheck disable=SC2086 # $solana_wallet should not be quoted
+exec $solana_wallet \
+  -l "$SOLANA_CONFIG_CLIENT_DIR"/leader.json -k "$client_id_path" --timeout 10 "$@"

net/.gitignore

@@ -0,0 +1,2 @@
+/config/
+/log/

net/README.md

@@ -0,0 +1,66 @@
+
+# Network Management
+This directory contains scripts useful for working with a test network.  It's
+intended to be both dev and CD friendly.
+
+### User Account Prerequisites
+
+Log in to GCP with:
+```bash
+$ gcloud auth login
+```
+
+Also ensure that `$(whoami)` is the name of an InfluxDB user account with enough
+access to create a new database.
+
+## Quick Start
+```bash
+$ cd net/
+$ ./gce.sh create -n 5 -c 1  #<-- Create a GCE testnet with 5 validators, 1 client (billing starts here)
+$ ./init-metrics.sh $(whoami)   #<-- Configure a metrics database for the testnet
+$ ./net.sh start             #<-- Deploy the network from the local workspace
+$ ./ssh.sh                   #<-- Details on how to ssh into any testnet node
+$ ./gce.sh delete            #<-- Dispose of the network (billing stops here)
+```
+
+## Tips
+
+### Running the network over public IP addresses
+By default private IP addresses are used with all instances in the same
+availability zone to avoid GCE network engress charges.  However to run the
+network over public IP addresses:
+```bash
+$ ./gce.sh create -P ...
+```
+
+### Deploying a Snap-based network
+To deploy the latest pre-built `edge` channel Snap (ie, latest from the `master`
+branch), once the testnet has been created run:
+
+```bash
+$ ./net.sh start -s edge
+```
+
+### Enabling CUDA
+First ensure the network instances are created with GPU enabled:
+```bash
+$ ./gce.sh create -g ...
+```
+
+If deploying a Snap-based network nothing further is required, as GPU presence
+is detected at runtime and the CUDA build is auto selected.
+
+If deploying a locally-built network, first run `./fetch-perf-libs.sh` then
+ensure the `cuda` feature is specified at network start:
+```bash
+$ ./net.sh start -f "cuda,erasure"
+```
+
+### How to interact with a CD testnet deployed by ci/testnet-deploy.sh
+
+Taking **master-testnet-solana-com** as an example, configure your workspace for
+the testnet using:
+```
+$ ./gce.sh config -p master-testnet-solana-com
+$ ./ssh.sh                                     # <-- Details on how to ssh into any testnet node
+```

net/common.sh

@@ -0,0 +1,58 @@
+# |source| this file
+#
+# Common utilities shared by other scripts in this directory
+#
+# The following directive disable complaints about unused variables in this
+# file:
+# shellcheck disable=2034
+#
+
+netDir=$(
+  cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+  echo "$PWD"
+)
+netConfigDir="$netDir"/config
+netLogDir="$netDir"/log
+mkdir -p "$netConfigDir" "$netLogDir"
+
+# shellcheck source=scripts/configure-metrics.sh
+source "$(dirname "${BASH_SOURCE[0]}")"/../scripts/configure-metrics.sh
+
+configFile="$netConfigDir/config"
+
+entrypointIp=
+publicNetwork=
+leaderIp=
+netBasename=
+sshPrivateKey=
+clientIpList=()
+sshOptions=()
+validatorIpList=()
+
+buildSshOptions() {
+  sshOptions=(
+    -o "BatchMode=yes"
+    -o "StrictHostKeyChecking=no"
+    -o "UserKnownHostsFile=/dev/null"
+    -o "User=solana"
+    -o "IdentityFile=$sshPrivateKey"
+    -o "LogLevel=ERROR"
+    -F /dev/null
+  )
+}
+
+loadConfigFile() {
+  [[ -r $configFile ]] || usage "Config file unreadable: $configFile"
+
+  # shellcheck source=/dev/null
+  source "$configFile"
+  [[ -n "$entrypointIp" ]] || usage "Config file invalid, entrypointIp unspecified: $configFile"
+  [[ -n "$publicNetwork" ]] || usage "Config file invalid, publicNetwork unspecified: $configFile"
+  [[ -n "$leaderIp" ]] || usage "Config file invalid, leaderIp unspecified: $configFile"
+  [[ -n "$netBasename" ]] || usage "Config file invalid, netBasename unspecified: $configFile"
+  [[ -n $sshPrivateKey ]] || usage "Config file invalid, sshPrivateKey unspecified: $configFile"
+  [[ ${#validatorIpList[@]} -gt 0 ]] || usage "Config file invalid, validatorIpList unspecified: $configFile"
+
+  buildSshOptions
+  configureMetrics
+}

net/gce.sh

@@ -0,0 +1,336 @@
+#!/bin/bash -e
+
+here=$(dirname "$0")
+# shellcheck source=net/scripts/gcloud.sh
+source "$here"/scripts/gcloud.sh
+# shellcheck source=net/common.sh
+source "$here"/common.sh
+
+prefix=testnet-dev-${USER//[^A-Za-z0-9]/}
+validatorNodeCount=5
+clientNodeCount=1
+leaderBootDiskSize=1TB
+leaderMachineType=n1-standard-16
+leaderAccelerator=
+validatorMachineType=n1-standard-4
+validatorBootDiskSize=$leaderBootDiskSize
+validatorAccelerator=
+clientMachineType=n1-standard-16
+clientBootDiskSize=40GB
+clientAccelerator=
+
+imageName="ubuntu-16-04-cuda-9-2-new"
+publicNetwork=false
+zone="us-west1-b"
+leaderAddress=
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [create|config|delete] [common options] [command-specific options]
+
+Configure a GCE-based testnet
+
+ create - create a new testnet (implies 'config')
+ config - configure the testnet and write a config file describing it
+ delete - delete the testnet
+
+ common options:
+   -p [prefix]      - Optional common prefix for instance names to avoid
+                      collisions (default: $prefix)
+
+ create-specific options:
+   -n [number]      - Number of validator nodes (default: $validatorNodeCount)
+   -c [number]      - Number of client nodes (default: $clientNodeCount)
+   -P               - Use public network IP addresses (default: $publicNetwork)
+   -z [zone]        - GCP Zone for the nodes (default: $zone)
+   -i [imageName]   - Existing image on GCE (default: $imageName)
+   -g               - Enable GPU
+   -a [address]     - Set the leader node's external IP address to this GCE address
+
+ config-specific options:
+   none
+
+ delete-specific options:
+   none
+
+EOF
+  exit $exitcode
+}
+
+
+command=$1
+[[ -n $command ]] || usage
+shift
+[[ $command = create || $command = config || $command = delete ]] || usage "Invalid command: $command"
+
+while getopts "h?p:Pi:n:c:z:ga:" opt; do
+  case $opt in
+  h | \?)
+    usage
+    ;;
+  p)
+    [[ ${OPTARG//[^A-Za-z0-9-]/} == "$OPTARG" ]] || usage "Invalid prefix: \"$OPTARG\", alphanumeric only"
+    prefix=$OPTARG
+    ;;
+  P)
+    publicNetwork=true
+    ;;
+  i)
+    imageName=$OPTARG
+    ;;
+  n)
+    validatorNodeCount=$OPTARG
+    ;;
+  c)
+    clientNodeCount=$OPTARG
+    ;;
+  z)
+    zone=$OPTARG
+    ;;
+  g)
+    leaderAccelerator="count=4,type=nvidia-tesla-k80"
+    ;;
+  a)
+    leaderAddress=$OPTARG
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+shift $((OPTIND - 1))
+
+[[ -z $1 ]] || usage "Unexpected argument: $1"
+sshPrivateKey="$netConfigDir/id_$prefix"
+
+prepareInstancesAndWriteConfigFile() {
+  $metricsWriteDatapoint "testnet-deploy net-config-begin=1"
+
+  cat >> "$configFile" <<EOF
+# autogenerated at $(date)
+netBasename=$prefix
+publicNetwork=$publicNetwork
+sshPrivateKey=$sshPrivateKey
+EOF
+
+  buildSshOptions
+
+  recordInstanceIp() {
+    declare name="$1"
+    declare publicIp="$3"
+    declare privateIp="$4"
+
+    declare arrayName="$6"
+
+    echo "$arrayName+=($publicIp)  # $name" >> "$configFile"
+    if [[ $arrayName = "leaderIp" ]]; then
+      if $publicNetwork; then
+        echo "entrypointIp=$publicIp" >> "$configFile"
+      else
+        echo "entrypointIp=$privateIp" >> "$configFile"
+      fi
+    fi
+  }
+
+  waitForStartupComplete() {
+    declare name="$1"
+    declare publicIp="$3"
+
+    echo "Waiting for $name to finish booting..."
+    (
+      for i in $(seq 1 30); do
+        if (set -x; ssh "${sshOptions[@]}" "$publicIp" "test -f /.gce-startup-complete"); then
+          break
+        fi
+        sleep 2
+        echo "Retry $i..."
+      done
+    )
+  }
+
+  echo "Looking for leader instance..."
+  gcloud_FindInstances "name=$prefix-leader" show
+  [[ ${#instances[@]} -eq 1 ]] || {
+    echo "Unable to find leader"
+    exit 1
+  }
+
+  echo "Fetching $sshPrivateKey from $leaderName"
+  (
+    rm -rf "$sshPrivateKey"{,pub}
+
+    declare leaderName
+    declare leaderZone
+    declare leaderIp
+    IFS=: read -r leaderName leaderZone leaderIp _ < <(echo "${instances[0]}")
+
+    set -x
+
+    # Try to ping the machine first.  There can be a delay between when the
+    # instance is reported as RUNNING and when it's reachable over the network
+    timeout 30s bash -c "set -o pipefail; until ping -c 3 $leaderIp | tr - _; do echo .; done"
+
+    # Try to scp in a couple times, sshd may not yet be up even though the
+    # machine can be pinged...
+    set -o pipefail
+    for i in $(seq 1 10); do
+      if gcloud compute scp --zone "$leaderZone" \
+          "$leaderName:/solana-id_ecdsa" "$sshPrivateKey"; then
+        break
+      fi
+      sleep 1
+      echo "Retry $i..."
+    done
+
+    chmod 400 "$sshPrivateKey"
+  )
+
+  echo "leaderIp=()" >> "$configFile"
+  gcloud_ForEachInstance recordInstanceIp leaderIp
+  gcloud_ForEachInstance waitForStartupComplete
+
+  echo "Looking for validator instances..."
+  gcloud_FindInstances "name~^$prefix-validator" show
+  [[ ${#instances[@]} -gt 0 ]] || {
+    echo "Unable to find validators"
+    exit 1
+  }
+  echo "validatorIpList=()" >> "$configFile"
+  gcloud_ForEachInstance recordInstanceIp validatorIpList
+  gcloud_ForEachInstance waitForStartupComplete
+
+  echo "clientIpList=()" >> "$configFile"
+  echo "Looking for client instances..."
+  gcloud_FindInstances "name~^$prefix-client" show
+  [[ ${#instances[@]} -eq 0 ]] || {
+    gcloud_ForEachInstance recordInstanceIp clientIpList
+    gcloud_ForEachInstance waitForStartupComplete
+  }
+
+  echo "Wrote $configFile"
+  $metricsWriteDatapoint "testnet-deploy net-config-complete=1"
+}
+
+case $command in
+delete)
+  $metricsWriteDatapoint "testnet-deploy net-delete-begin=1"
+
+  # Delete the leader node first to prevent unusual metrics on the dashboard
+  # during shutdown.
+  # TODO: It would be better to fully cut-off metrics reporting before any
+  # instances are deleted.
+  for filter in "^$prefix-leader" "^$prefix-"; do
+    gcloud_FindInstances "name~$filter"
+
+    if [[ ${#instances[@]} -eq 0 ]]; then
+      echo "No instances found matching '$filter'"
+    else
+      gcloud_DeleteInstances true
+    fi
+  done
+  rm -f "$configFile"
+
+  $metricsWriteDatapoint "testnet-deploy net-delete-complete=1"
+  ;;
+
+create)
+  [[ -n $validatorNodeCount ]] || usage "Need number of nodes"
+
+  $metricsWriteDatapoint "testnet-deploy net-create-begin=1"
+
+  rm -rf "$sshPrivateKey"{,.pub}
+  ssh-keygen -t ecdsa -N '' -f "$sshPrivateKey"
+
+  printNetworkInfo() {
+    cat <<EOF
+========================================================================================
+
+Network composition:
+  Leader = $leaderMachineType (GPU=${leaderAccelerator:-none})
+  Validators = $validatorNodeCount x $validatorMachineType (GPU=${validatorAccelerator:-none})
+  Client(s) = $clientNodeCount x $clientMachineType (GPU=${clientAccelerator:-none})
+
+========================================================================================
+
+EOF
+  }
+  printNetworkInfo
+
+  declare startupScript="$netConfigDir"/gce-startup-script.sh
+  cat > "$startupScript" <<EOF
+#!/bin/bash -ex
+# autogenerated at $(date)
+
+cat > /etc/motd <<EOM
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+  This instance has not been fully configured.
+  See "startup-script" log messages in /var/log/syslog for status:
+    $ sudo cat /var/log/syslog | grep startup-script
+
+  To block until setup is complete, run:
+    $ until [[ -f /.gce-startup-complete ]]; do sleep 1; done
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+EOM
+
+# Place the generated private key at /solana-id_ecdsa so it's retrievable by anybody
+# who is able to log into this machine
+cat > /solana-id_ecdsa <<EOK
+$(cat "$sshPrivateKey")
+EOK
+cat > /solana-id_ecdsa.pub <<EOK
+$(cat "$sshPrivateKey.pub")
+EOK
+chmod 444 /solana-id_ecdsa
+
+USER=\$(id -un)
+
+$(
+  cd "$here"/scripts/
+  cat \
+    disable-background-upgrades.sh \
+    create-solana-user.sh \
+    install-earlyoom.sh \
+    install-rsync.sh \
+    install-libssl-compatability.sh \
+)
+
+cat > /etc/motd <<EOM
+$(printNetworkInfo)
+EOM
+
+touch /.gce-startup-complete
+
+EOF
+
+  gcloud_CreateInstances "$prefix-leader" 1 "$zone" \
+    "$imageName" "$leaderMachineType" "$leaderBootDiskSize" "$leaderAccelerator" \
+    "$startupScript" "$leaderAddress"
+
+  gcloud_CreateInstances "$prefix-validator" "$validatorNodeCount" "$zone" \
+    "$imageName" "$validatorMachineType" "$validatorBootDiskSize" "$validatorAccelerator" \
+    "$startupScript" ""
+
+  if [[ $clientNodeCount -gt 0 ]]; then
+    gcloud_CreateInstances "$prefix-client" "$clientNodeCount" "$zone" \
+      "$imageName" "$clientMachineType" "$clientBootDiskSize" "$clientAccelerator" \
+      "$startupScript" ""
+  fi
+
+  $metricsWriteDatapoint "testnet-deploy net-create-complete=1"
+
+  prepareInstancesAndWriteConfigFile
+  ;;
+
+config)
+  prepareInstancesAndWriteConfigFile
+  ;;
+*)
+  usage "Unknown command: $command"
+esac

net/init-metrics.sh

@@ -0,0 +1,80 @@
+#!/bin/bash -e
+
+here=$(dirname "$0")
+# shellcheck source=net/common.sh
+source "$here"/common.sh
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [-e] [-d] [username]
+
+Creates a testnet dev metrics database
+
+  username        InfluxDB user with access to create a new database
+  -d              Delete the database instead of creating it
+  -e              Assume database already exists and SOLANA_METRICS_CONFIG is
+                  defined in the environment already
+
+EOF
+  exit $exitcode
+}
+
+loadConfigFile
+
+useEnv=false
+delete=false
+while getopts "hde" opt; do
+  case $opt in
+  h|\?)
+    usage
+    exit 0
+    ;;
+  d)
+    delete=true
+    ;;
+  e)
+    useEnv=true
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+shift $((OPTIND - 1))
+
+if $useEnv; then
+  [[ -n $SOLANA_METRICS_CONFIG ]] ||
+    usage "Error: SOLANA_METRICS_CONFIG is not defined in the environment"
+else
+  username=$1
+  [[ -n "$username" ]] || usage "username not specified"
+
+  read -rs -p "InfluxDB password for $username: " password
+  [[ -n $password ]] || { echo "Password not specified"; exit 1; }
+  echo
+
+  query() {
+    echo "$*"
+    curl -XPOST \
+      "https://metrics.solana.com:8086/query?u=${username}&p=${password}" \
+      --data-urlencode "q=$*"
+  }
+
+  query "DROP DATABASE \"$netBasename\""
+  ! $delete || exit 0
+  query "CREATE DATABASE \"$netBasename\""
+  query "ALTER RETENTION POLICY autogen ON \"$netBasename\" DURATION 7d"
+  query "GRANT READ ON \"$netBasename\" TO \"ro\""
+  query "GRANT WRITE ON \"$netBasename\" TO \"scratch_writer\""
+
+  SOLANA_METRICS_CONFIG="db=$netBasename,u=scratch_writer,p=topsecret"
+fi
+
+echo "export SOLANA_METRICS_CONFIG=\"$SOLANA_METRICS_CONFIG\"" >> "$configFile"
+
+exit 0

net/net.sh

@@ -0,0 +1,352 @@
+#!/bin/bash -e
+
+here=$(dirname "$0")
+SOLANA_ROOT="$(cd "$here"/..; pwd)"
+
+# shellcheck source=net/common.sh
+source "$here"/common.sh
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [start|stop|restart|sanity] [command-specific options]
+
+Operate a configured testnet
+
+ start    - Start the network
+ sanity   - Sanity check the network
+ stop     - Stop the network
+ restart  - Shortcut for stop then start
+
+ start-specific options:
+   -S [snapFilename]    - Deploy the specified Snap file
+   -s edge|beta|stable  - Deploy the latest Snap on the specified Snap release channel
+   -f [cargoFeatures]   - List of |cargo --feaures=| to activate
+                          (ignored if -s or -S is specified)
+
+   Note: if RUST_LOG is set in the environment it will be propogated into the
+         network nodes.
+
+ sanity/start-specific options:
+   -o noLedgerVerify    - Skip ledger verification
+   -o noValidatorSanity - Skip validator sanity
+
+ stop-specific options:
+   none
+
+EOF
+  exit $exitcode
+}
+
+snapChannel=
+snapFilename=
+deployMethod=local
+sanityExtraArgs=
+cargoFeatures=
+
+command=$1
+[[ -n $command ]] || usage
+shift
+
+while getopts "h?S:s:o:f:" opt; do
+  case $opt in
+  h | \?)
+    usage
+    ;;
+  S)
+    snapFilename=$OPTARG
+    [[ -f $snapFilename ]] || usage "Snap not readable: $snapFilename"
+    deployMethod=snap
+    ;;
+  s)
+    case $OPTARG in
+    edge|beta|stable)
+      snapChannel=$OPTARG
+      deployMethod=snap
+      ;;
+    *)
+      usage "Invalid snap channel: $OPTARG"
+      ;;
+    esac
+    ;;
+  f)
+    cargoFeatures=$OPTARG
+    ;;
+  o)
+    case $OPTARG in
+    noLedgerVerify|noValidatorSanity)
+      sanityExtraArgs="$sanityExtraArgs -o $OPTARG"
+      ;;
+    *)
+      echo "Error: unknown option: $OPTARG"
+      exit 1
+      ;;
+    esac
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+
+loadConfigFile
+expectedNodeCount=$((${#validatorIpList[@]} + 1))
+
+build() {
+  declare MAYBE_DOCKER=
+  if [[ $(uname) != Linux ]]; then
+    MAYBE_DOCKER="ci/docker-run.sh solanalabs/rust"
+  fi
+  SECONDS=0
+  (
+    cd "$SOLANA_ROOT"
+    echo "--- Build started at $(date)"
+
+    set -x
+    rm -rf farf
+    $MAYBE_DOCKER cargo install --features="$cargoFeatures" --root farf
+  )
+  echo "Build took $SECONDS seconds"
+}
+
+startCommon() {
+  declare ipAddress=$1
+  test -d "$SOLANA_ROOT"
+  ssh "${sshOptions[@]}" "$ipAddress" "mkdir -p ~/solana ~/.cargo/bin"
+  rsync -vPrc -e "ssh ${sshOptions[*]}" \
+    "$SOLANA_ROOT"/{fetch-perf-libs.sh,scripts,net,multinode-demo} \
+    "$ipAddress":~/solana/
+}
+
+startLeader() {
+  declare ipAddress=$1
+  declare logFile="$2"
+  echo "--- Starting leader: $leaderIp"
+  echo "start log: $logFile"
+
+  # Deploy local binaries to leader.  Validators and clients later fetch the
+  # binaries from the leader.
+  (
+    set -x
+    startCommon "$ipAddress" || exit 1
+    case $deployMethod in
+    snap)
+      rsync -vPrc -e "ssh ${sshOptions[*]}" "$snapFilename" "$ipAddress:~/solana/solana.snap"
+      ;;
+    local)
+      rsync -vPrc -e "ssh ${sshOptions[*]}" "$SOLANA_ROOT"/farf/bin/* "$ipAddress:~/.cargo/bin/"
+      ;;
+    *)
+      usage "Internal error: invalid deployMethod: $deployMethod"
+      ;;
+    esac
+
+    ssh "${sshOptions[@]}" -n "$ipAddress" \
+      "./solana/net/remote/remote-node.sh $deployMethod leader $publicNetwork $entrypointIp $expectedNodeCount \"$RUST_LOG\""
+  ) >> "$logFile" 2>&1 || {
+    cat "$logFile"
+    echo "^^^ +++"
+    exit 1
+  }
+}
+
+startValidator() {
+  declare ipAddress=$1
+  declare logFile="$netLogDir/validator-$ipAddress.log"
+
+  echo "--- Starting validator: $leaderIp"
+  echo "start log: $logFile"
+  (
+    set -x
+    startCommon "$ipAddress"
+    ssh "${sshOptions[@]}" -n "$ipAddress" \
+      "./solana/net/remote/remote-node.sh $deployMethod validator $publicNetwork $entrypointIp $expectedNodeCount \"$RUST_LOG\""
+  ) >> "$logFile" 2>&1 &
+  declare pid=$!
+  ln -sfT "validator-$ipAddress.log" "$netLogDir/validator-$pid.log"
+  pids+=("$pid")
+}
+
+startClient() {
+  declare ipAddress=$1
+  declare logFile="$2"
+  echo "--- Starting client: $ipAddress"
+  echo "start log: $logFile"
+  (
+    set -x
+    startCommon "$ipAddress"
+    ssh "${sshOptions[@]}" -f "$ipAddress" \
+      "./solana/net/remote/remote-client.sh $deployMethod $entrypointIp $expectedNodeCount \"$RUST_LOG\""
+  ) >> "$logFile" 2>&1 || {
+    cat "$logFile"
+    echo "^^^ +++"
+    exit 1
+  }
+}
+
+sanity() {
+  declare expectedNodeCount=$((${#validatorIpList[@]} + 1))
+  declare ok=true
+
+  echo "--- Sanity"
+  $metricsWriteDatapoint "testnet-deploy net-sanity-begin=1"
+
+  (
+    set -x
+    # shellcheck disable=SC2029 # remote-client.sh args are expanded on client side intentionally
+    ssh "${sshOptions[@]}" "$leaderIp" \
+      "./solana/net/remote/remote-sanity.sh $sanityExtraArgs"
+  ) || ok=false
+
+  $metricsWriteDatapoint "testnet-deploy net-sanity-complete=1"
+  $ok || exit 1
+}
+
+start() {
+  case $deployMethod in
+  snap)
+    if [[ -n $snapChannel ]]; then
+      rm -f "$SOLANA_ROOT"/solana_*.snap
+      if [[ $(uname) != Linux ]]; then
+        (
+          set -x
+          SOLANA_DOCKER_RUN_NOSETUID=1 "$SOLANA_ROOT"/ci/docker-run.sh ubuntu:18.04 bash -c "
+            set -ex;
+            apt-get -qq update;
+            apt-get -qq -y install snapd;
+            snap download --channel=$snapChannel solana;
+          "
+        )
+      else
+        (
+          cd "$SOLANA_ROOT"
+          snap download --channel="$snapChannel" solana
+        )
+      fi
+      snapFilename="$(echo "$SOLANA_ROOT"/solana_*.snap)"
+      [[ -r $snapFilename ]] || {
+        echo "Error: Snap not readable: $snapFilename"
+        exit 1
+      }
+    fi
+    ;;
+  local)
+    build
+    ;;
+  *)
+    usage "Internal error: invalid deployMethod: $deployMethod"
+    ;;
+  esac
+
+  echo "Deployment started at $(date)"
+  $metricsWriteDatapoint "testnet-deploy net-start-begin=1"
+
+  SECONDS=0
+  declare leaderDeployTime=
+  startLeader "$leaderIp" "$netLogDir/leader-$leaderIp.log"
+  leaderDeployTime=$SECONDS
+  $metricsWriteDatapoint "testnet-deploy net-leader-started=1"
+
+  SECONDS=0
+  pids=()
+  for ipAddress in "${validatorIpList[@]}"; do
+    startValidator "$ipAddress"
+  done
+
+  for pid in "${pids[@]}"; do
+    declare ok=true
+    wait "$pid" || ok=false
+    if ! $ok; then
+      cat "$netLogDir/validator-$pid.log"
+      echo ^^^ +++
+      exit 1
+    fi
+  done
+
+  $metricsWriteDatapoint "testnet-deploy net-validators-started=1"
+  validatorDeployTime=$SECONDS
+
+  sanity
+
+  SECONDS=0
+  for ipAddress in "${clientIpList[@]}"; do
+    startClient "$ipAddress" "$netLogDir/client-$ipAddress.log"
+  done
+  clientDeployTime=$SECONDS
+  $metricsWriteDatapoint "testnet-deploy net-start-complete=1"
+
+  if [[ $deployMethod = "snap" ]]; then
+    declare networkVersion=unknown
+    IFS=\  read -r _ networkVersion _ < <(
+      ssh "${sshOptions[@]}" "$leaderIp" \
+        "snap info solana | grep \"^installed:\""
+    )
+    networkVersion=${networkVersion/0+git./}
+    $metricsWriteDatapoint "testnet-deploy version=\"$networkVersion\""
+  fi
+
+  echo
+  echo "+++ Deployment Successful"
+  echo "Leader deployment took $leaderDeployTime seconds"
+  echo "Validator deployment (${#validatorIpList[@]} instances) took $validatorDeployTime seconds"
+  echo "Client deployment (${#clientIpList[@]} instances) took $clientDeployTime seconds"
+  echo "Network start logs in $netLogDir:"
+  ls -l "$netLogDir"
+}
+
+
+stopNode() {
+  local ipAddress=$1
+  echo "--- Stopping node: $ipAddress"
+  (
+    set -x
+    ssh "${sshOptions[@]}" "$ipAddress" "
+      set -x
+      if snap list solana; then
+        sudo snap set solana mode=
+        sudo snap remove solana
+      fi
+      ! tmux list-sessions || tmux kill-session
+      for pattern in solana- remote- oom-monitor net-stats; do
+        pkill -9 \$pattern
+      done
+    "
+  ) || true
+}
+
+stop() {
+  SECONDS=0
+  $metricsWriteDatapoint "testnet-deploy net-stop-begin=1"
+
+  stopNode "$leaderIp"
+
+  for ipAddress in "${validatorIpList[@]}" "${clientIpList[@]}"; do
+    stopNode "$ipAddress"
+  done
+
+  $metricsWriteDatapoint "testnet-deploy net-stop-complete=1"
+  echo "Stopping nodes took $SECONDS seconds"
+}
+
+case $command in
+restart)
+  stop
+  start
+  ;;
+start)
+  start
+  ;;
+sanity)
+  sanity
+  ;;
+stop)
+  stop
+  ;;
+*)
+  echo "Internal error: Unknown command: $command"
+  exit 1
+esac

net/remote/README.md

@@ -0,0 +1 @@
+Scripts that run on the remote testnet nodes

net/remote/remote-client.sh

@@ -0,0 +1,83 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")"/../..
+
+echo "$(date) | $0 $*" > client.log
+
+deployMethod="$1"
+entrypointIp="$2"
+numNodes="$3"
+RUST_LOG="$4"
+export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info
+
+missing() {
+  echo "Error: $1 not specified"
+  exit 1
+}
+
+[[ -n $deployMethod ]] || missing deployMethod
+[[ -n $entrypointIp ]] || missing entrypointIp
+[[ -n $numNodes ]]     || missing numNodes
+
+source net/common.sh
+loadConfigFile
+
+threadCount=$(nproc)
+if [[ $threadCount -gt 4 ]]; then
+  threadCount=4
+fi
+
+case $deployMethod in
+snap)
+  net/scripts/rsync-retry.sh -vPrc "$entrypointIp:~/solana/solana.snap" .
+  sudo snap install solana.snap --devmode --dangerous
+
+  solana_bench_tps=/snap/bin/solana.bench-tps
+  solana_keygen=/snap/bin/solana.keygen
+  ;;
+local)
+  PATH="$HOME"/.cargo/bin:"$PATH"
+  export USE_INSTALL=1
+  export SOLANA_DEFAULT_METRICS_RATE=1
+
+  net/scripts/rsync-retry.sh -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/
+  solana_bench_tps=solana-bench-tps
+  solana_keygen=solana-keygen
+  ;;
+*)
+  echo "Unknown deployment method: $deployMethod"
+  exit 1
+esac
+
+scripts/oom-monitor.sh > oom-monitor.log 2>&1 &
+scripts/net-stats.sh  > net-stats.log 2>&1 &
+
+! tmux list-sessions || tmux kill-session
+
+clientCommand="\
+  $solana_bench_tps \
+    --network $entrypointIp:8001 \
+    --identity client.json \
+    --num-nodes $numNodes \
+    --duration 600 \
+    --sustained \
+    --threads $threadCount \
+"
+
+keygenCommand="$solana_keygen -o client.json"
+tmux new -s solana-bench-tps -d "
+  [[ -r client.json ]] || {
+    echo '$ $keygenCommand'  | tee -a client.log
+    $keygenCommand >> client.log 2>&1
+  }
+
+  while true; do
+    echo === Client start: \$(date) | tee -a client.log
+    $metricsWriteDatapoint 'testnet-deploy client-begin=1'
+    echo '$ $clientCommand' | tee -a client.log
+    $clientCommand >> client.log 2>&1
+    $metricsWriteDatapoint 'testnet-deploy client-complete=1'
+  done
+"
+sleep 1
+tmux capture-pane -t solana-bench-tps -p -S -100

net/remote/remote-node.sh

@@ -0,0 +1,113 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")"/../..
+
+deployMethod="$1"
+nodeType="$2"
+publicNetwork="$3"
+entrypointIp="$4"
+numNodes="$5"
+RUST_LOG="$6"
+
+missing() {
+  echo "Error: $1 not specified"
+  exit 1
+}
+
+[[ -n $deployMethod ]]  || missing deployMethod
+[[ -n $nodeType ]]      || missing nodeType
+[[ -n $publicNetwork ]] || missing publicNetwork
+[[ -n $entrypointIp ]]  || missing entrypointIp
+[[ -n $numNodes ]]      || missing numNodes
+
+cat > deployConfig <<EOF
+deployMethod="$deployMethod"
+entrypointIp="$entrypointIp"
+numNodes="$numNodes"
+EOF
+
+source net/common.sh
+loadConfigFile
+
+if [[ $publicNetwork = true ]]; then
+  setupArgs="-p"
+else
+  setupArgs="-l"
+fi
+
+
+case $deployMethod in
+snap)
+  SECONDS=0
+  [[ $nodeType = leader ]] ||
+    net/scripts/rsync-retry.sh -vPrc "$entrypointIp:~/solana/solana.snap" .
+  sudo snap install solana.snap --devmode --dangerous
+
+  commonNodeConfig="\
+    leader-ip=$entrypointIp \
+    default-metrics-rate=1 \
+    metrics-config=$SOLANA_METRICS_CONFIG \
+    rust-log=$RUST_LOG \
+    setup-args=$setupArgs \
+  "
+
+  if [[ -e /dev/nvidia0 ]]; then
+    commonNodeConfig="$commonNodeConfig enable-cuda=1"
+  fi
+
+  if [[ $nodeType = leader ]]; then
+    nodeConfig="mode=leader+drone $commonNodeConfig"
+    ln -sf -T /var/snap/solana/current/leader/current leader.log
+    ln -sf -T /var/snap/solana/current/drone/current drone.log
+  else
+    nodeConfig="mode=validator $commonNodeConfig"
+    ln -sf -T /var/snap/solana/current/validator/current validator.log
+  fi
+
+  logmarker="solana deploy $(date)/$RANDOM"
+  logger "$logmarker"
+
+  # shellcheck disable=SC2086 # Don't want to double quote "$nodeConfig"
+  sudo snap set solana $nodeConfig
+  snap info solana
+  sudo snap get solana
+  echo Slight delay to get more syslog output
+  sleep 2
+  sudo grep -Pzo "$logmarker(.|\\n)*" /var/log/syslog
+
+  echo "Succeeded in ${SECONDS} seconds"
+  ;;
+local)
+  PATH="$HOME"/.cargo/bin:"$PATH"
+  export USE_INSTALL=1
+  export RUST_LOG
+  export SOLANA_DEFAULT_METRICS_RATE=1
+
+  ./fetch-perf-libs.sh
+  export LD_LIBRARY_PATH="$PWD/target/perf-libs:$LD_LIBRARY_PATH"
+
+  scripts/oom-monitor.sh  > oom-monitor.log 2>&1 &
+  scripts/net-stats.sh  > net-stats.log 2>&1 &
+
+  case $nodeType in
+  leader)
+    ./multinode-demo/setup.sh -t leader $setupArgs
+    ./multinode-demo/drone.sh > drone.log 2>&1 &
+    ./multinode-demo/leader.sh > leader.log 2>&1 &
+    ;;
+  validator)
+    net/scripts/rsync-retry.sh -vPrc "$entrypointIp:~/.cargo/bin/solana*" ~/.cargo/bin/
+
+    ./multinode-demo/setup.sh -t validator $setupArgs
+    ./multinode-demo/validator.sh "$entrypointIp":~/solana "$entrypointIp:8001" >validator.log 2>&1 &
+    ;;
+  *)
+    echo "Error: unknown node type: $nodeType"
+    exit 1
+    ;;
+  esac
+  ;;
+*)
+  echo "Unknown deployment method: $deployMethod"
+  exit 1
+esac

net/remote/remote-sanity.sh

@@ -0,0 +1,138 @@
+#!/bin/bash -e
+#
+# This script is to be run on the leader node
+#
+
+cd "$(dirname "$0")"/../..
+
+deployMethod=
+entrypointIp=
+numNodes=
+
+[[ -r deployConfig ]] || {
+  echo deployConfig missing
+  exit 1
+}
+# shellcheck source=/dev/null # deployConfig is written by remote-node.sh
+source deployConfig
+
+missing() {
+  echo "Error: $1 not specified"
+  exit 1
+}
+
+[[ -n $deployMethod ]] || missing deployMethod
+[[ -n $entrypointIp ]] || missing entrypointIp
+[[ -n $numNodes ]]     || missing numNodes
+
+ledgerVerify=true
+validatorSanity=true
+while [[ $1 = -o ]]; do
+  opt="$2"
+  shift 2
+  case $opt in
+  noLedgerVerify)
+    ledgerVerify=false
+    ;;
+  noValidatorSanity)
+    validatorSanity=false
+    ;;
+  *)
+    echo "Error: unknown option: $opt"
+    exit 1
+    ;;
+  esac
+done
+
+source net/common.sh
+loadConfigFile
+
+case $deployMethod in
+snap)
+  PATH="/snap/bin:$PATH"
+  export USE_SNAP=1
+  entrypointRsyncUrl="$entrypointIp"
+
+  solana_bench_tps=solana.bench-tps
+  solana_ledger_tool=solana.ledger-tool
+  solana_keygen=solana.keygen
+
+  ledger=/var/snap/solana/current/config/ledger
+  client_id=~/snap/solana/current/config/client-id.json
+
+  ;;
+local)
+  PATH="$HOME"/.cargo/bin:"$PATH"
+  export USE_INSTALL=1
+  entrypointRsyncUrl="$entrypointIp:~/solana"
+
+  solana_bench_tps=solana-bench-tps
+  solana_ledger_tool=solana-ledger-tool
+  solana_keygen=solana-keygen
+
+  ledger=config/ledger
+  client_id=config/client-id.json
+  ;;
+*)
+  echo "Unknown deployment method: $deployMethod"
+  exit 1
+esac
+
+
+echo "--- $entrypointIp: wallet sanity"
+(
+  set -x
+  scripts/wallet-sanity.sh "$entrypointRsyncUrl"
+)
+
+echo "+++ $entrypointIp: node count ($numNodes expected)"
+(
+  set -x
+  $solana_keygen -o "$client_id"
+  $solana_bench_tps --network "$entrypointIp:8001" --identity "$client_id" --num-nodes "$numNodes" --converge-only
+)
+
+echo "--- $entrypointIp: verify ledger"
+if $ledgerVerify; then
+  if [[ -d $ledger ]]; then
+    (
+      set -x
+      rm -rf /var/tmp/ledger-verify
+      du -hs "$ledger"
+      time cp -r "$ledger" /var/tmp/ledger-verify
+      time $solana_ledger_tool --ledger /var/tmp/ledger-verify verify
+    )
+  else
+    echo "^^^ +++"
+    echo "Ledger verify skipped: directory does not exist: $ledger"
+  fi
+else
+  echo "^^^ +++"
+  echo "Note: ledger verify disabled"
+fi
+
+
+echo "--- $entrypointIp: validator sanity"
+if $validatorSanity; then
+  (
+    set -ex -o pipefail
+    ./multinode-demo/setup.sh -t validator
+    timeout 10s ./multinode-demo/validator.sh "$entrypointRsyncUrl" "$entrypointIp:8001" 2>&1 | tee validator.log
+  ) || {
+    exitcode=$?
+    [[ $exitcode -eq 124 ]] || exit $exitcode
+  }
+  wc -l validator.log
+  if grep -C100 panic validator.log; then
+    echo "^^^ +++"
+    echo "Panic observed"
+    exit 1
+  else
+    echo "Validator log looks ok"
+  fi
+else
+  echo "^^^ +++"
+  echo "Note: validator sanity disabled"
+fi
+
+echo --- Pass

net/scripts/create-solana-user.sh

@@ -0,0 +1,27 @@
+#!/bin/bash -ex
+
+[[ $(uname) = Linux ]] || exit 1
+[[ $USER = root ]] || exit 1
+
+adduser solana --gecos "" --disabled-password --quiet
+adduser solana sudo
+echo "solana ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+id solana
+
+[[ -r /solana-id_ecdsa ]] || exit 1
+[[ -r /solana-id_ecdsa.pub ]] || exit 1
+
+sudo -u solana bash -c "
+  mkdir -p /home/solana/.ssh/
+  cd /home/solana/.ssh/
+  cp /solana-id_ecdsa.pub authorized_keys
+  umask 377
+  cp /solana-id_ecdsa id_ecdsa
+  echo \"
+    Host *
+    BatchMode yes
+    IdentityFile ~/.ssh/id_ecdsa
+    StrictHostKeyChecking no
+  \" > config
+"
+

net/scripts/disable-background-upgrades.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -ex
+#
+# Prevent background upgrades that block |apt-get|
+#
+# TODO: This approach is pretty uncompromising.  An alternative solution that
+#       doesn't involve deleting system files would be welcome.
+
+[[ $(uname) = Linux ]] || exit 1
+[[ $USER = root ]] || exit 1
+
+rm -rf /usr/lib/apt/apt.systemd.daily
+rm -rf /usr/bin/unattended-upgrade
+killall apt.systemd.daily || true
+killall unattended-upgrade || true
+
+while fuser /var/lib/dpkg/lock; do
+  echo Waiting for lock release...
+  sleep 1
+done
+

net/scripts/gcloud.sh

@@ -0,0 +1,187 @@
+# |source| this file
+#
+# Utilities for working with gcloud
+#
+
+
+#
+# gcloud_FindInstances [filter] [options]
+#
+# Find instances matching the specified pattern.
+#
+# For each matching instance, an entry in the `instances` array will be added with the
+# following information about the instance:
+#   "name:zone:public IP:private IP"
+#
+# filter   - The instances to filter on
+# options  - If set to the string "show", the list of instances will be echoed
+#            to stdout
+#
+# examples:
+#   $ gcloud_FindInstances "name=exact-machine-name"
+#   $ gcloud_FindInstances "name~^all-machines-with-a-common-machine-prefix"
+#
+gcloud_FindInstances() {
+  declare filter="$1"
+  declare options="$2"
+  instances=()
+
+  declare name zone publicIp privateIp status
+  while read -r name zone publicIp privateIp status; do
+    if [[ $status != RUNNING ]]; then
+      echo "Warning: $name is not RUNNING, ignoring it."
+      continue
+    fi
+    if [[ $options = show ]]; then
+      printf "%-30s | %-16s publicIp=%-16s privateIp=%s\n" "$name" "$zone" "$publicIp" "$privateIp"
+    fi
+
+    instances+=("$name:$zone:$publicIp:$privateIp")
+  done < <(gcloud compute instances list \
+             --filter="$filter" \
+             --format 'value(name,zone,networkInterfaces[0].accessConfigs[0].natIP,networkInterfaces[0].networkIP,status)')
+}
+
+#
+# gcloud_ForEachInstance [cmd] [extra args to cmd]
+#
+# Execute a command for each element in the `instances` array
+#
+#   cmd   - The command to execute on each instance
+#           The command will receive arguments followed by any
+#           additionl arguments supplied to gcloud_ForEachInstance:
+#               name     - name of the instance
+#               zone     - zone the instance is located in
+#               publicIp - The public IP address of this instance
+#               privateIp - The priate IP address of this instance
+#               count    - Monotonically increasing count for each
+#                          invocation of cmd, starting at 1
+#               ...      - Extra args to cmd..
+#
+#
+gcloud_ForEachInstance() {
+  declare cmd="$1"
+  shift
+  [[ -n $cmd ]] || { echo gcloud_ForEachInstance: cmd not specified; exit 1; }
+
+  declare count=1
+  for info in "${instances[@]}"; do
+    declare name zone publicIp privateIp
+    IFS=: read -r name zone publicIp privateIp < <(echo "$info")
+
+    eval "$cmd" "$name" "$zone" "$publicIp" "$privateIp" "$count" "$@"
+    count=$((count + 1))
+  done
+}
+
+#
+# gcloud_CreateInstances [namePrefix] [numNodes] [zone] [imageName]
+#                        [machineType] [bootDiskSize] [accelerator]
+#                        [startupScript] [address]
+#
+# Creates one more identical instances.
+#
+# namePrefix    - unique string to prefix all the instance names with
+# numNodes      - number of instances to create
+# zone          - zone to create the instances in
+# imageName     - Disk image for the instances
+# machineType   - GCE machine type
+# bootDiskSize  - Optional disk of the boot disk
+# accelerator   - Optional accelerator to attach to the instance(s), see
+#                 eg, request 4 K80 GPUs with "count=4,type=nvidia-tesla-k80"
+# startupScript - Optional startup script to execute when the instance boots
+# address       - Optional name of the GCE static IP address to attach to the
+#                 instance.  Requires that |numNodes| = 1 and that addressName
+#                 has been provisioned in the GCE region that is hosting |zone|
+#
+# Tip: use gcloud_FindInstances to locate the instances once this function
+#      returns
+gcloud_CreateInstances() {
+  declare namePrefix="$1"
+  declare numNodes="$2"
+  declare zone="$3"
+  declare imageName="$4"
+  declare machineType="$5"
+  declare optionalBootDiskSize="$6"
+  declare optionalAccelerator="$7"
+  declare optionalStartupScript="$8"
+  declare optionalAddress="$9"
+
+  declare nodes
+  if [[ $numNodes = 1 ]]; then
+    nodes=("$namePrefix")
+  else
+    read -ra nodes <<<$(seq -f "${namePrefix}%0${#numNodes}g" 1 "$numNodes")
+  fi
+
+  declare -a args
+  args=(
+    "--zone=$zone"
+    "--tags=testnet"
+    "--image=$imageName"
+    "--machine-type=$machineType"
+  )
+  if [[ -n $optionalBootDiskSize ]]; then
+    args+=(
+      "--boot-disk-size=$optionalBootDiskSize"
+    )
+  fi
+  if [[ -n $optionalAccelerator ]]; then
+    args+=(
+      "--accelerator=$optionalAccelerator"
+      --maintenance-policy TERMINATE
+      --restart-on-failure
+    )
+  fi
+  if [[ -n $optionalStartupScript ]]; then
+    args+=(
+      --metadata-from-file "startup-script=$optionalStartupScript"
+    )
+  fi
+
+  if [[ -n $optionalAddress ]]; then
+    [[ $numNodes = 1 ]] || {
+      echo "Error: address may not be supplied when provisioning multiple nodes: $optionalAddress"
+      exit 1
+    }
+    args+=(
+      "--address=$optionalAddress"
+    )
+  fi
+
+  (
+    set -x
+    gcloud beta compute instances create "${nodes[@]}" "${args[@]}"
+  )
+}
+
+#
+# gcloud_DeleteInstances [yes]
+#
+# Deletes all the instances listed in the `instances` array
+#
+# If yes = "true", skip the delete confirmation
+#
+gcloud_DeleteInstances() {
+  declare maybeQuiet=
+  if [[ $1 = true ]]; then
+    maybeQuiet=--quiet
+  fi
+
+  if [[ ${#instances[0]} -eq 0 ]]; then
+    echo No instances to delete
+    return
+  fi
+  declare names=("${instances[@]/:*/}")
+
+  # Assume all instances are in the same zone
+  # TODO: One day this assumption will be invalid
+  declare zone
+  IFS=: read -r _ zone _ < <(echo "${instances[0]}")
+
+  (
+    set -x
+    gcloud beta compute instances delete --zone "$zone" $maybeQuiet "${names[@]}"
+  )
+}
+

net/scripts/install-earlyoom.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -ex
+#
+# Install EarlyOOM
+#
+
+[[ $(uname) = Linux ]] || exit 1
+[[ $USER = root ]] || exit 1
+
+# 64 - enable signalling of processes (term, kill, oom-kill)
+# TODO: This setting will not persist across reboots
+sysctl -w kernel.sysrq=$(( $(cat /proc/sys/kernel/sysrq) | 64 ))
+
+if command -v earlyoom; then
+  systemctl status earlyoom
+else
+  wget http://ftp.us.debian.org/debian/pool/main/e/earlyoom/earlyoom_1.1-2_amd64.deb
+  apt install --quiet --yes ./earlyoom_1.1-2_amd64.deb
+
+  cat > earlyoom <<OOM
+  # use the kernel OOM killer, trigger at 20% available RAM,
+  EARLYOOM_ARGS="-k -m 20"
+OOM
+  cp earlyoom /etc/default/
+  rm earlyoom
+
+  systemctl stop earlyoom
+  systemctl enable earlyoom
+  systemctl start earlyoom
+fi
+

net/scripts/install-libssl-compatability.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -ex
+
+[[ $(uname) = Linux ]] || exit 1
+[[ $USER = root ]] || exit 1
+
+# Install libssl-dev to be compatible with binaries built on an Ubuntu machine...
+apt-get update
+apt-get --assume-yes install libssl-dev
+
+# Install libssl1.1 to be compatible with binaries built in the
+# solanalabs/rust docker image
+#
+# cc: https://github.com/solana-labs/solana/issues/1090
+# cc: https://packages.ubuntu.com/bionic/amd64/libssl1.1/download
+wget http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
+dpkg -i libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
+rm libssl1.1_1.1.0g-2ubuntu4.1_amd64.deb
+

net/scripts/install-rsync.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -ex
+#
+# Rsync setup for Snap builds
+#
+
+[[ $(uname) = Linux ]] || exit 1
+[[ $USER = root ]] || exit 1
+
+apt-get --assume-yes install rsync
+cat > /etc/rsyncd.conf <<-EOF
+[config]
+path = /var/snap/solana/current/config
+hosts allow = *
+read only = true
+EOF
+
+systemctl enable rsync
+systemctl start rsync
+

net/scripts/rsync-retry.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# rsync wrapper that retries a few times on failure
+#
+
+for i in $(seq 1 5); do
+  (
+    set -x
+    rsync "$@"
+  ) && exit 0
+  echo Retry "$i"...
+done

net/ssh.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+here=$(dirname "$0")
+# shellcheck source=net/common.sh
+source "$here"/common.sh
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [ipAddress] [extra ssh arguments]
+
+ssh into a node
+
+ ipAddress     - IP address of the desired node.
+
+If ipAddress is unspecified, a list of available nodes will be displayed.
+
+EOF
+  exit $exitcode
+}
+
+while getopts "h?" opt; do
+  case $opt in
+  h | \?)
+    usage
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+
+loadConfigFile
+
+ipAddress=$1
+shift
+if [[ -n "$ipAddress" ]]; then
+  set -x
+  exec ssh "${sshOptions[@]}" "$ipAddress" "$@"
+fi
+
+printNode() {
+  declare nodeType=$1
+  declare ip=$2
+  printf "  %-25s | For logs run: $0 $ip tail -f solana/$nodeType.log\n" "$0 $ip"
+}
+
+echo Leader:
+printNode leader "$leaderIp"
+echo
+echo Validators:
+for ipAddress in "${validatorIpList[@]}"; do
+  printNode validator "$ipAddress"
+done
+echo
+echo Clients:
+if [[ ${#clientIpList[@]} -eq 0 ]]; then
+  echo "  None"
+else
+  for ipAddress in "${clientIpList[@]}"; do
+    printNode client "$ipAddress"
+  done
+fi
+
+exit 0

rfcs/rfc-001-smart-contracts-engine.md

@@ -0,0 +1,211 @@
+# Smart Contracts Engine
+ 
+The goal of this RFC is to define a set of constraints for APIs and runtime such that we can execute our smart contracts safely on massively parallel hardware such as a GPU.  Our runtime is built around an OS *syscall* primitive.  The difference in blockchain is that now the OS does a cryptographic check of memory region ownership before accessing the memory in the Solana kernel.
+
+## Version
+
+version 0.2 
+
+## Toolchain Stack
+
+     +---------------------+       +---------------------+
+     |                     |       |                     |
+     |   +------------+    |       |   +------------+    |
+     |   |            |    |       |   |            |    |
+     |   |  frontend  |    |       |   |  verifier  |    |
+     |   |            |    |       |   |            |    |
+     |   +-----+------+    |       |   +-----+------+    |
+     |         |           |       |         |           |
+     |         |           |       |         |           |
+     |   +-----+------+    |       |   +-----+------+    |
+     |   |            |    |       |   |            |    |
+     |   |    llvm    |    |       |   |   loader   |    |
+     |   |            |    +------>+   |            |    |
+     |   +-----+------+    |       |   +-----+------+    |
+     |         |           |       |         |           |
+     |         |           |       |         |           |
+     |   +-----+------+    |       |   +-----+------+    |
+     |   |            |    |       |   |            |    |
+     |   |    ELF     |    |       |   |   runtime  |    |
+     |   |            |    |       |   |            |    |
+     |   +------------+    |       |   +------------+    |
+     |                     |       |                     |
+     |        client       |       |       solana        |
+     +---------------------+       +---------------------+
+
+                [Figure 1. Smart Contracts Stack]
+
+In Figure 1 an untrusted client, creates a program in the front-end language of her choice, (like C/C++/Rust/Lua), and compiles it with LLVM to a position independent shared object ELF, targeting BPF bytecode. Solana will safely load and execute the ELF.
+
+## Runtime
+
+The goal with the runtime is to have a general purpose execution environment that is highly parallelizeable and doesn't require dynamic resource management. The goal is to execute as many contracts as possible in parallel, and have them pass or fail without a destructive state change.
+
+
+### State
+
+State is addressed by an account which is at the moment simply the Pubkey.  Our goal is to eliminate memory allocation from within the smart contract itself.  Thus the client of the contract provides all the state that is necessary for the contract to execute in the transaction itself.  The runtime interacts with the contract through a state transition function, which takes a mapping of [(Pubkey,State)] and returns [(Pubkey, State')].  The State is an opeque type to the runtime, a `Vec<u8>`, the contents of which the contract has full control over.
+
+### Call Structure
+```
+/// Call definition
+/// Signed portion
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub struct CallData {
+    /// Each Pubkey in this vector is mapped to a corresponding `Page` that is loaded for contract execution
+    /// In a simple pay transaction `key[0]` is the token owner's key and `key[1]` is the recipient's key.
+    pub keys: Vec<Pubkey>,
+
+    /// The Pubkeys that are required to have a proof.  The proofs are a `Vec<Signature> which encoded along side this data structure
+    /// Each Signature signs the `required_proofs` vector as well as the `keys` vectors.  The transaction is valid if and only if all
+    /// the required signatures are present and the public key vector is unchanged between signatures.
+    pub required_proofs: Vec<u8>,
+
+    /// PoH data
+    /// last PoH hash observed by the sender
+    pub last_id: Hash,
+
+    /// Program
+    /// The address of the program we want to call.  ContractId is just a Pubkey that is the address of the loaded code that will execute this Call.
+    pub contract_id: ContractId,
+    /// OS scheduling fee
+    pub fee: i64,
+    /// struct version to prevent duplicate spends
+    /// Calls with a version <= Page.version are rejected
+    pub version: u64,
+    /// method to call in the contract
+    pub method: u8,
+    /// usedata in bytes
+    pub userdata: Vec<u8>,
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub struct Call {
+    /// Signatures and Keys
+    /// (signature, key index)
+    /// This vector contains a tuple of signatures, and the key index the signature is for
+    /// proofs[0] is always key[0]
+    pub proofs: Vec<Signature>,
+    pub data: CallData,
+}
+```
+
+At it's core, this is just a set of Pubkeys and Signatures with a bit of metadata.  The contract Pubkey routes this transaction into that contracts entry point.  `version` is used for dropping retransmitted requests.
+
+Contracts should be able to read any state that is part of runtime, but only write to state that the contract allocated.
+
+### Execution
+
+Calls batched and processed in a pipeline
+
+```
++-----------+    +-------------+    +--------------+    +--------------------+    
+| sigverify |--->| lock memory |--->| validate fee |--->| allocate new pages |--->
++-----------+    +-------------+    +--------------+    +--------------------+    
+                                
+    +------------+    +---------+    +--------------+    +-=------------+   
+--->| load pages |--->| execute |--->|unlock memory |--->| commit pages |   
+    +------------+    +---------+    +--------------+    +--------------+   
+
+```
+
+At the `execute` stage, the loaded pages have no data dependencies, so all the contracts can be executed in parallel. 
+## Memory Management
+```
+pub struct Page {
+    /// key that indexes this page
+    /// prove ownership of this key to spend from this Page
+    owner: Pubkey,
+    /// contract that owns this page
+    /// contract can write to the data that is in `memory` vector
+    contract: Pubkey,
+    /// balance that belongs to owner
+    balance: u64,
+    /// version of the structure, public for testing
+    version: u64,
+    /// hash of the page data
+    memhash: Hash,
+    /// The following could be in a separate structure
+    memory: Vec<u8>,
+}
+```
+
+The guarantee that runtime enforces:
+    1. The contract code is the only code that will modify the contents of `memory`
+    2. Total balances on all the pages is equal before and after exectuion of a call
+    3. Balances of each of the pages not owned by the contract must be equal to or greater after the call than before the call.
+
+## Entry Point
+Exectuion of the contract involves maping the contract's public key to an entry point which takes a pointer to the transaction, and an array of loaded pages.
+```
+// Find the method
+match (tx.contract, tx.method) {
+    // system interface
+    // everyone has the same reallocate
+    (_, 0) => system_0_realloc(&tx, &mut call_pages),
+    (_, 1) => system_1_assign(&tx, &mut call_pages),
+    // contract methods
+    (DEFAULT_CONTRACT, 128) => default_contract_128_move_funds(&tx, &mut call_pages),
+    (contract, method) => //... 
+```
+
+The first 127 methods are reserved for the system interface, which implements allocation and assignment of memory.  The rest, including the contract for moving funds are implemented by the contract itself.
+
+## System Interface
+```
+/// SYSTEM interface, same for very contract, methods 0 to 127
+/// method 0
+/// reallocate
+/// spend the funds from the call to the first recipient's
+pub fn system_0_realloc(call: &Call, pages: &mut Vec<Page>) {
+    if call.contract == DEFAULT_CONTRACT {
+        let size: u64 = deserialize(&call.userdata).unwrap();
+        pages[0].memory.resize(size as usize, 0u8);
+    }
+}
+/// method 1
+/// assign
+/// assign the page to a contract
+pub fn system_1_assign(call: &Call, pages: &mut Vec<Page>) {
+    let contract = deserialize(&call.userdata).unwrap();
+    if call.contract == DEFAULT_CONTRACT {
+        pages[0].contract = contract;
+        //zero out the memory in pages[0].memory
+        //Contracts need to own the state of that data otherwise a use could fabricate the state and
+        //manipulate the contract
+        pages[0].memory.clear();
+    }
+} 
+```
+The first method resizes the memory that is assosciated with the callers page.  The second system call assignes the page to the contract.  Both methods check if the current contract is 0, otherwise the method does nothing and the caller spent their fees.
+
+This ensures that when memory is assigned to the contract the initial state of all the bytes is 0, and the contract itself is the only thing that can modify that state.
+
+## Simplest contract
+```
+/// DEFAULT_CONTRACT interface
+/// All contracts start with 128
+/// method 128
+/// move_funds
+/// spend the funds from the call to the first recipient's
+pub fn default_contract_128_move_funds(call: &Call, pages: &mut Vec<Page>) {
+    let amount: u64 = deserialize(&call.userdata).unwrap();
+    if pages[0].balance >= amount  {
+        pages[0].balance -= amount;
+        pages[1].balance += amount;
+    }
+}
+``` 
+
+This simply moves the amount from page[0], which is the callers page, to page[1], which is the recipient's page.
+
+## Notes
+
+1. There is no dynamic memory allocation.
+2. Persistent Memory is allocated to a Key with ownership
+3. Contracts can `call` to update key owned state
+4. `call` is just a *syscall* that does a cryptographic check of memory ownership
+5. Kernel guarantees that when memory is assigned to the contract its state is 0
+6. Kernel guarantees that contract is the only thing that can modify memory that its assigned to
+7. Kernel guarantees that the contract can only spend tokens that are in pages that are assigned to it
+8. Kernel guarantees the balances belonging to pages are balanced before and after the call

rfcs/rfc-002-consensus.md

@@ -0,0 +1,122 @@
+# Consensus
+
+VERY WIP
+
+The goal of this RFC is to define the consensus algorithm used in solana.  This proposal covers a Proof of Stake algorithm that leverages Proof of History.  PoH is a permissionless clock for blockchain that is available before consensus.  This PoS approach leverages PoH to make strong assumptions about time between partitions.
+
+## Version
+
+version 0.1
+
+## Message Flow
+
+1. Transactions are ingested at the leader.
+2. Leader filters for valid transactions
+3. Leader executes valid transactions on its state
+4. Leader packages transactions into blobs
+5. Leader transmits blobs to validator nodes.
+    a. The set of supermajority + `M` by stake weight of nodes is rotated in round robin fashion.
+6. Validators retransmit blobs to peers in their set and to further downstream nodes.
+7. Validators validate the transactions and execute them on their state.
+8. Validators compute the hash of the state.
+9. Validators transmit votes to the leader.
+    a. Votes are signatures of the hash of the computed state.
+10. Leader executes the votes as any other transaction and broadcasts them out to the network
+11. Validators observe their votes, and all the votes from the network.
+12. Validators continue voting if the supermajority of stake is observed in the vote for the same hash.
+
+Supermajority is defined as `2/3rds + 1` vote of the PoS stakes.
+
+## Staking
+
+Validators `stake` some of their spendable sol into a staking account.  The stakes are not spendable and can only be used for voting.
+
+```
+CreateStake(
+    PoH count,
+    PoH hash,
+    source public key,
+    amount,
+    destination public key,
+    proof of ownership of destination public key,
+    signature of the message with the source keypair
+)
+```
+
+Creating the stake has a warmup period of TBD.  Unstaking requires the node to miss a certain amount of validation votes.
+
+## Validation Votes
+
+```
+Validate(
+    PoH count,
+    PoH hash,
+    stake public key,
+    signature of the state,
+    signature of the message with the stake keypair
+)
+```
+
+## Validator Slashing
+
+Validators `stake` some of their spendable sol into a staking account.  The stakes are not spendable and can only be used for voting.
+
+```
+Slash(Validate(
+    PoH count,
+    PoH hash,
+    stake public key,
+    ...
+    signature of the message with the stake keypair
+))
+```
+
+When the `Slash` vote is processed, validators should lookup `PoH hash` at `PoH count` and compare it with the message.  If they do not match, the stake at `stake public key` should be set to `0`.
+
+## Leader Slashing
+
+TBD.  The goal of this is to discourage leaders from generating multiple PoH streams.
+
+## Validation Vote Contract
+
+The goal of this contract is to simulate economic cost of mining on a shorter branch.
+
+1. With my signature I am certifying that I computed `state hash` at `PoH count` and `PoH hash`.
+2. I will not vote on a branch that doesn't contain this message for at least `N` counts, or until `PoH count` + `N` is reached by the PoH stream.
+3. I will not vote for any other branch below `PoH count`.
+    a. if there are other votes not present in this PoH history the validator may need to `cancel` them before creating this vote.
+
+## Leader Seed Generation
+
+Leader selection is decided via a random seed.  The process is as follows:
+
+1. Periodically at a specific `PoH count` select the first vote signatures that create a supermajority from the previous round.
+2. append them together
+3. hash the string for `N` counts via a similar process as PoH itself.
+4. The resulting hash is the random seed for `M` counts, where M > N
+
+## Leader Ranking and Rotation
+
+Leader's transmit for a count of `T`.  When `T` is reached all the validators should switch to the next ranked leader.  To rank leaders, the supermajority + `M` nodes are shuffled with the using the above calculated random seed.
+
+TBD: define a ranking for critical partitions without a node from supermajority + `M` set.
+
+## Partition selection
+
+Validators should select the first branch to reach finality, or the highest ranking leader.
+
+## Examples
+
+### Small Partition
+1. Network partition M occurs for 10% of the nodes
+2. The larger partition K, with 90% of the stake weight continues to operate as normal
+3. M cycles through the ranks until one of them is leader.
+4. M validators observe 10% of the vote pool, finality is not reached
+5. M and K re-connect.
+6. M validators cancel their votes on K which are below K's `PoH count`
+
+### Leader Timeout
+1. Next rank node observes a timeout.
+2. Nodes receiving both PoH streams pick the higher rank node.
+3. 2, causes a partition, since nodes can only vote for 1 leader.
+4. Partition is resolved just like in the [Small Partition](#small-parition)

rfcs/rfc-003-storage.md

@@ -0,0 +1,54 @@
+# Storage
+
+The goal of this RFC is to define a protocol for storing a very large ledger over a p2p network that is verified by solana validators.  At full capacity on a 1gbps network solana will generate 4 petabytes of data per year.  To prevent the network from centralizing around full nodes that have to store the full data set this protocol proposes a way for mining nodes to provide storage capacity for pieces of the network.
+
+# Version 
+
+version 0.1
+
+# Background
+
+The basic idea to Proof of Replication is encrypting a dataset with a public symmetric key using CBC encryption, then hash the encrypted dataset. The main problem with the naive approach is that a dishonest storage node can stream the encryption and delete the data as its hashed. The simple solution is to force the hash to be done on the reverse of the encryption, or perhaps with a random order. This ensures that all the data is present during the generation of the proof and it also requires the validator to have the entirety of the encrypted data present for verification of every proof of every identity. So the space required to validate is `(Number of Proofs)*(data size)`
+
+# Optimization with PoH 
+
+Our improvement on this approach is to randomly sample the encrypted blocks faster than it takes to encrypt, and record the hash of those samples into the PoH ledger. Thus the blocks stay in the exact same order for every PoRep and verification can stream the data and verify all the proofs in a single batch. This way we can verify multiple proofs concurrently, each one on its own CUDA core. With the current generation of graphics cards our network can support up to 14k replication identities or symmetric keys. The total space required for verification is `(2 CBC blocks) * (Number of Identities)`, with core count of equal to (Number of Identities). A CBC block is expected to be 1MB in size.
+
+# Network
+
+Validators for PoRep are the same validators that are verifying transactions. They have some stake that they have put up as collateral that ensures that their work is honest. If you can prove that a validator verified a fake PoRep, then the validators stake can be slashed.
+
+Replicators are specialized thin clients. They download a part of the ledger and store it, and provide PoReps of storing the ledger. For each verified PoRep replicators earn a reward of sol from the mining pool.
+
+# Constraints
+
+We have the following constraints:
+* At most 14k  replication identities can be used, because thats how many CUDA cores we can fit in a $5k box at the moment.
+* Verification requires generating the CBC blocks. That requires space of 2 blocks per identity, and 1 CUDA core per identity for the same dataset. So as many identities at once should be batched with as many proofs for those identities verified concurrently for the same dataset.
+
+# Validation and Replication Protocol
+
+1. Network sets the replication target number, lets say 1k. 1k PoRep identities are created from signatures of a PoH hash. So they are tied to a specific PoH hash. It doesn't matter who creates them, or simply the last 1k validation signatures we saw for the ledger at that count. This maybe just the initial batch of identities, because we want to stagger identity rotation.
+2. Any client can use any of these identities to create PoRep proofs. Replicator identities are the CBC encryption keys.
+3. Periodically at a specific PoH count, replicator that want to create PoRep proofs sign the PoH hash at that count. That signature is the seed used to pick the block and identity to replicate. A block is 1TB of ledger.
+4. Periodically at a specific PoH count, replicator submits PoRep proofs for their selected block. A signature of the PoH hash at that count is the seed used to sample the 1TB encrypted block, and hash it. This is done faster than it takes to encrypt the 1TB block with the original identity.
+5. Replicators must submit some number of fake proofs, which they can prove to be fake by providing the seed for the hash result.
+6. Periodically at a specific PoH count, validators sign the hash and use the signature to select the 1TB block that they need to validate. They batch all the identities and proofs and submit approval for all the verified ones.
+7. After #6, replicator client submit the proofs of fake proofs.
+
+For any random seed, we force everyone to use a signature that is derived from a PoH hash. Everyone must use the same count, so the same PoH hash is signed by every participant. The signatures are then each cryptographically tied to the keypair, which prevents a leader from grinding on the resulting value for more than 1 identity.
+
+We need to stagger the rotation of the identity keys. Once this gets going, the next identity could be generated by hashing itself with a PoH hash, or via some other process based on the validation signatures.
+
+Since there are many more client identities then encryption identities, we need to split the reward for multiple clients, and prevent Sybil attacks from generating many clients to acquire the same block of data. To remain BFT we want to avoid a single human entity from storing all the replications of a single chunk of the ledger.
+
+Our solution to this is to force the clients to continue using the same identity. If the first round is used to acquire the same block for many client identities, the second round for the same client identities will force a redistribution of the signatures, and therefore PoRep identities and blocks. Thus to get a reward for storage clients need to store the first block for free and the network can reward long lived client identities more than new ones.
+
+# Notes
+
+* We can reduce the costs of verification of PoRep by using PoH, and actually make it feasible to verify a large number of proofs for a global dataset.
+* We can eliminate grinding by forcing everyone to sign the same PoH hash and use the signatures as the seed
+* The game between validators and replicators is over random blocks and random encryption identities and random data samples. The goal of randomization is to prevent colluding groups from having overlap on data or validation.
+* Replicator clients fish for lazy validators by submitting fake proofs that they can prove are fake.
+* Replication identities are just symmetric encryption keys, the number of them on the network is our storage replication target. Many more client identities can exist than replicator identities, so unlimited number of clients can provide proofs of the same replicator identity.
+* To defend against Sybil client identities that try to store the same block we force the clients to store for multiple rounds before receiving a reward.

rfcs/rfc-004-tictactoe-program.md

@@ -0,0 +1,77 @@
+
+Two players want to play tic-tac-toe with each other on Solana.
+
+The tic-tac-toe program has already been provisioned on the network, and the
+program author has advertised the following information to potential gamers:
+* `tictactoe_publickey` - the program's public key
+* `tictactoe_gamestate_size` - the number of bytes needed to maintain the game state
+
+The game state is a well-documented data structure consisting of:
+- Player 1's public key
+- Player 2's public key
+- Game status. An 8-bit value where:
+  * 0 = game uninitialized
+  * 1 = Player 1's turn
+  * 2 = Player 2's turn
+  * 3 = Player 1 won
+  * 4 = Player 2 won
+- Current board configuration.  A 3x3 character array containing the values '\0', 'X' or 'O'
+
+### Game Setup
+
+1. Two players want to start a game.  Player 2 sends Player 1 their public key,
+`player2_publickey` off-chain (IM, email, etc)
+
+2. Player 1 creates a new keypair to represent the game state, `(gamestate_publickey,
+gamestate_privatekey)`.
+
+3. Player 1 issues an allocate_memory transaction, assigning that memory page to the
+tic-tac-toe program.  The `memory_fee` is used to *rent* the memory page for the
+duration of the game and is subtracted from current account balance of Player
+1:
+```
+allocate_memory(gamestate_publickey, tictactoe_publickey, tictactoe_gamestate_size, memory_fee)
+```
+
+
+4. Game state is then initialized by issuing a *new* call transaction to the
+tic-tac-toe program.  This transaction is signed by `gamestate_privatekey`, known only
+to Player 1.
+```
+call(tictactoe_publickey, gamestate_publickey, 'new', player1_publickey, player2_publickey)
+```
+
+5. Once the game is initialized, Player 1 shares `gamestate_publickey` with
+Player 2 off-chain (IM, email, etc)
+
+Note that it's likely each player prefer to generate a game-specific keypair
+rather than sharing their primary public key (`player1_publickey`,
+`player2_publickey`) with each other and the tic-tac-toe program.
+
+### Game Play
+
+Both players poll the network, via a **TBD off-chain RPC API**, to read the
+current game state from the `gamestate_publickey` memory page.
+
+When the *Game status* field indicates it's their turn, the player issues a
+*move* call transaction passing in the board position (1..9) that they want to
+mark as X or O:
+```
+call(tictactoe_publickey, gamestate_publickey, 'move', position)
+```
+The program will reject the transaction if it was not signed by the player whose
+turn it is.
+
+The outcome of the *move* call is also observed by polling the current game state via
+the **TBD off-chain RPC API**.
+
+### Game Cancellation
+
+At any time Player 1 may conclude the game by issuing:
+```
+call(tictactoe_publickey, gamestate_publickey, 'abort')
+```
+causing any remaining *rent* tokens assigned to the `gamestate_publickey` page
+to be transferred back to Player 1 by the tic-tac-toe program.  Lastly, the
+network recognizes the empty account and frees the `gamestate_publickey` memory
+page.

rfcs/rfc-005-branches-tags-and-channels.md

@@ -0,0 +1,59 @@
+```
+========================= master branch (edge channel) =======================>
+         \                      \                     \
+          \___v0.7.0 tag         \                     \
+           \                      \         v0.9.0 tag__\
+            \          v0.8.0 tag__\                     \
+ v0.7.1 tag__\                      \                 v0.9 branch (beta channel)
+              \___v0.7.2 tag         \___v0.8.1 tag
+               \                      \
+                \                      \
+           v0.7 branch         v0.8 branch (stable channel)
+
+```
+
+## Branches and Tags
+
+### master branch
+All new development occurs on the `master` branch.
+
+Bug fixes that affect a `vX.Y` branch are first made on `master`.  This is to
+allow a fix some soak time on `master` before it is applied to one or more
+stabilization branches.
+
+Merging to `master` first also helps ensure that fixes applied to one release
+are present for future releases.  (Sometimes the joy of landing a critical
+release blocker in a branch causes you to forget to propagate back to
+`master`!)"
+
+Once the bug fix lands on `master` it is cherry-picked into the `vX.Y` branch
+and potentially the `vX.Y-1` branch.  The exception to this rule is when a bug
+fix for `vX.Y` doesn't apply to `master` or `vX.Y-1`.
+
+Immediately after a new stabilization branch is forged, the `Cargo.toml` minor
+version (*Y*) in the `master` branch is incremented by the release engineer.
+Incrementing the major version of the `master` branch is outside the scope of
+this document.
+
+### v*X.Y* stabilization branches
+These are stabilization branches for a given milestone.  They are created off
+the `master` branch as late as possible prior to the milestone release.
+
+### v*X.Y.Z* release tag
+The release tags are created as desired by the owner of the given stabilization
+branch, and cause that *X.Y.Z* release to be shipped to https://crates.io,
+https://snapcraft.io/, and elsewhere.
+
+Immediately after a new v*X.Y.Z* branch tag has been created, the `Cargo.toml`
+patch version number (*Z*) of the stabilization branch is incremented by the
+release engineer.
+
+## Channels
+Channels are used by end-users (humans and bots) to consume the branches
+described in the previous section, so they may automatically update to the most
+recent version matching their desired stability.
+
+There are three release channels that map to branches as follows:
+* edge - tracks the `master` branch, least stable.
+* beta - tracks the largest (and latest) `vX.Y` stabilization branch, more stable.
+* stable - tracks the second largest `vX.Y` stabilization branch, most stable.

scripts/configure-metrics.sh

@@ -0,0 +1,51 @@
+# |source| this file
+#
+# The SOLANA_METRICS_CONFIG environment variable is formatted as a
+# comma-delimited list of parameters. All parameters are optional.
+#
+# Example:
+#   export SOLANA_METRICS_CONFIG="host=<metrics host>,db=<database name>,u=<username>,p=<password>"
+#
+# The following directive disable complaints about unused variables in this
+# file:
+# shellcheck disable=2034
+#
+metricsWriteDatapoint="$(dirname "${BASH_SOURCE[0]}")"/metrics-write-datapoint.sh
+
+configureMetrics() {
+  [[ -n $SOLANA_METRICS_CONFIG ]] || return 0
+
+  declare metricsParams
+  IFS=',' read -r -a metricsParams <<< "$SOLANA_METRICS_CONFIG"
+  for param in "${metricsParams[@]}"; do
+    IFS='=' read -r -a pair <<< "$param"
+    if [[ ${#pair[@]} != 2 ]]; then
+      echo Error: invalid metrics parameter: "$param" >&2
+    else
+      declare name="${pair[0]}"
+      declare value="${pair[1]}"
+      case "$name" in
+      host)
+        export INFLUX_HOST="$value"
+        echo INFLUX_HOST="$INFLUX_HOST" >&2
+        ;;
+      db)
+        export INFLUX_DATABASE="$value"
+        echo INFLUX_DATABASE="$INFLUX_DATABASE" >&2
+        ;;
+      u)
+        export INFLUX_USERNAME="$value"
+        echo INFLUX_USERNAME="$INFLUX_USERNAME" >&2
+        ;;
+      p)
+        export INFLUX_PASSWORD="$value"
+        echo INFLUX_PASSWORD="********" >&2
+        ;;
+      *)
+        echo Error: Unknown metrics parameter name: "$name" >&2
+        ;;
+      esac
+    fi
+  done
+}
+configureMetrics

scripts/metrics-write-datapoint.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+#
+# Send a metrics datapoint
+#
+
+point=$1
+if [[ -z $point ]]; then
+  echo "Data point not specified"
+  exit 1
+fi
+
+echo "Influx data point: $point"
+if [[ -z $INFLUX_DATABASE || -z $INFLUX_USERNAME || -z $INFLUX_PASSWORD ]]; then
+  echo Influx user credentials not found
+  exit 0
+fi
+
+echo "https://metrics.solana.com:8086/write?db=${INFLUX_DATABASE}&u=${INFLUX_USERNAME}&p=${INFLUX_PASSWORD}" \
+  | xargs curl --max-time 5 -XPOST --data-binary "$point"

scripts/net-stats.sh

@@ -0,0 +1,49 @@
+#!/bin/bash -e
+#
+# Reports network statistics
+#
+
+[[ $(uname) == Linux ]] || exit 0
+
+cd "$(dirname "$0")"
+
+# shellcheck source=scripts/configure-metrics.sh
+source configure-metrics.sh
+
+packets_received=0
+packets_received_diff=0
+receive_errors=0
+receive_errors_diff=0
+rcvbuf_errors=0
+rcvbuf_errors_diff=0
+
+update_netstat() {
+  declare net_stat
+  net_stat=$(netstat -suna)
+
+  declare stats
+  stats=$(echo "$net_stat" | awk 'BEGIN {tmp_var = 0} /packets received/ {tmp_var = $1} END { print tmp_var }')
+  packets_received_diff=$((stats - packets_received))
+  packets_received="$stats"
+
+  stats=$(echo "$net_stat" | awk 'BEGIN {tmp_var = 0} /packet receive errors/ {tmp_var = $1} END { print tmp_var }')
+  receive_errors_diff=$((stats - receive_errors))
+  receive_errors="$stats"
+
+  stats=$(echo "$net_stat" | awk 'BEGIN {tmp_var = 0} /RcvbufErrors/ {tmp_var = $2} END { print tmp_var }')
+  rcvbuf_errors_diff=$((stats - rcvbuf_errors))
+  rcvbuf_errors="$stats"
+}
+
+update_netstat
+
+while true; do
+  update_netstat
+  report="packets_received=$packets_received_diff,receive_errors=$receive_errors_diff,rcvbuf_errors=$rcvbuf_errors_diff"
+
+  echo "$report"
+  ./metrics-write-datapoint.sh "net-stats,hostname=$HOSTNAME $report"
+  sleep 1
+done
+
+exit 1

scripts/oom-monitor.sh

@@ -0,0 +1,35 @@
+#!/bin/bash -e
+#
+# Reports Linux OOM Killer activity
+#
+
+cd "$(dirname "$0")"
+
+# shellcheck source=scripts/oom-score-adj.sh
+source oom-score-adj.sh
+
+# shellcheck source=scripts/configure-metrics.sh
+source configure-metrics.sh
+
+[[ $(uname) = Linux ]] || exit 0
+
+syslog=/var/log/syslog
+[[ -r $syslog ]] || {
+  echo Unable to read $syslog
+  exit 1
+}
+
+# Adjust OOM score to reduce the chance that this script will be killed
+# during an Out of Memory event since the purpose of this script is to
+# report such events
+oom_score_adj "self" -500
+
+while read -r victim; do
+  echo "Out of memory event detected, $victim killed"
+  ./metrics-write-datapoint.sh "oom-killer,victim=$victim,hostname=$HOSTNAME killed=1"
+done < <( \
+  tail --follow=name --retry -n0 $syslog \
+  | sed --unbuffered -n 's/^.* Out of memory: Kill process [1-9][0-9]* (\([^)]*\)) .*/\1/p' \
+)
+
+exit 1

scripts/oom-score-adj.sh

@@ -0,0 +1,20 @@
+# |source| this file
+#
+# Adjusts the OOM score for the specified process.  Linux only
+#
+# usage: oom_score_adj [pid] [score]
+#
+oom_score_adj() {
+  declare pid=$1
+  declare score=$2
+  if [[ $(uname) != Linux ]]; then
+    return
+  fi
+
+  echo "$score" > "/proc/$pid/oom_score_adj" || true
+  declare currentScore
+  currentScore=$(cat "/proc/$pid/oom_score_adj" || true)
+  if [[ $score != "$currentScore" ]]; then
+    echo "Failed to set oom_score_adj to $score for pid $pid (current score: $currentScore)"
+  fi
+}

scripts/perf-plot.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+import matplotlib
+matplotlib.use('Agg')
+
+import matplotlib.pyplot as plt
+import json
+import sys
+
+stages_to_counters = {}
+stages_to_time = {}
+
+if len(sys.argv) != 2:
+    print("USAGE: {} <input file>".format(sys.argv[0]))
+    sys.exit(1)
+
+with open(sys.argv[1]) as fh:
+    for line in fh.readlines():
+        if "COUNTER" in line:
+            json_part = line[line.find("{"):]
+            x = json.loads(json_part)
+            counter = x['name']
+            if not (counter in stages_to_counters):
+                stages_to_counters[counter] = []
+                stages_to_time[counter] = []
+            stages_to_counters[counter].append(x['counts'])
+            stages_to_time[counter].append(x['now'])
+
+fig, ax = plt.subplots()
+
+for stage in stages_to_counters.keys():
+    plt.plot(stages_to_time[stage], stages_to_counters[stage], label=stage)
+
+plt.xlabel('ms')
+plt.ylabel('count')
+
+plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3,
+           ncol=2, mode="expand", borderaxespad=0.)
+
+plt.locator_params(axis='x', nbins=10)
+plt.grid(True)
+
+plt.savefig("perf.pdf")

scripts/perf-stats.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+
+stages_data = {}
+
+if len(sys.argv) != 2:
+    print("USAGE: {} <input file>".format(sys.argv[0]))
+    sys.exit(1)
+
+with open(sys.argv[1]) as fh:
+    for line in fh.readlines():
+        if "COUNTER" in line:
+            json_part = line[line.find("{"):]
+            x = json.loads(json_part)
+            counter = x['name']
+            if not (counter in stages_data):
+                stages_data[counter] = {'first_ts': x['now'], 'last_ts': x['now'], 'last_count': 0,
+                                        'data': [], 'max_speed': 0, 'min_speed': 9999999999.0,
+                                        'count': 0,
+                                        'max_speed_ts': 0, 'min_speed_ts': 0}
+            stages_data[counter]['count'] += 1
+            count_since_last = x['counts'] - stages_data[counter]['last_count']
+            time_since_last = float(x['now'] - stages_data[counter]['last_ts'])
+            if time_since_last > 1:
+                speed = 1000.0 * (count_since_last / time_since_last)
+                stages_data[counter]['data'].append(speed)
+                if speed > stages_data[counter]['max_speed']:
+                    stages_data[counter]['max_speed'] = speed
+                    stages_data[counter]['max_speed_ts'] = x['now']
+                if speed < stages_data[counter]['min_speed']:
+                    stages_data[counter]['min_speed'] = speed
+                    stages_data[counter]['min_speed_ts'] = x['now']
+            stages_data[counter]['last_ts'] = x['now']
+            stages_data[counter]['last_count'] = x['counts']
+
+for stage in stages_data.keys():
+    stages_data[stage]['data'].sort()
+    #mean_index = stages_data[stage]['count'] / 2
+    mean = 0
+    average = 0
+    eightieth = 0
+    data_len = len(stages_data[stage]['data'])
+    mean_index = int(data_len / 2)
+    eightieth_index = int(data_len * 0.8)
+    #print("mean idx: {} data.len: {}".format(mean_index, data_len))
+    if data_len > 0:
+        mean = stages_data[stage]['data'][mean_index]
+        average = float(sum(stages_data[stage]['data'])) / data_len
+        eightieth = stages_data[stage]['data'][eightieth_index]
+    print("stage: {} max: {:,.2f} min: {:.2f} count: {} total: {} mean: {:,.2f} average: {:,.2f} 80%: {:,.2f}".format(stage,
+                                                       stages_data[stage]['max_speed'],
+                                                       stages_data[stage]['min_speed'],
+                                                       stages_data[stage]['count'],
+                                                       stages_data[stage]['last_count'],
+                                                       mean, average, eightieth))
+    num = 5
+    idx = -1
+    if data_len >= num:
+        print("    top {}: ".format(num), end='')
+        for x in range(0, num):
+            print("{:,.2f}  ".format(stages_data[stage]['data'][idx]), end='')
+            idx -= 1
+            if stages_data[stage]['data'][idx] < average:
+                break
+        print("")
+    print("    max_ts: {} min_ts: {}".format(stages_data[stage]['max_speed_ts'], stages_data[stage]['min_speed_ts']))
+    print("\n")
+