Run release binary for leader node

We added them thinking it'd be a good stepping stone towards an
asynchronous thin client, but it's used inconsistently and where
it used, the function is still synchronous, which is just confusing.

.gitignore

@@ -1,3 +1,4 @@
 Cargo.lock
 /target/
 **/*.rs.bk
+.cargo

.travis.yml

@@ -1,22 +0,0 @@
-language: rust
-required: sudo
-services:
-- docker
-matrix:
-  allow_failures:
-  - rust: nightly
-  include:
-  - rust: stable
-  - rust: nightly
-    env:
-    - FEATURES='unstable'
-before_script: |
-  export PATH="$PATH:$HOME/.cargo/bin"
-  rustup component add rustfmt-preview
-script:
-- cargo fmt -- --write-mode=diff
-- cargo build --verbose --features "$FEATURES"
-- cargo test --verbose --features "$FEATURES"
-after_success: |
-  docker run -it --rm --security-opt seccomp=unconfined --volume "$PWD:/volume" elmtai/docker-rust-kcov
-  bash <(curl -s https://codecov.io/bash) -s target/cov

Cargo.toml

@@ -1,27 +1,28 @@
 [package]
 name = "solana"
-description = "High Performance Blockchain"
-version = "0.4.0"
+description = "Blockchain Rebuilt for Scale"
+version = "0.6.0"
 documentation = "https://docs.rs/solana"
-homepage = "http://loomprotocol.com/"
+homepage = "http://solana.com/"
 repository = "https://github.com/solana-labs/solana"
 authors = [
-    "Anatoly Yakovenko <anatoly@solana.co>",
-    "Greg Fitzgerald <greg@solana.co>",
+    "Anatoly Yakovenko <anatoly@solana.com>",
+    "Greg Fitzgerald <greg@solana.com>",
+    "Stephen Akridge <stephen@solana.com>",
 ]
 license = "Apache-2.0"
 
-[[bin]]
-name = "solana-historian-demo"
-path = "src/bin/historian-demo.rs"
-
 [[bin]]
 name = "solana-client-demo"
 path = "src/bin/client-demo.rs"
 
 [[bin]]
-name = "solana-testnode"
-path = "src/bin/testnode.rs"
+name = "solana-fullnode"
+path = "src/bin/fullnode.rs"
+
+[[bin]]
+name = "solana-fullnode-config"
+path = "src/bin/fullnode-config.rs"
 
 [[bin]]
 name = "solana-genesis"
@@ -35,12 +36,18 @@ path = "src/bin/genesis-demo.rs"
 name = "solana-mint"
 path = "src/bin/mint.rs"
 
+[[bin]]
+name = "solana-mint-demo"
+path = "src/bin/mint-demo.rs"
+
 [badges]
 codecov = { repository = "solana-labs/solana", branch = "master", service = "github" }
 
 [features]
 unstable = []
 ipv6 = []
+cuda = []
+erasure = []
 
 [dependencies]
 rayon = "1.0.0"
@@ -54,4 +61,11 @@ untrusted = "0.5.1"
 bincode = "1.0.0"
 chrono = { version = "0.4.0", features = ["serde"] }
 log = "^0.4.1"
+env_logger = "^0.4.1"
 matches = "^0.1.6"
+byteorder = "^1.2.1"
+libc = "^0.2.1"
+getopts = "^0.2"
+isatty = "0.1"
+rand = "0.4.2"
+pnet = "^0.21.0"

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2018 Anatoly Yakovenko <anatoly@loomprotocol.com> and Greg Fitzgerald <garious@gmail.com>
+Copyright 2018 Anatoly Yakovenko, Greg Fitzgerald and Stephen Akridge
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

README.md

@@ -1,6 +1,6 @@
 [![Solana crate](https://img.shields.io/crates/v/solana.svg)](https://crates.io/crates/solana)
 [![Solana documentation](https://docs.rs/solana/badge.svg)](https://docs.rs/solana)
-[![Build Status](https://travis-ci.org/solana-labs/solana.svg?branch=master)](https://travis-ci.org/solana-labs/solana)
+[![Build status](https://badge.buildkite.com/d4c4d7da9154e3a8fb7199325f430ccdb05be5fc1e92777e51.svg?branch=master)](https://buildkite.com/solana-labs/solana)
 [![codecov](https://codecov.io/gh/solana-labs/solana/branch/master/graph/badge.svg)](https://codecov.io/gh/solana-labs/solana)
 
 Disclaimer
@@ -8,15 +8,35 @@ Disclaimer
 
 All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort.  It is up to the reader to check and validate their accuracy and truthfulness.  Furthermore nothing in this project constitutes a solicitation for investment.
 
-Solana: High Performance Blockchain
+Solana: Blockchain Rebuilt for Scale
 ===
 
-Solana™ is a new architecture for a high performance blockchain. It aims to support
-over 700 thousand transactions per second on a gigabit network.
+Solana™ is a new blockchain architecture built from the ground up for scale. The architecture supports
+up to 710 thousand transactions per second on a gigabit network.
 
-Running the demo
+Introduction
 ===
 
+It's possible for a centralized database to process 710,000 transactions per second on a standard gigabit network if the transactions are, on average, no more than 176 bytes. A centralized database can also replicate itself and maintain high availability without significantly compromising that transaction rate using the distributed system technique known as Optimistic Concurrency Control [H.T.Kung, J.T.Robinson (1981)]. At Solana, we're demonstrating that these same theoretical limits apply just as well to blockchain on an adversarial network. The key ingredient? Finding a way to share time when nodes can't trust one-another. Once nodes can trust time, suddenly ~40 years of distributed systems research becomes applicable to blockchain! Furthermore, and much to our surprise, it can implemented using a mechanism that has existed in Bitcoin since day one. The Bitcoin feature is called nLocktime and it can be used to postdate transactions using block height instead of a timestamp. As a Bitcoin client, you'd use block height instead of a timestamp if you don't trust the network. Block height turns out to be an instance of what's being called a Verifiable Delay Function in cryptography circles. It's a cryptographically secure way to say time has passed. In Solana, we use a far more granular verifiable delay function, a SHA 256 hash chain, to checkpoint the ledger and coordinate consensus. With it, we implement Optimistic Concurrency Control and are now well in route towards that theoretical limit of 710,000 transactions per second.
+
+
+Testnet Demos
+===
+
+The Solana repo contains all the scripts you might need to spin up your own
+local testnet. Depending on what you're looking to achieve, you may want to
+run a different variation, as the full-fledged, performance-enhanced
+multinode testnet is considerably more complex to set up than a Rust-only,
+singlenode testnode.  If you are looking to develop high-level features, such
+as experimenting with smart contracts, save yourself some setup headaches and
+stick to the Rust-only singlenode demo.  If you're doing performance optimization
+of the transaction pipeline, consider the enhanced singlenode demo. If you're
+doing consensus work, you'll need at least a Rust-only multinode demo. If you want
+to reproduce our TPS metrics, run the enhanced multinode demo.
+
+For all four variations, you'd need the latest Rust toolchain and the Solana
+source code:
+
 First, install Rust's package manager Cargo.
 
 ```bash
@@ -24,54 +44,118 @@ $ curl https://sh.rustup.rs -sSf | sh
 $ source $HOME/.cargo/env
 ```
 
-The testnode server is initialized with a ledger from stdin and
+Now checkout the code from github:
+
+```bash
+$ git clone https://github.com/solana-labs/solana.git 
+$ cd solana
+```
+
+The demo code is sometimes broken between releases as we add new low-level
+features, so if this is your first time running the demo, you'll improve
+your odds of success if you check out the
+[latest release](https://github.com/solana-labs/solana/releases)
+before proceeding:
+
+```bash
+$ git checkout v0.6.0
+```
+
+Singlenode Testnet
+---
+
+The fullnode server is initialized with a ledger from stdin and
 generates new ledger entries on stdout. To create the input ledger, we'll need
 to create *the mint* and use it to generate a *genesis ledger*. It's done in
-two steps because the mint.json file contains a private key that will be
+two steps because the mint-demo.json file contains private keys that will be
 used later in this demo.
 
 ```bash
-    $ echo 1000000000 | cargo run --release --bin solana-mint | tee mint.json
-    $ cat mint.json | cargo run --release --bin solana-genesis | tee genesis.log
+$ echo 1000000000 | cargo run --release --bin solana-mint-demo > mint-demo.json
+$ cat mint-demo.json | cargo run --release --bin solana-genesis-demo > genesis.log
 ```
 
-Now you can start the server:
+Before you start a fullnode, make sure you know the IP address of the machine you
+want to be the leader for the demo, and make sure that udp ports 8000-10000 are
+open on all the machines you want to test with.
+
+Generate a leader configuration file with:
 
 ```bash
-    $ cat genesis.log | cargo run --release --bin solana-testnode | tee transactions0.log
+cargo run --release --bin solana-fullnode-config > leader.json
 ```
 
-Then, in a separate shell, let's execute some transactions. Note we pass in
+Now start the server:
+
+```bash
+$ cat ./multinode-demo/leader.sh
+#!/bin/bash
+export RUST_LOG=solana=info
+sudo sysctl -w net.core.rmem_max=26214400
+cargo run --release --bin solana-fullnode -- -l leader.json < genesis.log
+$ ./multinode-demo/leader.sh > leader-txs.log
+```
+
+To run a performance-enhanced fullnode on Linux, download `libcuda_verify_ed25519.a`. Enable
+it by adding `--features=cuda` to the line that runs `solana-fullnode` in `leader.sh`.
+
+```bash
+$ wget https://solana-build-artifacts.s3.amazonaws.com/v0.6.0/libcuda_verify_ed25519.a
+cargo run --release --features=cuda --bin solana-fullnode -- -l leader.json < genesis.log
+```
+
+Wait a few seconds for the server to initialize. It will print "Ready." when it's ready to
+receive transactions.
+
+Multinode Testnet
+---
+
+To run a multinode testnet, after starting a leader node, spin up some validator nodes:
+
+```bash
+$ cat ./multinode-demo/validator.sh
+#!/bin/bash
+rsync -v -e ssh $1/mint-demo.json .
+rsync -v -e ssh $1/leader.json .
+rsync -v -e ssh $1/genesis.log .
+export RUST_LOG=solana=info
+sudo sysctl -w net.core.rmem_max=26214400
+cargo run --release --bin solana-fullnode -- -l validator.json -v leader.json -b 9000 -d < genesis.log
+$ ./multinode-demo/validator.sh ubuntu@10.0.1.51:~/solana > validator-txs.log #The leader machine
+```
+
+As with the leader node, you can run a performance-enhanced validator fullnode by adding
+`--features=cuda` to the line that runs `solana-fullnode` in `validator.sh`.
+
+```bash
+cargo run --release --features=cuda --bin solana-fullnode -- -l validator.json -v leader.json -b 9000 -d < genesis.log
+```
+
+
+Testnet Client Demo
+---
+
+Now that your singlenode or multinode testnet is up and running, in a separate shell, let's send it some transactions! Note we pass in
 the JSON configuration file here, not the genesis ledger.
 
 ```bash
-    $ cat mint.json | cargo run --release --bin solana-client-demo
+$ cat ./multinode-demo/client.sh
+#!/bin/bash
+export RUST_LOG=solana=info
+rsync -v -e ssh $1/leader.json .
+rsync -v -e ssh $1/mint-demo.json .
+cat mint-demo.json | cargo run --release --bin solana-client-demo -- -l leader.json
+$ ./multinode-demo/client.sh ubuntu@10.0.1.51:~/solana #The leader machine
 ```
 
-Now kill the server with Ctrl-C, and take a look at the ledger. You should
-see something similar to:
-
-```json
-{"num_hashes":27,"id":[0, "..."],"event":"Tick"}
-{"num_hashes":3,"id":[67, "..."],"event":{"Transaction":{"tokens":42}}}
-{"num_hashes":27,"id":[0, "..."],"event":"Tick"}
-```
-
-Now restart the server from where we left off. Pass it both the genesis ledger, and
-the transaction ledger.
-
-```bash
-    $ cat genesis.log transactions0.log | cargo run --release --bin solana-testnode | tee transactions1.log
-```
-
-Lastly, run the client demo again, and verify that all funds were spent in the
-previous round, and so no additional transactions are added.
-
-```bash
-    $ cat mint.json | cargo run --release --bin solana-client-demo
-```
-
-Stop the server again, and verify there are only Tick entries, and no Transaction entries.
+What just happened? The client demo spins up several threads to send 500,000 transactions
+to the testnet as quickly as it can. The client then pings the testnet periodically to see
+how many transactions it processed in that time. Take note that the demo intentionally
+floods the network with UDP packets, such that the network will almost certainly drop a
+bunch of them. This ensures the testnet has an opportunity to reach 710k TPS. The client
+demo completes after it has convinced itself the testnet won't process any additional
+transactions. You should see several TPS measurements printed to the screen. In the
+multinode variation, you'll see TPS measurements for each validator node as well.
 
 Developing
 ===
@@ -87,6 +171,12 @@ $ source $HOME/.cargo/env
 $ rustup component add rustfmt-preview
 ```
 
+If your rustc version is lower than 1.26.1, please update it:
+
+```bash
+$ rustup update
+```
+
 Download the source code:
 
 ```bash
@@ -100,9 +190,26 @@ Testing
 Run the test suite:
 
 ```bash
-cargo test
+$ cargo test
 ```
 
+To emulate all the tests that will run on a Pull Request, run:
+```bash
+$ ./ci/run-local.sh
+```
+
+Debugging
+---
+
+There are some useful debug messages in the code, you can enable them on a per-module and per-level
+basis with the normal RUST\_LOG environment variable. Run the fullnode with this syntax:
+```bash
+$ RUST_LOG=solana::streamer=debug,solana::server=info cat genesis.log | ./target/release/solana-fullnode > transactions0.log
+```
+to see the debug and info sections for streamer and server respectively. Generally
+we are using debug for infrequent debug messages, trace for potentially frequent messages and
+info for performance-related logging.
+
 Benchmarking
 ---
 
@@ -117,3 +224,26 @@ Run the benchmarks:
 ```bash
 $ cargo +nightly bench --features="unstable"
 ```
+
+Code coverage
+---
+
+To generate code coverage statistics, run kcov via Docker:
+
+```bash
+$ ./ci/coverage.sh
+```
+The coverage report will be written to `./target/cov/index.html`
+
+
+Why coverage? While most see coverage as a code quality metric, we see it primarily as a developer
+productivity metric. When a developer makes a change to the codebase, presumably it's a *solution* to
+some problem.  Our unit-test suite is how we encode the set of *problems* the codebase solves. Running
+the test suite should indicate that your change didn't *infringe* on anyone else's solutions. Adding a
+test *protects* your solution from future changes. Say you don't understand why a line of code exists,
+try deleting it and running the unit-tests. The nearest test failure should tell you what problem
+was solved by that code. If no test fails, go ahead and submit a Pull Request that asks, "what
+problem is solved by this code?" On the other hand, if a test does fail and you can think of a
+better way to solve the same problem, a Pull Request with your solution would most certainly be
+welcome! Likewise, if rewriting a test can better communicate what code it's protecting, please
+send us that patch!

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-slate

build.rs

@@ -0,0 +1,15 @@
+use std::env;
+
+fn main() {
+    println!("cargo:rustc-link-search=native=.");
+    if !env::var("CARGO_FEATURE_CUDA").is_err() {
+        println!("cargo:rustc-link-lib=static=cuda_verify_ed25519");
+        println!("cargo:rustc-link-search=native=/usr/local/cuda/lib64");
+        println!("cargo:rustc-link-lib=dylib=cudart");
+        println!("cargo:rustc-link-lib=dylib=cuda");
+        println!("cargo:rustc-link-lib=dylib=cudadevrt");
+    }
+    if !env::var("CARGO_FEATURE_ERASURE").is_err() {
+        println!("cargo:rustc-link-lib=dylib=Jerasure");
+    }
+}

ci/.gitignore

@@ -0,0 +1,2 @@
+/node_modules/
+/package-lock.json

ci/buildkite.yml

@@ -0,0 +1,16 @@
+steps:
+  - command: "ci/coverage.sh"
+    name: "coverage [public]"
+  - command: "ci/docker-run.sh rust ci/test-stable.sh"
+    name: "stable [public]"
+  - command: "ci/docker-run.sh rustlang/rust:nightly ci/test-nightly.sh || true"
+    name: "nightly - FAILURES IGNORED [public]"
+  - command: "ci/docker-run.sh rust ci/test-ignored.sh"
+    name: "ignored [public]"
+  - command: "ci/test-cuda.sh"
+    name: "cuda"
+  - command: "ci/shellcheck.sh"
+    name: "shellcheck [public]"
+  - wait
+  - command: "ci/publish.sh"
+    name: "publish release artifacts"

ci/coverage.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+ci/docker-run.sh evilmachines/rust-cargo-kcov \
+  bash -exc "\
+    export RUST_BACKTRACE=1; \
+    cargo build --verbose; \
+    cargo kcov --lib --verbose; \
+  "
+
+echo Coverage report:
+ls -l target/cov/index.html
+
+if [[ -z "$CODECOV_TOKEN" ]]; then
+  echo CODECOV_TOKEN undefined
+else
+  bash <(curl -s https://codecov.io/bash)
+fi
+
+exit 0

ci/docker-run.sh

@@ -0,0 +1,41 @@
+#!/bin/bash -e
+
+usage() {
+  echo "Usage: $0 [docker image name] [command]"
+  echo
+  echo Runs command in the specified docker image with
+  echo a CI-appropriate environment
+  echo
+}
+
+cd "$(dirname "$0")/.."
+
+IMAGE="$1"
+if [[ -z "$IMAGE" ]]; then
+  echo Error: image not defined
+  exit 1
+fi
+
+docker pull "$IMAGE"
+shift
+
+ARGS=(--workdir /solana --volume "$PWD:/solana" --rm)
+
+ARGS+=(--env "CARGO_HOME=/solana/.cargo")
+
+# kcov tries to set the personality of the binary which docker
+# doesn't allow by default.
+ARGS+=(--security-opt "seccomp=unconfined")
+
+# Ensure files are created with the current host uid/gid
+ARGS+=(--user "$(id -u):$(id -g)")
+
+# Environment variables to propagate into the container
+ARGS+=(
+  --env BUILDKITE_TAG
+  --env CODECOV_TOKEN
+  --env CRATES_IO_TOKEN
+)
+
+set -x
+docker run "${ARGS[@]}" "$IMAGE" "$@"

ci/publish.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+if [[ -z "$BUILDKITE_TAG" ]]; then
+  # Skip publish if this is not a tagged release
+  exit 0
+fi
+
+if [[ -z "$CRATES_IO_TOKEN" ]]; then
+  echo CRATES_IO_TOKEN undefined
+  exit 1
+fi
+
+# TODO: Ensure the published version matches the contents of BUILDKITE_TAG
+ci/docker-run.sh rust \
+  bash -exc "cargo package; cargo publish --token $CRATES_IO_TOKEN"
+
+exit 0

ci/run-local.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+#
+# Run the entire buildkite CI pipeline locally for pre-testing before sending a
+# Github pull request
+#
+
+cd "$(dirname "$0")/.."
+BKRUN=ci/node_modules/.bin/bkrun
+
+if [[ ! -x $BKRUN ]]; then
+  (
+    set -x
+    cd ci/
+    npm install bkrun
+  )
+fi
+
+set -x
+./ci/node_modules/.bin/bkrun ci/buildkite.yml

ci/shellcheck.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -e
+#
+# Reference: https://github.com/koalaman/shellcheck/wiki/Directive
+
+cd "$(dirname "$0")/.."
+
+set -x
+find . -name "*.sh" -not -regex ".*/.cargo/.*" -not -regex ".*/node_modules/.*" -print0 \
+  | xargs -0 \
+      ci/docker-run.sh koalaman/shellcheck --color=always --external-sources --shell=bash
+exit 0

ci/test-cuda.sh

@@ -0,0 +1,22 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+LIB=libcuda_verify_ed25519.a
+if [[ ! -r $LIB ]]; then
+  if [[ -z "${libcuda_verify_ed25519_URL:-}" ]]; then
+    echo "$0 skipped.  Unable to locate $LIB"
+    exit 0
+  fi
+
+  export LD_LIBRARY_PATH=/usr/local/cuda/lib64
+  export PATH=$PATH:/usr/local/cuda/bin
+  curl -X GET -o $LIB "$libcuda_verify_ed25519_URL"
+fi
+
+# shellcheck disable=SC1090    # <-- shellcheck can't follow ~
+source ~/.cargo/env
+export RUST_BACKTRACE=1
+cargo test --features=cuda
+
+exit 0

ci/test-ignored.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+rustc --version
+cargo --version
+
+export RUST_BACKTRACE=1
+cargo test -- --ignored

ci/test-nightly.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+rustc --version
+cargo --version
+
+export RUST_BACKTRACE=1
+rustup component add rustfmt-preview
+cargo build --verbose --features unstable
+cargo test --verbose --features unstable
+cargo bench --verbose --features unstable
+
+exit 0

ci/test-stable.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+rustc --version
+cargo --version
+
+export RUST_BACKTRACE=1
+rustup component add rustfmt-preview
+cargo fmt -- --write-mode=diff
+cargo build --verbose
+cargo test --verbose
+
+exit 0

doc/historian.md

@@ -11,7 +11,7 @@ with by verifying each entry's hash can be generated from the hash in the previo
 extern crate solana;
 
 use solana::historian::Historian;
-use solana::ledger::{verify_slice, Entry, Hash};
+use solana::ledger::{Block, Entry, Hash};
 use solana::event::{generate_keypair, get_pubkey, sign_claim_data, Event};
 use std::thread::sleep;
 use std::time::Duration;
@@ -38,7 +38,7 @@ fn main() {
     }
     // Proof-of-History: Verify the historian learned about the events
     // in the same order they appear in the vector.
-    assert!(verify_slice(&entries, &seed));
+    assert!(entries[..].verify(&seed));
 }
 ```
 
@@ -56,7 +56,7 @@ Proof-of-History
 Take note of the last line:
 
 ```rust
-assert!(verify_slice(&entries, &seed));
+assert!(entries[..].verify(&seed));
 ```
 
 [It's a proof!](https://en.wikipedia.org/wiki/Curry–Howard_correspondence) For each entry returned by the

doc/historian.msc

@@ -14,5 +14,5 @@ msc {
   recorder=>historian [ label = "e2 = Entry{id: h6, n: 3, event: Tick}" ] ;
   client=>historian [ label = "collect()" ] ;
   historian=>client [ label = "entries = [e0, e1, e2]" ] ;
-  client=>client [ label = "verify_slice(entries, h0)" ] ;
+  client=>client [ label = "entries.verify(h0)" ] ;
 }

multinode-demo/client.sh

@@ -0,0 +1,16 @@
+#!/bin/bash -e
+
+if [[ -z "$1" ]]; then
+  echo "usage: $0 [network path to solana repo on leader machine]"
+  exit 1
+fi
+
+LEADER="$1"
+
+set -x
+export RUST_LOG=solana=info
+rsync -v -e ssh "$LEADER/leader.json" .
+rsync -v -e ssh "$LEADER/mint-demo.json" .
+
+cargo run --release --bin solana-client-demo -- \
+  -l leader.json < mint-demo.json 2>&1 | tee client.log

multinode-demo/leader.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+export RUST_LOG=solana=info
+sudo sysctl -w net.core.rmem_max=26214400
+cargo run --release --bin solana-fullnode -- -l leader.json < genesis.log

multinode-demo/validator.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -e
+
+if [[ -z "$1" ]]; then
+  echo "usage: $0 [network path to solana repo on leader machine]"
+  exit 1
+fi
+
+LEADER="$1"
+
+set -x
+
+rsync -v -e ssh "$LEADER/mint-demo.json" .
+rsync -v -e ssh "$LEADER/leader.json" .
+rsync -v -e ssh "$LEADER/genesis.log" .
+
+export RUST_LOG=solana=info
+
+sudo sysctl -w net.core.rmem_max=26214400
+
+cargo run --release --features=cuda --bin solana-fullnode -- \
+    -l validator.json -v leader.json -b 9000 -d < genesis.log

src/accountant.rs

@@ -1,399 +0,0 @@
-//! The `accountant` module tracks client balances, and the progress of pending
-//! transactions. It offers a high-level public API that signs transactions
-//! on behalf of the caller, and a private low-level API for when they have
-//! already been signed and verified.
-
-use chrono::prelude::*;
-use entry::Entry;
-use event::Event;
-use hash::Hash;
-use historian::Historian;
-use mint::Mint;
-use plan::{Plan, Witness};
-use recorder::Signal;
-use signature::{KeyPair, PublicKey, Signature};
-use std::collections::hash_map::Entry::Occupied;
-use std::collections::{HashMap, HashSet};
-use std::result;
-use std::sync::mpsc::SendError;
-use transaction::Transaction;
-
-#[derive(Debug, PartialEq, Eq)]
-pub enum AccountingError {
-    InsufficientFunds,
-    InvalidTransfer,
-    InvalidTransferSignature,
-    SendError,
-}
-
-pub type Result<T> = result::Result<T, AccountingError>;
-
-/// Commit funds to the 'to' party.
-fn complete_transaction(balances: &mut HashMap<PublicKey, i64>, plan: &Plan) {
-    if let Plan::Pay(ref payment) = *plan {
-        *balances.entry(payment.to).or_insert(0) += payment.tokens;
-    }
-}
-
-pub struct Accountant {
-    pub historian: Historian,
-    pub balances: HashMap<PublicKey, i64>,
-    pub first_id: Hash,
-    pending: HashMap<Signature, Plan>,
-    time_sources: HashSet<PublicKey>,
-    last_time: DateTime<Utc>,
-}
-
-impl Accountant {
-    /// Create an Accountant using an existing ledger.
-    pub fn new_from_entries<I>(entries: I, ms_per_tick: Option<u64>) -> Self
-    where
-        I: IntoIterator<Item = Entry>,
-    {
-        let mut entries = entries.into_iter();
-
-        // The first item in the ledger is required to be an entry with zero num_hashes,
-        // which implies its id can be used as the ledger's seed.
-        let entry0 = entries.next().unwrap();
-        let start_hash = entry0.id;
-
-        let hist = Historian::new(&start_hash, ms_per_tick);
-        let mut acc = Accountant {
-            historian: hist,
-            balances: HashMap::new(),
-            first_id: start_hash,
-            pending: HashMap::new(),
-            time_sources: HashSet::new(),
-            last_time: Utc.timestamp(0, 0),
-        };
-
-        // The second item in the ledger is a special transaction where the to and from
-        // fields are the same. That entry should be treated as a deposit, not a
-        // transfer to oneself.
-        let entry1 = entries.next().unwrap();
-        acc.process_verified_event(&entry1.events[0], true).unwrap();
-
-        for entry in entries {
-            for event in entry.events {
-                acc.process_verified_event(&event, false).unwrap();
-            }
-        }
-        acc
-    }
-
-    /// Create an Accountant with only a Mint. Typically used by unit tests.
-    pub fn new(mint: &Mint, ms_per_tick: Option<u64>) -> Self {
-        Self::new_from_entries(mint.create_entries(), ms_per_tick)
-    }
-
-    fn is_deposit(allow_deposits: bool, from: &PublicKey, plan: &Plan) -> bool {
-        if let Plan::Pay(ref payment) = *plan {
-            allow_deposits && *from == payment.to
-        } else {
-            false
-        }
-    }
-
-    /// Process and log the given Transaction.
-    pub fn log_verified_transaction(&mut self, tr: Transaction) -> Result<()> {
-        if self.get_balance(&tr.from).unwrap_or(0) < tr.tokens {
-            return Err(AccountingError::InsufficientFunds);
-        }
-
-        self.process_verified_transaction(&tr, false)?;
-        if let Err(SendError(_)) = self.historian
-            .sender
-            .send(Signal::Event(Event::Transaction(tr)))
-        {
-            return Err(AccountingError::SendError);
-        }
-
-        Ok(())
-    }
-
-    /// Verify and process the given Transaction.
-    pub fn log_transaction(&mut self, tr: Transaction) -> Result<()> {
-        if !tr.verify() {
-            return Err(AccountingError::InvalidTransfer);
-        }
-
-        self.log_verified_transaction(tr)
-    }
-
-    /// Process a Transaction that has already been verified.
-    fn process_verified_transaction(
-        self: &mut Self,
-        tr: &Transaction,
-        allow_deposits: bool,
-    ) -> Result<()> {
-        if !self.historian.reserve_signature(&tr.sig) {
-            return Err(AccountingError::InvalidTransferSignature);
-        }
-
-        if !Self::is_deposit(allow_deposits, &tr.from, &tr.plan) {
-            if let Some(x) = self.balances.get_mut(&tr.from) {
-                *x -= tr.tokens;
-            }
-        }
-
-        let mut plan = tr.plan.clone();
-        plan.apply_witness(&Witness::Timestamp(self.last_time));
-
-        if plan.is_complete() {
-            complete_transaction(&mut self.balances, &plan);
-        } else {
-            self.pending.insert(tr.sig, plan);
-        }
-
-        Ok(())
-    }
-
-    /// Process a Witness Signature that has already been verified.
-    fn process_verified_sig(&mut self, from: PublicKey, tx_sig: Signature) -> Result<()> {
-        if let Occupied(mut e) = self.pending.entry(tx_sig) {
-            e.get_mut().apply_witness(&Witness::Signature(from));
-            if e.get().is_complete() {
-                complete_transaction(&mut self.balances, e.get());
-                e.remove_entry();
-            }
-        };
-
-        Ok(())
-    }
-
-    /// Process a Witness Timestamp that has already been verified.
-    fn process_verified_timestamp(&mut self, from: PublicKey, dt: DateTime<Utc>) -> Result<()> {
-        // If this is the first timestamp we've seen, it probably came from the genesis block,
-        // so we'll trust it.
-        if self.last_time == Utc.timestamp(0, 0) {
-            self.time_sources.insert(from);
-        }
-
-        if self.time_sources.contains(&from) {
-            if dt > self.last_time {
-                self.last_time = dt;
-            }
-        } else {
-            return Ok(());
-        }
-
-        // Check to see if any timelocked transactions can be completed.
-        let mut completed = vec![];
-        for (key, plan) in &mut self.pending {
-            plan.apply_witness(&Witness::Timestamp(self.last_time));
-            if plan.is_complete() {
-                complete_transaction(&mut self.balances, plan);
-                completed.push(key.clone());
-            }
-        }
-
-        for key in completed {
-            self.pending.remove(&key);
-        }
-
-        Ok(())
-    }
-
-    /// Process an Transaction or Witness that has already been verified.
-    fn process_verified_event(self: &mut Self, event: &Event, allow_deposits: bool) -> Result<()> {
-        match *event {
-            Event::Transaction(ref tr) => self.process_verified_transaction(tr, allow_deposits),
-            Event::Signature { from, tx_sig, .. } => self.process_verified_sig(from, tx_sig),
-            Event::Timestamp { from, dt, .. } => self.process_verified_timestamp(from, dt),
-        }
-    }
-
-    /// Create, sign, and process a Transaction from `keypair` to `to` of
-    /// `n` tokens where `last_id` is the last Entry ID observed by the client.
-    pub fn transfer(
-        self: &mut Self,
-        n: i64,
-        keypair: &KeyPair,
-        to: PublicKey,
-        last_id: Hash,
-    ) -> Result<Signature> {
-        let tr = Transaction::new(keypair, to, n, last_id);
-        let sig = tr.sig;
-        self.log_transaction(tr).map(|_| sig)
-    }
-
-    /// Create, sign, and process a postdated Transaction from `keypair`
-    /// to `to` of `n` tokens on `dt` where `last_id` is the last Entry ID
-    /// observed by the client.
-    pub fn transfer_on_date(
-        self: &mut Self,
-        n: i64,
-        keypair: &KeyPair,
-        to: PublicKey,
-        dt: DateTime<Utc>,
-        last_id: Hash,
-    ) -> Result<Signature> {
-        let tr = Transaction::new_on_date(keypair, to, dt, n, last_id);
-        let sig = tr.sig;
-        self.log_transaction(tr).map(|_| sig)
-    }
-
-    pub fn get_balance(self: &Self, pubkey: &PublicKey) -> Option<i64> {
-        self.balances.get(pubkey).cloned()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use recorder::ExitReason;
-    use signature::KeyPairUtil;
-
-    #[test]
-    fn test_accountant() {
-        let alice = Mint::new(10_000);
-        let bob_pubkey = KeyPair::new().pubkey();
-        let mut acc = Accountant::new(&alice, Some(2));
-        acc.transfer(1_000, &alice.keypair(), bob_pubkey, alice.seed())
-            .unwrap();
-        assert_eq!(acc.get_balance(&bob_pubkey).unwrap(), 1_000);
-
-        acc.transfer(500, &alice.keypair(), bob_pubkey, alice.seed())
-            .unwrap();
-        assert_eq!(acc.get_balance(&bob_pubkey).unwrap(), 1_500);
-
-        drop(acc.historian.sender);
-        assert_eq!(
-            acc.historian.thread_hdl.join().unwrap(),
-            ExitReason::RecvDisconnected
-        );
-    }
-
-    #[test]
-    fn test_invalid_transfer() {
-        let alice = Mint::new(11_000);
-        let mut acc = Accountant::new(&alice, Some(2));
-        let bob_pubkey = KeyPair::new().pubkey();
-        acc.transfer(1_000, &alice.keypair(), bob_pubkey, alice.seed())
-            .unwrap();
-        assert_eq!(
-            acc.transfer(10_001, &alice.keypair(), bob_pubkey, alice.seed()),
-            Err(AccountingError::InsufficientFunds)
-        );
-
-        let alice_pubkey = alice.keypair().pubkey();
-        assert_eq!(acc.get_balance(&alice_pubkey).unwrap(), 10_000);
-        assert_eq!(acc.get_balance(&bob_pubkey).unwrap(), 1_000);
-
-        drop(acc.historian.sender);
-        assert_eq!(
-            acc.historian.thread_hdl.join().unwrap(),
-            ExitReason::RecvDisconnected
-        );
-    }
-
-    #[test]
-    fn test_overspend_attack() {
-        let alice = Mint::new(1);
-        let mut acc = Accountant::new(&alice, None);
-        let bob_pubkey = KeyPair::new().pubkey();
-        let mut tr = Transaction::new(&alice.keypair(), bob_pubkey, 1, alice.seed());
-        if let Plan::Pay(ref mut payment) = tr.plan {
-            payment.tokens = 2; // <-- attack!
-        }
-        assert_eq!(
-            acc.log_transaction(tr.clone()),
-            Err(AccountingError::InvalidTransfer)
-        );
-
-        // Also, ensure all branchs of the plan spend all tokens
-        if let Plan::Pay(ref mut payment) = tr.plan {
-            payment.tokens = 0; // <-- whoops!
-        }
-        assert_eq!(
-            acc.log_transaction(tr.clone()),
-            Err(AccountingError::InvalidTransfer)
-        );
-    }
-
-    #[test]
-    fn test_transfer_to_newb() {
-        let alice = Mint::new(10_000);
-        let mut acc = Accountant::new(&alice, Some(2));
-        let alice_keypair = alice.keypair();
-        let bob_pubkey = KeyPair::new().pubkey();
-        acc.transfer(500, &alice_keypair, bob_pubkey, alice.seed())
-            .unwrap();
-        assert_eq!(acc.get_balance(&bob_pubkey).unwrap(), 500);
-
-        drop(acc.historian.sender);
-        assert_eq!(
-            acc.historian.thread_hdl.join().unwrap(),
-            ExitReason::RecvDisconnected
-        );
-    }
-
-    #[test]
-    fn test_transfer_on_date() {
-        let alice = Mint::new(1);
-        let mut acc = Accountant::new(&alice, Some(2));
-        let alice_keypair = alice.keypair();
-        let bob_pubkey = KeyPair::new().pubkey();
-        let dt = Utc::now();
-        acc.transfer_on_date(1, &alice_keypair, bob_pubkey, dt, alice.seed())
-            .unwrap();
-
-        // Alice's balance will be zero because all funds are locked up.
-        assert_eq!(acc.get_balance(&alice.pubkey()), Some(0));
-
-        // Bob's balance will be None because the funds have not been
-        // sent.
-        assert_eq!(acc.get_balance(&bob_pubkey), None);
-
-        // Now, acknowledge the time in the condition occurred and
-        // that bob's funds are now available.
-        acc.process_verified_timestamp(alice.pubkey(), dt).unwrap();
-        assert_eq!(acc.get_balance(&bob_pubkey), Some(1));
-
-        acc.process_verified_timestamp(alice.pubkey(), dt).unwrap(); // <-- Attack! Attempt to process completed transaction.
-        assert_ne!(acc.get_balance(&bob_pubkey), Some(2));
-    }
-
-    #[test]
-    fn test_transfer_after_date() {
-        let alice = Mint::new(1);
-        let mut acc = Accountant::new(&alice, Some(2));
-        let alice_keypair = alice.keypair();
-        let bob_pubkey = KeyPair::new().pubkey();
-        let dt = Utc::now();
-        acc.process_verified_timestamp(alice.pubkey(), dt).unwrap();
-
-        // It's now past now, so this transfer should be processed immediately.
-        acc.transfer_on_date(1, &alice_keypair, bob_pubkey, dt, alice.seed())
-            .unwrap();
-
-        assert_eq!(acc.get_balance(&alice.pubkey()), Some(0));
-        assert_eq!(acc.get_balance(&bob_pubkey), Some(1));
-    }
-
-    #[test]
-    fn test_cancel_transfer() {
-        let alice = Mint::new(1);
-        let mut acc = Accountant::new(&alice, Some(2));
-        let alice_keypair = alice.keypair();
-        let bob_pubkey = KeyPair::new().pubkey();
-        let dt = Utc::now();
-        let sig = acc.transfer_on_date(1, &alice_keypair, bob_pubkey, dt, alice.seed())
-            .unwrap();
-
-        // Alice's balance will be zero because all funds are locked up.
-        assert_eq!(acc.get_balance(&alice.pubkey()), Some(0));
-
-        // Bob's balance will be None because the funds have not been
-        // sent.
-        assert_eq!(acc.get_balance(&bob_pubkey), None);
-
-        // Now, cancel the trancaction. Alice gets her funds back, Bob never sees them.
-        acc.process_verified_sig(alice.pubkey(), sig).unwrap();
-        assert_eq!(acc.get_balance(&alice.pubkey()), Some(1));
-        assert_eq!(acc.get_balance(&bob_pubkey), None);
-
-        acc.process_verified_sig(alice.pubkey(), sig).unwrap(); // <-- Attack! Attempt to cancel completed transaction.
-        assert_ne!(acc.get_balance(&alice.pubkey()), Some(2));
-    }
-}

src/accountant_skel.rs

@@ -1,189 +0,0 @@
-//! The `accountant_skel` module is a microservice that exposes the high-level
-//! Accountant API to the network. Its message encoding is currently
-//! in flux. Clients should use AccountantStub to interact with it.
-
-use accountant::Accountant;
-use bincode::{deserialize, serialize};
-use entry::Entry;
-use hash::Hash;
-use result::Result;
-use serde_json;
-use signature::PublicKey;
-use std::default::Default;
-use std::io::Write;
-use std::net::{SocketAddr, UdpSocket};
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::mpsc::channel;
-use std::sync::{Arc, Mutex};
-use std::thread::{spawn, JoinHandle};
-use std::time::Duration;
-use streamer;
-use transaction::Transaction;
-use rayon::prelude::*;
-
-pub struct AccountantSkel<W: Write + Send + 'static> {
-    pub acc: Accountant,
-    pub last_id: Hash,
-    writer: W,
-}
-
-#[cfg_attr(feature = "cargo-clippy", allow(large_enum_variant))]
-#[derive(Serialize, Deserialize, Debug)]
-pub enum Request {
-    Transaction(Transaction),
-    GetBalance { key: PublicKey },
-    GetId { is_last: bool },
-}
-
-impl Request {
-    /// Verify the request is valid.
-    pub fn verify(&self) -> bool {
-        match *self {
-            Request::Transaction(ref tr) => tr.verify(),
-            _ => true,
-        }
-    }
-}
-
-/// Parallel verfication of a batch of requests.
-fn filter_valid_requests(reqs: Vec<(Request, SocketAddr)>) -> Vec<(Request, SocketAddr)> {
-    reqs.into_par_iter().filter({ |x| x.0.verify() }).collect()
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub enum Response {
-    Balance { key: PublicKey, val: Option<i64> },
-    Entries { entries: Vec<Entry> },
-    Id { id: Hash, is_last: bool },
-}
-
-impl<W: Write + Send + 'static> AccountantSkel<W> {
-    /// Create a new AccountantSkel that wraps the given Accountant.
-    pub fn new(acc: Accountant, w: W) -> Self {
-        let last_id = acc.first_id;
-        AccountantSkel {
-            acc,
-            last_id,
-            writer: w,
-        }
-    }
-
-    /// Process any Entry items that have been published by the Historian.
-    pub fn sync(&mut self) -> Hash {
-        while let Ok(entry) = self.acc.historian.receiver.try_recv() {
-            self.last_id = entry.id;
-            writeln!(self.writer, "{}", serde_json::to_string(&entry).unwrap()).unwrap();
-        }
-        self.last_id
-    }
-
-    /// Process Request items sent by clients.
-    pub fn log_verified_request(&mut self, msg: Request) -> Option<Response> {
-        match msg {
-            Request::Transaction(tr) => {
-                if let Err(err) = self.acc.log_verified_transaction(tr) {
-                    eprintln!("Transaction error: {:?}", err);
-                }
-                None
-            }
-            Request::GetBalance { key } => {
-                let val = self.acc.get_balance(&key);
-                Some(Response::Balance { key, val })
-            }
-            Request::GetId { is_last } => Some(Response::Id {
-                id: if is_last {
-                    self.sync()
-                } else {
-                    self.acc.first_id
-                },
-                is_last,
-            }),
-        }
-    }
-
-    fn process(
-        obj: &Arc<Mutex<AccountantSkel<W>>>,
-        r_reader: &streamer::Receiver,
-        s_responder: &streamer::Responder,
-        packet_recycler: &streamer::PacketRecycler,
-        response_recycler: &streamer::ResponseRecycler,
-    ) -> Result<()> {
-        let timer = Duration::new(1, 0);
-        let msgs = r_reader.recv_timeout(timer)?;
-        let msgs_ = msgs.clone();
-        let rsps = streamer::allocate(response_recycler);
-        let rsps_ = rsps.clone();
-        {
-            let mut reqs = vec![];
-            for packet in &msgs.read().unwrap().packets {
-                let rsp_addr = packet.meta.get_addr();
-                let sz = packet.meta.size;
-                let req = deserialize(&packet.data[0..sz])?;
-                reqs.push((req, rsp_addr));
-            }
-            let reqs = filter_valid_requests(reqs);
-
-            let mut num = 0;
-            let mut ursps = rsps.write().unwrap();
-            for (req, rsp_addr) in reqs {
-                if let Some(resp) = obj.lock().unwrap().log_verified_request(req) {
-                    if ursps.responses.len() <= num {
-                        ursps
-                            .responses
-                            .resize((num + 1) * 2, streamer::Response::default());
-                    }
-                    let rsp = &mut ursps.responses[num];
-                    let v = serialize(&resp)?;
-                    let len = v.len();
-                    rsp.data[..len].copy_from_slice(&v);
-                    rsp.meta.size = len;
-                    rsp.meta.set_addr(&rsp_addr);
-                    num += 1;
-                }
-            }
-            ursps.responses.resize(num, streamer::Response::default());
-        }
-        s_responder.send(rsps_)?;
-        streamer::recycle(packet_recycler, msgs_);
-        Ok(())
-    }
-
-    /// Create a UDP microservice that forwards messages the given AccountantSkel.
-    /// Set `exit` to shutdown its threads.
-    pub fn serve(
-        obj: Arc<Mutex<AccountantSkel<W>>>,
-        addr: &str,
-        exit: Arc<AtomicBool>,
-    ) -> Result<Vec<JoinHandle<()>>> {
-        let read = UdpSocket::bind(addr)?;
-        // make sure we are on the same interface
-        let mut local = read.local_addr()?;
-        local.set_port(0);
-        let write = UdpSocket::bind(local)?;
-
-        let packet_recycler = Arc::new(Mutex::new(Vec::new()));
-        let response_recycler = Arc::new(Mutex::new(Vec::new()));
-        let (s_reader, r_reader) = channel();
-        let t_receiver = streamer::receiver(read, exit.clone(), packet_recycler.clone(), s_reader)?;
-
-        let (s_responder, r_responder) = channel();
-        let t_responder =
-            streamer::responder(write, exit.clone(), response_recycler.clone(), r_responder);
-
-        let skel = obj.clone();
-        let t_server = spawn(move || loop {
-            let e = AccountantSkel::process(
-                &skel,
-                &r_reader,
-                &s_responder,
-                &packet_recycler,
-                &response_recycler,
-            );
-            if e.is_err() && exit.load(Ordering::Relaxed) {
-                break;
-            }
-        });
-
-        Ok(vec![t_receiver, t_responder, t_server])
-    }
-}

src/accountant_stub.rs

@@ -1,126 +0,0 @@
-//! The `accountant_stub` module is a client-side object that interfaces with a server-side Accountant
-//! object via the network interface exposed by AccountantSkel. Client code should use
-//! this object instead of writing messages to the network directly. The binary
-//! encoding of its messages are unstable and may change in future releases.
-
-use accountant_skel::{Request, Response};
-use bincode::{deserialize, serialize};
-use hash::Hash;
-use signature::{KeyPair, PublicKey, Signature};
-use std::io;
-use std::net::UdpSocket;
-use transaction::Transaction;
-
-pub struct AccountantStub {
-    pub addr: String,
-    pub socket: UdpSocket,
-}
-
-impl AccountantStub {
-    /// Create a new AccountantStub that will interface with AccountantSkel
-    /// over `socket`. To receive responses, the caller must bind `socket`
-    /// to a public address before invoking AccountantStub methods.
-    pub fn new(addr: &str, socket: UdpSocket) -> Self {
-        AccountantStub {
-            addr: addr.to_string(),
-            socket,
-        }
-    }
-
-    /// Send a signed Transaction to the server for processing. This method
-    /// does not wait for a response.
-    pub fn transfer_signed(&self, tr: Transaction) -> io::Result<usize> {
-        let req = Request::Transaction(tr);
-        let data = serialize(&req).unwrap();
-        self.socket.send_to(&data, &self.addr)
-    }
-
-    /// Creates, signs, and processes a Transaction. Useful for writing unit-tests.
-    pub fn transfer(
-        &self,
-        n: i64,
-        keypair: &KeyPair,
-        to: PublicKey,
-        last_id: &Hash,
-    ) -> io::Result<Signature> {
-        let tr = Transaction::new(keypair, to, n, *last_id);
-        let sig = tr.sig;
-        self.transfer_signed(tr).map(|_| sig)
-    }
-
-    /// Request the balance of the user holding `pubkey`. This method blocks
-    /// until the server sends a response. If the response packet is dropped
-    /// by the network, this method will hang indefinitely.
-    pub fn get_balance(&self, pubkey: &PublicKey) -> io::Result<Option<i64>> {
-        let req = Request::GetBalance { key: *pubkey };
-        let data = serialize(&req).expect("serialize GetBalance");
-        self.socket.send_to(&data, &self.addr)?;
-        let mut buf = vec![0u8; 1024];
-        self.socket.recv_from(&mut buf)?;
-        let resp = deserialize(&buf).expect("deserialize balance");
-        if let Response::Balance { key, val } = resp {
-            assert_eq!(key, *pubkey);
-            return Ok(val);
-        }
-        Ok(None)
-    }
-
-    /// Request the first or last Entry ID from the server.
-    fn get_id(&self, is_last: bool) -> io::Result<Hash> {
-        let req = Request::GetId { is_last };
-        let data = serialize(&req).expect("serialize GetId");
-        self.socket.send_to(&data, &self.addr)?;
-        let mut buf = vec![0u8; 1024];
-        self.socket.recv_from(&mut buf)?;
-        let resp = deserialize(&buf).expect("deserialize Id");
-        if let Response::Id { id, .. } = resp {
-            return Ok(id);
-        }
-        Ok(Default::default())
-    }
-
-    /// Request the last Entry ID from the server. This method blocks
-    /// until the server sends a response. At the time of this writing,
-    /// it also has the side-effect of causing the server to log any
-    /// entries that have been published by the Historian.
-    pub fn get_last_id(&self) -> io::Result<Hash> {
-        self.get_id(true)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use accountant::Accountant;
-    use accountant_skel::AccountantSkel;
-    use mint::Mint;
-    use signature::{KeyPair, KeyPairUtil};
-    use std::io::sink;
-    use std::sync::atomic::{AtomicBool, Ordering};
-    use std::sync::{Arc, Mutex};
-    use std::thread::sleep;
-    use std::time::Duration;
-
-    // TODO: Figure out why this test sometimes hangs on TravisCI.
-    #[test]
-    fn test_accountant_stub() {
-        let addr = "127.0.0.1:9000";
-        let send_addr = "127.0.0.1:9001";
-        let alice = Mint::new(10_000);
-        let acc = Accountant::new(&alice, Some(30));
-        let bob_pubkey = KeyPair::new().pubkey();
-        let exit = Arc::new(AtomicBool::new(false));
-        let acc = Arc::new(Mutex::new(AccountantSkel::new(acc, sink())));
-        let _threads = AccountantSkel::serve(acc, addr, exit.clone()).unwrap();
-        sleep(Duration::from_millis(300));
-
-        let socket = UdpSocket::bind(send_addr).unwrap();
-
-        let acc = AccountantStub::new(addr, socket);
-        let last_id = acc.get_last_id().unwrap();
-        let _sig = acc.transfer(500, &alice.keypair(), bob_pubkey, &last_id)
-            .unwrap();
-        assert_eq!(acc.get_balance(&bob_pubkey).unwrap().unwrap(), 500);
-        exit.store(true, Ordering::Relaxed);
-    }
-}

src/bank.rs

@@ -0,0 +1,651 @@
+//! The `bank` module tracks client balances, and the progress of pending
+//! transactions. It offers a high-level public API that signs transactions
+//! on behalf of the caller, and a private low-level API for when they have
+//! already been signed and verified.
+
+extern crate libc;
+
+use chrono::prelude::*;
+use entry::Entry;
+use hash::Hash;
+use mint::Mint;
+use payment_plan::{Payment, PaymentPlan, Witness};
+use rayon::prelude::*;
+use signature::{KeyPair, PublicKey, Signature};
+use std::collections::hash_map::Entry::Occupied;
+use std::collections::{HashMap, HashSet, VecDeque};
+use std::result;
+use std::sync::atomic::{AtomicIsize, AtomicUsize, Ordering};
+use std::sync::RwLock;
+use transaction::{Instruction, Plan, Transaction};
+
+pub const MAX_ENTRY_IDS: usize = 1024 * 4;
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum BankError {
+    AccountNotFound(PublicKey),
+    InsufficientFunds(PublicKey),
+    DuplicateSiganture(Signature),
+    LastIdNotFound(Hash),
+    NegativeTokens,
+}
+
+pub type Result<T> = result::Result<T, BankError>;
+
+pub struct Bank {
+    balances: RwLock<HashMap<PublicKey, AtomicIsize>>,
+    pending: RwLock<HashMap<Signature, Plan>>,
+    last_ids: RwLock<VecDeque<(Hash, RwLock<HashSet<Signature>>)>>,
+    time_sources: RwLock<HashSet<PublicKey>>,
+    last_time: RwLock<DateTime<Utc>>,
+    transaction_count: AtomicUsize,
+}
+
+impl Bank {
+    /// Create an Bank using a deposit.
+    pub fn new_from_deposit(deposit: &Payment) -> Self {
+        let bank = Bank {
+            balances: RwLock::new(HashMap::new()),
+            pending: RwLock::new(HashMap::new()),
+            last_ids: RwLock::new(VecDeque::new()),
+            time_sources: RwLock::new(HashSet::new()),
+            last_time: RwLock::new(Utc.timestamp(0, 0)),
+            transaction_count: AtomicUsize::new(0),
+        };
+        bank.apply_payment(deposit);
+        bank
+    }
+
+    /// Create an Bank with only a Mint. Typically used by unit tests.
+    pub fn new(mint: &Mint) -> Self {
+        let deposit = Payment {
+            to: mint.pubkey(),
+            tokens: mint.tokens,
+        };
+        let bank = Self::new_from_deposit(&deposit);
+        bank.register_entry_id(&mint.last_id());
+        bank
+    }
+
+    /// Commit funds to the 'to' party.
+    fn apply_payment(&self, payment: &Payment) {
+        // First we check balances with a read lock to maximize potential parallelization.
+        if self.balances
+            .read()
+            .expect("'balances' read lock in apply_payment")
+            .contains_key(&payment.to)
+        {
+            let bals = self.balances.read().expect("'balances' read lock");
+            bals[&payment.to].fetch_add(payment.tokens as isize, Ordering::Relaxed);
+        } else {
+            // Now we know the key wasn't present a nanosecond ago, but it might be there
+            // by the time we aquire a write lock, so we'll have to check again.
+            let mut bals = self.balances.write().expect("'balances' write lock");
+            if bals.contains_key(&payment.to) {
+                bals[&payment.to].fetch_add(payment.tokens as isize, Ordering::Relaxed);
+            } else {
+                bals.insert(payment.to, AtomicIsize::new(payment.tokens as isize));
+            }
+        }
+    }
+
+    /// Return the last entry ID registered
+    pub fn last_id(&self) -> Hash {
+        let last_ids = self.last_ids.read().expect("'last_ids' read lock");
+        let last_item = last_ids.iter().last().expect("empty 'last_ids' list");
+        last_item.0
+    }
+
+    fn reserve_signature(signatures: &RwLock<HashSet<Signature>>, sig: &Signature) -> Result<()> {
+        if signatures
+            .read()
+            .expect("'signatures' read lock")
+            .contains(sig)
+        {
+            return Err(BankError::DuplicateSiganture(*sig));
+        }
+        signatures
+            .write()
+            .expect("'signatures' write lock")
+            .insert(*sig);
+        Ok(())
+    }
+
+    fn forget_signature(signatures: &RwLock<HashSet<Signature>>, sig: &Signature) {
+        signatures
+            .write()
+            .expect("'signatures' write lock in forget_signature")
+            .remove(sig);
+    }
+
+    fn forget_signature_with_last_id(&self, sig: &Signature, last_id: &Hash) {
+        if let Some(entry) = self.last_ids
+            .read()
+            .expect("'last_ids' read lock in forget_signature_with_last_id")
+            .iter()
+            .rev()
+            .find(|x| x.0 == *last_id)
+        {
+            Self::forget_signature(&entry.1, sig);
+        }
+    }
+
+    fn reserve_signature_with_last_id(&self, sig: &Signature, last_id: &Hash) -> Result<()> {
+        if let Some(entry) = self.last_ids
+            .read()
+            .expect("'last_ids' read lock in reserve_signature_with_last_id")
+            .iter()
+            .rev()
+            .find(|x| x.0 == *last_id)
+        {
+            return Self::reserve_signature(&entry.1, sig);
+        }
+        Err(BankError::LastIdNotFound(*last_id))
+    }
+
+    /// Tell the bank which Entry IDs exist on the ledger. This function
+    /// assumes subsequent calls correspond to later entries, and will boot
+    /// the oldest ones once its internal cache is full. Once boot, the
+    /// bank will reject transactions using that `last_id`.
+    pub fn register_entry_id(&self, last_id: &Hash) {
+        let mut last_ids = self.last_ids
+            .write()
+            .expect("'last_ids' write lock in register_entry_id");
+        if last_ids.len() >= MAX_ENTRY_IDS {
+            last_ids.pop_front();
+        }
+        last_ids.push_back((*last_id, RwLock::new(HashSet::new())));
+    }
+
+    /// Deduct tokens from the 'from' address the account has sufficient
+    /// funds and isn't a duplicate.
+    fn apply_debits(&self, tx: &Transaction) -> Result<()> {
+        if let Instruction::NewContract(contract) = &tx.instruction {
+            trace!("Transaction {}", contract.tokens);
+            if contract.tokens < 0 {
+                return Err(BankError::NegativeTokens);
+            }
+        }
+        let bals = self.balances
+            .read()
+            .expect("'balances' read lock in apply_debits");
+        let option = bals.get(&tx.from);
+
+        if option.is_none() {
+            return Err(BankError::AccountNotFound(tx.from));
+        }
+
+        self.reserve_signature_with_last_id(&tx.sig, &tx.last_id)?;
+
+        loop {
+            let result = if let Instruction::NewContract(contract) = &tx.instruction {
+                let bal = option.expect("assignment of option to bal");
+                let current = bal.load(Ordering::Relaxed) as i64;
+
+                if current < contract.tokens {
+                    self.forget_signature_with_last_id(&tx.sig, &tx.last_id);
+                    return Err(BankError::InsufficientFunds(tx.from));
+                }
+
+                bal.compare_exchange(
+                    current as isize,
+                    (current - contract.tokens) as isize,
+                    Ordering::Relaxed,
+                    Ordering::Relaxed,
+                )
+            } else {
+                Ok(0)
+            };
+
+            match result {
+                Ok(_) => {
+                    self.transaction_count.fetch_add(1, Ordering::Relaxed);
+                    return Ok(());
+                }
+                Err(_) => continue,
+            };
+        }
+    }
+
+    fn apply_credits(&self, tx: &Transaction) {
+        match &tx.instruction {
+            Instruction::NewContract(contract) => {
+                let mut plan = contract.plan.clone();
+                plan.apply_witness(&Witness::Timestamp(*self.last_time
+                    .read()
+                    .expect("timestamp creation in apply_credits")));
+
+                if let Some(ref payment) = plan.final_payment() {
+                    self.apply_payment(payment);
+                } else {
+                    let mut pending = self.pending
+                        .write()
+                        .expect("'pending' write lock in apply_credits");
+                    pending.insert(tx.sig, plan);
+                }
+            }
+            Instruction::ApplyTimestamp(dt) => {
+                let _ = self.apply_timestamp(tx.from, *dt);
+            }
+            Instruction::ApplySignature(tx_sig) => {
+                let _ = self.apply_signature(tx.from, *tx_sig);
+            }
+        }
+    }
+
+    /// Process a Transaction.
+    fn process_transaction(&self, tx: &Transaction) -> Result<()> {
+        self.apply_debits(tx)?;
+        self.apply_credits(tx);
+        Ok(())
+    }
+
+    /// Process a batch of transactions.
+    pub fn process_transactions(&self, txs: Vec<Transaction>) -> Vec<Result<Transaction>> {
+        // Run all debits first to filter out any transactions that can't be processed
+        // in parallel deterministically.
+        info!("processing Transactions {}", txs.len());
+        let results: Vec<_> = txs.into_par_iter()
+            .map(|tx| self.apply_debits(&tx).map(|_| tx))
+            .collect(); // Calling collect() here forces all debits to complete before moving on.
+
+        results
+            .into_par_iter()
+            .map(|result| {
+                result.map(|tx| {
+                    self.apply_credits(&tx);
+                    tx
+                })
+            })
+            .collect()
+    }
+
+    pub fn process_entries<I>(&self, entries: I) -> Result<()>
+    where
+        I: IntoIterator<Item = Entry>,
+    {
+        for entry in entries {
+            for result in self.process_transactions(entry.transactions) {
+                result?;
+            }
+            self.register_entry_id(&entry.id);
+        }
+        Ok(())
+    }
+
+    /// Process a Witness Signature.
+    fn apply_signature(&self, from: PublicKey, tx_sig: Signature) -> Result<()> {
+        if let Occupied(mut e) = self.pending
+            .write()
+            .expect("write() in apply_signature")
+            .entry(tx_sig)
+        {
+            e.get_mut().apply_witness(&Witness::Signature(from));
+            if let Some(payment) = e.get().final_payment() {
+                self.apply_payment(&payment);
+                e.remove_entry();
+            }
+        };
+
+        Ok(())
+    }
+
+    /// Process a Witness Timestamp.
+    fn apply_timestamp(&self, from: PublicKey, dt: DateTime<Utc>) -> Result<()> {
+        // If this is the first timestamp we've seen, it probably came from the genesis block,
+        // so we'll trust it.
+        if *self.last_time
+            .read()
+            .expect("'last_time' read lock on first timestamp check")
+            == Utc.timestamp(0, 0)
+        {
+            self.time_sources
+                .write()
+                .expect("'time_sources' write lock on first timestamp")
+                .insert(from);
+        }
+
+        if self.time_sources
+            .read()
+            .expect("'time_sources' read lock")
+            .contains(&from)
+        {
+            if dt > *self.last_time.read().expect("'last_time' read lock") {
+                *self.last_time.write().expect("'last_time' write lock") = dt;
+            }
+        } else {
+            return Ok(());
+        }
+
+        // Check to see if any timelocked transactions can be completed.
+        let mut completed = vec![];
+
+        // Hold 'pending' write lock until the end of this function. Otherwise another thread can
+        // double-spend if it enters before the modified plan is removed from 'pending'.
+        let mut pending = self.pending
+            .write()
+            .expect("'pending' write lock in apply_timestamp");
+        for (key, plan) in pending.iter_mut() {
+            plan.apply_witness(&Witness::Timestamp(*self.last_time
+                .read()
+                .expect("'last_time' read lock when creating timestamp")));
+            if let Some(ref payment) = plan.final_payment() {
+                self.apply_payment(payment);
+                completed.push(key.clone());
+            }
+        }
+
+        for key in completed {
+            pending.remove(&key);
+        }
+
+        Ok(())
+    }
+
+    /// Create, sign, and process a Transaction from `keypair` to `to` of
+    /// `n` tokens where `last_id` is the last Entry ID observed by the client.
+    pub fn transfer(
+        &self,
+        n: i64,
+        keypair: &KeyPair,
+        to: PublicKey,
+        last_id: Hash,
+    ) -> Result<Signature> {
+        let tx = Transaction::new(keypair, to, n, last_id);
+        let sig = tx.sig;
+        self.process_transaction(&tx).map(|_| sig)
+    }
+
+    /// Create, sign, and process a postdated Transaction from `keypair`
+    /// to `to` of `n` tokens on `dt` where `last_id` is the last Entry ID
+    /// observed by the client.
+    pub fn transfer_on_date(
+        &self,
+        n: i64,
+        keypair: &KeyPair,
+        to: PublicKey,
+        dt: DateTime<Utc>,
+        last_id: Hash,
+    ) -> Result<Signature> {
+        let tx = Transaction::new_on_date(keypair, to, dt, n, last_id);
+        let sig = tx.sig;
+        self.process_transaction(&tx).map(|_| sig)
+    }
+
+    pub fn get_balance(&self, pubkey: &PublicKey) -> Option<i64> {
+        let bals = self.balances
+            .read()
+            .expect("'balances' read lock in get_balance");
+        bals.get(pubkey).map(|x| x.load(Ordering::Relaxed) as i64)
+    }
+
+    pub fn transaction_count(&self) -> usize {
+        self.transaction_count.load(Ordering::Relaxed)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bincode::serialize;
+    use hash::hash;
+    use signature::KeyPairUtil;
+
+    #[test]
+    fn test_bank() {
+        let mint = Mint::new(10_000);
+        let pubkey = KeyPair::new().pubkey();
+        let bank = Bank::new(&mint);
+        assert_eq!(bank.last_id(), mint.last_id());
+
+        bank.transfer(1_000, &mint.keypair(), pubkey, mint.last_id())
+            .unwrap();
+        assert_eq!(bank.get_balance(&pubkey).unwrap(), 1_000);
+
+        bank.transfer(500, &mint.keypair(), pubkey, mint.last_id())
+            .unwrap();
+        assert_eq!(bank.get_balance(&pubkey).unwrap(), 1_500);
+        assert_eq!(bank.transaction_count(), 2);
+    }
+
+    #[test]
+    fn test_invalid_tokens() {
+        let mint = Mint::new(1);
+        let pubkey = KeyPair::new().pubkey();
+        let bank = Bank::new(&mint);
+        assert_eq!(
+            bank.transfer(-1, &mint.keypair(), pubkey, mint.last_id()),
+            Err(BankError::NegativeTokens)
+        );
+        assert_eq!(bank.transaction_count(), 0);
+    }
+
+    #[test]
+    fn test_account_not_found() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let keypair = KeyPair::new();
+        assert_eq!(
+            bank.transfer(1, &keypair, mint.pubkey(), mint.last_id()),
+            Err(BankError::AccountNotFound(keypair.pubkey()))
+        );
+        assert_eq!(bank.transaction_count(), 0);
+    }
+
+    #[test]
+    fn test_invalid_transfer() {
+        let mint = Mint::new(11_000);
+        let bank = Bank::new(&mint);
+        let pubkey = KeyPair::new().pubkey();
+        bank.transfer(1_000, &mint.keypair(), pubkey, mint.last_id())
+            .unwrap();
+        assert_eq!(bank.transaction_count(), 1);
+        assert_eq!(
+            bank.transfer(10_001, &mint.keypair(), pubkey, mint.last_id()),
+            Err(BankError::InsufficientFunds(mint.pubkey()))
+        );
+        assert_eq!(bank.transaction_count(), 1);
+
+        let mint_pubkey = mint.keypair().pubkey();
+        assert_eq!(bank.get_balance(&mint_pubkey).unwrap(), 10_000);
+        assert_eq!(bank.get_balance(&pubkey).unwrap(), 1_000);
+    }
+
+    #[test]
+    fn test_transfer_to_newb() {
+        let mint = Mint::new(10_000);
+        let bank = Bank::new(&mint);
+        let pubkey = KeyPair::new().pubkey();
+        bank.transfer(500, &mint.keypair(), pubkey, mint.last_id())
+            .unwrap();
+        assert_eq!(bank.get_balance(&pubkey).unwrap(), 500);
+    }
+
+    #[test]
+    fn test_transfer_on_date() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let pubkey = KeyPair::new().pubkey();
+        let dt = Utc::now();
+        bank.transfer_on_date(1, &mint.keypair(), pubkey, dt, mint.last_id())
+            .unwrap();
+
+        // Mint's balance will be zero because all funds are locked up.
+        assert_eq!(bank.get_balance(&mint.pubkey()), Some(0));
+
+        // tx count is 1, because debits were applied.
+        assert_eq!(bank.transaction_count(), 1);
+
+        // pubkey's balance will be None because the funds have not been
+        // sent.
+        assert_eq!(bank.get_balance(&pubkey), None);
+
+        // Now, acknowledge the time in the condition occurred and
+        // that pubkey's funds are now available.
+        bank.apply_timestamp(mint.pubkey(), dt).unwrap();
+        assert_eq!(bank.get_balance(&pubkey), Some(1));
+
+        // tx count is still 1, because we chose not to count timestamp transactions
+        // tx count.
+        assert_eq!(bank.transaction_count(), 1);
+
+        bank.apply_timestamp(mint.pubkey(), dt).unwrap(); // <-- Attack! Attempt to process completed transaction.
+        assert_ne!(bank.get_balance(&pubkey), Some(2));
+    }
+
+    #[test]
+    fn test_transfer_after_date() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let pubkey = KeyPair::new().pubkey();
+        let dt = Utc::now();
+        bank.apply_timestamp(mint.pubkey(), dt).unwrap();
+
+        // It's now past now, so this transfer should be processed immediately.
+        bank.transfer_on_date(1, &mint.keypair(), pubkey, dt, mint.last_id())
+            .unwrap();
+
+        assert_eq!(bank.get_balance(&mint.pubkey()), Some(0));
+        assert_eq!(bank.get_balance(&pubkey), Some(1));
+    }
+
+    #[test]
+    fn test_cancel_transfer() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let pubkey = KeyPair::new().pubkey();
+        let dt = Utc::now();
+        let sig = bank.transfer_on_date(1, &mint.keypair(), pubkey, dt, mint.last_id())
+            .unwrap();
+
+        // Assert the debit counts as a transaction.
+        assert_eq!(bank.transaction_count(), 1);
+
+        // Mint's balance will be zero because all funds are locked up.
+        assert_eq!(bank.get_balance(&mint.pubkey()), Some(0));
+
+        // pubkey's balance will be None because the funds have not been
+        // sent.
+        assert_eq!(bank.get_balance(&pubkey), None);
+
+        // Now, cancel the trancaction. Mint gets her funds back, pubkey never sees them.
+        bank.apply_signature(mint.pubkey(), sig).unwrap();
+        assert_eq!(bank.get_balance(&mint.pubkey()), Some(1));
+        assert_eq!(bank.get_balance(&pubkey), None);
+
+        // Assert cancel doesn't cause count to go backward.
+        assert_eq!(bank.transaction_count(), 1);
+
+        bank.apply_signature(mint.pubkey(), sig).unwrap(); // <-- Attack! Attempt to cancel completed transaction.
+        assert_ne!(bank.get_balance(&mint.pubkey()), Some(2));
+    }
+
+    #[test]
+    fn test_duplicate_transaction_signature() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let sig = Signature::default();
+        assert!(
+            bank.reserve_signature_with_last_id(&sig, &mint.last_id())
+                .is_ok()
+        );
+        assert_eq!(
+            bank.reserve_signature_with_last_id(&sig, &mint.last_id()),
+            Err(BankError::DuplicateSiganture(sig))
+        );
+    }
+
+    #[test]
+    fn test_forget_signature() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let sig = Signature::default();
+        bank.reserve_signature_with_last_id(&sig, &mint.last_id())
+            .unwrap();
+        bank.forget_signature_with_last_id(&sig, &mint.last_id());
+        assert!(
+            bank.reserve_signature_with_last_id(&sig, &mint.last_id())
+                .is_ok()
+        );
+    }
+
+    #[test]
+    fn test_max_entry_ids() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let sig = Signature::default();
+        for i in 0..MAX_ENTRY_IDS {
+            let last_id = hash(&serialize(&i).unwrap()); // Unique hash
+            bank.register_entry_id(&last_id);
+        }
+        // Assert we're no longer able to use the oldest entry ID.
+        assert_eq!(
+            bank.reserve_signature_with_last_id(&sig, &mint.last_id()),
+            Err(BankError::LastIdNotFound(mint.last_id()))
+        );
+    }
+
+    #[test]
+    fn test_debits_before_credits() {
+        let mint = Mint::new(2);
+        let bank = Bank::new(&mint);
+        let keypair = KeyPair::new();
+        let tx0 = Transaction::new(&mint.keypair(), keypair.pubkey(), 2, mint.last_id());
+        let tx1 = Transaction::new(&keypair, mint.pubkey(), 1, mint.last_id());
+        let txs = vec![tx0, tx1];
+        let results = bank.process_transactions(txs);
+        assert!(results[1].is_err());
+
+        // Assert bad transactions aren't counted.
+        assert_eq!(bank.transaction_count(), 1);
+    }
+}
+
+#[cfg(all(feature = "unstable", test))]
+mod bench {
+    extern crate test;
+    use self::test::Bencher;
+    use bank::*;
+    use bincode::serialize;
+    use hash::hash;
+    use signature::KeyPairUtil;
+
+    #[bench]
+    fn bench_process_transaction(bencher: &mut Bencher) {
+        let mint = Mint::new(100_000_000);
+        let bank = Bank::new(&mint);
+        // Create transactions between unrelated parties.
+        let transactions: Vec<_> = (0..4096)
+            .into_par_iter()
+            .map(|i| {
+                // Seed the 'from' account.
+                let rando0 = KeyPair::new();
+                let tx = Transaction::new(&mint.keypair(), rando0.pubkey(), 1_000, mint.last_id());
+                bank.process_transaction(&tx).unwrap();
+
+                // Seed the 'to' account and a cell for its signature.
+                let last_id = hash(&serialize(&i).unwrap()); // Unique hash
+                bank.register_entry_id(&last_id);
+
+                let rando1 = KeyPair::new();
+                let tx = Transaction::new(&rando0, rando1.pubkey(), 1, last_id);
+                bank.process_transaction(&tx).unwrap();
+
+                // Finally, return a transaction that's unique
+                Transaction::new(&rando0, rando1.pubkey(), 1, last_id)
+            })
+            .collect();
+        bencher.iter(|| {
+            // Since benchmarker runs this multiple times, we need to clear the signatures.
+            for sigs in bank.last_ids.read().unwrap().iter() {
+                sigs.1.write().unwrap().clear();
+            }
+
+            assert!(
+                bank.process_transactions(transactions.clone())
+                    .iter()
+                    .all(|x| x.is_ok())
+            );
+        });
+    }
+}

src/banking_stage.rs

@@ -0,0 +1,303 @@
+//! The `banking_stage` processes Transaction messages.
+
+use bank::Bank;
+use bincode::deserialize;
+use packet;
+use packet::SharedPackets;
+use rayon::prelude::*;
+use record_stage::Signal;
+use result::Result;
+use std::net::SocketAddr;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{channel, Receiver, Sender};
+use std::sync::Arc;
+use std::thread::{Builder, JoinHandle};
+use std::time::Duration;
+use std::time::Instant;
+use timing;
+use transaction::Transaction;
+
+pub struct BankingStage {
+    pub thread_hdl: JoinHandle<()>,
+    pub signal_receiver: Receiver<Signal>,
+}
+
+impl BankingStage {
+    pub fn new(
+        bank: Arc<Bank>,
+        exit: Arc<AtomicBool>,
+        verified_receiver: Receiver<Vec<(SharedPackets, Vec<u8>)>>,
+        packet_recycler: packet::PacketRecycler,
+    ) -> Self {
+        let (signal_sender, signal_receiver) = channel();
+        let thread_hdl = Builder::new()
+            .name("solana-banking-stage".to_string())
+            .spawn(move || loop {
+                let e = Self::process_packets(
+                    bank.clone(),
+                    &verified_receiver,
+                    &signal_sender,
+                    &packet_recycler,
+                );
+                if e.is_err() {
+                    if exit.load(Ordering::Relaxed) {
+                        break;
+                    }
+                }
+            })
+            .unwrap();
+        BankingStage {
+            thread_hdl,
+            signal_receiver,
+        }
+    }
+
+    fn deserialize_transactions(p: &packet::Packets) -> Vec<Option<(Transaction, SocketAddr)>> {
+        p.packets
+            .par_iter()
+            .map(|x| {
+                deserialize(&x.data[0..x.meta.size])
+                    .map(|req| (req, x.meta.addr()))
+                    .ok()
+            })
+            .collect()
+    }
+
+    fn process_packets(
+        bank: Arc<Bank>,
+        verified_receiver: &Receiver<Vec<(SharedPackets, Vec<u8>)>>,
+        signal_sender: &Sender<Signal>,
+        packet_recycler: &packet::PacketRecycler,
+    ) -> Result<()> {
+        let timer = Duration::new(1, 0);
+        let recv_start = Instant::now();
+        let mms = verified_receiver.recv_timeout(timer)?;
+        let mut reqs_len = 0;
+        let mms_len = mms.len();
+        info!(
+            "@{:?} process start stalled for: {:?}ms batches: {}",
+            timing::timestamp(),
+            timing::duration_as_ms(&recv_start.elapsed()),
+            mms.len(),
+        );
+        let proc_start = Instant::now();
+        for (msgs, vers) in mms {
+            let transactions = Self::deserialize_transactions(&msgs.read().unwrap());
+            reqs_len += transactions.len();
+            let transactions = transactions
+                .into_iter()
+                .zip(vers)
+                .filter_map(|(tx, ver)| match tx {
+                    None => None,
+                    Some((tx, _addr)) => if tx.verify_plan() && ver != 0 {
+                        Some(tx)
+                    } else {
+                        None
+                    },
+                })
+                .collect();
+
+            debug!("process_transactions");
+            let results = bank.process_transactions(transactions);
+            let transactions = results.into_iter().filter_map(|x| x.ok()).collect();
+            signal_sender.send(Signal::Events(transactions))?;
+            debug!("done process_transactions");
+
+            packet_recycler.recycle(msgs);
+        }
+        let total_time_s = timing::duration_as_s(&proc_start.elapsed());
+        let total_time_ms = timing::duration_as_ms(&proc_start.elapsed());
+        info!(
+            "@{:?} done processing transaction batches: {} time: {:?}ms reqs: {} reqs/s: {}",
+            timing::timestamp(),
+            mms_len,
+            total_time_ms,
+            reqs_len,
+            (reqs_len as f32) / (total_time_s)
+        );
+        Ok(())
+    }
+}
+
+// TODO: When banking is pulled out of RequestStage, add this test back in.
+
+//use bank::Bank;
+//use entry::Entry;
+//use hash::Hash;
+//use record_stage::RecordStage;
+//use record_stage::Signal;
+//use result::Result;
+//use std::sync::mpsc::{channel, Sender};
+//use std::sync::{Arc, Mutex};
+//use std::time::Duration;
+//use transaction::Transaction;
+//
+//#[cfg(test)]
+//mod tests {
+//    use bank::Bank;
+//    use mint::Mint;
+//    use signature::{KeyPair, KeyPairUtil};
+//    use transaction::Transaction;
+//
+//    #[test]
+//    // TODO: Move this test banking_stage. Calling process_transactions() directly
+//    // defeats the purpose of this test.
+//    fn test_banking_sequential_consistency() {
+//        // In this attack we'll demonstrate that a verifier can interpret the ledger
+//        // differently if either the server doesn't signal the ledger to add an
+//        // Entry OR if the verifier tries to parallelize across multiple Entries.
+//        let mint = Mint::new(2);
+//        let bank = Bank::new(&mint);
+//        let banking_stage = EventProcessor::new(bank, &mint.last_id(), None);
+//
+//        // Process a batch that includes a transaction that receives two tokens.
+//        let alice = KeyPair::new();
+//        let tx = Transaction::new(&mint.keypair(), alice.pubkey(), 2, mint.last_id());
+//        let transactions = vec![tx];
+//        let entry0 = banking_stage.process_transactions(transactions).unwrap();
+//
+//        // Process a second batch that spends one of those tokens.
+//        let tx = Transaction::new(&alice, mint.pubkey(), 1, mint.last_id());
+//        let transactions = vec![tx];
+//        let entry1 = banking_stage.process_transactions(transactions).unwrap();
+//
+//        // Collect the ledger and feed it to a new bank.
+//        let entries = vec![entry0, entry1];
+//
+//        // Assert the user holds one token, not two. If the server only output one
+//        // entry, then the second transaction will be rejected, because it drives
+//        // the account balance below zero before the credit is added.
+//        let bank = Bank::new(&mint);
+//        for entry in entries {
+//            assert!(
+//                bank
+//                    .process_transactions(entry.transactions)
+//                    .into_iter()
+//                    .all(|x| x.is_ok())
+//            );
+//        }
+//        assert_eq!(bank.get_balance(&alice.pubkey()), Some(1));
+//    }
+//}
+//
+//#[cfg(all(feature = "unstable", test))]
+//mod bench {
+//    extern crate test;
+//    use self::test::Bencher;
+//    use bank::{Bank, MAX_ENTRY_IDS};
+//    use bincode::serialize;
+//    use hash::hash;
+//    use mint::Mint;
+//    use rayon::prelude::*;
+//    use signature::{KeyPair, KeyPairUtil};
+//    use std::collections::HashSet;
+//    use std::time::Instant;
+//    use transaction::Transaction;
+//
+//    #[bench]
+//    fn bench_process_transactions(_bencher: &mut Bencher) {
+//        let mint = Mint::new(100_000_000);
+//        let bank = Bank::new(&mint);
+//        // Create transactions between unrelated parties.
+//        let txs = 100_000;
+//        let last_ids: Mutex<HashSet<Hash>> = Mutex::new(HashSet::new());
+//        let transactions: Vec<_> = (0..txs)
+//            .into_par_iter()
+//            .map(|i| {
+//                // Seed the 'to' account and a cell for its signature.
+//                let dummy_id = i % (MAX_ENTRY_IDS as i32);
+//                let last_id = hash(&serialize(&dummy_id).unwrap()); // Semi-unique hash
+//                {
+//                    let mut last_ids = last_ids.lock().unwrap();
+//                    if !last_ids.contains(&last_id) {
+//                        last_ids.insert(last_id);
+//                        bank.register_entry_id(&last_id);
+//                    }
+//                }
+//
+//                // Seed the 'from' account.
+//                let rando0 = KeyPair::new();
+//                let tx = Transaction::new(&mint.keypair(), rando0.pubkey(), 1_000, last_id);
+//                bank.process_transaction(&tx).unwrap();
+//
+//                let rando1 = KeyPair::new();
+//                let tx = Transaction::new(&rando0, rando1.pubkey(), 2, last_id);
+//                bank.process_transaction(&tx).unwrap();
+//
+//                // Finally, return a transaction that's unique
+//                Transaction::new(&rando0, rando1.pubkey(), 1, last_id)
+//            })
+//            .collect();
+//
+//        let banking_stage = EventProcessor::new(bank, &mint.last_id(), None);
+//
+//        let now = Instant::now();
+//        assert!(banking_stage.process_transactions(transactions).is_ok());
+//        let duration = now.elapsed();
+//        let sec = duration.as_secs() as f64 + duration.subsec_nanos() as f64 / 1_000_000_000.0;
+//        let tps = txs as f64 / sec;
+//
+//        // Ensure that all transactions were successfully logged.
+//        drop(banking_stage.historian_input);
+//        let entries: Vec<Entry> = banking_stage.output.lock().unwrap().iter().collect();
+//        assert_eq!(entries.len(), 1);
+//        assert_eq!(entries[0].transactions.len(), txs as usize);
+//
+//        println!("{} tps", tps);
+//    }
+//}
+
+#[cfg(all(feature = "unstable", test))]
+mod bench {
+    extern crate test;
+    use self::test::Bencher;
+    use bank::*;
+    use banking_stage::BankingStage;
+    use mint::Mint;
+    use packet::{to_packets, PacketRecycler};
+    use record_stage::Signal;
+    use signature::{KeyPair, KeyPairUtil};
+    use std::iter;
+    use std::sync::mpsc::channel;
+    use std::sync::Arc;
+    use transaction::Transaction;
+
+    #[bench]
+    fn bench_stage(bencher: &mut Bencher) {
+        let tx = 100_usize;
+        let mint = Mint::new(1_000_000_000);
+        let pubkey = KeyPair::new().pubkey();
+
+        let transactions: Vec<_> = (0..tx)
+            .map(|i| Transaction::new(&mint.keypair(), pubkey, i as i64, mint.last_id()))
+            .collect();
+
+        let (verified_sender, verified_receiver) = channel();
+        let (signal_sender, signal_receiver) = channel();
+        let packet_recycler = PacketRecycler::default();
+        let verified: Vec<_> = to_packets(&packet_recycler, transactions)
+            .into_iter()
+            .map(|x| {
+                let len = (*x).read().unwrap().packets.len();
+                (x, iter::repeat(1).take(len).collect())
+            })
+            .collect();
+
+        bencher.iter(move || {
+            let bank = Arc::new(Bank::new(&mint));
+            verified_sender.send(verified.clone()).unwrap();
+            BankingStage::process_packets(
+                bank.clone(),
+                &verified_receiver,
+                &signal_sender,
+                &packet_recycler,
+            ).unwrap();
+            let signal = signal_receiver.recv().unwrap();
+            if let Signal::Events(ref transactions) = signal {
+                assert_eq!(transactions.len(), tx);
+            } else {
+                assert!(false);
+            }
+        });
+    }
+}

src/bin/client-demo.rs

@@ -1,41 +1,151 @@
+extern crate getopts;
+extern crate isatty;
+extern crate pnet;
 extern crate rayon;
 extern crate serde_json;
 extern crate solana;
 
-use solana::accountant_stub::AccountantStub;
-use solana::mint::Mint;
-use solana::signature::{KeyPair, KeyPairUtil};
-use solana::transaction::Transaction;
-use std::io::stdin;
-use std::net::UdpSocket;
-use std::time::{Duration, Instant};
-use std::thread::sleep;
+use getopts::Options;
+use isatty::stdin_isatty;
+use pnet::datalink;
 use rayon::prelude::*;
+use solana::crdt::{Crdt, ReplicatedData};
+use solana::data_replicator::DataReplicator;
+use solana::mint::MintDemo;
+use solana::signature::{GenKeys, KeyPair, KeyPairUtil};
+use solana::streamer::default_window;
+use solana::thin_client::ThinClient;
+use solana::transaction::Transaction;
+use std::env;
+use std::fs::File;
+use std::io::{stdin, Read};
+use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
+use std::process::exit;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, RwLock};
+use std::thread::sleep;
+use std::thread::JoinHandle;
+use std::time::Duration;
+use std::time::Instant;
+
+fn print_usage(program: &str, opts: Options) {
+    let mut brief = format!("Usage: cat <mint.json> | {} [options]\n\n", program);
+    brief += "  Solana client demo creates a number of transactions and\n";
+    brief += "  sends them to a target node.";
+    brief += "  Takes json formatted mint file to stdin.";
+
+    print!("{}", opts.usage(&brief));
+}
+
+fn get_ip_addr() -> Option<IpAddr> {
+    for iface in datalink::interfaces() {
+        for p in iface.ips {
+            if !p.ip().is_loopback() && !p.ip().is_multicast() {
+                return Some(p.ip());
+            }
+        }
+    }
+    None
+}
 
 fn main() {
-    let addr = "127.0.0.1:8000";
-    let send_addr = "127.0.0.1:8001";
+    let mut threads = 4usize;
+    let mut num_nodes = 1usize;
 
-    let mint: Mint = serde_json::from_reader(stdin()).unwrap();
-    let mint_keypair = mint.keypair();
-    let mint_pubkey = mint.pubkey();
+    let mut opts = Options::new();
+    opts.optopt("l", "", "leader", "leader.json");
+    opts.optopt("c", "", "client port", "port");
+    opts.optopt("t", "", "number of threads", &format!("{}", threads));
+    opts.optflag("d", "dyn", "detect network address dynamically");
+    opts.optopt(
+        "n",
+        "",
+        "number of nodes to converge to",
+        &format!("{}", num_nodes),
+    );
+    opts.optflag("h", "help", "print help");
+    let args: Vec<String> = env::args().collect();
+    let matches = match opts.parse(&args[1..]) {
+        Ok(m) => m,
+        Err(e) => {
+            eprintln!("{}", e);
+            exit(1);
+        }
+    };
 
-    let socket = UdpSocket::bind(send_addr).unwrap();
-    let acc = AccountantStub::new(addr, socket);
-    let last_id = acc.get_last_id().unwrap();
+    if matches.opt_present("h") {
+        let program = args[0].clone();
+        print_usage(&program, opts);
+        return;
+    }
+    let mut addr: SocketAddr = "0.0.0.0:8100".parse().unwrap();
+    if matches.opt_present("c") {
+        let port = matches.opt_str("c").unwrap().parse().unwrap();
+        addr.set_port(port);
+    }
+    if matches.opt_present("d") {
+        addr.set_ip(get_ip_addr().unwrap());
+    }
+    let client_addr: Arc<RwLock<SocketAddr>> = Arc::new(RwLock::new(addr));
+    if matches.opt_present("t") {
+        threads = matches.opt_str("t").unwrap().parse().expect("integer");
+    }
+    if matches.opt_present("n") {
+        num_nodes = matches.opt_str("n").unwrap().parse().expect("integer");
+    }
 
-    let mint_balance = acc.get_balance(&mint_pubkey).unwrap().unwrap();
-    println!("Mint's Initial Balance {}", mint_balance);
+    let leader = if matches.opt_present("l") {
+        read_leader(matches.opt_str("l").unwrap())
+    } else {
+        let server_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8000);
+        ReplicatedData::new_leader(&server_addr)
+    };
+
+    let signal = Arc::new(AtomicBool::new(false));
+    let mut c_threads = vec![];
+    let validators = converge(
+        &client_addr,
+        &leader,
+        signal.clone(),
+        num_nodes + 2,
+        &mut c_threads,
+    );
+
+    if stdin_isatty() {
+        eprintln!("nothing found on stdin, expected a json file");
+        exit(1);
+    }
+
+    let mut buffer = String::new();
+    let num_bytes = stdin().read_to_string(&mut buffer).unwrap();
+    if num_bytes == 0 {
+        eprintln!("empty file on stdin, expected a json file");
+        exit(1);
+    }
+
+    println!("Parsing stdin...");
+    let demo: MintDemo = serde_json::from_str(&buffer).unwrap_or_else(|e| {
+        eprintln!("failed to parse json: {}", e);
+        exit(1);
+    });
+    let mut client = mk_client(&client_addr, &leader);
+
+    println!("Get last ID...");
+    let last_id = client.get_last_id();
+    println!("Got last ID {:?}", last_id);
+
+    let rnd = GenKeys::new(demo.mint.keypair().public_key_bytes());
+
+    println!("Creating keypairs...");
+    let txs = demo.num_accounts / 2;
+    let keypairs = rnd.gen_n_keypairs(demo.num_accounts);
+    let keypair_pairs: Vec<_> = keypairs.chunks(2).collect();
 
     println!("Signing transactions...");
-    let txs = 100_000;
     let now = Instant::now();
-    let transactions: Vec<_> = (0..txs)
+    let transactions: Vec<_> = keypair_pairs
         .into_par_iter()
-        .map(|_| {
-            let rando_pubkey = KeyPair::new().pubkey();
-            Transaction::new(&mint_keypair, rando_pubkey, 1, last_id)
-        })
+        .map(|chunk| Transaction::new(&chunk[0], chunk[1].pubkey(), 1, last_id))
         .collect();
     let duration = now.elapsed();
     let ns = duration.as_secs() * 1_000_000_000 + u64::from(duration.subsec_nanos());
@@ -47,27 +157,134 @@ fn main() {
         nsps / 1_000_f64
     );
 
-    println!("Transferring 1 unit {} times...", txs);
-    let now = Instant::now();
-    let mut _sig = Default::default();
-    for tr in transactions {
-        _sig = tr.sig;
-        acc.transfer_signed(tr).unwrap();
-    }
-    println!("Waiting for last transaction to be confirmed...",);
-    let mut val = mint_balance;
-    let mut prev = 0;
-    while val != prev {
-        sleep(Duration::from_millis(20));
-        prev = val;
-        val = acc.get_balance(&mint_pubkey).unwrap().unwrap();
-    }
-    println!("Mint's Final Balance {}", val);
-    let txs = mint_balance - val;
-    println!("Successful transactions {}", txs);
+    let first_count = client.transaction_count();
+    println!("initial count {}", first_count);
 
-    let duration = now.elapsed();
-    let ns = duration.as_secs() * 1_000_000_000 + u64::from(duration.subsec_nanos());
-    let tps = (txs * 1_000_000_000) as f64 / ns as f64;
-    println!("Done. {} tps!", tps);
+    println!("Transfering {} transactions in {} batches", txs, threads);
+    let sz = transactions.len() / threads;
+    let chunks: Vec<_> = transactions.chunks(sz).collect();
+    chunks.into_par_iter().for_each(|txs| {
+        println!(
+            "Transferring 1 unit {} times... to {:?}",
+            txs.len(),
+            leader.transactions_addr
+        );
+        let client = mk_client(&client_addr, &leader);
+        for tx in txs {
+            client.transfer_signed(tx.clone()).unwrap();
+        }
+    });
+
+    println!("Sampling tps every second...",);
+    validators.into_par_iter().for_each(|val| {
+        let mut client = mk_client(&client_addr, &val);
+        let mut now = Instant::now();
+        let mut initial_tx_count = client.transaction_count();
+        for i in 0..100 {
+            let tx_count = client.transaction_count();
+            let duration = now.elapsed();
+            now = Instant::now();
+            let sample = tx_count - initial_tx_count;
+            initial_tx_count = tx_count;
+            println!(
+                "{}: Transactions processed {}",
+                val.transactions_addr, sample
+            );
+            let ns = duration.as_secs() * 1_000_000_000 + u64::from(duration.subsec_nanos());
+            let tps = (sample * 1_000_000_000) as f64 / ns as f64;
+            println!("{}: {} tps", val.transactions_addr, tps);
+            let total = tx_count - first_count;
+            println!(
+                "{}: Total Transactions processed {}",
+                val.transactions_addr, total
+            );
+            if total == transactions.len() as u64 {
+                break;
+            }
+            if i > 20 && sample == 0 {
+                break;
+            }
+            sleep(Duration::new(1, 0));
+        }
+    });
+    signal.store(true, Ordering::Relaxed);
+    for t in c_threads {
+        t.join().unwrap();
+    }
+}
+
+fn mk_client(locked_addr: &Arc<RwLock<SocketAddr>>, r: &ReplicatedData) -> ThinClient {
+    let mut addr = locked_addr.write().unwrap();
+    let port = addr.port();
+    let transactions_socket = UdpSocket::bind(addr.clone()).unwrap();
+    addr.set_port(port + 1);
+    let requests_socket = UdpSocket::bind(addr.clone()).unwrap();
+    addr.set_port(port + 2);
+    ThinClient::new(
+        r.requests_addr,
+        requests_socket,
+        r.transactions_addr,
+        transactions_socket,
+    )
+}
+
+fn spy_node(client_addr: &Arc<RwLock<SocketAddr>>) -> (ReplicatedData, UdpSocket) {
+    let mut addr = client_addr.write().unwrap();
+    let port = addr.port();
+    let gossip = UdpSocket::bind(addr.clone()).unwrap();
+    addr.set_port(port + 1);
+    let daddr = "0.0.0.0:0".parse().unwrap();
+    let pubkey = KeyPair::new().pubkey();
+    let node = ReplicatedData::new(pubkey, gossip.local_addr().unwrap(), daddr, daddr, daddr);
+    (node, gossip)
+}
+
+fn converge(
+    client_addr: &Arc<RwLock<SocketAddr>>,
+    leader: &ReplicatedData,
+    exit: Arc<AtomicBool>,
+    num_nodes: usize,
+    threads: &mut Vec<JoinHandle<()>>,
+) -> Vec<ReplicatedData> {
+    //lets spy on the network
+    let daddr = "0.0.0.0:0".parse().unwrap();
+    let (spy, spy_gossip) = spy_node(client_addr);
+    let mut spy_crdt = Crdt::new(spy);
+    spy_crdt.insert(&leader);
+    spy_crdt.set_leader(leader.id);
+    let spy_ref = Arc::new(RwLock::new(spy_crdt));
+    let window = default_window();
+    let gossip_send_socket = UdpSocket::bind("0.0.0.0:0").expect("bind 0");
+    let data_replicator = DataReplicator::new(
+        spy_ref.clone(),
+        window.clone(),
+        spy_gossip,
+        gossip_send_socket,
+        exit.clone(),
+    ).expect("DataReplicator::new");
+    //wait for the network to converge
+    for _ in 0..30 {
+        let min = spy_ref.read().unwrap().convergence();
+        if num_nodes as u64 == min {
+            println!("converged!");
+            break;
+        }
+        sleep(Duration::new(1, 0));
+    }
+    threads.extend(data_replicator.thread_hdls.into_iter());
+    let v: Vec<ReplicatedData> = spy_ref
+        .read()
+        .unwrap()
+        .table
+        .values()
+        .into_iter()
+        .filter(|x| x.requests_addr != daddr)
+        .map(|x| x.clone())
+        .collect();
+    v.clone()
+}
+
+fn read_leader(path: String) -> ReplicatedData {
+    let file = File::open(path).expect("file");
+    serde_json::from_reader(file).expect("parse")
 }

src/bin/fullnode-config.rs

@@ -0,0 +1,52 @@
+extern crate getopts;
+extern crate serde_json;
+extern crate solana;
+
+use getopts::Options;
+use solana::crdt::{get_ip_addr, parse_port_or_addr, ReplicatedData};
+use std::env;
+use std::io;
+use std::net::SocketAddr;
+use std::process::exit;
+
+fn print_usage(program: &str, opts: Options) {
+    let mut brief = format!("Usage: {} [options]\n\n", program);
+    brief += "  Create a solana fullnode config file\n";
+
+    print!("{}", opts.usage(&brief));
+}
+
+fn main() {
+    let mut opts = Options::new();
+    opts.optopt("b", "", "bind", "bind to port or address");
+    opts.optflag("d", "dyn", "detect network address dynamically");
+    opts.optflag("h", "help", "print help");
+    let args: Vec<String> = env::args().collect();
+    let matches = match opts.parse(&args[1..]) {
+        Ok(m) => m,
+        Err(e) => {
+            eprintln!("{}", e);
+            exit(1);
+        }
+    };
+    if matches.opt_present("h") {
+        let program = args[0].clone();
+        print_usage(&program, opts);
+        return;
+    }
+
+    let bind_addr: SocketAddr = {
+        let mut bind_addr = parse_port_or_addr(matches.opt_str("b"));
+        if matches.opt_present("d") {
+            let ip = get_ip_addr().unwrap();
+            bind_addr.set_ip(ip);
+        }
+        bind_addr
+    };
+
+    // we need all the receiving sockets to be bound within the expected
+    // port range that we open on aws
+    let repl_data = ReplicatedData::new_leader(&bind_addr);
+    let stdout = io::stdout();
+    serde_json::to_writer(stdout, &repl_data).expect("serialize");
+}

src/bin/fullnode.rs

@@ -0,0 +1,160 @@
+extern crate env_logger;
+extern crate getopts;
+extern crate isatty;
+extern crate serde_json;
+extern crate solana;
+#[macro_use]
+extern crate log;
+
+use getopts::Options;
+use isatty::stdin_isatty;
+use solana::bank::Bank;
+use solana::crdt::ReplicatedData;
+use solana::entry::Entry;
+use solana::payment_plan::PaymentPlan;
+use solana::server::Server;
+use solana::transaction::Instruction;
+use std::env;
+use std::fs::File;
+use std::io::{stdin, Read};
+use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket};
+use std::process::exit;
+use std::sync::atomic::AtomicBool;
+use std::sync::Arc;
+//use std::time::Duration;
+
+fn print_usage(program: &str, opts: Options) {
+    let mut brief = format!("Usage: cat <transaction.log> | {} [options]\n\n", program);
+    brief += "  Run a Solana node to handle transactions and\n";
+    brief += "  write a new transaction log to stdout.\n";
+    brief += "  Takes existing transaction log from stdin.";
+
+    print!("{}", opts.usage(&brief));
+}
+
+fn main() {
+    env_logger::init().unwrap();
+    let mut opts = Options::new();
+    opts.optopt("l", "", "load", "load my identity to path.json");
+    opts.optflag("h", "help", "print help");
+    opts.optopt(
+        "v",
+        "",
+        "validator",
+        "run as replicate with path to leader.json",
+    );
+    let args: Vec<String> = env::args().collect();
+    let matches = match opts.parse(&args[1..]) {
+        Ok(m) => m,
+        Err(e) => {
+            eprintln!("{}", e);
+            exit(1);
+        }
+    };
+    if matches.opt_present("h") {
+        let program = args[0].clone();
+        print_usage(&program, opts);
+        return;
+    }
+    if stdin_isatty() {
+        eprintln!("nothing found on stdin, expected a log file");
+        exit(1);
+    }
+
+    let mut buffer = String::new();
+    let num_bytes = stdin().read_to_string(&mut buffer).unwrap();
+    if num_bytes == 0 {
+        eprintln!("empty file on stdin, expected a log file");
+        exit(1);
+    }
+
+    eprintln!("Initializing...");
+    let mut entries = buffer.lines().map(|line| {
+        serde_json::from_str(&line).unwrap_or_else(|e| {
+            eprintln!("failed to parse json: {}", e);
+            exit(1);
+        })
+    });
+
+    eprintln!("done parsing...");
+
+    // The first item in the ledger is required to be an entry with zero num_hashes,
+    // which implies its id can be used as the ledger's seed.
+    let entry0 = entries.next().unwrap();
+
+    // The second item in the ledger is a special transaction where the to and from
+    // fields are the same. That entry should be treated as a deposit, not a
+    // transfer to oneself.
+    let entry1: Entry = entries.next().unwrap();
+    let tx = &entry1.transactions[0];
+    let deposit = if let Instruction::NewContract(contract) = &tx.instruction {
+        contract.plan.final_payment()
+    } else {
+        None
+    };
+
+    eprintln!("creating bank...");
+
+    let bank = Bank::new_from_deposit(&deposit.unwrap());
+    bank.register_entry_id(&entry0.id);
+    bank.register_entry_id(&entry1.id);
+
+    eprintln!("processing entries...");
+    bank.process_entries(entries).expect("process_entries");
+
+    eprintln!("creating networking stack...");
+
+    let exit = Arc::new(AtomicBool::new(false));
+    let bind_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8000);
+    let mut repl_data = ReplicatedData::new_leader(&bind_addr);
+    if matches.opt_present("l") {
+        let path = matches.opt_str("l").unwrap();
+        if let Ok(file) = File::open(path.clone()) {
+            if let Ok(data) = serde_json::from_reader(file) {
+                repl_data = data;
+            } else {
+                warn!("failed to parse leader {}, generating new identity", path);
+            }
+        }
+    }
+    let threads = if matches.opt_present("v") {
+        eprintln!("starting validator... {}", repl_data.requests_addr);
+        let path = matches.opt_str("v").unwrap();
+        let file = File::open(path).expect("file");
+        let leader = serde_json::from_reader(file).expect("parse");
+        let s = Server::new_validator(
+            bank,
+            repl_data.clone(),
+            UdpSocket::bind(repl_data.requests_addr).unwrap(),
+            UdpSocket::bind("0.0.0.0:0").unwrap(),
+            UdpSocket::bind(repl_data.replicate_addr).unwrap(),
+            UdpSocket::bind(repl_data.gossip_addr).unwrap(),
+            leader,
+            exit.clone(),
+        );
+        s.thread_hdls
+    } else {
+        eprintln!("starting leader... {}", repl_data.requests_addr);
+        repl_data.current_leader_id = repl_data.id.clone();
+        let file = File::create("leader.log").expect("leader.log create");
+        let server = Server::new_leader(
+            bank,
+            //Some(Duration::from_millis(1000)),
+            None,
+            repl_data.clone(),
+            UdpSocket::bind(repl_data.requests_addr).unwrap(),
+            UdpSocket::bind(repl_data.transactions_addr).unwrap(),
+            UdpSocket::bind("0.0.0.0:0").unwrap(),
+            UdpSocket::bind("0.0.0.0:0").unwrap(),
+            UdpSocket::bind(repl_data.gossip_addr).unwrap(),
+            exit.clone(),
+            file,
+        );
+        server.thread_hdls
+    };
+    eprintln!("Ready. Listening on {}", repl_data.transactions_addr);
+
+    for t in threads {
+        t.join().expect("join");
+    }
+}

src/bin/genesis-demo.rs

@@ -1,30 +1,77 @@
+extern crate isatty;
+extern crate rayon;
 extern crate serde_json;
 extern crate solana;
 
-use solana::entry::create_entry;
-use solana::event::Event;
-use solana::hash::Hash;
-use solana::mint::Mint;
-use solana::signature::{KeyPair, KeyPairUtil, PublicKey};
+use isatty::stdin_isatty;
+use rayon::prelude::*;
+use solana::bank::MAX_ENTRY_IDS;
+use solana::entry::{next_entry, Entry};
+use solana::mint::MintDemo;
+use solana::signature::{GenKeys, KeyPairUtil};
 use solana::transaction::Transaction;
-use std::io::stdin;
-
-fn transfer(from: &KeyPair, (to, tokens): (PublicKey, i64), last_id: Hash) -> Event {
-    Event::Transaction(Transaction::new(from, to, tokens, last_id))
-}
+use std::io::{stdin, Read};
+use std::process::exit;
 
+// Generate a ledger with lots and lots of accounts.
 fn main() {
-    let mint: Mint = serde_json::from_reader(stdin()).unwrap();
-    let mut entries = mint.create_entries();
+    if stdin_isatty() {
+        eprintln!("nothing found on stdin, expected a json file");
+        exit(1);
+    }
 
-    let from = mint.keypair();
-    let seed = mint.seed();
-    let alice = (KeyPair::new().pubkey(), 200);
-    let bob = (KeyPair::new().pubkey(), 100);
-    let events = vec![transfer(&from, alice, seed), transfer(&from, bob, seed)];
-    entries.push(create_entry(&seed, 0, events));
+    let mut buffer = String::new();
+    let num_bytes = stdin().read_to_string(&mut buffer).unwrap();
+    if num_bytes == 0 {
+        eprintln!("empty file on stdin, expected a json file");
+        exit(1);
+    }
 
-    for entry in entries {
+    let demo: MintDemo = serde_json::from_str(&buffer).unwrap_or_else(|e| {
+        eprintln!("failed to parse json: {}", e);
+        exit(1);
+    });
+
+    let rnd = GenKeys::new(demo.mint.keypair().public_key_bytes());
+    let num_accounts = demo.num_accounts;
+    let tokens_per_user = 1_000;
+
+    let keypairs = rnd.gen_n_keypairs(num_accounts);
+
+    let mint_keypair = demo.mint.keypair();
+    let last_id = demo.mint.last_id();
+
+    for entry in demo.mint.create_entries() {
         println!("{}", serde_json::to_string(&entry).unwrap());
     }
+
+    eprintln!("Creating {} empty entries...", MAX_ENTRY_IDS);
+
+    // Offer client lots of entry IDs to use for each transaction's last_id.
+    let mut last_id = last_id;
+    let mut last_ids = vec![];
+    for _ in 0..MAX_ENTRY_IDS {
+        let entry = next_entry(&last_id, 1, vec![]);
+        last_id = entry.id;
+        last_ids.push(last_id);
+        let serialized = serde_json::to_string(&entry).unwrap_or_else(|e| {
+            eprintln!("failed to serialize: {}", e);
+            exit(1);
+        });
+        println!("{}", serialized);
+    }
+
+    eprintln!("Creating {} transactions...", num_accounts);
+    let transactions: Vec<_> = keypairs
+        .into_par_iter()
+        .enumerate()
+        .map(|(i, rando)| {
+            let last_id = last_ids[i % MAX_ENTRY_IDS];
+            Transaction::new(&mint_keypair, rando.pubkey(), tokens_per_user, last_id)
+        })
+        .collect();
+
+    eprintln!("Logging the creation of {} accounts...", num_accounts);
+    let entry = Entry::new(&last_id, 0, transactions);
+    println!("{}", serde_json::to_string(&entry).unwrap());
 }

src/bin/genesis.rs

@@ -1,14 +1,36 @@
 //! A command-line executable for generating the chain's genesis block.
 
+extern crate isatty;
 extern crate serde_json;
 extern crate solana;
 
+use isatty::stdin_isatty;
 use solana::mint::Mint;
-use std::io::stdin;
+use std::io::{stdin, Read};
+use std::process::exit;
 
 fn main() {
-    let mint: Mint = serde_json::from_reader(stdin()).unwrap();
+    if stdin_isatty() {
+        eprintln!("nothing found on stdin, expected a json file");
+        exit(1);
+    }
+
+    let mut buffer = String::new();
+    let num_bytes = stdin().read_to_string(&mut buffer).unwrap();
+    if num_bytes == 0 {
+        eprintln!("empty file on stdin, expected a json file");
+        exit(1);
+    }
+
+    let mint: Mint = serde_json::from_str(&buffer).unwrap_or_else(|e| {
+        eprintln!("failed to parse json: {}", e);
+        exit(1);
+    });
     for x in mint.create_entries() {
-        println!("{}", serde_json::to_string(&x).unwrap());
+        let serialized = serde_json::to_string(&x).unwrap_or_else(|e| {
+            eprintln!("failed to serialize: {}", e);
+            exit(1);
+        });
+        println!("{}", serialized);
     }
 }

src/bin/historian-demo.rs

@@ -1,37 +0,0 @@
-extern crate solana;
-
-use solana::entry::Entry;
-use solana::event::Event;
-use solana::hash::Hash;
-use solana::historian::Historian;
-use solana::ledger::verify_slice;
-use solana::recorder::Signal;
-use solana::signature::{KeyPair, KeyPairUtil};
-use solana::transaction::Transaction;
-use std::sync::mpsc::SendError;
-use std::thread::sleep;
-use std::time::Duration;
-
-fn create_ledger(hist: &Historian, seed: &Hash) -> Result<(), SendError<Signal>> {
-    sleep(Duration::from_millis(15));
-    let keypair = KeyPair::new();
-    let tr = Transaction::new(&keypair, keypair.pubkey(), 42, *seed);
-    let signal0 = Signal::Event(Event::Transaction(tr));
-    hist.sender.send(signal0)?;
-    sleep(Duration::from_millis(10));
-    Ok(())
-}
-
-fn main() {
-    let seed = Hash::default();
-    let hist = Historian::new(&seed, Some(10));
-    create_ledger(&hist, &seed).expect("send error");
-    drop(hist.sender);
-    let entries: Vec<Entry> = hist.receiver.iter().collect();
-    for entry in &entries {
-        println!("{:?}", entry);
-    }
-    // Proof-of-History: Verify the historian learned about the events
-    // in the same order they appear in the vector.
-    assert!(verify_slice(&entries, &seed));
-}

src/bin/mint-demo.rs

@@ -0,0 +1,21 @@
+extern crate rayon;
+extern crate ring;
+extern crate serde_json;
+extern crate solana;
+
+use solana::mint::{Mint, MintDemo};
+use std::io;
+
+fn main() {
+    let mut input_text = String::new();
+    io::stdin().read_line(&mut input_text).unwrap();
+    let trimmed = input_text.trim();
+    let tokens = trimmed.parse::<i64>().unwrap();
+
+    let mint = Mint::new(tokens);
+    let tokens_per_user = 1_000;
+    let num_accounts = tokens / tokens_per_user;
+
+    let demo = MintDemo { mint, num_accounts };
+    println!("{}", serde_json::to_string(&demo).unwrap());
+}

src/bin/mint.rs

@@ -1,15 +1,29 @@
+extern crate isatty;
 extern crate serde_json;
 extern crate solana;
 
+use isatty::stdin_isatty;
 use solana::mint::Mint;
 use std::io;
+use std::process::exit;
 
 fn main() {
     let mut input_text = String::new();
+    if stdin_isatty() {
+        eprintln!("nothing found on stdin, expected a token number");
+        exit(1);
+    }
+
     io::stdin().read_line(&mut input_text).unwrap();
     let trimmed = input_text.trim();
-    let tokens = trimmed.parse::<i64>().unwrap();
-
+    let tokens = trimmed.parse::<i64>().unwrap_or_else(|e| {
+        eprintln!("{}", e);
+        exit(1);
+    });
     let mint = Mint::new(tokens);
-    println!("{}", serde_json::to_string(&mint).unwrap());
+    let serialized = serde_json::to_string(&mint).unwrap_or_else(|e| {
+        eprintln!("failed to serialize: {}", e);
+        exit(1);
+    });
+    println!("{}", serialized);
 }

src/bin/testnode.rs

@@ -1,25 +0,0 @@
-extern crate serde_json;
-extern crate solana;
-
-use solana::accountant::Accountant;
-use solana::accountant_skel::AccountantSkel;
-use std::io::{self, stdout, BufRead};
-use std::sync::atomic::AtomicBool;
-use std::sync::{Arc, Mutex};
-
-fn main() {
-    let addr = "127.0.0.1:8000";
-    let stdin = io::stdin();
-    let entries = stdin
-        .lock()
-        .lines()
-        .map(|line| serde_json::from_str(&line.unwrap()).unwrap());
-    let acc = Accountant::new_from_entries(entries, Some(1000));
-    let exit = Arc::new(AtomicBool::new(false));
-    let skel = Arc::new(Mutex::new(AccountantSkel::new(acc, stdout())));
-    eprintln!("Listening on {}", addr);
-    let threads = AccountantSkel::serve(skel, addr, exit.clone()).unwrap();
-    for t in threads {
-        t.join().expect("join");
-    }
-}

src/budget.rs

@@ -1,17 +1,13 @@
-//! The `plan` module provides a domain-specific language for payment plans. Users create Plan objects that
-//! are given to an interpreter. The interpreter listens for `Witness` events,
-//! which it uses to reduce the payment plan. When the plan is reduced to a
+//! The `budget` module provides a domain-specific language for payment plans. Users create Budget objects that
+//! are given to an interpreter. The interpreter listens for `Witness` transactions,
+//! which it uses to reduce the payment plan. When the budget is reduced to a
 //! `Payment`, the payment is executed.
 
 use chrono::prelude::*;
+use payment_plan::{Payment, PaymentPlan, Witness};
 use signature::PublicKey;
 use std::mem;
 
-pub enum Witness {
-    Timestamp(DateTime<Utc>),
-    Signature(PublicKey),
-}
-
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
 pub enum Condition {
     Timestamp(DateTime<Utc>),
@@ -29,36 +25,31 @@ impl Condition {
     }
 }
 
+#[repr(C)]
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
-pub struct Payment {
-    pub tokens: i64,
-    pub to: PublicKey,
-}
-
-#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
-pub enum Plan {
+pub enum Budget {
     Pay(Payment),
     After(Condition, Payment),
     Race((Condition, Payment), (Condition, Payment)),
 }
 
-impl Plan {
-    /// Create the simplest spending plan - one that pays `tokens` to PublicKey.
+impl Budget {
+    /// Create the simplest budget - one that pays `tokens` to PublicKey.
     pub fn new_payment(tokens: i64, to: PublicKey) -> Self {
-        Plan::Pay(Payment { tokens, to })
+        Budget::Pay(Payment { tokens, to })
     }
 
-    /// Create a spending plan that pays `tokens` to `to` after being witnessed by `from`.
+    /// Create a budget that pays `tokens` to `to` after being witnessed by `from`.
     pub fn new_authorized_payment(from: PublicKey, tokens: i64, to: PublicKey) -> Self {
-        Plan::After(Condition::Signature(from), Payment { tokens, to })
+        Budget::After(Condition::Signature(from), Payment { tokens, to })
     }
 
-    /// Create a spending plan that pays `tokens` to `to` after the given DateTime.
+    /// Create a budget that pays `tokens` to `to` after the given DateTime.
     pub fn new_future_payment(dt: DateTime<Utc>, tokens: i64, to: PublicKey) -> Self {
-        Plan::After(Condition::Timestamp(dt), Payment { tokens, to })
+        Budget::After(Condition::Timestamp(dt), Payment { tokens, to })
     }
 
-    /// Create a spending plan that pays `tokens` to `to` after the given DateTime
+    /// Create a budget that pays `tokens` to `to` after the given DateTime
     /// unless cancelled by `from`.
     pub fn new_cancelable_future_payment(
         dt: DateTime<Utc>,
@@ -66,44 +57,46 @@ impl Plan {
         tokens: i64,
         to: PublicKey,
     ) -> Self {
-        Plan::Race(
+        Budget::Race(
             (Condition::Timestamp(dt), Payment { tokens, to }),
             (Condition::Signature(from), Payment { tokens, to: from }),
         )
     }
+}
 
-    /// Return true if the spending plan requires no additional Witnesses.
-    pub fn is_complete(&self) -> bool {
+impl PaymentPlan for Budget {
+    /// Return Payment if the budget requires no additional Witnesses.
+    fn final_payment(&self) -> Option<Payment> {
         match *self {
-            Plan::Pay(_) => true,
-            _ => false,
+            Budget::Pay(ref payment) => Some(payment.clone()),
+            _ => None,
         }
     }
 
-    /// Return true if the plan spends exactly `spendable_tokens`.
-    pub fn verify(&self, spendable_tokens: i64) -> bool {
+    /// Return true if the budget spends exactly `spendable_tokens`.
+    fn verify(&self, spendable_tokens: i64) -> bool {
         match *self {
-            Plan::Pay(ref payment) | Plan::After(_, ref payment) => {
+            Budget::Pay(ref payment) | Budget::After(_, ref payment) => {
                 payment.tokens == spendable_tokens
             }
-            Plan::Race(ref a, ref b) => {
+            Budget::Race(ref a, ref b) => {
                 a.1.tokens == spendable_tokens && b.1.tokens == spendable_tokens
             }
         }
     }
 
-    /// Apply a witness to the spending plan to see if the plan can be reduced.
-    /// If so, modify the plan in-place.
-    pub fn apply_witness(&mut self, witness: &Witness) {
+    /// Apply a witness to the budget to see if the budget can be reduced.
+    /// If so, modify the budget in-place.
+    fn apply_witness(&mut self, witness: &Witness) {
         let new_payment = match *self {
-            Plan::After(ref cond, ref payment) if cond.is_satisfied(witness) => Some(payment),
-            Plan::Race((ref cond, ref payment), _) if cond.is_satisfied(witness) => Some(payment),
-            Plan::Race(_, (ref cond, ref payment)) if cond.is_satisfied(witness) => Some(payment),
+            Budget::After(ref cond, ref payment) if cond.is_satisfied(witness) => Some(payment),
+            Budget::Race((ref cond, ref payment), _) if cond.is_satisfied(witness) => Some(payment),
+            Budget::Race(_, (ref cond, ref payment)) if cond.is_satisfied(witness) => Some(payment),
             _ => None,
         }.cloned();
 
         if let Some(payment) = new_payment {
-            mem::replace(self, Plan::Pay(payment));
+            mem::replace(self, Budget::Pay(payment));
         }
     }
 }
@@ -128,14 +121,14 @@ mod tests {
     }
 
     #[test]
-    fn test_verify_plan() {
+    fn test_verify() {
         let dt = Utc.ymd(2014, 11, 14).and_hms(8, 9, 10);
         let from = PublicKey::default();
         let to = PublicKey::default();
-        assert!(Plan::new_payment(42, to).verify(42));
-        assert!(Plan::new_authorized_payment(from, 42, to).verify(42));
-        assert!(Plan::new_future_payment(dt, 42, to).verify(42));
-        assert!(Plan::new_cancelable_future_payment(dt, from, 42, to).verify(42));
+        assert!(Budget::new_payment(42, to).verify(42));
+        assert!(Budget::new_authorized_payment(from, 42, to).verify(42));
+        assert!(Budget::new_future_payment(dt, 42, to).verify(42));
+        assert!(Budget::new_cancelable_future_payment(dt, from, 42, to).verify(42));
     }
 
     #[test]
@@ -143,9 +136,9 @@ mod tests {
         let from = PublicKey::default();
         let to = PublicKey::default();
 
-        let mut plan = Plan::new_authorized_payment(from, 42, to);
-        plan.apply_witness(&Witness::Signature(from));
-        assert_eq!(plan, Plan::new_payment(42, to));
+        let mut budget = Budget::new_authorized_payment(from, 42, to);
+        budget.apply_witness(&Witness::Signature(from));
+        assert_eq!(budget, Budget::new_payment(42, to));
     }
 
     #[test]
@@ -153,9 +146,9 @@ mod tests {
         let dt = Utc.ymd(2014, 11, 14).and_hms(8, 9, 10);
         let to = PublicKey::default();
 
-        let mut plan = Plan::new_future_payment(dt, 42, to);
-        plan.apply_witness(&Witness::Timestamp(dt));
-        assert_eq!(plan, Plan::new_payment(42, to));
+        let mut budget = Budget::new_future_payment(dt, 42, to);
+        budget.apply_witness(&Witness::Timestamp(dt));
+        assert_eq!(budget, Budget::new_payment(42, to));
     }
 
     #[test]
@@ -164,12 +157,12 @@ mod tests {
         let from = PublicKey::default();
         let to = PublicKey::default();
 
-        let mut plan = Plan::new_cancelable_future_payment(dt, from, 42, to);
-        plan.apply_witness(&Witness::Timestamp(dt));
-        assert_eq!(plan, Plan::new_payment(42, to));
+        let mut budget = Budget::new_cancelable_future_payment(dt, from, 42, to);
+        budget.apply_witness(&Witness::Timestamp(dt));
+        assert_eq!(budget, Budget::new_payment(42, to));
 
-        let mut plan = Plan::new_cancelable_future_payment(dt, from, 42, to);
-        plan.apply_witness(&Witness::Signature(from));
-        assert_eq!(plan, Plan::new_payment(42, from));
+        let mut budget = Budget::new_cancelable_future_payment(dt, from, 42, to);
+        budget.apply_witness(&Witness::Signature(from));
+        assert_eq!(budget, Budget::new_payment(42, from));
     }
 }

src/crdt.rs

@@ -0,0 +1,788 @@
+//! The `crdt` module defines a data structure that is shared by all the nodes in the network over
+//! a gossip control plane.  The goal is to share small bits of off-chain information and detect and
+//! repair partitions.
+//!
+//! This CRDT only supports a very limited set of types.  A map of PublicKey -> Versioned Struct.
+//! The last version is always picked during an update.
+//!
+//! The network is arranged in layers:
+//!
+//! * layer 0 - Leader.
+//! * layer 1 - As many nodes as we can fit
+//! * layer 2 - Everyone else, if layer 1 is `2^10`, layer 2 should be able to fit `2^20` number of nodes.
+//!
+//! Bank needs to provide an interface for us to query the stake weight
+
+use bincode::{deserialize, serialize};
+use byteorder::{LittleEndian, ReadBytesExt};
+use hash::Hash;
+use packet::{to_blob, Blob, BlobRecycler, SharedBlob, BLOB_SIZE};
+use pnet::datalink;
+use rayon::prelude::*;
+use result::{Error, Result};
+use ring::rand::{SecureRandom, SystemRandom};
+use signature::{KeyPair, KeyPairUtil};
+use signature::{PublicKey, Signature};
+use std;
+use std::collections::HashMap;
+use std::collections::VecDeque;
+use std::io::Cursor;
+use std::net::{IpAddr, SocketAddr, UdpSocket};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, RwLock};
+use std::thread::{sleep, Builder, JoinHandle};
+use std::time::Duration;
+use streamer::{BlobReceiver, BlobSender};
+
+pub fn parse_port_or_addr(optstr: Option<String>) -> SocketAddr {
+    let daddr: SocketAddr = "0.0.0.0:8000".parse().expect("default socket address");
+    if let Some(addrstr) = optstr {
+        if let Ok(port) = addrstr.parse() {
+            let mut addr = daddr.clone();
+            addr.set_port(port);
+            addr
+        } else if let Ok(addr) = addrstr.parse() {
+            addr
+        } else {
+            daddr
+        }
+    } else {
+        daddr
+    }
+}
+
+pub fn get_ip_addr() -> Option<IpAddr> {
+    for iface in datalink::interfaces() {
+        for p in iface.ips {
+            if !p.ip().is_loopback() && !p.ip().is_multicast() {
+                return Some(p.ip());
+            }
+        }
+    }
+    None
+}
+
+/// Structure to be replicated by the network
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+pub struct ReplicatedData {
+    pub id: PublicKey,
+    sig: Signature,
+    /// should always be increasing
+    pub version: u64,
+    /// address to connect to for gossip
+    pub gossip_addr: SocketAddr,
+    /// address to connect to for replication
+    pub replicate_addr: SocketAddr,
+    /// address to connect to when this node is leader
+    pub requests_addr: SocketAddr,
+    /// transactions address
+    pub transactions_addr: SocketAddr,
+    /// current leader identity
+    pub current_leader_id: PublicKey,
+    /// last verified hash that was submitted to the leader
+    last_verified_hash: Hash,
+    /// last verified count, always increasing
+    last_verified_count: u64,
+}
+
+impl ReplicatedData {
+    pub fn new(
+        id: PublicKey,
+        gossip_addr: SocketAddr,
+        replicate_addr: SocketAddr,
+        requests_addr: SocketAddr,
+        transactions_addr: SocketAddr,
+    ) -> ReplicatedData {
+        ReplicatedData {
+            id,
+            sig: Signature::default(),
+            version: 0,
+            gossip_addr,
+            replicate_addr,
+            requests_addr,
+            transactions_addr,
+            current_leader_id: PublicKey::default(),
+            last_verified_hash: Hash::default(),
+            last_verified_count: 0,
+        }
+    }
+
+    fn next_port(addr: &SocketAddr, nxt: u16) -> SocketAddr {
+        let mut nxt_addr = addr.clone();
+        nxt_addr.set_port(addr.port() + nxt);
+        nxt_addr
+    }
+
+    pub fn new_leader(bind_addr: &SocketAddr) -> Self {
+        let transactions_addr = bind_addr.clone();
+        let gossip_addr = Self::next_port(&bind_addr, 1);
+        let replicate_addr = Self::next_port(&bind_addr, 2);
+        let requests_addr = Self::next_port(&bind_addr, 3);
+        let pubkey = KeyPair::new().pubkey();
+        ReplicatedData::new(
+            pubkey,
+            gossip_addr,
+            replicate_addr,
+            requests_addr,
+            transactions_addr,
+        )
+    }
+}
+
+/// `Crdt` structure keeps a table of `ReplicatedData` structs
+/// # Properties
+/// * `table` - map of public id's to versioned and signed ReplicatedData structs
+/// * `local` - map of public id's to what `self.update_index` `self.table` was updated
+/// * `remote` - map of public id's to the `remote.update_index` was sent
+/// * `update_index` - my update index
+/// # Remarks
+/// This implements two services, `gossip` and `listen`.
+/// * `gossip` - asynchronously ask nodes to send updates
+/// * `listen` - listen for requests and responses
+/// No attempt to keep track of timeouts or dropped requests is made, or should be.
+pub struct Crdt {
+    pub table: HashMap<PublicKey, ReplicatedData>,
+    /// Value of my update index when entry in table was updated.
+    /// Nodes will ask for updates since `update_index`, and this node
+    /// should respond with all the identities that are greater then the
+    /// request's `update_index` in this list
+    local: HashMap<PublicKey, u64>,
+    /// The value of the remote update index that I have last seen
+    /// This Node will ask external nodes for updates since the value in this list
+    pub remote: HashMap<PublicKey, u64>,
+    pub update_index: u64,
+    pub me: PublicKey,
+    timeout: Duration,
+}
+// TODO These messages should be signed, and go through the gpu pipeline for spam filtering
+#[derive(Serialize, Deserialize)]
+enum Protocol {
+    /// forward your own latest data structure when requesting an update
+    /// this doesn't update the `remote` update index, but it allows the
+    /// recepient of this request to add knowledge of this node to the network
+    RequestUpdates(u64, ReplicatedData),
+    //TODO might need a since?
+    /// from id, form's last update index, ReplicatedData
+    ReceiveUpdates(PublicKey, u64, Vec<ReplicatedData>),
+    /// ask for a missing index
+    RequestWindowIndex(ReplicatedData, u64),
+}
+
+impl Crdt {
+    pub fn new(me: ReplicatedData) -> Crdt {
+        assert_eq!(me.version, 0);
+        let mut g = Crdt {
+            table: HashMap::new(),
+            local: HashMap::new(),
+            remote: HashMap::new(),
+            me: me.id,
+            update_index: 1,
+            timeout: Duration::from_millis(100),
+        };
+        g.local.insert(me.id, g.update_index);
+        g.table.insert(me.id, me);
+        g
+    }
+    pub fn my_data(&self) -> &ReplicatedData {
+        &self.table[&self.me]
+    }
+    pub fn leader_data(&self) -> &ReplicatedData {
+        &self.table[&self.table[&self.me].current_leader_id]
+    }
+
+    pub fn set_leader(&mut self, key: PublicKey) -> () {
+        let mut me = self.my_data().clone();
+        me.current_leader_id = key;
+        me.version += 1;
+        self.insert(&me);
+    }
+
+    pub fn insert(&mut self, v: &ReplicatedData) {
+        // TODO check that last_verified types are always increasing
+        if self.table.get(&v.id).is_none() || (v.version > self.table[&v.id].version) {
+            //somehow we signed a message for our own identity with a higher version that
+            // we have stored ourselves
+            trace!(
+                "me: {:?} v.id: {:?} version: {}",
+                &self.me[..4],
+                &v.id[..4],
+                v.version
+            );
+            self.update_index += 1;
+            let _ = self.table.insert(v.id.clone(), v.clone());
+            let _ = self.local.insert(v.id, self.update_index);
+        } else {
+            trace!(
+                "INSERT FAILED me: {:?} data: {:?} new.version: {} me.version: {}",
+                &self.me[..4],
+                &v.id[..4],
+                v.version,
+                self.table[&v.id].version
+            );
+        }
+    }
+
+    /// broadcast messages from the leader to layer 1 nodes
+    /// # Remarks
+    /// We need to avoid having obj locked while doing any io, such as the `send_to`
+    pub fn broadcast(
+        obj: &Arc<RwLock<Self>>,
+        blobs: &Vec<SharedBlob>,
+        s: &UdpSocket,
+        transmit_index: &mut u64,
+    ) -> Result<()> {
+        let (me, table): (ReplicatedData, Vec<ReplicatedData>) = {
+            // copy to avoid locking during IO
+            let robj = obj.read().expect("'obj' read lock in pub fn broadcast");
+            trace!("broadcast table {}", robj.table.len());
+            let cloned_table: Vec<ReplicatedData> = robj.table.values().cloned().collect();
+            (robj.table[&robj.me].clone(), cloned_table)
+        };
+        let daddr = "0.0.0.0:0".parse().unwrap();
+        let nodes: Vec<&ReplicatedData> = table
+            .iter()
+            .filter(|v| {
+                if me.id == v.id {
+                    //filter myself
+                    false
+                } else if v.replicate_addr == daddr {
+                    //filter nodes that are not listening
+                    false
+                } else {
+                    trace!("broadcast node {}", v.replicate_addr);
+                    true
+                }
+            })
+            .collect();
+        if nodes.len() < 1 {
+            warn!("crdt too small");
+            return Err(Error::CrdtTooSmall);
+        }
+        trace!("nodes table {}", nodes.len());
+        trace!("blobs table {}", blobs.len());
+        // enumerate all the blobs, those are the indices
+        // transmit them to nodes, starting from a different node
+        let orders: Vec<_> = blobs
+            .iter()
+            .enumerate()
+            .zip(
+                nodes
+                    .iter()
+                    .cycle()
+                    .skip((*transmit_index as usize) % nodes.len()),
+            )
+            .collect();
+        trace!("orders table {}", orders.len());
+        let errs: Vec<_> = orders
+            .into_iter()
+            .map(|((i, b), v)| {
+                // only leader should be broadcasting
+                assert!(me.current_leader_id != v.id);
+                let mut blob = b.write().expect("'b' write lock in pub fn broadcast");
+                blob.set_id(me.id).expect("set_id in pub fn broadcast");
+                blob.set_index(*transmit_index + i as u64)
+                    .expect("set_index in pub fn broadcast");
+                //TODO profile this, may need multiple sockets for par_iter
+                trace!("broadcast {} to {}", blob.meta.size, v.replicate_addr);
+                assert!(blob.meta.size < BLOB_SIZE);
+                let e = s.send_to(&blob.data[..blob.meta.size], &v.replicate_addr);
+                trace!("done broadcast {} to {}", blob.meta.size, v.replicate_addr);
+                e
+            })
+            .collect();
+        trace!("broadcast results {}", errs.len());
+        for e in errs {
+            match e {
+                Err(e) => {
+                    error!("broadcast result {:?}", e);
+                    return Err(Error::IO(e));
+                }
+                _ => (),
+            }
+            *transmit_index += 1;
+        }
+        Ok(())
+    }
+
+    /// retransmit messages from the leader to layer 1 nodes
+    /// # Remarks
+    /// We need to avoid having obj locked while doing any io, such as the `send_to`
+    pub fn retransmit(obj: &Arc<RwLock<Self>>, blob: &SharedBlob, s: &UdpSocket) -> Result<()> {
+        let (me, table): (ReplicatedData, Vec<ReplicatedData>) = {
+            // copy to avoid locking during IO
+            let s = obj.read().expect("'obj' read lock in pub fn retransmit");
+            (s.table[&s.me].clone(), s.table.values().cloned().collect())
+        };
+        blob.write()
+            .unwrap()
+            .set_id(me.id)
+            .expect("set_id in pub fn retransmit");
+        let rblob = blob.read().unwrap();
+        let daddr = "0.0.0.0:0".parse().unwrap();
+        let orders: Vec<_> = table
+            .iter()
+            .filter(|v| {
+                if me.id == v.id {
+                    false
+                } else if me.current_leader_id == v.id {
+                    trace!("skip retransmit to leader {:?}", v.id);
+                    false
+                } else if v.replicate_addr == daddr {
+                    trace!("skip nodes that are not listening {:?}", v.id);
+                    false
+                } else {
+                    true
+                }
+            })
+            .collect();
+        let errs: Vec<_> = orders
+            .par_iter()
+            .map(|v| {
+                trace!(
+                    "retransmit blob {} to {}",
+                    rblob.get_index().unwrap(),
+                    v.replicate_addr
+                );
+                //TODO profile this, may need multiple sockets for par_iter
+                assert!(rblob.meta.size < BLOB_SIZE);
+                s.send_to(&rblob.data[..rblob.meta.size], &v.replicate_addr)
+            })
+            .collect();
+        for e in errs {
+            match e {
+                Err(e) => {
+                    info!("retransmit error {:?}", e);
+                    return Err(Error::IO(e));
+                }
+                _ => (),
+            }
+        }
+        Ok(())
+    }
+
+    // max number of nodes that we could be converged to
+    pub fn convergence(&self) -> u64 {
+        let max = self.remote.values().len() as u64 + 1;
+        self.remote.values().fold(max, |a, b| std::cmp::min(a, *b))
+    }
+
+    fn random() -> u64 {
+        let rnd = SystemRandom::new();
+        let mut buf = [0u8; 8];
+        rnd.fill(&mut buf).expect("rnd.fill in pub fn random");
+        let mut rdr = Cursor::new(&buf);
+        rdr.read_u64::<LittleEndian>()
+            .expect("rdr.read_u64 in fn random")
+    }
+    fn get_updates_since(&self, v: u64) -> (PublicKey, u64, Vec<ReplicatedData>) {
+        //trace!("get updates since {}", v);
+        let data = self.table
+            .values()
+            .filter(|x| self.local[&x.id] > v)
+            .cloned()
+            .collect();
+        let id = self.me;
+        let ups = self.update_index;
+        (id, ups, data)
+    }
+
+    pub fn window_index_request(&self, ix: u64) -> Result<(SocketAddr, Vec<u8>)> {
+        let daddr = "0.0.0.0:0".parse().unwrap();
+        let valid: Vec<_> = self.table
+            .values()
+            .filter(|r| r.id != self.me && r.replicate_addr != daddr)
+            .collect();
+        if valid.is_empty() {
+            return Err(Error::CrdtTooSmall);
+        }
+        let n = (Self::random() as usize) % valid.len();
+        let addr = valid[n].gossip_addr.clone();
+        let req = Protocol::RequestWindowIndex(self.table[&self.me].clone(), ix);
+        let out = serialize(&req)?;
+        Ok((addr, out))
+    }
+
+    /// Create a random gossip request
+    /// # Returns
+    /// (A,B)
+    /// * A - Address to send to
+    /// * B - RequestUpdates protocol message
+    fn gossip_request(&self) -> Result<(SocketAddr, Protocol)> {
+        let options: Vec<_> = self.table.values().filter(|v| v.id != self.me).collect();
+        if options.len() < 1 {
+            trace!(
+                "crdt too small for gossip {:?} {}",
+                &self.me[..4],
+                self.table.len()
+            );
+            return Err(Error::CrdtTooSmall);
+        }
+        let n = (Self::random() as usize) % options.len();
+        let v = options[n].clone();
+        let remote_update_index = *self.remote.get(&v.id).unwrap_or(&0);
+        let req = Protocol::RequestUpdates(remote_update_index, self.table[&self.me].clone());
+        trace!(
+            "created gossip request from {:?} to {:?} {}",
+            &self.me[..4],
+            &v.id[..4],
+            v.gossip_addr
+        );
+        Ok((v.gossip_addr, req))
+    }
+
+    /// At random pick a node and try to get updated changes from them
+    fn run_gossip(
+        obj: &Arc<RwLock<Self>>,
+        blob_sender: &BlobSender,
+        blob_recycler: &BlobRecycler,
+    ) -> Result<()> {
+        //TODO we need to keep track of stakes and weight the selection by stake size
+        //TODO cache sockets
+
+        // Lock the object only to do this operation and not for any longer
+        // especially not when doing the `sock.send_to`
+        let (remote_gossip_addr, req) = obj.read()
+            .expect("'obj' read lock in fn run_gossip")
+            .gossip_request()?;
+        // TODO this will get chatty, so we need to first ask for number of updates since
+        // then only ask for specific data that we dont have
+        let blob = to_blob(req, remote_gossip_addr, blob_recycler)?;
+        let mut q: VecDeque<SharedBlob> = VecDeque::new();
+        q.push_back(blob);
+        blob_sender.send(q)?;
+        Ok(())
+    }
+
+    /// Apply updates that we received from the identity `from`
+    /// # Arguments
+    /// * `from` - identity of the sender of the updates
+    /// * `update_index` - the number of updates that `from` has completed and this set of `data` represents
+    /// * `data` - the update data
+    fn apply_updates(&mut self, from: PublicKey, update_index: u64, data: &[ReplicatedData]) {
+        trace!("got updates {}", data.len());
+        // TODO we need to punish/spam resist here
+        // sig verify the whole update and slash anyone who sends a bad update
+        for v in data {
+            self.insert(&v);
+        }
+        *self.remote.entry(from).or_insert(update_index) = update_index;
+    }
+
+    /// randomly pick a node and ask them for updates asynchronously
+    pub fn gossip(
+        obj: Arc<RwLock<Self>>,
+        blob_recycler: BlobRecycler,
+        blob_sender: BlobSender,
+        exit: Arc<AtomicBool>,
+    ) -> JoinHandle<()> {
+        Builder::new()
+            .name("solana-gossip".to_string())
+            .spawn(move || loop {
+                let _ = Self::run_gossip(&obj, &blob_sender, &blob_recycler);
+                if exit.load(Ordering::Relaxed) {
+                    return;
+                }
+                //TODO this should be a tuned parameter
+                sleep(
+                    obj.read()
+                        .expect("'obj' read lock in pub fn gossip")
+                        .timeout,
+                );
+            })
+            .unwrap()
+    }
+    fn run_window_request(
+        window: &Arc<RwLock<Vec<Option<SharedBlob>>>>,
+        from: &ReplicatedData,
+        ix: u64,
+        blob_recycler: &BlobRecycler,
+    ) -> Option<SharedBlob> {
+        let pos = (ix as usize) % window.read().unwrap().len();
+        if let &Some(ref blob) = &window.read().unwrap()[pos] {
+            let rblob = blob.read().unwrap();
+            let blob_ix = rblob.get_index().expect("run_window_request get_index");
+            if blob_ix == ix {
+                let out = blob_recycler.allocate();
+                // copy to avoid doing IO inside the lock
+                {
+                    let mut outblob = out.write().unwrap();
+                    let sz = rblob.meta.size;
+                    outblob.meta.size = sz;
+                    outblob.data[..sz].copy_from_slice(&rblob.data[..sz]);
+                    outblob.meta.set_addr(&from.replicate_addr);
+                    //TODO, set the sender id to the requester so we dont retransmit
+                    //come up with a cleaner solution for this when sender signatures are checked
+                    outblob.set_id(from.id).expect("blob set_id");
+                }
+                return Some(out);
+            }
+        } else {
+            assert!(window.read().unwrap()[pos].is_none());
+            info!("failed RequestWindowIndex {} {}", ix, from.replicate_addr);
+        }
+        None
+    }
+
+    //TODO we should first coalesce all the requests
+    fn handle_blob(
+        obj: &Arc<RwLock<Self>>,
+        window: &Arc<RwLock<Vec<Option<SharedBlob>>>>,
+        blob_recycler: &BlobRecycler,
+        blob: &Blob,
+    ) -> Option<SharedBlob> {
+        match deserialize(&blob.data[..blob.meta.size]) {
+            // TODO sigverify these
+            Ok(Protocol::RequestUpdates(v, reqdata)) => {
+                trace!("RequestUpdates {}", v);
+                let addr = reqdata.gossip_addr;
+                // only lock for this call, dont lock during IO `sock.send_to` or `sock.recv_from`
+                let (from, ups, data) = obj.read()
+                    .expect("'obj' read lock in RequestUpdates")
+                    .get_updates_since(v);
+                trace!("get updates since response {} {}", v, data.len());
+                let len = data.len();
+                let rsp = Protocol::ReceiveUpdates(from, ups, data);
+                obj.write().unwrap().insert(&reqdata);
+                if len < 1 {
+                    let me = obj.read().unwrap();
+                    trace!(
+                        "no updates me {:?} ix {} since {}",
+                        &me.me[..4],
+                        me.update_index,
+                        v
+                    );
+                    None
+                } else if let Ok(r) = to_blob(rsp, addr, &blob_recycler) {
+                    trace!(
+                        "sending updates me {:?} len {} to {:?} {}",
+                        &obj.read().unwrap().me[..4],
+                        len,
+                        &reqdata.id[..4],
+                        addr,
+                    );
+                    Some(r)
+                } else {
+                    warn!("to_blob failed");
+                    None
+                }
+            }
+            Ok(Protocol::ReceiveUpdates(from, ups, data)) => {
+                trace!("ReceivedUpdates {:?} {} {}", &from[0..4], ups, data.len());
+                obj.write()
+                    .expect("'obj' write lock in ReceiveUpdates")
+                    .apply_updates(from, ups, &data);
+                None
+            }
+            Ok(Protocol::RequestWindowIndex(from, ix)) => {
+                //TODO verify from is signed
+                obj.write().unwrap().insert(&from);
+                let me = obj.read().unwrap().my_data().clone();
+                trace!(
+                    "received RequestWindowIndex {} {} myaddr {}",
+                    ix,
+                    from.replicate_addr,
+                    me.replicate_addr
+                );
+                assert_ne!(from.replicate_addr, me.replicate_addr);
+                Self::run_window_request(&window, &from, ix, blob_recycler)
+            }
+            Err(_) => {
+                warn!("deserialize crdt packet failed");
+                None
+            }
+        }
+    }
+
+    /// Process messages from the network
+    fn run_listen(
+        obj: &Arc<RwLock<Self>>,
+        window: &Arc<RwLock<Vec<Option<SharedBlob>>>>,
+        blob_recycler: &BlobRecycler,
+        requests_receiver: &BlobReceiver,
+        response_sender: &BlobSender,
+    ) -> Result<()> {
+        //TODO cache connections
+        let timeout = Duration::new(1, 0);
+        let mut reqs = requests_receiver.recv_timeout(timeout)?;
+        while let Ok(mut more) = requests_receiver.try_recv() {
+            reqs.append(&mut more);
+        }
+        let resp: VecDeque<_> = reqs.iter()
+            .filter_map(|b| Self::handle_blob(obj, window, blob_recycler, &b.read().unwrap()))
+            .collect();
+        response_sender.send(resp)?;
+        while let Some(r) = reqs.pop_front() {
+            blob_recycler.recycle(r);
+        }
+        Ok(())
+    }
+    pub fn listen(
+        obj: Arc<RwLock<Self>>,
+        window: Arc<RwLock<Vec<Option<SharedBlob>>>>,
+        blob_recycler: BlobRecycler,
+        requests_receiver: BlobReceiver,
+        response_sender: BlobSender,
+        exit: Arc<AtomicBool>,
+    ) -> JoinHandle<()> {
+        Builder::new()
+            .name("solana-listen".to_string())
+            .spawn(move || loop {
+                let e = Self::run_listen(
+                    &obj,
+                    &window,
+                    &blob_recycler,
+                    &requests_receiver,
+                    &response_sender,
+                );
+                if e.is_err() {
+                    info!(
+                        "run_listen timeout, table size: {}",
+                        obj.read().unwrap().table.len()
+                    );
+                }
+                if exit.load(Ordering::Relaxed) {
+                    return;
+                }
+            })
+            .unwrap()
+    }
+}
+
+pub struct Sockets {
+    pub gossip: UdpSocket,
+    pub gossip_send: UdpSocket,
+    pub requests: UdpSocket,
+    pub replicate: UdpSocket,
+    pub transaction: UdpSocket,
+    pub respond: UdpSocket,
+    pub broadcast: UdpSocket,
+}
+
+pub struct TestNode {
+    pub data: ReplicatedData,
+    pub sockets: Sockets,
+}
+
+impl TestNode {
+    pub fn new() -> TestNode {
+        let gossip = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let gossip_send = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let requests = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let transaction = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let replicate = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let respond = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let broadcast = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let pubkey = KeyPair::new().pubkey();
+        let data = ReplicatedData::new(
+            pubkey,
+            gossip.local_addr().unwrap(),
+            replicate.local_addr().unwrap(),
+            requests.local_addr().unwrap(),
+            transaction.local_addr().unwrap(),
+        );
+        TestNode {
+            data: data,
+            sockets: Sockets {
+                gossip,
+                gossip_send,
+                requests,
+                replicate,
+                transaction,
+                respond,
+                broadcast,
+            },
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crdt::{parse_port_or_addr, Crdt, ReplicatedData};
+    use signature::{KeyPair, KeyPairUtil};
+
+    #[test]
+    fn test_parse_port_or_addr() {
+        let p1 = parse_port_or_addr(Some("9000".to_string()));
+        assert_eq!(p1.port(), 9000);
+        let p2 = parse_port_or_addr(Some("127.0.0.1:7000".to_string()));
+        assert_eq!(p2.port(), 7000);
+        let p3 = parse_port_or_addr(None);
+        assert_eq!(p3.port(), 8000);
+    }
+
+    /// Test that insert drops messages that are older
+    #[test]
+    fn insert_test() {
+        let mut d = ReplicatedData::new(
+            KeyPair::new().pubkey(),
+            "127.0.0.1:1234".parse().unwrap(),
+            "127.0.0.1:1235".parse().unwrap(),
+            "127.0.0.1:1236".parse().unwrap(),
+            "127.0.0.1:1237".parse().unwrap(),
+        );
+        assert_eq!(d.version, 0);
+        let mut crdt = Crdt::new(d.clone());
+        assert_eq!(crdt.table[&d.id].version, 0);
+        d.version = 2;
+        crdt.insert(&d);
+        assert_eq!(crdt.table[&d.id].version, 2);
+        d.version = 1;
+        crdt.insert(&d);
+        assert_eq!(crdt.table[&d.id].version, 2);
+    }
+    fn sorted(ls: &Vec<ReplicatedData>) -> Vec<ReplicatedData> {
+        let mut copy: Vec<_> = ls.iter().cloned().collect();
+        copy.sort_by(|x, y| x.id.cmp(&y.id));
+        copy
+    }
+    #[test]
+    fn update_test() {
+        let d1 = ReplicatedData::new(
+            KeyPair::new().pubkey(),
+            "127.0.0.1:1234".parse().unwrap(),
+            "127.0.0.1:1235".parse().unwrap(),
+            "127.0.0.1:1236".parse().unwrap(),
+            "127.0.0.1:1237".parse().unwrap(),
+        );
+        let d2 = ReplicatedData::new(
+            KeyPair::new().pubkey(),
+            "127.0.0.1:1234".parse().unwrap(),
+            "127.0.0.1:1235".parse().unwrap(),
+            "127.0.0.1:1236".parse().unwrap(),
+            "127.0.0.1:1237".parse().unwrap(),
+        );
+        let d3 = ReplicatedData::new(
+            KeyPair::new().pubkey(),
+            "127.0.0.1:1234".parse().unwrap(),
+            "127.0.0.1:1235".parse().unwrap(),
+            "127.0.0.1:1236".parse().unwrap(),
+            "127.0.0.1:1237".parse().unwrap(),
+        );
+        let mut crdt = Crdt::new(d1.clone());
+        let (key, ix, ups) = crdt.get_updates_since(0);
+        assert_eq!(key, d1.id);
+        assert_eq!(ix, 1);
+        assert_eq!(ups.len(), 1);
+        assert_eq!(sorted(&ups), sorted(&vec![d1.clone()]));
+        crdt.insert(&d2);
+        let (key, ix, ups) = crdt.get_updates_since(0);
+        assert_eq!(key, d1.id);
+        assert_eq!(ix, 2);
+        assert_eq!(ups.len(), 2);
+        assert_eq!(sorted(&ups), sorted(&vec![d1.clone(), d2.clone()]));
+        crdt.insert(&d3);
+        let (key, ix, ups) = crdt.get_updates_since(0);
+        assert_eq!(key, d1.id);
+        assert_eq!(ix, 3);
+        assert_eq!(ups.len(), 3);
+        assert_eq!(sorted(&ups), sorted(&vec![d2.clone(), d1, d3]));
+        let mut crdt2 = Crdt::new(d2.clone());
+        crdt2.apply_updates(key, ix, &ups);
+        assert_eq!(crdt2.table.values().len(), 3);
+        assert_eq!(
+            sorted(&crdt2.table.values().map(|x| x.clone()).collect()),
+            sorted(&crdt.table.values().map(|x| x.clone()).collect())
+        );
+    }
+
+}

src/data_replicator.rs

@@ -0,0 +1,84 @@
+use crdt;
+use packet;
+use result::Result;
+use std::net::UdpSocket;
+use std::sync::atomic::AtomicBool;
+use std::sync::mpsc::channel;
+use std::sync::{Arc, RwLock};
+use std::thread::JoinHandle;
+use streamer;
+
+pub struct DataReplicator {
+    pub thread_hdls: Vec<JoinHandle<()>>,
+}
+
+impl DataReplicator {
+    pub fn new(
+        crdt: Arc<RwLock<crdt::Crdt>>,
+        window: Arc<RwLock<Vec<Option<packet::SharedBlob>>>>,
+        gossip_listen_socket: UdpSocket,
+        gossip_send_socket: UdpSocket,
+        exit: Arc<AtomicBool>,
+    ) -> Result<DataReplicator> {
+        let blob_recycler = packet::BlobRecycler::default();
+        let (request_sender, request_receiver) = channel();
+        trace!(
+            "DataReplicator: id: {:?}, listening on: {:?}",
+            &crdt.read().unwrap().me[..4],
+            gossip_listen_socket.local_addr().unwrap()
+        );
+        let t_receiver = streamer::blob_receiver(
+            exit.clone(),
+            blob_recycler.clone(),
+            gossip_listen_socket,
+            request_sender,
+        )?;
+        let (response_sender, response_receiver) = channel();
+        let t_responder = streamer::responder(
+            gossip_send_socket,
+            exit.clone(),
+            blob_recycler.clone(),
+            response_receiver,
+        );
+        let t_listen = crdt::Crdt::listen(
+            crdt.clone(),
+            window,
+            blob_recycler.clone(),
+            request_receiver,
+            response_sender.clone(),
+            exit.clone(),
+        );
+        let t_gossip = crdt::Crdt::gossip(crdt.clone(), blob_recycler, response_sender, exit);
+        let thread_hdls = vec![t_receiver, t_responder, t_listen, t_gossip];
+        Ok(DataReplicator { thread_hdls })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crdt::{Crdt, TestNode};
+    use data_replicator::DataReplicator;
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::{Arc, RwLock};
+
+    #[test]
+    // test that stage will exit when flag is set
+    fn test_exit() {
+        let exit = Arc::new(AtomicBool::new(false));
+        let tn = TestNode::new();
+        let crdt = Crdt::new(tn.data.clone());
+        let c = Arc::new(RwLock::new(crdt));
+        let w = Arc::new(RwLock::new(vec![]));
+        let d = DataReplicator::new(
+            c.clone(),
+            w,
+            tn.sockets.gossip,
+            tn.sockets.gossip_send,
+            exit.clone(),
+        ).unwrap();
+        exit.store(true, Ordering::Relaxed);
+        for t in d.thread_hdls {
+            t.join().expect("thread join");
+        }
+    }
+}

src/entry.rs

@@ -2,13 +2,13 @@
 //! unique ID that is the hash of the Entry before it, plus the hash of the
 //! transactions within it. Entries cannot be reordered, and its field `num_hashes`
 //! represents an approximate amount of time since the last Entry was created.
-use event::Event;
 use hash::{extend_and_hash, hash, Hash};
 use rayon::prelude::*;
+use transaction::Transaction;
 
 /// Each Entry contains three pieces of data. The `num_hashes` field is the number
 /// of hashes performed since the previous entry.  The `id` field is the result
-/// of hashing `id` from the previous entry `num_hashes` times.  The `events`
+/// of hashing `id` from the previous entry `num_hashes` times.  The `transactions`
 /// field points to Events that took place shortly after `id` was generated.
 ///
 /// If you divide `num_hashes` by the amount of time it takes to generate a new hash, you
@@ -21,86 +21,94 @@ use rayon::prelude::*;
 pub struct Entry {
     pub num_hashes: u64,
     pub id: Hash,
-    pub events: Vec<Event>,
+    pub transactions: Vec<Transaction>,
 }
 
 impl Entry {
-    /// Creates a Entry from the number of hashes `num_hashes` since the previous event
+    /// Creates the next Entry `num_hashes` after `start_hash`.
+    pub fn new(start_hash: &Hash, cur_hashes: u64, transactions: Vec<Transaction>) -> Self {
+        let num_hashes = cur_hashes + if transactions.is_empty() { 0 } else { 1 };
+        let id = next_hash(start_hash, 0, &transactions);
+        Entry {
+            num_hashes,
+            id,
+            transactions,
+        }
+    }
+
+    /// Creates the next Tick Entry `num_hashes` after `start_hash`.
+    pub fn new_mut(
+        start_hash: &mut Hash,
+        cur_hashes: &mut u64,
+        transactions: Vec<Transaction>,
+    ) -> Self {
+        let entry = Self::new(start_hash, *cur_hashes, transactions);
+        *start_hash = entry.id;
+        *cur_hashes = 0;
+        entry
+    }
+
+    /// Creates a Entry from the number of hashes `num_hashes` since the previous transaction
     /// and that resulting `id`.
     pub fn new_tick(num_hashes: u64, id: &Hash) -> Self {
         Entry {
             num_hashes,
             id: *id,
-            events: vec![],
+            transactions: vec![],
         }
     }
 
     /// Verifies self.id is the result of hashing a `start_hash` `self.num_hashes` times.
-    /// If the event is not a Tick, then hash that as well.
+    /// If the transaction is not a Tick, then hash that as well.
     pub fn verify(&self, start_hash: &Hash) -> bool {
-        self.events.par_iter().all(|event| event.verify())
-            && self.id == next_hash(start_hash, self.num_hashes, &self.events)
+        self.transactions.par_iter().all(|tx| tx.verify_plan())
+            && self.id == next_hash(start_hash, self.num_hashes, &self.transactions)
     }
 }
 
-/// Creates the hash `num_hashes` after `start_hash`. If the event contains
-/// signature, the final hash will be a hash of both the previous ID and
+fn add_transaction_data(hash_data: &mut Vec<u8>, tx: &Transaction) {
+    hash_data.push(0u8);
+    hash_data.extend_from_slice(&tx.sig);
+}
+
+/// Creates the hash `num_hashes` after `start_hash`. If the transaction contains
+/// a signature, the final hash will be a hash of both the previous ID and
 /// the signature.
-pub fn next_hash(start_hash: &Hash, num_hashes: u64, events: &[Event]) -> Hash {
+pub fn next_hash(start_hash: &Hash, num_hashes: u64, transactions: &[Transaction]) -> Hash {
     let mut id = *start_hash;
     for _ in 1..num_hashes {
         id = hash(&id);
     }
 
-    // Hash all the event data
+    // Hash all the transaction data
     let mut hash_data = vec![];
-    for event in events {
-        let sig = event.get_signature();
-        if let Some(sig) = sig {
-            hash_data.extend_from_slice(&sig);
-        }
+    for tx in transactions {
+        add_transaction_data(&mut hash_data, tx);
     }
 
     if !hash_data.is_empty() {
-        return extend_and_hash(&id, &hash_data);
-    }
-
-    id
-}
-
-/// Creates the next Entry `num_hashes` after `start_hash`.
-pub fn create_entry(start_hash: &Hash, cur_hashes: u64, events: Vec<Event>) -> Entry {
-    let num_hashes = cur_hashes + if events.is_empty() { 0 } else { 1 };
-    let id = next_hash(start_hash, 0, &events);
-    Entry {
-        num_hashes,
-        id,
-        events,
+        extend_and_hash(&id, &hash_data)
+    } else if num_hashes != 0 {
+        hash(&id)
+    } else {
+        id
     }
 }
 
-/// Creates the next Tick Entry `num_hashes` after `start_hash`.
-pub fn create_entry_mut(start_hash: &mut Hash, cur_hashes: &mut u64, events: Vec<Event>) -> Entry {
-    let entry = create_entry(start_hash, *cur_hashes, events);
-    *start_hash = entry.id;
-    *cur_hashes = 0;
-    entry
-}
-
-/// Creates the next Tick Entry `num_hashes` after `start_hash`.
-pub fn next_tick(start_hash: &Hash, num_hashes: u64) -> Entry {
+/// Creates the next Tick or Event Entry `num_hashes` after `start_hash`.
+pub fn next_entry(start_hash: &Hash, num_hashes: u64, transactions: Vec<Transaction>) -> Entry {
     Entry {
         num_hashes,
-        id: next_hash(start_hash, num_hashes, &[]),
-        events: vec![],
+        id: next_hash(start_hash, num_hashes, &transactions),
+        transactions,
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use entry::create_entry;
-    use event::Event;
+    use chrono::prelude::*;
+    use entry::Entry;
     use hash::hash;
     use signature::{KeyPair, KeyPairUtil};
     use transaction::Transaction;
@@ -111,30 +119,49 @@ mod tests {
         let one = hash(&zero);
         assert!(Entry::new_tick(0, &zero).verify(&zero)); // base case
         assert!(!Entry::new_tick(0, &zero).verify(&one)); // base case, bad
-        assert!(next_tick(&zero, 1).verify(&zero)); // inductive step
-        assert!(!next_tick(&zero, 1).verify(&one)); // inductive step, bad
+        assert!(next_entry(&zero, 1, vec![]).verify(&zero)); // inductive step
+        assert!(!next_entry(&zero, 1, vec![]).verify(&one)); // inductive step, bad
     }
 
     #[test]
-    fn test_event_reorder_attack() {
+    fn test_transaction_reorder_attack() {
         let zero = Hash::default();
 
         // First, verify entries
         let keypair = KeyPair::new();
-        let tr0 = Event::Transaction(Transaction::new(&keypair, keypair.pubkey(), 0, zero));
-        let tr1 = Event::Transaction(Transaction::new(&keypair, keypair.pubkey(), 1, zero));
-        let mut e0 = create_entry(&zero, 0, vec![tr0.clone(), tr1.clone()]);
+        let tx0 = Transaction::new(&keypair, keypair.pubkey(), 0, zero);
+        let tx1 = Transaction::new(&keypair, keypair.pubkey(), 1, zero);
+        let mut e0 = Entry::new(&zero, 0, vec![tx0.clone(), tx1.clone()]);
         assert!(e0.verify(&zero));
 
-        // Next, swap two events and ensure verification fails.
-        e0.events[0] = tr1; // <-- attack
-        e0.events[1] = tr0;
+        // Next, swap two transactions and ensure verification fails.
+        e0.transactions[0] = tx1; // <-- attack
+        e0.transactions[1] = tx0;
         assert!(!e0.verify(&zero));
     }
 
     #[test]
-    fn test_next_tick() {
+    fn test_witness_reorder_attack() {
         let zero = Hash::default();
-        assert_eq!(next_tick(&zero, 1).num_hashes, 1)
+
+        // First, verify entries
+        let keypair = KeyPair::new();
+        let tx0 = Transaction::new_timestamp(&keypair, Utc::now(), zero);
+        let tx1 = Transaction::new_signature(&keypair, Default::default(), zero);
+        let mut e0 = Entry::new(&zero, 0, vec![tx0.clone(), tx1.clone()]);
+        assert!(e0.verify(&zero));
+
+        // Next, swap two witness transactions and ensure verification fails.
+        e0.transactions[0] = tx1; // <-- attack
+        e0.transactions[1] = tx0;
+        assert!(!e0.verify(&zero));
+    }
+
+    #[test]
+    fn test_next_entry() {
+        let zero = Hash::default();
+        let tick = next_entry(&zero, 1, vec![]);
+        assert_eq!(tick.num_hashes, 1);
+        assert_ne!(tick.id, zero);
     }
 }

src/entry_writer.rs

@@ -0,0 +1,80 @@
+//! The `entry_writer` module helps implement the TPU's write stage.
+
+use bank::Bank;
+use entry::Entry;
+use ledger::Block;
+use packet;
+use result::Result;
+use serde_json;
+use std::collections::VecDeque;
+use std::io::sink;
+use std::io::Write;
+use std::sync::mpsc::Receiver;
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+use streamer;
+
+pub struct EntryWriter<'a> {
+    bank: &'a Bank,
+}
+
+impl<'a> EntryWriter<'a> {
+    /// Create a new Tpu that wraps the given Bank.
+    pub fn new(bank: &'a Bank) -> Self {
+        EntryWriter { bank }
+    }
+
+    fn write_entry<W: Write>(&self, writer: &Mutex<W>, entry: &Entry) {
+        trace!("write_entry entry");
+        self.bank.register_entry_id(&entry.id);
+        writeln!(
+            writer.lock().expect("'writer' lock in fn fn write_entry"),
+            "{}",
+            serde_json::to_string(&entry).expect("'entry' to_strong in fn write_entry")
+        ).expect("writeln! in fn write_entry");
+    }
+
+    fn write_entries<W: Write>(
+        &self,
+        writer: &Mutex<W>,
+        entry_receiver: &Receiver<Entry>,
+    ) -> Result<Vec<Entry>> {
+        //TODO implement a serialize for channel that does this without allocations
+        let mut l = vec![];
+        let entry = entry_receiver.recv_timeout(Duration::new(1, 0))?;
+        self.write_entry(writer, &entry);
+        l.push(entry);
+        while let Ok(entry) = entry_receiver.try_recv() {
+            self.write_entry(writer, &entry);
+            l.push(entry);
+        }
+        Ok(l)
+    }
+
+    /// Process any Entry items that have been published by the Historian.
+    /// continuosly broadcast blobs of entries out
+    pub fn write_and_send_entries<W: Write>(
+        &self,
+        broadcast: &streamer::BlobSender,
+        blob_recycler: &packet::BlobRecycler,
+        writer: &Mutex<W>,
+        entry_receiver: &Receiver<Entry>,
+    ) -> Result<()> {
+        let mut q = VecDeque::new();
+        let list = self.write_entries(writer, entry_receiver)?;
+        trace!("New blobs? {}", list.len());
+        list.to_blobs(blob_recycler, &mut q);
+        if !q.is_empty() {
+            trace!("broadcasting {}", q.len());
+            broadcast.send(q)?;
+        }
+        Ok(())
+    }
+
+    /// Process any Entry items that have been published by the Historian.
+    /// continuosly broadcast blobs of entries out
+    pub fn drain_entries(&self, entry_receiver: &Receiver<Entry>) -> Result<()> {
+        self.write_entries(&Arc::new(Mutex::new(sink())), entry_receiver)?;
+        Ok(())
+    }
+}

src/erasure.rs

@@ -0,0 +1,430 @@
+// Support erasure coding
+
+use packet::{BlobRecycler, SharedBlob};
+use std::result;
+
+//TODO(sakridge) pick these values
+const NUM_CODED: usize = 10;
+const MAX_MISSING: usize = 2;
+const NUM_DATA: usize = NUM_CODED - MAX_MISSING;
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum ErasureError {
+    NotEnoughBlocksToDecode,
+    DecodeError,
+    InvalidBlockSize,
+}
+
+pub type Result<T> = result::Result<T, ErasureError>;
+
+// k = number of data devices
+// m = number of coding devices
+// w = word size
+
+extern "C" {
+    fn jerasure_matrix_encode(
+        k: i32,
+        m: i32,
+        w: i32,
+        matrix: *const i32,
+        data_ptrs: *const *const u8,
+        coding_ptrs: *const *mut u8,
+        size: i32,
+    );
+    fn jerasure_matrix_decode(
+        k: i32,
+        m: i32,
+        w: i32,
+        matrix: *const i32,
+        row_k_ones: i32,
+        erasures: *const i32,
+        data_ptrs: *const *mut u8,
+        coding_ptrs: *const *const u8,
+        size: i32,
+    ) -> i32;
+    fn galois_single_divide(a: i32, b: i32, w: i32) -> i32;
+}
+
+fn get_matrix(m: i32, k: i32, w: i32) -> Vec<i32> {
+    let mut matrix = vec![0; (m * k) as usize];
+    for i in 0..m {
+        for j in 0..k {
+            unsafe {
+                matrix[(i * k + j) as usize] = galois_single_divide(1, i ^ (m + j), w);
+            }
+        }
+    }
+    matrix
+}
+
+pub const ERASURE_W: i32 = 32;
+
+// Generate coding blocks into coding
+//   There are some alignment restrictions, blocks should be aligned by 16 bytes
+//   which means their size should be >= 16 bytes
+pub fn generate_coding_blocks(coding: &mut [&mut [u8]], data: &[&[u8]]) -> Result<()> {
+    if data.len() == 0 {
+        return Ok(());
+    }
+    let m = coding.len() as i32;
+    let block_len = data[0].len();
+    let matrix: Vec<i32> = get_matrix(m, data.len() as i32, ERASURE_W);
+    let mut coding_arg = Vec::new();
+    let mut data_arg = Vec::new();
+    for block in data {
+        if block_len != block.len() {
+            return Err(ErasureError::InvalidBlockSize);
+        }
+        data_arg.push(block.as_ptr());
+    }
+    for mut block in coding {
+        if block_len != block.len() {
+            return Err(ErasureError::InvalidBlockSize);
+        }
+        coding_arg.push(block.as_mut_ptr());
+    }
+
+    unsafe {
+        jerasure_matrix_encode(
+            data.len() as i32,
+            m,
+            ERASURE_W,
+            matrix.as_ptr(),
+            data_arg.as_ptr(),
+            coding_arg.as_ptr(),
+            data[0].len() as i32,
+        );
+    }
+    Ok(())
+}
+
+// Recover data + coding blocks into data blocks
+//   data: array of blocks to recover into
+//   coding: arry of coding blocks
+//   erasures: list of indices in data where blocks should be recovered
+pub fn decode_blocks(data: &mut [&mut [u8]], coding: &[&[u8]], erasures: &[i32]) -> Result<()> {
+    if data.len() == 0 {
+        return Ok(());
+    }
+    let block_len = data[0].len();
+    let matrix: Vec<i32> = get_matrix(coding.len() as i32, data.len() as i32, ERASURE_W);
+
+    // generate coding pointers, blocks should be the same size
+    let mut coding_arg: Vec<*const u8> = Vec::new();
+    for x in coding.iter() {
+        if x.len() != block_len {
+            return Err(ErasureError::InvalidBlockSize);
+        }
+        coding_arg.push(x.as_ptr());
+    }
+
+    // generate data pointers, blocks should be the same size
+    let mut data_arg: Vec<*mut u8> = Vec::new();
+    for x in data.iter_mut() {
+        if x.len() != block_len {
+            return Err(ErasureError::InvalidBlockSize);
+        }
+        data_arg.push(x.as_mut_ptr());
+    }
+    unsafe {
+        let ret = jerasure_matrix_decode(
+            data.len() as i32,
+            coding.len() as i32,
+            ERASURE_W,
+            matrix.as_ptr(),
+            0,
+            erasures.as_ptr(),
+            data_arg.as_ptr(),
+            coding_arg.as_ptr(),
+            data[0].len() as i32,
+        );
+        trace!("jerasure_matrix_decode ret: {}", ret);
+        for x in data[erasures[0] as usize][0..8].iter() {
+            trace!("{} ", x)
+        }
+        trace!("");
+        if ret < 0 {
+            return Err(ErasureError::DecodeError);
+        }
+    }
+    Ok(())
+}
+
+// Generate coding blocks in window from consumed to consumed+NUM_DATA
+pub fn generate_coding(
+    re: &BlobRecycler,
+    window: &mut Vec<SharedBlob>,
+    consumed: usize,
+) -> Result<()> {
+    let mut data_blobs = Vec::new();
+    let mut coding_blobs = Vec::new();
+    let mut data_locks = Vec::new();
+    let mut data_ptrs: Vec<&[u8]> = Vec::new();
+    let mut coding_locks = Vec::new();
+    let mut coding_ptrs: Vec<&mut [u8]> = Vec::new();
+    for i in consumed..consumed + NUM_DATA {
+        let n = i % window.len();
+        data_blobs.push(
+            window[n]
+                .clone()
+                .expect("'data_blobs' arr in pub fn generate_coding"),
+        );
+    }
+    for b in &data_blobs {
+        data_locks.push(b.write().expect("'b' write lock in pub fn generate_coding"));
+    }
+    for (i, l) in data_locks.iter_mut().enumerate() {
+        trace!("i: {} data: {}", i, l.data[0]);
+        data_ptrs.push(&l.data);
+    }
+
+    // generate coding ptr array
+    let coding_start = consumed + NUM_DATA;
+    let coding_end = consumed + NUM_CODED;
+    for i in coding_start..coding_end {
+        let n = i % window.len();
+        window[n] = re.allocate();
+        coding_blobs.push(
+            window[n]
+                .clone()
+                .expect("'coding_blobs' arr in pub fn generate_coding"),
+        );
+    }
+    for b in &coding_blobs {
+        coding_locks.push(
+            b.write()
+                .expect("'coding_locks' arr in pub fn generate_coding"),
+        );
+    }
+    for (i, l) in coding_locks.iter_mut().enumerate() {
+        trace!("i: {} data: {}", i, l.data[0]);
+        coding_ptrs.push(&mut l.data);
+    }
+
+    generate_coding_blocks(coding_ptrs.as_mut_slice(), &data_ptrs)?;
+    trace!("consumed: {}", consumed);
+    Ok(())
+}
+
+// Recover missing blocks into window
+//   missing blocks should be None, will use re
+//   to allocate new ones. Returns err if not enough
+//   coding blocks are present to restore
+pub fn recover(
+    re: &BlobRecycler,
+    window: &mut Vec<Option<SharedBlob>>,
+    consumed: usize,
+) -> Result<()> {
+    //recover with erasure coding
+    let mut data_missing = 0;
+    let mut coded_missing = 0;
+    let coding_start = consumed + NUM_DATA;
+    let coding_end = consumed + NUM_CODED;
+    for i in consumed..coding_end {
+        let n = i % window.len();
+        if window[n].is_none() {
+            if i >= coding_start {
+                coded_missing += 1;
+            } else {
+                data_missing += 1;
+            }
+        }
+    }
+    trace!("missing: data: {} coding: {}", data_missing, coded_missing);
+    if data_missing > 0 {
+        if (data_missing + coded_missing) <= MAX_MISSING {
+            let mut blobs: Vec<SharedBlob> = Vec::new();
+            let mut locks = Vec::new();
+            let mut data_ptrs: Vec<&mut [u8]> = Vec::new();
+            let mut coding_ptrs: Vec<&[u8]> = Vec::new();
+            let mut erasures: Vec<i32> = Vec::new();
+            for i in consumed..coding_end {
+                let j = i % window.len();
+                let mut b = &mut window[j];
+                if b.is_some() {
+                    blobs.push(b.clone().expect("'blobs' arr in pb fn recover"));
+                    continue;
+                }
+                let n = re.allocate();
+                *b = Some(n.clone());
+                //mark the missing memory
+                blobs.push(n);
+                erasures.push((i - consumed) as i32);
+            }
+            erasures.push(-1);
+            trace!("erasures: {:?}", erasures);
+            //lock everything
+            for b in &blobs {
+                locks.push(b.write().expect("'locks' arr in pb fn recover"));
+            }
+            for (i, l) in locks.iter_mut().enumerate() {
+                if i >= NUM_DATA {
+                    trace!("pushing coding: {}", i);
+                    coding_ptrs.push(&l.data);
+                } else {
+                    trace!("pushing data: {}", i);
+                    data_ptrs.push(&mut l.data);
+                }
+            }
+            trace!(
+                "coding_ptrs.len: {} data_ptrs.len {}",
+                coding_ptrs.len(),
+                data_ptrs.len()
+            );
+            decode_blocks(data_ptrs.as_mut_slice(), &coding_ptrs, &erasures)?;
+        } else {
+            return Err(ErasureError::NotEnoughBlocksToDecode);
+        }
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod test {
+    use erasure;
+    use packet::{BlobRecycler, SharedBlob, PACKET_DATA_SIZE};
+
+    #[test]
+    pub fn test_coding() {
+        let zero_vec = vec![0; 16];
+        let mut vs: Vec<Vec<u8>> = (0..4).map(|i| (i..(16 + i)).collect()).collect();
+        let v_orig: Vec<u8> = vs[0].clone();
+
+        let m = 2;
+        let mut coding_blocks: Vec<_> = (0..m).map(|_| vec![0u8; 16]).collect();
+
+        {
+            let mut coding_blocks_slices: Vec<_> =
+                coding_blocks.iter_mut().map(|x| x.as_mut_slice()).collect();
+            let v_slices: Vec<_> = vs.iter().map(|x| x.as_slice()).collect();
+
+            assert!(
+                erasure::generate_coding_blocks(
+                    coding_blocks_slices.as_mut_slice(),
+                    v_slices.as_slice()
+                ).is_ok()
+            );
+        }
+        trace!("coding blocks:");
+        for b in &coding_blocks {
+            trace!("{:?}", b);
+        }
+        let erasure: i32 = 1;
+        let erasures = vec![erasure, -1];
+        // clear an entry
+        vs[erasure as usize].copy_from_slice(zero_vec.as_slice());
+
+        {
+            let coding_blocks_slices: Vec<_> = coding_blocks.iter().map(|x| x.as_slice()).collect();
+            let mut v_slices: Vec<_> = vs.iter_mut().map(|x| x.as_mut_slice()).collect();
+
+            assert!(
+                erasure::decode_blocks(
+                    v_slices.as_mut_slice(),
+                    coding_blocks_slices.as_slice(),
+                    erasures.as_slice(),
+                ).is_ok()
+            );
+        }
+
+        trace!("vs:");
+        for v in &vs {
+            trace!("{:?}", v);
+        }
+        assert_eq!(v_orig, vs[0]);
+    }
+
+    fn print_window(window: &Vec<Option<SharedBlob>>) {
+        for (i, w) in window.iter().enumerate() {
+            print!("window({}): ", i);
+            if w.is_some() {
+                let window_lock = w.clone().unwrap();
+                let window_data = window_lock.read().unwrap().data;
+                for i in 0..8 {
+                    print!("{} ", window_data[i]);
+                }
+            } else {
+                print!("null");
+            }
+            println!("");
+        }
+    }
+
+    #[test]
+    pub fn test_window_recover() {
+        let mut window = Vec::new();
+        let blob_recycler = BlobRecycler::default();
+        let offset = 4;
+        for i in 0..(4 * erasure::NUM_CODED + 1) {
+            let b = blob_recycler.allocate();
+            let b_ = b.clone();
+            let data_len = b.read().unwrap().data.len();
+            let mut w = b.write().unwrap();
+            w.set_index(i as u64).unwrap();
+            assert_eq!(i as u64, w.get_index().unwrap());
+            w.meta.size = PACKET_DATA_SIZE;
+            for k in 0..data_len {
+                w.data[k] = (k + i) as u8;
+            }
+            window.push(Some(b_));
+        }
+        println!("** after-gen:");
+        print_window(&window);
+        assert!(erasure::generate_coding(&blob_recycler, &mut window, offset).is_ok());
+        assert!(
+            erasure::generate_coding(&blob_recycler, &mut window, offset + erasure::NUM_CODED)
+                .is_ok()
+        );
+        assert!(
+            erasure::generate_coding(
+                &blob_recycler,
+                &mut window,
+                offset + (2 * erasure::NUM_CODED)
+            ).is_ok()
+        );
+        assert!(
+            erasure::generate_coding(
+                &blob_recycler,
+                &mut window,
+                offset + (3 * erasure::NUM_CODED)
+            ).is_ok()
+        );
+        println!("** after-coding:");
+        print_window(&window);
+        let refwindow = window[offset + 1].clone();
+        window[offset + 1] = None;
+        window[offset + 2] = None;
+        window[offset + erasure::NUM_CODED + 3] = None;
+        window[offset + (2 * erasure::NUM_CODED) + 0] = None;
+        window[offset + (2 * erasure::NUM_CODED) + 1] = None;
+        window[offset + (2 * erasure::NUM_CODED) + 2] = None;
+        let window_l0 = &(window[offset + (3 * erasure::NUM_CODED)]).clone().unwrap();
+        window_l0.write().unwrap().data[0] = 55;
+        println!("** after-nulling:");
+        print_window(&window);
+        assert!(erasure::recover(&blob_recycler, &mut window, offset).is_ok());
+        assert!(erasure::recover(&blob_recycler, &mut window, offset + erasure::NUM_CODED).is_ok());
+        assert!(
+            erasure::recover(
+                &blob_recycler,
+                &mut window,
+                offset + (2 * erasure::NUM_CODED)
+            ).is_err()
+        );
+        assert!(
+            erasure::recover(
+                &blob_recycler,
+                &mut window,
+                offset + (3 * erasure::NUM_CODED)
+            ).is_ok()
+        );
+        println!("** after-restore:");
+        print_window(&window);
+        let window_l = window[offset + 1].clone().unwrap();
+        let ref_l = refwindow.clone().unwrap();
+        assert_eq!(
+            window_l.read().unwrap().data.to_vec(),
+            ref_l.read().unwrap().data.to_vec()
+        );
+    }
+}

src/event.rs

@@ -1,54 +0,0 @@
-//! The `event` module handles events, which may be a `Transaction`, or a `Witness` used to process a pending
-//! Transaction.
-
-use bincode::serialize;
-use chrono::prelude::*;
-use signature::{KeyPair, KeyPairUtil, PublicKey, Signature, SignatureUtil};
-use transaction::Transaction;
-
-#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
-pub enum Event {
-    Transaction(Transaction),
-    Signature {
-        from: PublicKey,
-        tx_sig: Signature,
-        sig: Signature,
-    },
-    Timestamp {
-        from: PublicKey,
-        dt: DateTime<Utc>,
-        sig: Signature,
-    },
-}
-
-impl Event {
-    /// Create and sign a new Witness Timestamp. Used for unit-testing.
-    pub fn new_timestamp(from: &KeyPair, dt: DateTime<Utc>) -> Self {
-        let sign_data = serialize(&dt).unwrap();
-        let sig = Signature::clone_from_slice(from.sign(&sign_data).as_ref());
-        Event::Timestamp {
-            from: from.pubkey(),
-            dt,
-            sig,
-        }
-    }
-
-    // TODO: Rename this to transaction_signature().
-    /// If the Event is a Transaction, return its Signature.
-    pub fn get_signature(&self) -> Option<Signature> {
-        match *self {
-            Event::Transaction(ref tr) => Some(tr.sig),
-            Event::Signature { .. } | Event::Timestamp { .. } => None,
-        }
-    }
-
-    /// Verify the Event's signature's are valid and if a transaction, that its
-    /// spending plan is valid.
-    pub fn verify(&self) -> bool {
-        match *self {
-            Event::Transaction(ref tr) => tr.verify(),
-            Event::Signature { from, tx_sig, sig } => sig.verify(&from, &tx_sig),
-            Event::Timestamp { from, dt, sig } => sig.verify(&from, &serialize(&dt).unwrap()),
-        }
-    }
-}

src/fetch_stage.rs

@@ -0,0 +1,31 @@
+//! The `fetch_stage` batches input from a UDP socket and sends it to a channel.
+
+use packet;
+use std::net::UdpSocket;
+use std::sync::atomic::AtomicBool;
+use std::sync::mpsc::channel;
+use std::sync::Arc;
+use std::thread::JoinHandle;
+use streamer;
+
+pub struct FetchStage {
+    pub packet_receiver: streamer::PacketReceiver,
+    pub thread_hdl: JoinHandle<()>,
+}
+
+impl FetchStage {
+    pub fn new(
+        socket: UdpSocket,
+        exit: Arc<AtomicBool>,
+        packet_recycler: packet::PacketRecycler,
+    ) -> Self {
+        let (packet_sender, packet_receiver) = channel();
+        let thread_hdl =
+            streamer::receiver(socket, exit.clone(), packet_recycler.clone(), packet_sender);
+
+        FetchStage {
+            packet_receiver,
+            thread_hdl,
+        }
+    }
+}

src/hash.rs

@@ -1,7 +1,7 @@
 //! The `hash` module provides functions for creating SHA-256 hashes.
 
-use generic_array::GenericArray;
 use generic_array::typenum::U32;
+use generic_array::GenericArray;
 use sha2::{Digest, Sha256};
 
 pub type Hash = GenericArray<u8, U32>;

src/historian.rs

@@ -1,135 +0,0 @@
-//! The `historian` module provides a microservice for generating a Proof of History.
-//! It manages a thread containing a Proof of History Recorder.
-
-use entry::Entry;
-use hash::Hash;
-use recorder::{ExitReason, Recorder, Signal};
-use signature::Signature;
-use std::collections::HashSet;
-use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
-use std::thread::{spawn, JoinHandle};
-use std::time::Instant;
-
-pub struct Historian {
-    pub sender: SyncSender<Signal>,
-    pub receiver: Receiver<Entry>,
-    pub thread_hdl: JoinHandle<ExitReason>,
-    pub signatures: HashSet<Signature>,
-}
-
-impl Historian {
-    pub fn new(start_hash: &Hash, ms_per_tick: Option<u64>) -> Self {
-        let (sender, event_receiver) = sync_channel(1000);
-        let (entry_sender, receiver) = sync_channel(1000);
-        let thread_hdl =
-            Historian::create_recorder(*start_hash, ms_per_tick, event_receiver, entry_sender);
-        let signatures = HashSet::new();
-        Historian {
-            sender,
-            receiver,
-            thread_hdl,
-            signatures,
-        }
-    }
-
-    pub fn reserve_signature(&mut self, sig: &Signature) -> bool {
-        if self.signatures.contains(sig) {
-            return false;
-        }
-        self.signatures.insert(*sig);
-        true
-    }
-
-    /// A background thread that will continue tagging received Event messages and
-    /// sending back Entry messages until either the receiver or sender channel is closed.
-    fn create_recorder(
-        start_hash: Hash,
-        ms_per_tick: Option<u64>,
-        receiver: Receiver<Signal>,
-        sender: SyncSender<Entry>,
-    ) -> JoinHandle<ExitReason> {
-        spawn(move || {
-            let mut recorder = Recorder::new(receiver, sender, start_hash);
-            let now = Instant::now();
-            loop {
-                if let Err(err) = recorder.process_events(now, ms_per_tick) {
-                    return err;
-                }
-                if ms_per_tick.is_some() {
-                    recorder.hash();
-                }
-            }
-        })
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use ledger::*;
-    use std::thread::sleep;
-    use std::time::Duration;
-
-    #[test]
-    fn test_historian() {
-        let zero = Hash::default();
-        let hist = Historian::new(&zero, None);
-
-        hist.sender.send(Signal::Tick).unwrap();
-        sleep(Duration::new(0, 1_000_000));
-        hist.sender.send(Signal::Tick).unwrap();
-        sleep(Duration::new(0, 1_000_000));
-        hist.sender.send(Signal::Tick).unwrap();
-
-        let entry0 = hist.receiver.recv().unwrap();
-        let entry1 = hist.receiver.recv().unwrap();
-        let entry2 = hist.receiver.recv().unwrap();
-
-        assert_eq!(entry0.num_hashes, 0);
-        assert_eq!(entry1.num_hashes, 0);
-        assert_eq!(entry2.num_hashes, 0);
-
-        drop(hist.sender);
-        assert_eq!(
-            hist.thread_hdl.join().unwrap(),
-            ExitReason::RecvDisconnected
-        );
-
-        assert!(verify_slice(&[entry0, entry1, entry2], &zero));
-    }
-
-    #[test]
-    fn test_historian_closed_sender() {
-        let zero = Hash::default();
-        let hist = Historian::new(&zero, None);
-        drop(hist.receiver);
-        hist.sender.send(Signal::Tick).unwrap();
-        assert_eq!(
-            hist.thread_hdl.join().unwrap(),
-            ExitReason::SendDisconnected
-        );
-    }
-
-    #[test]
-    fn test_duplicate_event_signature() {
-        let zero = Hash::default();
-        let mut hist = Historian::new(&zero, None);
-        let sig = Signature::default();
-        assert!(hist.reserve_signature(&sig));
-        assert!(!hist.reserve_signature(&sig));
-    }
-
-    #[test]
-    fn test_ticking_historian() {
-        let zero = Hash::default();
-        let hist = Historian::new(&zero, Some(20));
-        sleep(Duration::from_millis(30));
-        hist.sender.send(Signal::Tick).unwrap();
-        drop(hist.sender);
-        let entries: Vec<Entry> = hist.receiver.iter().collect();
-        assert!(entries.len() > 1);
-
-        // Ensure the ID is not the seed.
-        assert_ne!(entries[0].id, zero);
-    }
-}

src/ledger.rs

@@ -1,46 +1,188 @@
 //! The `ledger` module provides functions for parallel verification of the
 //! Proof of History ledger.
 
-use entry::{next_tick, Entry};
+use bincode::{deserialize, serialize_into};
+use entry::{next_entry, Entry};
 use hash::Hash;
+use packet;
+use packet::{SharedBlob, BLOB_DATA_SIZE, BLOB_SIZE};
 use rayon::prelude::*;
+use std::cmp::min;
+use std::collections::VecDeque;
+use std::io::Cursor;
+use std::mem::size_of;
+use transaction::Transaction;
 
-/// Verifies the hashes and counts of a slice of events are all consistent.
-pub fn verify_slice(entries: &[Entry], start_hash: &Hash) -> bool {
-    let genesis = [Entry::new_tick(Default::default(), start_hash)];
-    let entry_pairs = genesis.par_iter().chain(entries).zip(entries);
-    entry_pairs.all(|(x0, x1)| x1.verify(&x0.id))
+pub trait Block {
+    /// Verifies the hashes and counts of a slice of transactions are all consistent.
+    fn verify(&self, start_hash: &Hash) -> bool;
+    fn to_blobs(&self, blob_recycler: &packet::BlobRecycler, q: &mut VecDeque<SharedBlob>);
 }
 
-/// Create a vector of Ticks of length `len` from `start_hash` hash and `num_hashes`.
-pub fn next_ticks(start_hash: &Hash, num_hashes: u64, len: usize) -> Vec<Entry> {
-    let mut id = *start_hash;
-    let mut ticks = vec![];
-    for _ in 0..len {
-        let entry = next_tick(&id, num_hashes);
-        id = entry.id;
-        ticks.push(entry);
+impl Block for [Entry] {
+    fn verify(&self, start_hash: &Hash) -> bool {
+        let genesis = [Entry::new_tick(0, start_hash)];
+        let entry_pairs = genesis.par_iter().chain(self).zip(self);
+        entry_pairs.all(|(x0, x1)| x1.verify(&x0.id))
     }
-    ticks
+
+    fn to_blobs(&self, blob_recycler: &packet::BlobRecycler, q: &mut VecDeque<SharedBlob>) {
+        let mut start = 0;
+        let mut end = 0;
+        while start < self.len() {
+            let mut entries: Vec<Vec<Entry>> = Vec::new();
+            let mut total = 0;
+            for i in &self[start..] {
+                total += size_of::<Transaction>() * i.transactions.len();
+                total += size_of::<Entry>();
+                if total >= BLOB_DATA_SIZE {
+                    break;
+                }
+                end += 1;
+            }
+            // See if we need to split the transactions
+            if end <= start {
+                let mut transaction_start = 0;
+                let num_transactions_per_blob = BLOB_DATA_SIZE / size_of::<Transaction>();
+                let total_entry_chunks = (self[end].transactions.len() + num_transactions_per_blob
+                    - 1) / num_transactions_per_blob;
+                trace!(
+                    "splitting transactions end: {} total_chunks: {}",
+                    end,
+                    total_entry_chunks
+                );
+                for _ in 0..total_entry_chunks {
+                    let transaction_end = min(
+                        transaction_start + num_transactions_per_blob,
+                        self[end].transactions.len(),
+                    );
+                    let mut entry = Entry {
+                        num_hashes: self[end].num_hashes,
+                        id: self[end].id,
+                        transactions: self[end].transactions[transaction_start..transaction_end]
+                            .to_vec(),
+                    };
+                    entries.push(vec![entry]);
+                    transaction_start = transaction_end;
+                }
+                end += 1;
+            } else {
+                entries.push(self[start..end].to_vec());
+            }
+
+            for entry in entries {
+                let b = blob_recycler.allocate();
+                let pos = {
+                    let mut bd = b.write().unwrap();
+                    let mut out = Cursor::new(bd.data_mut());
+                    serialize_into(&mut out, &entry).expect("failed to serialize output");
+                    out.position() as usize
+                };
+                assert!(pos < BLOB_SIZE);
+                b.write().unwrap().set_size(pos);
+                q.push_back(b);
+            }
+            start = end;
+        }
+    }
+}
+
+/// Create a vector of Entries of length `transaction_batches.len()` from `start_hash` hash, `num_hashes`, and `transaction_batches`.
+pub fn next_entries(
+    start_hash: &Hash,
+    num_hashes: u64,
+    transaction_batches: Vec<Vec<Transaction>>,
+) -> Vec<Entry> {
+    let mut id = *start_hash;
+    let mut entries = vec![];
+    for transactions in &transaction_batches {
+        let transactions = transactions.clone();
+        let entry = next_entry(&id, num_hashes, transactions);
+        id = entry.id;
+        entries.push(entry);
+    }
+    entries
+}
+
+pub fn reconstruct_entries_from_blobs(blobs: &VecDeque<SharedBlob>) -> Vec<Entry> {
+    let mut entries_to_apply: Vec<Entry> = Vec::new();
+    let mut last_id = Hash::default();
+    for msgs in blobs {
+        let blob = msgs.read().unwrap();
+        let entries: Vec<Entry> = deserialize(&blob.data()[..blob.meta.size]).unwrap();
+        for entry in entries {
+            if entry.id == last_id {
+                if let Some(last_entry) = entries_to_apply.last_mut() {
+                    last_entry.transactions.extend(entry.transactions);
+                }
+            } else {
+                last_id = entry.id;
+                entries_to_apply.push(entry);
+            }
+        }
+        //TODO respond back to leader with hash of the state
+    }
+    entries_to_apply
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use hash::hash;
+    use packet::BlobRecycler;
+    use signature::{KeyPair, KeyPairUtil};
+    use transaction::Transaction;
 
     #[test]
     fn test_verify_slice() {
         let zero = Hash::default();
         let one = hash(&zero);
-        assert!(verify_slice(&vec![], &zero)); // base case
-        assert!(verify_slice(&vec![Entry::new_tick(0, &zero)], &zero)); // singleton case 1
-        assert!(!verify_slice(&vec![Entry::new_tick(0, &zero)], &one)); // singleton case 2, bad
-        assert!(verify_slice(&next_ticks(&zero, 0, 2), &zero)); // inductive step
+        assert!(vec![][..].verify(&zero)); // base case
+        assert!(vec![Entry::new_tick(0, &zero)][..].verify(&zero)); // singleton case 1
+        assert!(!vec![Entry::new_tick(0, &zero)][..].verify(&one)); // singleton case 2, bad
+        assert!(next_entries(&zero, 0, vec![vec![]; 2])[..].verify(&zero)); // inductive step
 
-        let mut bad_ticks = next_ticks(&zero, 0, 2);
+        let mut bad_ticks = next_entries(&zero, 0, vec![vec![]; 2]);
         bad_ticks[1].id = one;
-        assert!(!verify_slice(&bad_ticks, &zero)); // inductive step, bad
+        assert!(!bad_ticks.verify(&zero)); // inductive step, bad
+    }
+
+    #[test]
+    fn test_entry_to_blobs() {
+        let zero = Hash::default();
+        let one = hash(&zero);
+        let keypair = KeyPair::new();
+        let tx0 = Transaction::new(&keypair, keypair.pubkey(), 1, one);
+        let transactions = vec![tx0.clone(); 10000];
+        let e0 = Entry::new(&zero, 0, transactions);
+
+        let entries = vec![e0.clone(); 1];
+        let blob_recycler = BlobRecycler::default();
+        let mut blob_q = VecDeque::new();
+        entries.to_blobs(&blob_recycler, &mut blob_q);
+
+        assert_eq!(reconstruct_entries_from_blobs(&blob_q), entries);
+    }
+
+    #[test]
+    fn test_next_entries() {
+        let mut id = Hash::default();
+        let next_id = hash(&id);
+        let keypair = KeyPair::new();
+        let tx0 = Transaction::new(&keypair, keypair.pubkey(), 1, next_id);
+        let transactions = vec![tx0.clone(); 5];
+        let transaction_batches = vec![transactions.clone(); 5];
+        let entries0 = next_entries(&id, 0, transaction_batches);
+
+        assert_eq!(entries0.len(), 5);
+
+        let mut entries1 = vec![];
+        for _ in 0..5 {
+            let entry = next_entry(&id, 0, transactions.clone());
+            id = entry.id;
+            entries1.push(entry);
+        }
+        assert_eq!(entries0, entries1);
     }
 }
 
@@ -51,11 +193,11 @@ mod bench {
     use ledger::*;
 
     #[bench]
-    fn event_bench(bencher: &mut Bencher) {
-        let start_hash = Default::default();
-        let events = next_ticks(&start_hash, 10_000, 8);
+    fn bench_next_entries(bencher: &mut Bencher) {
+        let start_hash = Hash::default();
+        let entries = next_entries(&start_hash, 10_000, vec![vec![]; 8]);
         bencher.iter(|| {
-            assert!(verify_slice(&events, &start_hash));
+            assert!(entries.verify(&start_hash));
         });
     }
 }

src/lib.rs

@@ -1,22 +1,44 @@
 #![cfg_attr(feature = "unstable", feature(test))]
-pub mod accountant;
-pub mod accountant_skel;
-pub mod accountant_stub;
+pub mod bank;
+pub mod banking_stage;
+pub mod budget;
+pub mod crdt;
+pub mod data_replicator;
 pub mod entry;
-pub mod event;
+pub mod entry_writer;
+#[cfg(feature = "erasure")]
+pub mod erasure;
+pub mod fetch_stage;
 pub mod hash;
-pub mod historian;
 pub mod ledger;
+pub mod logger;
 pub mod mint;
-pub mod plan;
+pub mod packet;
+pub mod payment_plan;
+pub mod record_stage;
 pub mod recorder;
+pub mod replicate_stage;
+pub mod request;
+pub mod request_processor;
+pub mod request_stage;
 pub mod result;
+pub mod rpu;
+pub mod server;
 pub mod signature;
+pub mod sigverify;
+pub mod sigverify_stage;
 pub mod streamer;
+pub mod thin_client;
+pub mod timing;
+pub mod tpu;
 pub mod transaction;
+pub mod tvu;
+pub mod write_stage;
 extern crate bincode;
+extern crate byteorder;
 extern crate chrono;
 extern crate generic_array;
+extern crate libc;
 #[macro_use]
 extern crate log;
 extern crate rayon;
@@ -24,6 +46,7 @@ extern crate ring;
 extern crate serde;
 #[macro_use]
 extern crate serde_derive;
+extern crate pnet;
 extern crate serde_json;
 extern crate sha2;
 extern crate untrusted;
@@ -31,3 +54,5 @@ extern crate untrusted;
 #[cfg(test)]
 #[macro_use]
 extern crate matches;
+
+extern crate rand;

src/logger.rs

@@ -0,0 +1,11 @@
+use std::sync::{Once, ONCE_INIT};
+extern crate env_logger;
+
+static INIT: Once = ONCE_INIT;
+
+/// Setup function that is only run once, even if called multiple times.
+pub fn setup() {
+    INIT.call_once(|| {
+        let _ = env_logger::init();
+    });
+}

src/mint.rs

@@ -1,8 +1,6 @@
 //! The `mint` module is a library for generating the chain's genesis block.
 
 use entry::Entry;
-use entry::create_entry;
-use event::Event;
 use hash::{hash, Hash};
 use ring::rand::SystemRandom;
 use signature::{KeyPair, KeyPairUtil, PublicKey};
@@ -19,8 +17,11 @@ pub struct Mint {
 impl Mint {
     pub fn new(tokens: i64) -> Self {
         let rnd = SystemRandom::new();
-        let pkcs8 = KeyPair::generate_pkcs8(&rnd).unwrap().to_vec();
-        let keypair = KeyPair::from_pkcs8(Input::from(&pkcs8)).unwrap();
+        let pkcs8 = KeyPair::generate_pkcs8(&rnd)
+            .expect("generate_pkcs8 in mint pub fn new")
+            .to_vec();
+        let keypair =
+            KeyPair::from_pkcs8(Input::from(&pkcs8)).expect("from_pkcs8 in mint pub fn new");
         let pubkey = keypair.pubkey();
         Mint {
             pkcs8,
@@ -33,47 +34,59 @@ impl Mint {
         hash(&self.pkcs8)
     }
 
+    pub fn last_id(&self) -> Hash {
+        self.create_entries()[1].id
+    }
+
     pub fn keypair(&self) -> KeyPair {
-        KeyPair::from_pkcs8(Input::from(&self.pkcs8)).unwrap()
+        KeyPair::from_pkcs8(Input::from(&self.pkcs8)).expect("from_pkcs8 in mint pub fn keypair")
     }
 
     pub fn pubkey(&self) -> PublicKey {
         self.pubkey
     }
 
-    pub fn create_events(&self) -> Vec<Event> {
+    pub fn create_transactions(&self) -> Vec<Transaction> {
         let keypair = self.keypair();
-        let tr = Transaction::new(&keypair, self.pubkey(), self.tokens, self.seed());
-        vec![Event::Transaction(tr)]
+        let tx = Transaction::new(&keypair, self.pubkey(), self.tokens, self.seed());
+        vec![tx]
     }
 
     pub fn create_entries(&self) -> Vec<Entry> {
-        let e0 = create_entry(&self.seed(), 0, vec![]);
-        let e1 = create_entry(&e0.id, 0, self.create_events());
+        let e0 = Entry::new(&self.seed(), 0, vec![]);
+        let e1 = Entry::new(&e0.id, 0, self.create_transactions());
         vec![e0, e1]
     }
 }
 
+#[derive(Serialize, Deserialize, Debug)]
+pub struct MintDemo {
+    pub mint: Mint,
+    pub num_accounts: i64,
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
-    use ledger::verify_slice;
-    use plan::Plan;
+    use budget::Budget;
+    use ledger::Block;
+    use transaction::{Instruction, Plan};
 
     #[test]
-    fn test_create_events() {
-        let mut events = Mint::new(100).create_events().into_iter();
-        if let Event::Transaction(tr) = events.next().unwrap() {
-            if let Plan::Pay(payment) = tr.plan {
-                assert_eq!(tr.from, payment.to);
+    fn test_create_transactions() {
+        let mut transactions = Mint::new(100).create_transactions().into_iter();
+        let tx = transactions.next().unwrap();
+        if let Instruction::NewContract(contract) = tx.instruction {
+            if let Plan::Budget(Budget::Pay(payment)) = contract.plan {
+                assert_eq!(tx.from, payment.to);
             }
         }
-        assert_eq!(events.next(), None);
+        assert_eq!(transactions.next(), None);
     }
 
     #[test]
     fn test_verify_entries() {
         let entries = Mint::new(100).create_entries();
-        assert!(verify_slice(&entries, &entries[0].id));
+        assert!(entries[..].verify(&entries[0].id));
     }
 }

src/packet.rs

@@ -0,0 +1,491 @@
+//! The `packet` module defines data structures and methods to pull data from the network.
+use bincode::{deserialize, serialize};
+use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
+use result::{Error, Result};
+use serde::Serialize;
+use signature::PublicKey;
+use std::collections::VecDeque;
+use std::fmt;
+use std::io;
+use std::mem::size_of;
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, UdpSocket};
+use std::sync::{Arc, Mutex, RwLock};
+
+pub type SharedPackets = Arc<RwLock<Packets>>;
+pub type SharedBlob = Arc<RwLock<Blob>>;
+pub type PacketRecycler = Recycler<Packets>;
+pub type BlobRecycler = Recycler<Blob>;
+
+pub const NUM_PACKETS: usize = 1024 * 8;
+pub const BLOB_SIZE: usize = 64 * 1024;
+pub const BLOB_DATA_SIZE: usize = BLOB_SIZE - BLOB_ID_END;
+pub const PACKET_DATA_SIZE: usize = 256;
+pub const NUM_BLOBS: usize = (NUM_PACKETS * PACKET_DATA_SIZE) / BLOB_SIZE;
+
+#[derive(Clone, Default)]
+#[repr(C)]
+pub struct Meta {
+    pub size: usize,
+    pub addr: [u16; 8],
+    pub port: u16,
+    pub v6: bool,
+}
+
+#[derive(Clone)]
+#[repr(C)]
+pub struct Packet {
+    pub data: [u8; PACKET_DATA_SIZE],
+    pub meta: Meta,
+}
+
+impl fmt::Debug for Packet {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "Packet {{ size: {:?}, addr: {:?} }}",
+            self.meta.size,
+            self.meta.addr()
+        )
+    }
+}
+
+impl Default for Packet {
+    fn default() -> Packet {
+        Packet {
+            data: [0u8; PACKET_DATA_SIZE],
+            meta: Meta::default(),
+        }
+    }
+}
+
+impl Meta {
+    pub fn addr(&self) -> SocketAddr {
+        if !self.v6 {
+            let addr = [
+                self.addr[0] as u8,
+                self.addr[1] as u8,
+                self.addr[2] as u8,
+                self.addr[3] as u8,
+            ];
+            let ipv4: Ipv4Addr = From::<[u8; 4]>::from(addr);
+            SocketAddr::new(IpAddr::V4(ipv4), self.port)
+        } else {
+            let ipv6: Ipv6Addr = From::<[u16; 8]>::from(self.addr);
+            SocketAddr::new(IpAddr::V6(ipv6), self.port)
+        }
+    }
+
+    pub fn set_addr(&mut self, a: &SocketAddr) {
+        match *a {
+            SocketAddr::V4(v4) => {
+                let ip = v4.ip().octets();
+                self.addr[0] = u16::from(ip[0]);
+                self.addr[1] = u16::from(ip[1]);
+                self.addr[2] = u16::from(ip[2]);
+                self.addr[3] = u16::from(ip[3]);
+                self.port = a.port();
+            }
+            SocketAddr::V6(v6) => {
+                self.addr = v6.ip().segments();
+                self.port = a.port();
+                self.v6 = true;
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Packets {
+    pub packets: Vec<Packet>,
+}
+
+//auto derive doesn't support large arrays
+impl Default for Packets {
+    fn default() -> Packets {
+        Packets {
+            packets: vec![Packet::default(); NUM_PACKETS],
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct Blob {
+    pub data: [u8; BLOB_SIZE],
+    pub meta: Meta,
+}
+
+impl fmt::Debug for Blob {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "Blob {{ size: {:?}, addr: {:?} }}",
+            self.meta.size,
+            self.meta.addr()
+        )
+    }
+}
+
+//auto derive doesn't support large arrays
+impl Default for Blob {
+    fn default() -> Blob {
+        Blob {
+            data: [0u8; BLOB_SIZE],
+            meta: Meta::default(),
+        }
+    }
+}
+
+pub struct Recycler<T> {
+    gc: Arc<Mutex<Vec<Arc<RwLock<T>>>>>,
+}
+
+impl<T: Default> Default for Recycler<T> {
+    fn default() -> Recycler<T> {
+        Recycler {
+            gc: Arc::new(Mutex::new(vec![])),
+        }
+    }
+}
+
+impl<T: Default> Clone for Recycler<T> {
+    fn clone(&self) -> Recycler<T> {
+        Recycler {
+            gc: self.gc.clone(),
+        }
+    }
+}
+
+impl<T: Default> Recycler<T> {
+    pub fn allocate(&self) -> Arc<RwLock<T>> {
+        let mut gc = self.gc.lock().expect("recycler lock in pb fn allocate");
+        gc.pop()
+            .unwrap_or_else(|| Arc::new(RwLock::new(Default::default())))
+    }
+    pub fn recycle(&self, msgs: Arc<RwLock<T>>) {
+        let mut gc = self.gc.lock().expect("recycler lock in pub fn recycle");
+        gc.push(msgs);
+    }
+}
+
+impl Packets {
+    fn run_read_from(&mut self, socket: &UdpSocket) -> Result<usize> {
+        self.packets.resize(NUM_PACKETS, Packet::default());
+        let mut i = 0;
+        //DOCUMENTED SIDE-EFFECT
+        //Performance out of the IO without poll
+        //  * block on the socket until it's readable
+        //  * set the socket to non blocking
+        //  * read until it fails
+        //  * set it back to blocking before returning
+        socket.set_nonblocking(false)?;
+        for p in &mut self.packets {
+            p.meta.size = 0;
+            trace!("receiving on {}", socket.local_addr().unwrap());
+            match socket.recv_from(&mut p.data) {
+                Err(_) if i > 0 => {
+                    debug!("got {:?} messages on {}", i, socket.local_addr().unwrap());
+                    break;
+                }
+                Err(e) => {
+                    trace!("recv_from err {:?}", e);
+                    return Err(Error::IO(e));
+                }
+                Ok((nrecv, from)) => {
+                    p.meta.size = nrecv;
+                    p.meta.set_addr(&from);
+                    if i == 0 {
+                        socket.set_nonblocking(true)?;
+                    }
+                }
+            }
+            i += 1;
+        }
+        Ok(i)
+    }
+    pub fn recv_from(&mut self, socket: &UdpSocket) -> Result<()> {
+        let sz = self.run_read_from(socket)?;
+        self.packets.resize(sz, Packet::default());
+        debug!("recv_from: {}", sz);
+        Ok(())
+    }
+    pub fn send_to(&self, socket: &UdpSocket) -> Result<()> {
+        for p in &self.packets {
+            let a = p.meta.addr();
+            socket.send_to(&p.data[..p.meta.size], &a)?;
+        }
+        Ok(())
+    }
+}
+
+pub fn to_packets<T: Serialize>(r: &PacketRecycler, xs: Vec<T>) -> Vec<SharedPackets> {
+    let mut out = vec![];
+    for x in xs.chunks(NUM_PACKETS) {
+        let p = r.allocate();
+        p.write()
+            .unwrap()
+            .packets
+            .resize(x.len(), Default::default());
+        for (i, o) in x.iter().zip(p.write().unwrap().packets.iter_mut()) {
+            let v = serialize(&i).expect("serialize request");
+            let len = v.len();
+            o.data[..len].copy_from_slice(&v);
+            o.meta.size = len;
+        }
+        out.push(p);
+    }
+    return out;
+}
+
+pub fn to_blob<T: Serialize>(
+    resp: T,
+    rsp_addr: SocketAddr,
+    blob_recycler: &BlobRecycler,
+) -> Result<SharedBlob> {
+    let blob = blob_recycler.allocate();
+    {
+        let mut b = blob.write().unwrap();
+        let v = serialize(&resp)?;
+        let len = v.len();
+        // TODO: we are not using .data_mut() method here because
+        // the raw bytes are being serialized and sent, this isn't the
+        // right interface, and we should create a separate path for
+        // sending request responses in the RPU
+        assert!(len < BLOB_SIZE);
+        b.data[..len].copy_from_slice(&v);
+        b.meta.size = len;
+        b.meta.set_addr(&rsp_addr);
+    }
+    Ok(blob)
+}
+
+pub fn to_blobs<T: Serialize>(
+    rsps: Vec<(T, SocketAddr)>,
+    blob_recycler: &BlobRecycler,
+) -> Result<VecDeque<SharedBlob>> {
+    let mut blobs = VecDeque::new();
+    for (resp, rsp_addr) in rsps {
+        blobs.push_back(to_blob(resp, rsp_addr, blob_recycler)?);
+    }
+    Ok(blobs)
+}
+
+const BLOB_INDEX_END: usize = size_of::<u64>();
+const BLOB_ID_END: usize = BLOB_INDEX_END + size_of::<usize>() + size_of::<PublicKey>();
+
+impl Blob {
+    pub fn get_index(&self) -> Result<u64> {
+        let mut rdr = io::Cursor::new(&self.data[0..BLOB_INDEX_END]);
+        let r = rdr.read_u64::<LittleEndian>()?;
+        Ok(r)
+    }
+    pub fn set_index(&mut self, ix: u64) -> Result<()> {
+        let mut wtr = vec![];
+        wtr.write_u64::<LittleEndian>(ix)?;
+        self.data[..BLOB_INDEX_END].clone_from_slice(&wtr);
+        Ok(())
+    }
+    /// sender id, we use this for identifying if its a blob from the leader that we should
+    /// retransmit.  eventually blobs should have a signature that we can use ffor spam filtering
+    pub fn get_id(&self) -> Result<PublicKey> {
+        let e = deserialize(&self.data[BLOB_INDEX_END..BLOB_ID_END])?;
+        Ok(e)
+    }
+
+    pub fn set_id(&mut self, id: PublicKey) -> Result<()> {
+        let wtr = serialize(&id)?;
+        self.data[BLOB_INDEX_END..BLOB_ID_END].clone_from_slice(&wtr);
+        Ok(())
+    }
+
+    pub fn data(&self) -> &[u8] {
+        &self.data[BLOB_ID_END..]
+    }
+    pub fn data_mut(&mut self) -> &mut [u8] {
+        &mut self.data[BLOB_ID_END..]
+    }
+    pub fn set_size(&mut self, size: usize) {
+        self.meta.size = size + BLOB_ID_END;
+    }
+    pub fn recv_from(re: &BlobRecycler, socket: &UdpSocket) -> Result<VecDeque<SharedBlob>> {
+        let mut v = VecDeque::new();
+        //DOCUMENTED SIDE-EFFECT
+        //Performance out of the IO without poll
+        //  * block on the socket until it's readable
+        //  * set the socket to non blocking
+        //  * read until it fails
+        //  * set it back to blocking before returning
+        socket.set_nonblocking(false)?;
+        for i in 0..NUM_BLOBS {
+            let r = re.allocate();
+            {
+                let mut p = r.write().expect("'r' write lock in pub fn recv_from");
+                trace!("receiving on {}", socket.local_addr().unwrap());
+                match socket.recv_from(&mut p.data) {
+                    Err(_) if i > 0 => {
+                        trace!("got {:?} messages on {}", i, socket.local_addr().unwrap());
+                        break;
+                    }
+                    Err(e) => {
+                        if e.kind() != io::ErrorKind::WouldBlock {
+                            info!("recv_from err {:?}", e);
+                        }
+                        return Err(Error::IO(e));
+                    }
+                    Ok((nrecv, from)) => {
+                        p.meta.size = nrecv;
+                        p.meta.set_addr(&from);
+                        if i == 0 {
+                            socket.set_nonblocking(true)?;
+                        }
+                    }
+                }
+            }
+            v.push_back(r);
+        }
+        Ok(v)
+    }
+    pub fn send_to(
+        re: &BlobRecycler,
+        socket: &UdpSocket,
+        v: &mut VecDeque<SharedBlob>,
+    ) -> Result<()> {
+        while let Some(r) = v.pop_front() {
+            {
+                let p = r.read().expect("'r' read lock in pub fn send_to");
+                let a = p.meta.addr();
+                socket.send_to(&p.data[..p.meta.size], &a)?;
+            }
+            re.recycle(r);
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use packet::{to_packets, Blob, BlobRecycler, Packet, PacketRecycler, Packets, NUM_PACKETS};
+    use request::Request;
+    use std::collections::VecDeque;
+    use std::io;
+    use std::io::Write;
+    use std::net::UdpSocket;
+
+    #[test]
+    pub fn packet_recycler_test() {
+        let r = PacketRecycler::default();
+        let p = r.allocate();
+        r.recycle(p);
+        assert_eq!(r.gc.lock().unwrap().len(), 1);
+        let _ = r.allocate();
+        assert_eq!(r.gc.lock().unwrap().len(), 0);
+    }
+    #[test]
+    pub fn blob_recycler_test() {
+        let r = BlobRecycler::default();
+        let p = r.allocate();
+        r.recycle(p);
+        assert_eq!(r.gc.lock().unwrap().len(), 1);
+        let _ = r.allocate();
+        assert_eq!(r.gc.lock().unwrap().len(), 0);
+    }
+    #[test]
+    pub fn packet_send_recv() {
+        let reader = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let addr = reader.local_addr().unwrap();
+        let sender = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let saddr = sender.local_addr().unwrap();
+        let r = PacketRecycler::default();
+        let p = r.allocate();
+        p.write().unwrap().packets.resize(10, Packet::default());
+        for m in p.write().unwrap().packets.iter_mut() {
+            m.meta.set_addr(&addr);
+            m.meta.size = 256;
+        }
+        p.read().unwrap().send_to(&sender).unwrap();
+        p.write().unwrap().recv_from(&reader).unwrap();
+        for m in p.write().unwrap().packets.iter_mut() {
+            assert_eq!(m.meta.size, 256);
+            assert_eq!(m.meta.addr(), saddr);
+        }
+
+        r.recycle(p);
+    }
+
+    #[test]
+    fn test_to_packets() {
+        let tx = Request::GetTransactionCount;
+        let re = PacketRecycler::default();
+        let rv = to_packets(&re, vec![tx.clone(); 1]);
+        assert_eq!(rv.len(), 1);
+        assert_eq!(rv[0].read().unwrap().packets.len(), 1);
+
+        let rv = to_packets(&re, vec![tx.clone(); NUM_PACKETS]);
+        assert_eq!(rv.len(), 1);
+        assert_eq!(rv[0].read().unwrap().packets.len(), NUM_PACKETS);
+
+        let rv = to_packets(&re, vec![tx.clone(); NUM_PACKETS + 1]);
+        assert_eq!(rv.len(), 2);
+        assert_eq!(rv[0].read().unwrap().packets.len(), NUM_PACKETS);
+        assert_eq!(rv[1].read().unwrap().packets.len(), 1);
+    }
+
+    #[test]
+    pub fn blob_send_recv() {
+        trace!("start");
+        let reader = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let addr = reader.local_addr().unwrap();
+        let sender = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let r = BlobRecycler::default();
+        let p = r.allocate();
+        p.write().unwrap().meta.set_addr(&addr);
+        p.write().unwrap().meta.size = 1024;
+        let mut v = VecDeque::new();
+        v.push_back(p);
+        assert_eq!(v.len(), 1);
+        Blob::send_to(&r, &sender, &mut v).unwrap();
+        trace!("send_to");
+        assert_eq!(v.len(), 0);
+        let mut rv = Blob::recv_from(&r, &reader).unwrap();
+        trace!("recv_from");
+        assert_eq!(rv.len(), 1);
+        let rp = rv.pop_front().unwrap();
+        assert_eq!(rp.write().unwrap().meta.size, 1024);
+        r.recycle(rp);
+    }
+
+    #[cfg(all(feature = "ipv6", test))]
+    #[test]
+    pub fn blob_ipv6_send_recv() {
+        let reader = UdpSocket::bind("[::1]:0").expect("bind");
+        let addr = reader.local_addr().unwrap();
+        let sender = UdpSocket::bind("[::1]:0").expect("bind");
+        let r = BlobRecycler::default();
+        let p = r.allocate();
+        p.write().unwrap().meta.set_addr(&addr);
+        p.write().unwrap().meta.size = 1024;
+        let mut v = VecDeque::default();
+        v.push_back(p);
+        Blob::send_to(&r, &sender, &mut v).unwrap();
+        let mut rv = Blob::recv_from(&r, &reader).unwrap();
+        let rp = rv.pop_front().unwrap();
+        assert_eq!(rp.write().unwrap().meta.size, 1024);
+        r.recycle(rp);
+    }
+
+    #[test]
+    pub fn debug_trait() {
+        write!(io::sink(), "{:?}", Packet::default()).unwrap();
+        write!(io::sink(), "{:?}", Packets::default()).unwrap();
+        write!(io::sink(), "{:?}", Blob::default()).unwrap();
+    }
+    #[test]
+    pub fn blob_test() {
+        let mut b = Blob::default();
+        b.set_index(<u64>::max_value()).unwrap();
+        assert_eq!(b.get_index().unwrap(), <u64>::max_value());
+        b.data_mut()[0] = 1;
+        assert_eq!(b.data()[0], 1);
+        assert_eq!(b.get_index().unwrap(), <u64>::max_value());
+    }
+
+}

src/payment_plan.rs

@@ -0,0 +1,31 @@
+//! The `plan` module provides a domain-specific language for payment plans. Users create Budget objects that
+//! are given to an interpreter. The interpreter listens for `Witness` transactions,
+//! which it uses to reduce the payment plan. When the plan is reduced to a
+//! `Payment`, the payment is executed.
+
+use chrono::prelude::*;
+use signature::PublicKey;
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub enum Witness {
+    Timestamp(DateTime<Utc>),
+    Signature(PublicKey),
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub struct Payment {
+    pub tokens: i64,
+    pub to: PublicKey,
+}
+
+pub trait PaymentPlan {
+    /// Return Payment if the payment plan requires no additional Witnesses.
+    fn final_payment(&self) -> Option<Payment>;
+
+    /// Return true if the plan spends exactly `spendable_tokens`.
+    fn verify(&self, spendable_tokens: i64) -> bool;
+
+    /// Apply a witness to the payment plan to see if the plan can be reduced.
+    /// If so, modify the plan in-place.
+    fn apply_witness(&mut self, witness: &Witness);
+}

src/record_stage.rs

@@ -0,0 +1,204 @@
+//! The `record_stage` module provides an object for generating a Proof of History.
+//! It records Event items on behalf of its users. It continuously generates
+//! new hashes, only stopping to check if it has been sent an Event item. It
+//! tags each Event with an Entry, and sends it back. The Entry includes the
+//! Event, the latest hash, and the number of hashes since the last transaction.
+//! The resulting stream of entries represents ordered transactions in time.
+
+use entry::Entry;
+use hash::Hash;
+use recorder::Recorder;
+use std::sync::mpsc::{channel, Receiver, RecvError, Sender, TryRecvError};
+use std::thread::{Builder, JoinHandle};
+use std::time::{Duration, Instant};
+use transaction::Transaction;
+
+#[cfg_attr(feature = "cargo-clippy", allow(large_enum_variant))]
+pub enum Signal {
+    Tick,
+    Events(Vec<Transaction>),
+}
+
+pub struct RecordStage {
+    pub entry_receiver: Receiver<Entry>,
+    pub thread_hdl: JoinHandle<()>,
+}
+
+impl RecordStage {
+    /// A background thread that will continue tagging received Event messages and
+    /// sending back Entry messages until either the receiver or sender channel is closed.
+    pub fn new(signal_receiver: Receiver<Signal>, start_hash: &Hash) -> Self {
+        let (entry_sender, entry_receiver) = channel();
+        let start_hash = start_hash.clone();
+
+        let thread_hdl = Builder::new()
+            .name("solana-record-stage".to_string())
+            .spawn(move || {
+                let mut recorder = Recorder::new(start_hash);
+                let _ = Self::process_signals(&mut recorder, &signal_receiver, &entry_sender);
+            })
+            .unwrap();
+
+        RecordStage {
+            entry_receiver,
+            thread_hdl,
+        }
+    }
+
+    /// Same as `RecordStage::new`, but will automatically produce entries every `tick_duration`.
+    pub fn new_with_clock(
+        signal_receiver: Receiver<Signal>,
+        start_hash: &Hash,
+        tick_duration: Duration,
+    ) -> Self {
+        let (entry_sender, entry_receiver) = channel();
+        let start_hash = start_hash.clone();
+
+        let thread_hdl = Builder::new()
+            .name("solana-record-stage".to_string())
+            .spawn(move || {
+                let mut recorder = Recorder::new(start_hash);
+                let start_time = Instant::now();
+                loop {
+                    if let Err(_) = Self::try_process_signals(
+                        &mut recorder,
+                        start_time,
+                        tick_duration,
+                        &signal_receiver,
+                        &entry_sender,
+                    ) {
+                        return;
+                    }
+                    recorder.hash();
+                }
+            })
+            .unwrap();
+
+        RecordStage {
+            entry_receiver,
+            thread_hdl,
+        }
+    }
+
+    fn process_signal(
+        signal: Signal,
+        recorder: &mut Recorder,
+        sender: &Sender<Entry>,
+    ) -> Result<(), ()> {
+        let txs = if let Signal::Events(txs) = signal {
+            txs
+        } else {
+            vec![]
+        };
+        let entry = recorder.record(txs);
+        sender.send(entry).map_err(|_| ())
+    }
+
+    fn process_signals(
+        recorder: &mut Recorder,
+        receiver: &Receiver<Signal>,
+        sender: &Sender<Entry>,
+    ) -> Result<(), ()> {
+        loop {
+            match receiver.recv() {
+                Ok(signal) => Self::process_signal(signal, recorder, sender)?,
+                Err(RecvError) => return Err(()),
+            }
+        }
+    }
+
+    fn try_process_signals(
+        recorder: &mut Recorder,
+        start_time: Instant,
+        tick_duration: Duration,
+        receiver: &Receiver<Signal>,
+        sender: &Sender<Entry>,
+    ) -> Result<(), ()> {
+        loop {
+            if let Some(entry) = recorder.tick(start_time, tick_duration) {
+                sender.send(entry).or(Err(()))?;
+            }
+            match receiver.try_recv() {
+                Ok(signal) => Self::process_signal(signal, recorder, sender)?,
+                Err(TryRecvError::Empty) => return Ok(()),
+                Err(TryRecvError::Disconnected) => return Err(()),
+            };
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use ledger::Block;
+    use signature::{KeyPair, KeyPairUtil};
+    use std::sync::mpsc::channel;
+    use std::thread::sleep;
+
+    #[test]
+    fn test_historian() {
+        let (tx_sender, tx_receiver) = channel();
+        let zero = Hash::default();
+        let record_stage = RecordStage::new(tx_receiver, &zero);
+
+        tx_sender.send(Signal::Tick).unwrap();
+        sleep(Duration::new(0, 1_000_000));
+        tx_sender.send(Signal::Tick).unwrap();
+        sleep(Duration::new(0, 1_000_000));
+        tx_sender.send(Signal::Tick).unwrap();
+
+        let entry0 = record_stage.entry_receiver.recv().unwrap();
+        let entry1 = record_stage.entry_receiver.recv().unwrap();
+        let entry2 = record_stage.entry_receiver.recv().unwrap();
+
+        assert_eq!(entry0.num_hashes, 0);
+        assert_eq!(entry1.num_hashes, 0);
+        assert_eq!(entry2.num_hashes, 0);
+
+        drop(tx_sender);
+        assert_eq!(record_stage.thread_hdl.join().unwrap(), ());
+
+        assert!([entry0, entry1, entry2].verify(&zero));
+    }
+
+    #[test]
+    fn test_historian_closed_sender() {
+        let (tx_sender, tx_receiver) = channel();
+        let zero = Hash::default();
+        let record_stage = RecordStage::new(tx_receiver, &zero);
+        drop(record_stage.entry_receiver);
+        tx_sender.send(Signal::Tick).unwrap();
+        assert_eq!(record_stage.thread_hdl.join().unwrap(), ());
+    }
+
+    #[test]
+    fn test_transactions() {
+        let (tx_sender, signal_receiver) = channel();
+        let zero = Hash::default();
+        let record_stage = RecordStage::new(signal_receiver, &zero);
+        let alice_keypair = KeyPair::new();
+        let bob_pubkey = KeyPair::new().pubkey();
+        let tx0 = Transaction::new(&alice_keypair, bob_pubkey, 1, zero);
+        let tx1 = Transaction::new(&alice_keypair, bob_pubkey, 2, zero);
+        tx_sender.send(Signal::Events(vec![tx0, tx1])).unwrap();
+        drop(tx_sender);
+        let entries: Vec<_> = record_stage.entry_receiver.iter().collect();
+        assert_eq!(entries.len(), 1);
+    }
+
+    #[test]
+    fn test_clock() {
+        let (tx_sender, tx_receiver) = channel();
+        let zero = Hash::default();
+        let record_stage =
+            RecordStage::new_with_clock(tx_receiver, &zero, Duration::from_millis(20));
+        sleep(Duration::from_millis(900));
+        tx_sender.send(Signal::Tick).unwrap();
+        drop(tx_sender);
+        let entries: Vec<Entry> = record_stage.entry_receiver.iter().collect();
+        assert!(entries.len() > 1);
+
+        // Ensure the ID is not the seed.
+        assert_ne!(entries[0].id, zero);
+    }
+}

src/recorder.rs

@@ -1,45 +1,21 @@
 //! The `recorder` module provides an object for generating a Proof of History.
-//! It records Event items on behalf of its users. It continuously generates
-//! new hashes, only stopping to check if it has been sent an Event item. It
-//! tags each Event with an Entry, and sends it back. The Entry includes the
-//! Event, the latest hash, and the number of hashes since the last event.
-//! The resulting stream of entries represents ordered events in time.
+//! It records Event items on behalf of its users.
 
-use entry::{create_entry_mut, Entry};
-use event::Event;
+use entry::Entry;
 use hash::{hash, Hash};
-use std::mem;
-use std::sync::mpsc::{Receiver, SyncSender, TryRecvError};
 use std::time::{Duration, Instant};
-
-#[cfg_attr(feature = "cargo-clippy", allow(large_enum_variant))]
-pub enum Signal {
-    Tick,
-    Event(Event),
-}
-
-#[derive(Debug, PartialEq, Eq)]
-pub enum ExitReason {
-    RecvDisconnected,
-    SendDisconnected,
-}
+use transaction::Transaction;
 
 pub struct Recorder {
-    sender: SyncSender<Entry>,
-    receiver: Receiver<Signal>,
     last_hash: Hash,
-    events: Vec<Event>,
     num_hashes: u64,
-    num_ticks: u64,
+    num_ticks: u32,
 }
 
 impl Recorder {
-    pub fn new(receiver: Receiver<Signal>, sender: SyncSender<Entry>, start_hash: Hash) -> Self {
+    pub fn new(last_hash: Hash) -> Self {
         Recorder {
-            receiver,
-            sender,
-            last_hash: start_hash,
-            events: vec![],
+            last_hash,
             num_hashes: 0,
             num_ticks: 0,
         }
@@ -50,40 +26,17 @@ impl Recorder {
         self.num_hashes += 1;
     }
 
-    pub fn record_entry(&mut self) -> Result<(), ExitReason> {
-        let events = mem::replace(&mut self.events, vec![]);
-        let entry = create_entry_mut(&mut self.last_hash, &mut self.num_hashes, events);
-        self.sender
-            .send(entry)
-            .or(Err(ExitReason::SendDisconnected))?;
-        Ok(())
+    pub fn record(&mut self, transactions: Vec<Transaction>) -> Entry {
+        Entry::new_mut(&mut self.last_hash, &mut self.num_hashes, transactions)
     }
 
-    pub fn process_events(
-        &mut self,
-        epoch: Instant,
-        ms_per_tick: Option<u64>,
-    ) -> Result<(), ExitReason> {
-        loop {
-            if let Some(ms) = ms_per_tick {
-                if epoch.elapsed() > Duration::from_millis((self.num_ticks + 1) * ms) {
-                    self.record_entry()?;
-                    self.num_ticks += 1;
-                }
-            }
-
-            match self.receiver.try_recv() {
-                Ok(signal) => match signal {
-                    Signal::Tick => {
-                        self.record_entry()?;
-                    }
-                    Signal::Event(event) => {
-                        self.events.push(event);
-                    }
-                },
-                Err(TryRecvError::Empty) => return Ok(()),
-                Err(TryRecvError::Disconnected) => return Err(ExitReason::RecvDisconnected),
-            };
+    pub fn tick(&mut self, start_time: Instant, tick_duration: Duration) -> Option<Entry> {
+        if start_time.elapsed() > tick_duration * (self.num_ticks + 1) {
+            // TODO: don't let this overflow u32
+            self.num_ticks += 1;
+            Some(self.record(vec![]))
+        } else {
+            None
         }
     }
 }

src/replicate_stage.rs

@@ -0,0 +1,55 @@
+//! The `replicate_stage` replicates transactions broadcast by the leader.
+
+use bank::Bank;
+use ledger;
+use packet;
+use result::Result;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::thread::{Builder, JoinHandle};
+use std::time::Duration;
+use streamer;
+
+pub struct ReplicateStage {
+    pub thread_hdl: JoinHandle<()>,
+}
+
+impl ReplicateStage {
+    /// Process verified blobs, already in order
+    fn replicate_requests(
+        bank: &Arc<Bank>,
+        verified_receiver: &streamer::BlobReceiver,
+        blob_recycler: &packet::BlobRecycler,
+    ) -> Result<()> {
+        let timer = Duration::new(1, 0);
+        let blobs = verified_receiver.recv_timeout(timer)?;
+        let entries = ledger::reconstruct_entries_from_blobs(&blobs);
+        let res = bank.process_entries(entries);
+        if res.is_err() {
+            error!("process_entries {} {:?}", blobs.len(), res);
+        }
+        res?;
+        for blob in blobs {
+            blob_recycler.recycle(blob);
+        }
+        Ok(())
+    }
+
+    pub fn new(
+        bank: Arc<Bank>,
+        exit: Arc<AtomicBool>,
+        window_receiver: streamer::BlobReceiver,
+        blob_recycler: packet::BlobRecycler,
+    ) -> Self {
+        let thread_hdl = Builder::new()
+            .name("solana-replicate-stage".to_string())
+            .spawn(move || loop {
+                let e = Self::replicate_requests(&bank, &window_receiver, &blob_recycler);
+                if e.is_err() && exit.load(Ordering::Relaxed) {
+                    break;
+                }
+            })
+            .unwrap();
+        ReplicateStage { thread_hdl }
+    }
+}

src/request.rs

@@ -0,0 +1,26 @@
+//! The `request` module defines the messages for the thin client.
+
+use hash::Hash;
+use signature::PublicKey;
+
+#[cfg_attr(feature = "cargo-clippy", allow(large_enum_variant))]
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub enum Request {
+    GetBalance { key: PublicKey },
+    GetLastId,
+    GetTransactionCount,
+}
+
+impl Request {
+    /// Verify the request is valid.
+    pub fn verify(&self) -> bool {
+        true
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub enum Response {
+    Balance { key: PublicKey, val: Option<i64> },
+    LastId { id: Hash },
+    TransactionCount { transaction_count: u64 },
+}

src/request_processor.rs

@@ -0,0 +1,54 @@
+//! The `request_processor` processes thin client Request messages.
+
+use bank::Bank;
+use request::{Request, Response};
+use std::net::SocketAddr;
+use std::sync::Arc;
+
+pub struct RequestProcessor {
+    bank: Arc<Bank>,
+}
+
+impl RequestProcessor {
+    /// Create a new Tpu that wraps the given Bank.
+    pub fn new(bank: Arc<Bank>) -> Self {
+        RequestProcessor { bank }
+    }
+
+    /// Process Request items sent by clients.
+    fn process_request(
+        &self,
+        msg: Request,
+        rsp_addr: SocketAddr,
+    ) -> Option<(Response, SocketAddr)> {
+        match msg {
+            Request::GetBalance { key } => {
+                let val = self.bank.get_balance(&key);
+                let rsp = (Response::Balance { key, val }, rsp_addr);
+                info!("Response::Balance {:?}", rsp);
+                Some(rsp)
+            }
+            Request::GetLastId => {
+                let id = self.bank.last_id();
+                let rsp = (Response::LastId { id }, rsp_addr);
+                info!("Response::LastId {:?}", rsp);
+                Some(rsp)
+            }
+            Request::GetTransactionCount => {
+                let transaction_count = self.bank.transaction_count() as u64;
+                let rsp = (Response::TransactionCount { transaction_count }, rsp_addr);
+                info!("Response::TransactionCount {:?}", rsp);
+                Some(rsp)
+            }
+        }
+    }
+
+    pub fn process_requests(
+        &self,
+        reqs: Vec<(Request, SocketAddr)>,
+    ) -> Vec<(Response, SocketAddr)> {
+        reqs.into_iter()
+            .filter_map(|(req, rsp_addr)| self.process_request(req, rsp_addr))
+            .collect()
+    }
+}

src/request_stage.rs

@@ -0,0 +1,116 @@
+//! The `request_stage` processes thin client Request messages.
+
+use bincode::deserialize;
+use packet;
+use packet::SharedPackets;
+use rayon::prelude::*;
+use request::Request;
+use request_processor::RequestProcessor;
+use result::Result;
+use std::net::SocketAddr;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{channel, Receiver};
+use std::sync::Arc;
+use std::thread::{Builder, JoinHandle};
+use std::time::Instant;
+use streamer;
+use timing;
+
+pub struct RequestStage {
+    pub thread_hdl: JoinHandle<()>,
+    pub blob_receiver: streamer::BlobReceiver,
+    pub request_processor: Arc<RequestProcessor>,
+}
+
+impl RequestStage {
+    pub fn deserialize_requests(p: &packet::Packets) -> Vec<Option<(Request, SocketAddr)>> {
+        p.packets
+            .par_iter()
+            .map(|x| {
+                deserialize(&x.data[0..x.meta.size])
+                    .map(|req| (req, x.meta.addr()))
+                    .ok()
+            })
+            .collect()
+    }
+
+    pub fn process_request_packets(
+        request_processor: &RequestProcessor,
+        packet_receiver: &Receiver<SharedPackets>,
+        blob_sender: &streamer::BlobSender,
+        packet_recycler: &packet::PacketRecycler,
+        blob_recycler: &packet::BlobRecycler,
+    ) -> Result<()> {
+        let (batch, batch_len) = streamer::recv_batch(packet_receiver)?;
+
+        info!(
+            "@{:?} request_stage: processing: {}",
+            timing::timestamp(),
+            batch_len
+        );
+
+        let mut reqs_len = 0;
+        let proc_start = Instant::now();
+        for msgs in batch {
+            let reqs: Vec<_> = Self::deserialize_requests(&msgs.read().unwrap())
+                .into_iter()
+                .filter_map(|x| x)
+                .collect();
+            reqs_len += reqs.len();
+
+            let rsps = request_processor.process_requests(reqs);
+
+            let blobs = packet::to_blobs(rsps, blob_recycler)?;
+            if !blobs.is_empty() {
+                info!("process: sending blobs: {}", blobs.len());
+                //don't wake up the other side if there is nothing
+                blob_sender.send(blobs)?;
+            }
+            packet_recycler.recycle(msgs);
+        }
+        let total_time_s = timing::duration_as_s(&proc_start.elapsed());
+        let total_time_ms = timing::duration_as_ms(&proc_start.elapsed());
+        info!(
+            "@{:?} done process batches: {} time: {:?}ms reqs: {} reqs/s: {}",
+            timing::timestamp(),
+            batch_len,
+            total_time_ms,
+            reqs_len,
+            (reqs_len as f32) / (total_time_s)
+        );
+        Ok(())
+    }
+    pub fn new(
+        request_processor: RequestProcessor,
+        exit: Arc<AtomicBool>,
+        packet_receiver: Receiver<SharedPackets>,
+        packet_recycler: packet::PacketRecycler,
+        blob_recycler: packet::BlobRecycler,
+    ) -> Self {
+        let request_processor = Arc::new(request_processor);
+        let request_processor_ = request_processor.clone();
+        let (blob_sender, blob_receiver) = channel();
+        let thread_hdl = Builder::new()
+            .name("solana-request-stage".to_string())
+            .spawn(move || loop {
+                let e = Self::process_request_packets(
+                    &request_processor_,
+                    &packet_receiver,
+                    &blob_sender,
+                    &packet_recycler,
+                    &blob_recycler,
+                );
+                if e.is_err() {
+                    if exit.load(Ordering::Relaxed) {
+                        break;
+                    }
+                }
+            })
+            .unwrap();
+        RequestStage {
+            thread_hdl,
+            blob_receiver,
+            request_processor,
+        }
+    }
+}

src/result.rs

@@ -1,5 +1,6 @@
 //! The `result` module exposes a Result type that propagates one of many different Error types.
 
+use bank;
 use bincode;
 use serde_json;
 use std;
@@ -14,8 +15,11 @@ pub enum Error {
     RecvError(std::sync::mpsc::RecvError),
     RecvTimeoutError(std::sync::mpsc::RecvTimeoutError),
     Serialize(std::boxed::Box<bincode::ErrorKind>),
+    BankError(bank::BankError),
     SendError,
     Services,
+    CrdtTooSmall,
+    GenericError,
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -30,6 +34,11 @@ impl std::convert::From<std::sync::mpsc::RecvTimeoutError> for Error {
         Error::RecvTimeoutError(e)
     }
 }
+impl std::convert::From<bank::BankError> for Error {
+    fn from(e: bank::BankError) -> Error {
+        Error::BankError(e)
+    }
+}
 impl<T> std::convert::From<std::sync::mpsc::SendError<T>> for Error {
     fn from(_e: std::sync::mpsc::SendError<T>) -> Error {
         Error::SendError
@@ -70,9 +79,10 @@ mod tests {
     use std::io;
     use std::io::Write;
     use std::net::SocketAddr;
+    use std::panic;
+    use std::sync::mpsc::channel;
     use std::sync::mpsc::RecvError;
     use std::sync::mpsc::RecvTimeoutError;
-    use std::sync::mpsc::channel;
     use std::thread;
 
     fn addr_parse_error() -> Result<SocketAddr> {
@@ -81,6 +91,7 @@ mod tests {
     }
 
     fn join_error() -> Result<()> {
+        panic::set_hook(Box::new(|_info| {}));
         let r = thread::spawn(|| panic!("hi")).join()?;
         Ok(r)
     }

src/rpu.rs

@@ -0,0 +1,55 @@
+//! The `rpu` module implements the Request Processing Unit, a
+//! 5-stage transaction processing pipeline in software.
+
+use bank::Bank;
+use packet;
+use request_processor::RequestProcessor;
+use request_stage::RequestStage;
+use std::net::UdpSocket;
+use std::sync::atomic::AtomicBool;
+use std::sync::mpsc::channel;
+use std::sync::Arc;
+use std::thread::JoinHandle;
+use streamer;
+
+pub struct Rpu {
+    pub thread_hdls: Vec<JoinHandle<()>>,
+}
+
+impl Rpu {
+    pub fn new(
+        bank: Arc<Bank>,
+        requests_socket: UdpSocket,
+        respond_socket: UdpSocket,
+        exit: Arc<AtomicBool>,
+    ) -> Self {
+        let packet_recycler = packet::PacketRecycler::default();
+        let (packet_sender, packet_receiver) = channel();
+        let t_receiver = streamer::receiver(
+            requests_socket,
+            exit.clone(),
+            packet_recycler.clone(),
+            packet_sender,
+        );
+
+        let blob_recycler = packet::BlobRecycler::default();
+        let request_processor = RequestProcessor::new(bank.clone());
+        let request_stage = RequestStage::new(
+            request_processor,
+            exit.clone(),
+            packet_receiver,
+            packet_recycler.clone(),
+            blob_recycler.clone(),
+        );
+
+        let t_responder = streamer::responder(
+            respond_socket,
+            exit.clone(),
+            blob_recycler.clone(),
+            request_stage.blob_receiver,
+        );
+
+        let thread_hdls = vec![t_receiver, t_responder, request_stage.thread_hdl];
+        Rpu { thread_hdls }
+    }
+}

src/server.rs

@@ -0,0 +1,100 @@
+//! The `server` module hosts all the server microservices.
+
+use bank::Bank;
+use crdt::{Crdt, ReplicatedData};
+use data_replicator::DataReplicator;
+use packet;
+use rpu::Rpu;
+use std::io::Write;
+use std::net::UdpSocket;
+use std::sync::atomic::AtomicBool;
+use std::sync::{Arc, RwLock};
+use std::thread::JoinHandle;
+use std::time::Duration;
+use streamer;
+use tpu::Tpu;
+use tvu::Tvu;
+
+pub struct Server {
+    pub thread_hdls: Vec<JoinHandle<()>>,
+}
+
+impl Server {
+    pub fn new_leader<W: Write + Send + 'static>(
+        bank: Bank,
+        tick_duration: Option<Duration>,
+        me: ReplicatedData,
+        requests_socket: UdpSocket,
+        transactions_socket: UdpSocket,
+        broadcast_socket: UdpSocket,
+        respond_socket: UdpSocket,
+        gossip_socket: UdpSocket,
+        exit: Arc<AtomicBool>,
+        writer: W,
+    ) -> Self {
+        let bank = Arc::new(bank);
+        let mut thread_hdls = vec![];
+        let rpu = Rpu::new(bank.clone(), requests_socket, respond_socket, exit.clone());
+        thread_hdls.extend(rpu.thread_hdls);
+
+        let blob_recycler = packet::BlobRecycler::default();
+        let tpu = Tpu::new(
+            bank.clone(),
+            tick_duration,
+            transactions_socket,
+            blob_recycler.clone(),
+            exit.clone(),
+            writer,
+        );
+        thread_hdls.extend(tpu.thread_hdls);
+
+        let crdt = Arc::new(RwLock::new(Crdt::new(me)));
+        let window = streamer::default_window();
+        let gossip_send_socket = UdpSocket::bind("0.0.0.0:0").expect("bind 0");
+        let data_replicator = DataReplicator::new(
+            crdt.clone(),
+            window.clone(),
+            gossip_socket,
+            gossip_send_socket,
+            exit.clone(),
+        ).expect("DataReplicator::new");
+        thread_hdls.extend(data_replicator.thread_hdls);
+
+        let t_broadcast = streamer::broadcaster(
+            broadcast_socket,
+            exit.clone(),
+            crdt,
+            window,
+            blob_recycler.clone(),
+            tpu.blob_receiver,
+        );
+        thread_hdls.extend(vec![t_broadcast]);
+
+        Server { thread_hdls }
+    }
+    pub fn new_validator(
+        bank: Bank,
+        me: ReplicatedData,
+        requests_socket: UdpSocket,
+        respond_socket: UdpSocket,
+        replicate_socket: UdpSocket,
+        gossip_socket: UdpSocket,
+        leader_repl_data: ReplicatedData,
+        exit: Arc<AtomicBool>,
+    ) -> Self {
+        let bank = Arc::new(bank);
+        let mut thread_hdls = vec![];
+        let rpu = Rpu::new(bank.clone(), requests_socket, respond_socket, exit.clone());
+        thread_hdls.extend(rpu.thread_hdls);
+        let tvu = Tvu::new(
+            bank.clone(),
+            me,
+            gossip_socket,
+            replicate_socket,
+            leader_repl_data,
+            exit.clone(),
+        );
+        thread_hdls.extend(tvu.thread_hdls);
+        Server { thread_hdls }
+    }
+}

src/signature.rs

@@ -1,9 +1,14 @@
 //! The `signature` module provides functionality for public, and private keys.
 
-use generic_array::GenericArray;
 use generic_array::typenum::{U32, U64};
+use generic_array::GenericArray;
+use rand::{ChaChaRng, Rng, SeedableRng};
+use rayon::prelude::*;
+use ring::error::Unspecified;
+use ring::rand::SecureRandom;
 use ring::signature::Ed25519KeyPair;
 use ring::{rand, signature};
+use std::cell::RefCell;
 use untrusted;
 
 pub type KeyPair = Ed25519KeyPair;
@@ -19,8 +24,10 @@ impl KeyPairUtil for Ed25519KeyPair {
     /// Return a new ED25519 keypair
     fn new() -> Self {
         let rng = rand::SystemRandom::new();
-        let pkcs8_bytes = signature::Ed25519KeyPair::generate_pkcs8(&rng).unwrap();
-        signature::Ed25519KeyPair::from_pkcs8(untrusted::Input::from(&pkcs8_bytes)).unwrap()
+        let pkcs8_bytes = signature::Ed25519KeyPair::generate_pkcs8(&rng)
+            .expect("generate_pkcs8 in signature pb fn new");
+        signature::Ed25519KeyPair::from_pkcs8(untrusted::Input::from(&pkcs8_bytes))
+            .expect("from_pcks8 in signature pb fn new")
     }
 
     /// Return the public key for the given keypair
@@ -41,3 +48,92 @@ impl SignatureUtil for GenericArray<u8, U64> {
         signature::verify(&signature::ED25519, peer_public_key, msg, sig).is_ok()
     }
 }
+
+pub struct GenKeys {
+    // This is necessary because the rng needs to mutate its state to remain
+    // deterministic, and the fill trait requires an immuatble reference to self
+    generator: RefCell<ChaChaRng>,
+}
+
+impl GenKeys {
+    pub fn new(seed: &[u8]) -> GenKeys {
+        let seed32: Vec<_> = seed.iter().map(|&x| x as u32).collect();
+        let rng = ChaChaRng::from_seed(&seed32);
+        GenKeys {
+            generator: RefCell::new(rng),
+        }
+    }
+
+    pub fn new_key(&self) -> Vec<u8> {
+        KeyPair::generate_pkcs8(self).unwrap().to_vec()
+    }
+
+    pub fn gen_n_seeds(&self, n: i64) -> Vec<[u8; 16]> {
+        let mut rng = self.generator.borrow_mut();
+        (0..n).map(|_| rng.gen()).collect()
+    }
+
+    pub fn gen_n_keypairs(&self, n: i64) -> Vec<KeyPair> {
+        self.gen_n_seeds(n)
+            .into_par_iter()
+            .map(|seed| {
+                let pkcs8 = GenKeys::new(&seed).new_key();
+                KeyPair::from_pkcs8(untrusted::Input::from(&pkcs8)).unwrap()
+            })
+            .collect()
+    }
+}
+
+impl SecureRandom for GenKeys {
+    fn fill(&self, dest: &mut [u8]) -> Result<(), Unspecified> {
+        let mut rng = self.generator.borrow_mut();
+        rng.fill_bytes(dest);
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::HashSet;
+
+    #[test]
+    fn test_new_key_is_deterministic() {
+        let seed = [1, 2, 3, 4];
+        let rng0 = GenKeys::new(&seed);
+        let rng1 = GenKeys::new(&seed);
+
+        for _ in 0..100 {
+            assert_eq!(rng0.new_key(), rng1.new_key());
+        }
+    }
+
+    fn gen_n_pubkeys(seed: &[u8], n: i64) -> HashSet<PublicKey> {
+        GenKeys::new(&seed)
+            .gen_n_keypairs(n)
+            .into_iter()
+            .map(|x| x.pubkey())
+            .collect()
+    }
+
+    #[test]
+    fn test_gen_n_pubkeys_deterministic() {
+        let seed = [1, 2, 3, 4];
+        assert_eq!(gen_n_pubkeys(&seed, 50), gen_n_pubkeys(&seed, 50));
+    }
+}
+
+#[cfg(all(feature = "unstable", test))]
+mod bench {
+    extern crate test;
+
+    use self::test::Bencher;
+    use super::*;
+
+    #[bench]
+    fn bench_gen_keys(b: &mut Bencher) {
+        let seed: &[_] = &[1, 2, 3, 4];
+        let rnd = GenKeys::new(seed);
+        b.iter(|| rnd.gen_n_keypairs(1000));
+    }
+}

src/sigverify.rs

@@ -0,0 +1,214 @@
+use packet::{Packet, SharedPackets};
+use std::mem::size_of;
+use transaction::{PUB_KEY_OFFSET, SIGNED_DATA_OFFSET, SIG_OFFSET};
+
+pub const TX_OFFSET: usize = 0;
+
+#[cfg(feature = "cuda")]
+#[repr(C)]
+struct Elems {
+    elems: *const Packet,
+    num: u32,
+}
+
+#[cfg(feature = "cuda")]
+#[link(name = "cuda_verify_ed25519")]
+extern "C" {
+    fn ed25519_verify_many(
+        vecs: *const Elems,
+        num: u32,          //number of vecs
+        message_size: u32, //size of each element inside the elems field of the vec
+        public_key_offset: u32,
+        signature_offset: u32,
+        signed_message_offset: u32,
+        signed_message_len_offset: u32,
+        out: *mut u8, //combined length of all the items in vecs
+    ) -> u32;
+}
+
+#[cfg(not(feature = "cuda"))]
+fn verify_packet(packet: &Packet) -> u8 {
+    use ring::signature;
+    use signature::{PublicKey, Signature};
+    use untrusted;
+
+    let msg_start = TX_OFFSET + SIGNED_DATA_OFFSET;
+    let sig_start = TX_OFFSET + SIG_OFFSET;
+    let sig_end = sig_start + size_of::<Signature>();
+    let pub_key_start = TX_OFFSET + PUB_KEY_OFFSET;
+    let pub_key_end = pub_key_start + size_of::<PublicKey>();
+
+    if packet.meta.size <= msg_start {
+        return 0;
+    }
+
+    let msg_end = packet.meta.size;
+    signature::verify(
+        &signature::ED25519,
+        untrusted::Input::from(&packet.data[pub_key_start..pub_key_end]),
+        untrusted::Input::from(&packet.data[msg_start..msg_end]),
+        untrusted::Input::from(&packet.data[sig_start..sig_end]),
+    ).is_ok() as u8
+}
+
+fn batch_size(batches: &Vec<SharedPackets>) -> usize {
+    batches
+        .iter()
+        .map(|p| p.read().unwrap().packets.len())
+        .sum()
+}
+
+#[cfg(not(feature = "cuda"))]
+pub fn ed25519_verify(batches: &Vec<SharedPackets>) -> Vec<Vec<u8>> {
+    use rayon::prelude::*;
+    info!("CPU ECDSA for {}", batch_size(batches));
+    batches
+        .into_par_iter()
+        .map(|p| {
+            p.read()
+                .expect("'p' read lock in ed25519_verify")
+                .packets
+                .par_iter()
+                .map(verify_packet)
+                .collect()
+        })
+        .collect()
+}
+
+#[cfg(feature = "cuda")]
+pub fn ed25519_verify(batches: &Vec<SharedPackets>) -> Vec<Vec<u8>> {
+    use packet::PACKET_DATA_SIZE;
+
+    info!("CUDA ECDSA for {}", batch_size(batches));
+    let mut out = Vec::new();
+    let mut elems = Vec::new();
+    let mut locks = Vec::new();
+    let mut rvs = Vec::new();
+
+    for packets in batches {
+        locks.push(
+            packets
+                .read()
+                .expect("'packets' read lock in pub fn ed25519_verify"),
+        );
+    }
+    let mut num = 0;
+    for p in locks {
+        elems.push(Elems {
+            elems: p.packets.as_ptr(),
+            num: p.packets.len() as u32,
+        });
+        let mut v = Vec::new();
+        v.resize(p.packets.len(), 0);
+        rvs.push(v);
+        num += p.packets.len();
+    }
+    out.resize(num, 0);
+    trace!("Starting verify num packets: {}", num);
+    trace!("elem len: {}", elems.len() as u32);
+    trace!("packet sizeof: {}", size_of::<Packet>() as u32);
+    trace!("pub key: {}", (TX_OFFSET + PUB_KEY_OFFSET) as u32);
+    trace!("sig offset: {}", (TX_OFFSET + SIG_OFFSET) as u32);
+    trace!("sign data: {}", (TX_OFFSET + SIGNED_DATA_OFFSET) as u32);
+    trace!("len offset: {}", PACKET_DATA_SIZE as u32);
+    unsafe {
+        let res = ed25519_verify_many(
+            elems.as_ptr(),
+            elems.len() as u32,
+            size_of::<Packet>() as u32,
+            (TX_OFFSET + PUB_KEY_OFFSET) as u32,
+            (TX_OFFSET + SIG_OFFSET) as u32,
+            (TX_OFFSET + SIGNED_DATA_OFFSET) as u32,
+            PACKET_DATA_SIZE as u32,
+            out.as_mut_ptr(),
+        );
+        if res != 0 {
+            trace!("RETURN!!!: {}", res);
+        }
+    }
+    trace!("done verify");
+    let mut num = 0;
+    for vs in rvs.iter_mut() {
+        for mut v in vs.iter_mut() {
+            *v = out[num];
+            if *v != 0 {
+                trace!("VERIFIED PACKET!!!!!");
+            }
+            num += 1;
+        }
+    }
+    rvs
+}
+
+#[cfg(test)]
+mod tests {
+    use bincode::serialize;
+    use packet::{Packet, Packets, SharedPackets};
+    use sigverify;
+    use std::sync::RwLock;
+    use transaction::Transaction;
+    use transaction::{memfind, test_tx};
+
+    #[test]
+    fn test_layout() {
+        let tx = test_tx();
+        let tx_bytes = serialize(&tx).unwrap();
+        let packet = serialize(&tx).unwrap();
+        assert_matches!(memfind(&packet, &tx_bytes), Some(sigverify::TX_OFFSET));
+        assert_matches!(memfind(&packet, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), None);
+    }
+
+    fn make_packet_from_transaction(tx: Transaction) -> Packet {
+        let tx_bytes = serialize(&tx).unwrap();
+        let mut packet = Packet::default();
+        packet.meta.size = tx_bytes.len();
+        packet.data[..packet.meta.size].copy_from_slice(&tx_bytes);
+        return packet;
+    }
+
+    fn test_verify_n(n: usize, modify_data: bool) {
+        let tx = test_tx();
+        let mut packet = make_packet_from_transaction(tx);
+
+        // jumble some data to test failure
+        if modify_data {
+            packet.data[20] = 10;
+        }
+
+        // generate packet vector
+        let mut packets = Packets::default();
+        packets.packets = Vec::new();
+        for _ in 0..n {
+            packets.packets.push(packet.clone());
+        }
+        let shared_packets = SharedPackets::new(RwLock::new(packets));
+        let batches = vec![shared_packets.clone(), shared_packets.clone()];
+
+        // verify packets
+        let ans = sigverify::ed25519_verify(&batches);
+
+        // check result
+        let ref_ans = if modify_data { 0u8 } else { 1u8 };
+        assert_eq!(ans, vec![vec![ref_ans; n], vec![ref_ans; n]]);
+    }
+
+    #[test]
+    fn test_verify_zero() {
+        test_verify_n(0, false);
+    }
+
+    #[test]
+    fn test_verify_one() {
+        test_verify_n(1, false);
+    }
+
+    #[test]
+    fn test_verify_seventy_one() {
+        test_verify_n(71, false);
+    }
+
+    #[test]
+    fn test_verify_fail() {
+        test_verify_n(5, true);
+    }
+}

src/sigverify_stage.rs

@@ -0,0 +1,96 @@
+//! The `sigverify_stage` implements the signature verification stage of the TPU.
+
+use packet::SharedPackets;
+use rand::{thread_rng, Rng};
+use result::Result;
+use sigverify;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{channel, Receiver, Sender};
+use std::sync::{Arc, Mutex};
+use std::thread::{spawn, JoinHandle};
+use std::time::Instant;
+use streamer;
+use timing;
+
+pub struct SigVerifyStage {
+    pub verified_receiver: Receiver<Vec<(SharedPackets, Vec<u8>)>>,
+    pub thread_hdls: Vec<JoinHandle<()>>,
+}
+
+impl SigVerifyStage {
+    pub fn new(exit: Arc<AtomicBool>, packet_receiver: Receiver<SharedPackets>) -> Self {
+        let (verified_sender, verified_receiver) = channel();
+        let thread_hdls = Self::verifier_services(exit, packet_receiver, verified_sender);
+        SigVerifyStage {
+            thread_hdls,
+            verified_receiver,
+        }
+    }
+
+    fn verify_batch(batch: Vec<SharedPackets>) -> Vec<(SharedPackets, Vec<u8>)> {
+        let r = sigverify::ed25519_verify(&batch);
+        batch.into_iter().zip(r).collect()
+    }
+
+    fn verifier(
+        recvr: &Arc<Mutex<streamer::PacketReceiver>>,
+        sendr: &Arc<Mutex<Sender<Vec<(SharedPackets, Vec<u8>)>>>>,
+    ) -> Result<()> {
+        let (batch, len) =
+            streamer::recv_batch(&recvr.lock().expect("'recvr' lock in fn verifier"))?;
+
+        let now = Instant::now();
+        let batch_len = batch.len();
+        let rand_id = thread_rng().gen_range(0, 100);
+        info!(
+            "@{:?} verifier: verifying: {} id: {}",
+            timing::timestamp(),
+            batch.len(),
+            rand_id
+        );
+
+        let verified_batch = Self::verify_batch(batch);
+        sendr
+            .lock()
+            .expect("lock in fn verify_batch in tpu")
+            .send(verified_batch)?;
+
+        let total_time_ms = timing::duration_as_ms(&now.elapsed());
+        let total_time_s = timing::duration_as_s(&now.elapsed());
+        info!(
+            "@{:?} verifier: done. batches: {} total verify time: {:?} id: {} verified: {} v/s {}",
+            timing::timestamp(),
+            batch_len,
+            total_time_ms,
+            rand_id,
+            len,
+            (len as f32 / total_time_s)
+        );
+        Ok(())
+    }
+
+    fn verifier_service(
+        exit: Arc<AtomicBool>,
+        packet_receiver: Arc<Mutex<streamer::PacketReceiver>>,
+        verified_sender: Arc<Mutex<Sender<Vec<(SharedPackets, Vec<u8>)>>>>,
+    ) -> JoinHandle<()> {
+        spawn(move || loop {
+            let e = Self::verifier(&packet_receiver.clone(), &verified_sender.clone());
+            if e.is_err() && exit.load(Ordering::Relaxed) {
+                break;
+            }
+        })
+    }
+
+    fn verifier_services(
+        exit: Arc<AtomicBool>,
+        packet_receiver: streamer::PacketReceiver,
+        verified_sender: Sender<Vec<(SharedPackets, Vec<u8>)>>,
+    ) -> Vec<JoinHandle<()>> {
+        let sender = Arc::new(Mutex::new(verified_sender));
+        let receiver = Arc::new(Mutex::new(packet_receiver));
+        (0..4)
+            .map(|_| Self::verifier_service(exit.clone(), receiver.clone(), sender.clone()))
+            .collect()
+    }
+}

src/thin_client.rs

@@ -0,0 +1,292 @@
+//! The `thin_client` module is a client-side object that interfaces with
+//! a server-side TPU.  Client code should use this object instead of writing
+//! messages to the network directly. The binary encoding of its messages are
+//! unstable and may change in future releases.
+
+use bincode::{deserialize, serialize};
+use hash::Hash;
+use request::{Request, Response};
+use signature::{KeyPair, PublicKey, Signature};
+use std::collections::HashMap;
+use std::io;
+use std::net::{SocketAddr, UdpSocket};
+use transaction::Transaction;
+
+pub struct ThinClient {
+    requests_addr: SocketAddr,
+    requests_socket: UdpSocket,
+    transactions_addr: SocketAddr,
+    transactions_socket: UdpSocket,
+    last_id: Option<Hash>,
+    transaction_count: u64,
+    balances: HashMap<PublicKey, Option<i64>>,
+}
+
+impl ThinClient {
+    /// Create a new ThinClient that will interface with Rpu
+    /// over `requests_socket` and `transactions_socket`. To receive responses, the caller must bind `socket`
+    /// to a public address before invoking ThinClient methods.
+    pub fn new(
+        requests_addr: SocketAddr,
+        requests_socket: UdpSocket,
+        transactions_addr: SocketAddr,
+        transactions_socket: UdpSocket,
+    ) -> Self {
+        let client = ThinClient {
+            requests_addr,
+            requests_socket,
+            transactions_addr,
+            transactions_socket,
+            last_id: None,
+            transaction_count: 0,
+            balances: HashMap::new(),
+        };
+        client
+    }
+
+    pub fn recv_response(&self) -> io::Result<Response> {
+        let mut buf = vec![0u8; 1024];
+        trace!("start recv_from");
+        self.requests_socket.recv_from(&mut buf)?;
+        trace!("end recv_from");
+        let resp = deserialize(&buf).expect("deserialize balance in thin_client");
+        Ok(resp)
+    }
+
+    pub fn process_response(&mut self, resp: Response) {
+        match resp {
+            Response::Balance { key, val } => {
+                trace!("Response balance {:?} {:?}", key, val);
+                self.balances.insert(key, val);
+            }
+            Response::LastId { id } => {
+                info!("Response last_id {:?}", id);
+                self.last_id = Some(id);
+            }
+            Response::TransactionCount { transaction_count } => {
+                info!("Response transaction count {:?}", transaction_count);
+                self.transaction_count = transaction_count;
+            }
+        }
+    }
+
+    /// Send a signed Transaction to the server for processing. This method
+    /// does not wait for a response.
+    pub fn transfer_signed(&self, tx: Transaction) -> io::Result<usize> {
+        let data = serialize(&tx).expect("serialize Transaction in pub fn transfer_signed");
+        self.transactions_socket
+            .send_to(&data, &self.transactions_addr)
+    }
+
+    /// Creates, signs, and processes a Transaction. Useful for writing unit-tests.
+    pub fn transfer(
+        &self,
+        n: i64,
+        keypair: &KeyPair,
+        to: PublicKey,
+        last_id: &Hash,
+    ) -> io::Result<Signature> {
+        let tx = Transaction::new(keypair, to, n, *last_id);
+        let sig = tx.sig;
+        self.transfer_signed(tx).map(|_| sig)
+    }
+
+    /// Request the balance of the user holding `pubkey`. This method blocks
+    /// until the server sends a response. If the response packet is dropped
+    /// by the network, this method will hang indefinitely.
+    pub fn get_balance(&mut self, pubkey: &PublicKey) -> io::Result<i64> {
+        trace!("get_balance");
+        let req = Request::GetBalance { key: *pubkey };
+        let data = serialize(&req).expect("serialize GetBalance in pub fn get_balance");
+        self.requests_socket
+            .send_to(&data, &self.requests_addr)
+            .expect("buffer error in pub fn get_balance");
+        let mut done = false;
+        while !done {
+            let resp = self.recv_response()?;
+            trace!("recv_response {:?}", resp);
+            if let &Response::Balance { ref key, .. } = &resp {
+                done = key == pubkey;
+            }
+            self.process_response(resp);
+        }
+        self.balances[pubkey].ok_or(io::Error::new(io::ErrorKind::Other, "nokey"))
+    }
+
+    /// Request the transaction count.  If the response packet is dropped by the network,
+    /// this method will hang.
+    pub fn transaction_count(&mut self) -> u64 {
+        info!("transaction_count");
+        let req = Request::GetTransactionCount;
+        let data =
+            serialize(&req).expect("serialize GetTransactionCount in pub fn transaction_count");
+        self.requests_socket
+            .send_to(&data, &self.requests_addr)
+            .expect("buffer error in pub fn transaction_count");
+        let mut done = false;
+        while !done {
+            let resp = self.recv_response().expect("transaction count dropped");
+            info!("recv_response {:?}", resp);
+            if let &Response::TransactionCount { .. } = &resp {
+                done = true;
+            }
+            self.process_response(resp);
+        }
+        self.transaction_count
+    }
+
+    /// Request the last Entry ID from the server. This method blocks
+    /// until the server sends a response.
+    pub fn get_last_id(&mut self) -> Hash {
+        info!("get_last_id");
+        let req = Request::GetLastId;
+        let data = serialize(&req).expect("serialize GetLastId in pub fn get_last_id");
+        self.requests_socket
+            .send_to(&data, &self.requests_addr)
+            .expect("buffer error in pub fn get_last_id");
+        let mut done = false;
+        while !done {
+            let resp = self.recv_response().expect("get_last_id response");
+            if let &Response::LastId { .. } = &resp {
+                done = true;
+            }
+            self.process_response(resp);
+        }
+        self.last_id.expect("some last_id")
+    }
+
+    pub fn poll_get_balance(&mut self, pubkey: &PublicKey) -> io::Result<i64> {
+        use std::time::Instant;
+
+        let mut balance;
+        let now = Instant::now();
+        loop {
+            balance = self.get_balance(pubkey);
+            if balance.is_ok() || now.elapsed().as_secs() > 1 {
+                break;
+            }
+        }
+
+        balance
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bank::Bank;
+    use budget::Budget;
+    use crdt::TestNode;
+    use logger;
+    use mint::Mint;
+    use server::Server;
+    use signature::{KeyPair, KeyPairUtil};
+    use std::io::sink;
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::Arc;
+    use std::thread::sleep;
+    use std::time::Duration;
+    use transaction::{Instruction, Plan};
+
+    #[test]
+    fn test_thin_client() {
+        logger::setup();
+        let leader = TestNode::new();
+
+        let alice = Mint::new(10_000);
+        let bank = Bank::new(&alice);
+        let bob_pubkey = KeyPair::new().pubkey();
+        let exit = Arc::new(AtomicBool::new(false));
+
+        let server = Server::new_leader(
+            bank,
+            Some(Duration::from_millis(30)),
+            leader.data.clone(),
+            leader.sockets.requests,
+            leader.sockets.transaction,
+            leader.sockets.broadcast,
+            leader.sockets.respond,
+            leader.sockets.gossip,
+            exit.clone(),
+            sink(),
+        );
+        sleep(Duration::from_millis(900));
+
+        let requests_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let transactions_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+
+        let mut client = ThinClient::new(
+            leader.data.requests_addr,
+            requests_socket,
+            leader.data.transactions_addr,
+            transactions_socket,
+        );
+        let last_id = client.get_last_id();
+        let _sig = client
+            .transfer(500, &alice.keypair(), bob_pubkey, &last_id)
+            .unwrap();
+        let balance = client.poll_get_balance(&bob_pubkey);
+        assert_eq!(balance.unwrap(), 500);
+        exit.store(true, Ordering::Relaxed);
+        for t in server.thread_hdls {
+            t.join().unwrap();
+        }
+    }
+
+    #[test]
+    fn test_bad_sig() {
+        logger::setup();
+        let leader = TestNode::new();
+        let alice = Mint::new(10_000);
+        let bank = Bank::new(&alice);
+        let bob_pubkey = KeyPair::new().pubkey();
+        let exit = Arc::new(AtomicBool::new(false));
+
+        let server = Server::new_leader(
+            bank,
+            Some(Duration::from_millis(30)),
+            leader.data.clone(),
+            leader.sockets.requests,
+            leader.sockets.transaction,
+            leader.sockets.broadcast,
+            leader.sockets.respond,
+            leader.sockets.gossip,
+            exit.clone(),
+            sink(),
+        );
+        sleep(Duration::from_millis(300));
+
+        let requests_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+        requests_socket
+            .set_read_timeout(Some(Duration::new(5, 0)))
+            .unwrap();
+        let transactions_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let mut client = ThinClient::new(
+            leader.data.requests_addr,
+            requests_socket,
+            leader.data.transactions_addr,
+            transactions_socket,
+        );
+        let last_id = client.get_last_id();
+
+        let tx = Transaction::new(&alice.keypair(), bob_pubkey, 500, last_id);
+
+        let _sig = client.transfer_signed(tx).unwrap();
+
+        let last_id = client.get_last_id();
+
+        let mut tr2 = Transaction::new(&alice.keypair(), bob_pubkey, 501, last_id);
+        if let Instruction::NewContract(contract) = &mut tr2.instruction {
+            contract.tokens = 502;
+            contract.plan = Plan::Budget(Budget::new_payment(502, bob_pubkey));
+        }
+        let _sig = client.transfer_signed(tr2).unwrap();
+
+        let balance = client.poll_get_balance(&bob_pubkey);
+        assert_eq!(balance.unwrap(), 500);
+        exit.store(true, Ordering::Relaxed);
+        for t in server.thread_hdls {
+            t.join().unwrap();
+        }
+    }
+}

src/timing.rs

@@ -0,0 +1,17 @@
+use std::time::Duration;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+pub fn duration_as_ms(d: &Duration) -> u64 {
+    return (d.as_secs() * 1000) + (d.subsec_nanos() as u64 / 1_000_000);
+}
+
+pub fn duration_as_s(d: &Duration) -> f32 {
+    return d.as_secs() as f32 + (d.subsec_nanos() as f32 / 1_000_000_000.0);
+}
+
+pub fn timestamp() -> u64 {
+    let now = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .expect("create timestamp in timing");
+    return duration_as_ms(&now);
+}

src/tpu.rs

@@ -0,0 +1,75 @@
+//! The `tpu` module implements the Transaction Processing Unit, a
+//! 5-stage transaction processing pipeline in software.
+
+use bank::Bank;
+use banking_stage::BankingStage;
+use fetch_stage::FetchStage;
+use packet::{BlobRecycler, PacketRecycler};
+use record_stage::RecordStage;
+use sigverify_stage::SigVerifyStage;
+use std::io::Write;
+use std::net::UdpSocket;
+use std::sync::atomic::AtomicBool;
+use std::sync::{Arc, Mutex};
+use std::thread::JoinHandle;
+use std::time::Duration;
+use streamer::BlobReceiver;
+use write_stage::WriteStage;
+
+pub struct Tpu {
+    pub blob_receiver: BlobReceiver,
+    pub thread_hdls: Vec<JoinHandle<()>>,
+}
+
+impl Tpu {
+    pub fn new<W: Write + Send + 'static>(
+        bank: Arc<Bank>,
+        tick_duration: Option<Duration>,
+        transactions_socket: UdpSocket,
+        blob_recycler: BlobRecycler,
+        exit: Arc<AtomicBool>,
+        writer: W,
+    ) -> Self {
+        let packet_recycler = PacketRecycler::default();
+
+        let fetch_stage =
+            FetchStage::new(transactions_socket, exit.clone(), packet_recycler.clone());
+
+        let sigverify_stage = SigVerifyStage::new(exit.clone(), fetch_stage.packet_receiver);
+
+        let banking_stage = BankingStage::new(
+            bank.clone(),
+            exit.clone(),
+            sigverify_stage.verified_receiver,
+            packet_recycler.clone(),
+        );
+
+        let record_stage = match tick_duration {
+            Some(tick_duration) => RecordStage::new_with_clock(
+                banking_stage.signal_receiver,
+                &bank.last_id(),
+                tick_duration,
+            ),
+            None => RecordStage::new(banking_stage.signal_receiver, &bank.last_id()),
+        };
+
+        let write_stage = WriteStage::new(
+            bank.clone(),
+            exit.clone(),
+            blob_recycler.clone(),
+            Mutex::new(writer),
+            record_stage.entry_receiver,
+        );
+        let mut thread_hdls = vec![
+            fetch_stage.thread_hdl,
+            banking_stage.thread_hdl,
+            record_stage.thread_hdl,
+            write_stage.thread_hdl,
+        ];
+        thread_hdls.extend(sigverify_stage.thread_hdls.into_iter());
+        Tpu {
+            blob_receiver: write_stage.blob_receiver,
+            thread_hdls,
+        }
+    }
+}

src/transaction.rs

@@ -1,35 +1,116 @@
 //! The `transaction` module provides functionality for creating log transactions.
 
 use bincode::serialize;
+use budget::{Budget, Condition};
 use chrono::prelude::*;
-use rayon::prelude::*;
 use hash::Hash;
-use plan::{Condition, Payment, Plan};
+use payment_plan::{Payment, PaymentPlan, Witness};
 use signature::{KeyPair, KeyPairUtil, PublicKey, Signature, SignatureUtil};
 
+pub const SIGNED_DATA_OFFSET: usize = 112;
+pub const SIG_OFFSET: usize = 8;
+pub const PUB_KEY_OFFSET: usize = 80;
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub enum Plan {
+    Budget(Budget),
+}
+
+// A proxy for the underlying DSL.
+impl PaymentPlan for Plan {
+    fn final_payment(&self) -> Option<Payment> {
+        match self {
+            Plan::Budget(budget) => budget.final_payment(),
+        }
+    }
+
+    fn verify(&self, spendable_tokens: i64) -> bool {
+        match self {
+            Plan::Budget(budget) => budget.verify(spendable_tokens),
+        }
+    }
+
+    fn apply_witness(&mut self, witness: &Witness) {
+        match self {
+            Plan::Budget(budget) => budget.apply_witness(witness),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub struct Contract {
+    pub tokens: i64,
+    pub plan: Plan,
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub enum Instruction {
+    NewContract(Contract),
+    ApplyTimestamp(DateTime<Utc>),
+    ApplySignature(Signature),
+}
+
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
 pub struct Transaction {
-    pub from: PublicKey,
-    pub plan: Plan,
-    pub tokens: i64,
-    pub last_id: Hash,
     pub sig: Signature,
+    pub from: PublicKey,
+    pub instruction: Instruction,
+    pub last_id: Hash,
+    pub fee: i64,
 }
 
 impl Transaction {
+    fn new_from_instruction(
+        from_keypair: &KeyPair,
+        instruction: Instruction,
+        last_id: Hash,
+        fee: i64,
+    ) -> Self {
+        let from = from_keypair.pubkey();
+        let mut tx = Transaction {
+            sig: Signature::default(),
+            instruction,
+            last_id,
+            from,
+            fee,
+        };
+        tx.sign(from_keypair);
+        tx
+    }
+
+    /// Create and sign a new Transaction. Used for unit-testing.
+    pub fn new_taxed(
+        from_keypair: &KeyPair,
+        to: PublicKey,
+        tokens: i64,
+        fee: i64,
+        last_id: Hash,
+    ) -> Self {
+        let payment = Payment {
+            tokens: tokens - fee,
+            to,
+        };
+        let budget = Budget::Pay(payment);
+        let plan = Plan::Budget(budget);
+        let instruction = Instruction::NewContract(Contract { plan, tokens });
+        Self::new_from_instruction(from_keypair, instruction, last_id, fee)
+    }
+
     /// Create and sign a new Transaction. Used for unit-testing.
     pub fn new(from_keypair: &KeyPair, to: PublicKey, tokens: i64, last_id: Hash) -> Self {
-        let from = from_keypair.pubkey();
-        let plan = Plan::Pay(Payment { tokens, to });
-        let mut tr = Transaction {
-            from,
-            plan,
-            tokens,
-            last_id,
-            sig: Signature::default(),
-        };
-        tr.sign(from_keypair);
-        tr
+        Self::new_taxed(from_keypair, to, tokens, 0, last_id)
+    }
+
+    /// Create and sign a new Witness Timestamp. Used for unit-testing.
+    pub fn new_timestamp(from_keypair: &KeyPair, dt: DateTime<Utc>, last_id: Hash) -> Self {
+        let instruction = Instruction::ApplyTimestamp(dt);
+        Self::new_from_instruction(from_keypair, instruction, last_id, 0)
+    }
+
+    /// Create and sign a new Witness Signature. Used for unit-testing.
+    pub fn new_signature(from_keypair: &KeyPair, tx_sig: Signature, last_id: Hash) -> Self {
+        let instruction = Instruction::ApplySignature(tx_sig);
+        Self::new_from_instruction(from_keypair, instruction, last_id, 0)
     }
 
     /// Create and sign a postdated Transaction. Used for unit-testing.
@@ -41,23 +122,24 @@ impl Transaction {
         last_id: Hash,
     ) -> Self {
         let from = from_keypair.pubkey();
-        let plan = Plan::Race(
+        let budget = Budget::Race(
             (Condition::Timestamp(dt), Payment { tokens, to }),
             (Condition::Signature(from), Payment { tokens, to: from }),
         );
-        let mut tr = Transaction {
-            from,
-            plan,
-            tokens,
-            last_id,
-            sig: Signature::default(),
-        };
-        tr.sign(from_keypair);
-        tr
+        let plan = Plan::Budget(budget);
+        let instruction = Instruction::NewContract(Contract { plan, tokens });
+        Self::new_from_instruction(from_keypair, instruction, last_id, 0)
     }
 
     fn get_sign_data(&self) -> Vec<u8> {
-        serialize(&(&self.from, &self.plan, &self.tokens, &self.last_id)).unwrap()
+        let mut data = serialize(&(&self.instruction)).expect("serialize Contract");
+        let last_id_data = serialize(&(&self.last_id)).expect("serialize last_id");
+        data.extend_from_slice(&last_id_data);
+
+        let fee_data = serialize(&(&self.fee)).expect("serialize last_id");
+        data.extend_from_slice(&fee_data);
+
+        data
     }
 
     /// Sign this transaction.
@@ -66,25 +148,39 @@ impl Transaction {
         self.sig = Signature::clone_from_slice(keypair.sign(&sign_data).as_ref());
     }
 
-    /// Verify this transaction's signature and its spending plan.
-    pub fn verify(&self) -> bool {
-        self.sig.verify(&self.from, &self.get_sign_data()) && self.plan.verify(self.tokens)
+    pub fn verify_sig(&self) -> bool {
+        warn!("transaction signature verification called");
+        self.sig.verify(&self.from, &self.get_sign_data())
+    }
+
+    pub fn verify_plan(&self) -> bool {
+        if let Instruction::NewContract(contract) = &self.instruction {
+            self.fee >= 0 && self.fee <= contract.tokens
+                && contract.plan.verify(contract.tokens - self.fee)
+        } else {
+            true
+        }
     }
 }
 
-/// Verify a batch of signatures.
-pub fn verify_signatures(transactions: &[Transaction]) -> bool {
-    transactions.par_iter().all(|tr| tr.verify())
+#[cfg(test)]
+pub fn test_tx() -> Transaction {
+    let keypair1 = KeyPair::new();
+    let pubkey1 = keypair1.pubkey();
+    let zero = Hash::default();
+    Transaction::new(&keypair1, pubkey1, 42, zero)
 }
 
-/// Verify a batch of spending plans.
-pub fn verify_plans(transactions: &[Transaction]) -> bool {
-    transactions.par_iter().all(|tr| tr.plan.verify(tr.tokens))
-}
-
-/// Verify a batch of transactions.
-pub fn verify_transactions(transactions: &[Transaction]) -> bool {
-    verify_signatures(transactions) && verify_plans(transactions)
+#[cfg(test)]
+pub fn memfind<A: Eq>(a: &[A], b: &[A]) -> Option<usize> {
+    assert!(a.len() >= b.len());
+    let end = a.len() - b.len() + 1;
+    for i in 0..end {
+        if a[i..i + b.len()] == b[..] {
+            return Some(i);
+        }
+    }
+    None
 }
 
 #[cfg(test)]
@@ -96,8 +192,8 @@ mod tests {
     fn test_claim() {
         let keypair = KeyPair::new();
         let zero = Hash::default();
-        let tr0 = Transaction::new(&keypair, keypair.pubkey(), 42, zero);
-        assert!(tr0.verify());
+        let tx0 = Transaction::new(&keypair, keypair.pubkey(), 42, zero);
+        assert!(tx0.verify_plan());
     }
 
     #[test]
@@ -106,22 +202,34 @@ mod tests {
         let keypair0 = KeyPair::new();
         let keypair1 = KeyPair::new();
         let pubkey1 = keypair1.pubkey();
-        let tr0 = Transaction::new(&keypair0, pubkey1, 42, zero);
-        assert!(tr0.verify());
+        let tx0 = Transaction::new(&keypair0, pubkey1, 42, zero);
+        assert!(tx0.verify_plan());
+    }
+
+    #[test]
+    fn test_transfer_with_fee() {
+        let zero = Hash::default();
+        let keypair0 = KeyPair::new();
+        let pubkey1 = KeyPair::new().pubkey();
+        assert!(Transaction::new_taxed(&keypair0, pubkey1, 1, 1, zero).verify_plan());
+        assert!(!Transaction::new_taxed(&keypair0, pubkey1, 1, 2, zero).verify_plan());
+        assert!(!Transaction::new_taxed(&keypair0, pubkey1, 1, -1, zero).verify_plan());
     }
 
     #[test]
     fn test_serialize_claim() {
-        let plan = Plan::Pay(Payment {
+        let budget = Budget::Pay(Payment {
             tokens: 0,
             to: Default::default(),
         });
+        let plan = Plan::Budget(budget);
+        let instruction = Instruction::NewContract(Contract { plan, tokens: 0 });
         let claim0 = Transaction {
+            instruction,
             from: Default::default(),
-            plan,
-            tokens: 0,
             last_id: Default::default(),
             sig: Default::default(),
+            fee: 0,
         };
         let buf = serialize(&claim0).unwrap();
         let claim1: Transaction = deserialize(&buf).unwrap();
@@ -129,14 +237,19 @@ mod tests {
     }
 
     #[test]
-    fn test_bad_event_signature() {
+    fn test_token_attack() {
         let zero = Hash::default();
         let keypair = KeyPair::new();
         let pubkey = keypair.pubkey();
-        let mut tr = Transaction::new(&keypair, pubkey, 42, zero);
-        tr.sign(&keypair);
-        tr.tokens = 1_000_000; // <-- attack!
-        assert!(!tr.verify());
+        let mut tx = Transaction::new(&keypair, pubkey, 42, zero);
+        if let Instruction::NewContract(contract) = &mut tx.instruction {
+            contract.tokens = 1_000_000; // <-- attack, part 1!
+            if let Plan::Budget(Budget::Pay(ref mut payment)) = contract.plan {
+                payment.tokens = contract.tokens; // <-- attack, part 2!
+            }
+        }
+        assert!(tx.verify_plan());
+        assert!(!tx.verify_sig());
     }
 
     #[test]
@@ -146,46 +259,44 @@ mod tests {
         let thief_keypair = KeyPair::new();
         let pubkey1 = keypair1.pubkey();
         let zero = Hash::default();
-        let mut tr = Transaction::new(&keypair0, pubkey1, 42, zero);
-        tr.sign(&keypair0);
-        if let Plan::Pay(ref mut payment) = tr.plan {
-            payment.to = thief_keypair.pubkey(); // <-- attack!
-        };
-        assert!(!tr.verify());
+        let mut tx = Transaction::new(&keypair0, pubkey1, 42, zero);
+        if let Instruction::NewContract(contract) = &mut tx.instruction {
+            if let Plan::Budget(Budget::Pay(ref mut payment)) = contract.plan {
+                payment.to = thief_keypair.pubkey(); // <-- attack!
+            }
+        }
+        assert!(tx.verify_plan());
+        assert!(!tx.verify_sig());
+    }
+    #[test]
+    fn test_layout() {
+        let tx = test_tx();
+        let sign_data = tx.get_sign_data();
+        let tx_bytes = serialize(&tx).unwrap();
+        assert_matches!(memfind(&tx_bytes, &sign_data), Some(SIGNED_DATA_OFFSET));
+        assert_matches!(memfind(&tx_bytes, &tx.sig), Some(SIG_OFFSET));
+        assert_matches!(memfind(&tx_bytes, &tx.from), Some(PUB_KEY_OFFSET));
     }
 
     #[test]
-    fn test_verify_transactions() {
-        let alice_keypair = KeyPair::new();
-        let bob_pubkey = KeyPair::new().pubkey();
-        let carol_pubkey = KeyPair::new().pubkey();
-        let last_id = Hash::default();
-        let tr0 = Transaction::new(&alice_keypair, bob_pubkey, 1, last_id);
-        let tr1 = Transaction::new(&alice_keypair, carol_pubkey, 1, last_id);
-        let transactions = vec![tr0, tr1];
-        assert!(verify_transactions(&transactions));
-    }
-}
-
-#[cfg(all(feature = "unstable", test))]
-mod bench {
-    extern crate test;
-    use self::test::Bencher;
-    use transaction::*;
-
-    #[bench]
-    fn verify_signatures_bench(bencher: &mut Bencher) {
-        let alice_keypair = KeyPair::new();
-        let last_id = Hash::default();
-        let transactions: Vec<_> = (0..64)
-            .into_par_iter()
-            .map(|_| {
-                let rando_pubkey = KeyPair::new().pubkey();
-                Transaction::new(&alice_keypair, rando_pubkey, 1, last_id)
-            })
-            .collect();
-        bencher.iter(|| {
-            assert!(verify_signatures(&transactions));
-        });
+    fn test_overspend_attack() {
+        let keypair0 = KeyPair::new();
+        let keypair1 = KeyPair::new();
+        let zero = Hash::default();
+        let mut tx = Transaction::new(&keypair0, keypair1.pubkey(), 1, zero);
+        if let Instruction::NewContract(contract) = &mut tx.instruction {
+            if let Plan::Budget(Budget::Pay(ref mut payment)) = contract.plan {
+                payment.tokens = 2; // <-- attack!
+            }
+        }
+        assert!(!tx.verify_plan());
+
+        // Also, ensure all branchs of the plan spend all tokens
+        if let Instruction::NewContract(contract) = &mut tx.instruction {
+            if let Plan::Budget(Budget::Pay(ref mut payment)) = contract.plan {
+                payment.tokens = 0; // <-- whoops!
+            }
+        }
+        assert!(!tx.verify_plan());
     }
 }

src/tvu.rs

@@ -0,0 +1,295 @@
+//! The `tvu` module implements the Transaction Validation Unit, a
+//! 5-stage transaction validation pipeline in software.
+//! 1. streamer
+//! - Incoming blobs are picked up from the replicate socket.
+//! 2. verifier
+//! - TODO Blobs are sent to the GPU, and while the memory is there the PoH stream is verified
+//! along with the ecdsa signature for the blob and each signature in all the transactions.  Blobs
+//! with errors are dropped, or marked for slashing.
+//! 3.a retransmit
+//! - Blobs originating from the parent (leader, at the moment, is the only parent), are retransmit to all the
+//! peers in the crdt.  Peers is everyone who is not me or the leader that has a known replicate
+//! address.
+//! 3.b window
+//! - Verified blobs are placed into a window, indexed by the counter set by the leader.sockets. This could
+//! be the PoH counter if its monotonically increasing in each blob.  Erasure coding is used to
+//! recover any missing packets, and requests are made at random to peers and parents to retransmit
+//! a missing packet.
+//! 4. accountant
+//! - Contigous blobs are sent to the accountant for processing transactions
+//! 5. validator
+//! - TODO Validation messages are sent back to the leader
+
+use bank::Bank;
+use crdt::{Crdt, ReplicatedData};
+use data_replicator::DataReplicator;
+use packet;
+use replicate_stage::ReplicateStage;
+use std::net::UdpSocket;
+use std::sync::atomic::AtomicBool;
+use std::sync::mpsc::channel;
+use std::sync::{Arc, RwLock};
+use std::thread::JoinHandle;
+use streamer;
+
+pub struct Tvu {
+    pub thread_hdls: Vec<JoinHandle<()>>,
+}
+
+impl Tvu {
+    /// This service receives messages from a leader in the network and processes the transactions
+    /// on the bank state.
+    /// # Arguments
+    /// * `bank` - The bank state.
+    /// * `me` - my configuration
+    /// * `gossip` - my gossisp socket
+    /// * `replicate` - my replicate socket
+    /// * `leader` - leader configuration
+    /// * `exit` - The exit signal.
+    pub fn new(
+        bank: Arc<Bank>,
+        me: ReplicatedData,
+        gossip_listen_socket: UdpSocket,
+        replicate: UdpSocket,
+        leader: ReplicatedData,
+        exit: Arc<AtomicBool>,
+    ) -> Self {
+        //replicate pipeline
+        let crdt = Arc::new(RwLock::new(Crdt::new(me)));
+        crdt.write()
+            .expect("'crdt' write lock in pub fn replicate")
+            .set_leader(leader.id);
+        crdt.write()
+            .expect("'crdt' write lock before insert() in pub fn replicate")
+            .insert(&leader);
+        let window = streamer::default_window();
+        let gossip_send_socket = UdpSocket::bind("0.0.0.0:0").expect("bind 0");
+        let data_replicator = DataReplicator::new(
+            crdt.clone(),
+            window.clone(),
+            gossip_listen_socket,
+            gossip_send_socket,
+            exit.clone(),
+        ).expect("DataReplicator::new");
+
+        // TODO pull this socket out through the public interface
+        // make sure we are on the same interface
+        let mut local = replicate.local_addr().expect("tvu: get local address");
+        local.set_port(0);
+        let write = UdpSocket::bind(local).expect("tvu: bind to local socket");
+
+        let blob_recycler = packet::BlobRecycler::default();
+        let (blob_sender, blob_receiver) = channel();
+        let t_blob_receiver = streamer::blob_receiver(
+            exit.clone(),
+            blob_recycler.clone(),
+            replicate,
+            blob_sender.clone(),
+        ).expect("tvu: blob receiver creation");
+        let (window_sender, window_receiver) = channel();
+        let (retransmit_sender, retransmit_receiver) = channel();
+
+        let t_retransmit = streamer::retransmitter(
+            write,
+            exit.clone(),
+            crdt.clone(),
+            blob_recycler.clone(),
+            retransmit_receiver,
+        );
+
+        //TODO
+        //the packets coming out of blob_receiver need to be sent to the GPU and verified
+        //then sent to the window, which does the erasure coding reconstruction
+        let t_window = streamer::window(
+            exit.clone(),
+            crdt.clone(),
+            window,
+            blob_recycler.clone(),
+            blob_receiver,
+            window_sender,
+            retransmit_sender,
+        );
+
+        let replicate_stage = ReplicateStage::new(
+            bank.clone(),
+            exit.clone(),
+            window_receiver,
+            blob_recycler.clone(),
+        );
+
+        let mut threads = vec![
+            //replicate threads
+            t_blob_receiver,
+            t_retransmit,
+            t_window,
+            replicate_stage.thread_hdl,
+        ];
+        threads.extend(data_replicator.thread_hdls.into_iter());
+        Tvu {
+            thread_hdls: threads,
+        }
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use bank::Bank;
+    use bincode::serialize;
+    use crdt::{Crdt, TestNode};
+    use data_replicator::DataReplicator;
+    use entry::Entry;
+    use hash::{hash, Hash};
+    use logger;
+    use mint::Mint;
+    use packet::BlobRecycler;
+    use result::Result;
+    use signature::{KeyPair, KeyPairUtil};
+    use std::collections::VecDeque;
+    use std::net::UdpSocket;
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::mpsc::channel;
+    use std::sync::{Arc, RwLock};
+    use std::time::Duration;
+    use streamer;
+    use transaction::Transaction;
+    use tvu::Tvu;
+
+    fn new_replicator(
+        crdt: Arc<RwLock<Crdt>>,
+        listen: UdpSocket,
+        exit: Arc<AtomicBool>,
+    ) -> Result<DataReplicator> {
+        let window = streamer::default_window();
+        let send_sock = UdpSocket::bind("0.0.0.0:0").expect("bind 0");
+        DataReplicator::new(crdt, window, listen, send_sock, exit)
+    }
+    /// Test that message sent from leader to target1 and replicated to target2
+    #[test]
+    fn test_replicate() {
+        logger::setup();
+        let leader = TestNode::new();
+        let target1 = TestNode::new();
+        let target2 = TestNode::new();
+        let exit = Arc::new(AtomicBool::new(false));
+
+        //start crdt_leader
+        let mut crdt_l = Crdt::new(leader.data.clone());
+        crdt_l.set_leader(leader.data.id);
+
+        let cref_l = Arc::new(RwLock::new(crdt_l));
+        let dr_l = new_replicator(cref_l, leader.sockets.gossip, exit.clone()).unwrap();
+
+        //start crdt2
+        let mut crdt2 = Crdt::new(target2.data.clone());
+        crdt2.insert(&leader.data);
+        crdt2.set_leader(leader.data.id);
+        let leader_id = leader.data.id;
+        let cref2 = Arc::new(RwLock::new(crdt2));
+        let dr_2 = new_replicator(cref2, target2.sockets.gossip, exit.clone()).unwrap();
+
+        // setup some blob services to send blobs into the socket
+        // to simulate the source peer and get blobs out of the socket to
+        // simulate target peer
+        let recv_recycler = BlobRecycler::default();
+        let resp_recycler = BlobRecycler::default();
+        let (s_reader, r_reader) = channel();
+        let t_receiver = streamer::blob_receiver(
+            exit.clone(),
+            recv_recycler.clone(),
+            target2.sockets.replicate,
+            s_reader,
+        ).unwrap();
+
+        // simulate leader sending messages
+        let (s_responder, r_responder) = channel();
+        let t_responder = streamer::responder(
+            leader.sockets.requests,
+            exit.clone(),
+            resp_recycler.clone(),
+            r_responder,
+        );
+
+        let starting_balance = 10_000;
+        let mint = Mint::new(starting_balance);
+        let replicate_addr = target1.data.replicate_addr;
+        let bank = Arc::new(Bank::new(&mint));
+        let tvu = Tvu::new(
+            bank.clone(),
+            target1.data,
+            target1.sockets.gossip,
+            target1.sockets.replicate,
+            leader.data,
+            exit.clone(),
+        );
+
+        let mut alice_ref_balance = starting_balance;
+        let mut msgs = VecDeque::new();
+        let mut cur_hash = Hash::default();
+        let num_blobs = 10;
+        let transfer_amount = 501;
+        let bob_keypair = KeyPair::new();
+        for i in 0..num_blobs {
+            let b = resp_recycler.allocate();
+            let b_ = b.clone();
+            let mut w = b.write().unwrap();
+            w.set_index(i).unwrap();
+            w.set_id(leader_id).unwrap();
+
+            let entry0 = Entry::new(&cur_hash, i, vec![]);
+            bank.register_entry_id(&cur_hash);
+            cur_hash = hash(&cur_hash);
+
+            let tx0 = Transaction::new(
+                &mint.keypair(),
+                bob_keypair.pubkey(),
+                transfer_amount,
+                cur_hash,
+            );
+            bank.register_entry_id(&cur_hash);
+            cur_hash = hash(&cur_hash);
+            let entry1 = Entry::new(&cur_hash, i + num_blobs, vec![tx0]);
+            bank.register_entry_id(&cur_hash);
+            cur_hash = hash(&cur_hash);
+
+            alice_ref_balance -= transfer_amount;
+
+            let serialized_entry = serialize(&vec![entry0, entry1]).unwrap();
+
+            w.data_mut()[..serialized_entry.len()].copy_from_slice(&serialized_entry);
+            w.set_size(serialized_entry.len());
+            w.meta.set_addr(&replicate_addr);
+            drop(w);
+            msgs.push_back(b_);
+        }
+
+        // send the blobs into the socket
+        s_responder.send(msgs).expect("send");
+
+        // receive retransmitted messages
+        let timer = Duration::new(1, 0);
+        let mut msgs: Vec<_> = Vec::new();
+        while let Ok(msg) = r_reader.recv_timeout(timer) {
+            trace!("msg: {:?}", msg);
+            msgs.push(msg);
+        }
+
+        let alice_balance = bank.get_balance(&mint.keypair().pubkey()).unwrap();
+        assert_eq!(alice_balance, alice_ref_balance);
+
+        let bob_balance = bank.get_balance(&bob_keypair.pubkey()).unwrap();
+        assert_eq!(bob_balance, starting_balance - alice_ref_balance);
+
+        exit.store(true, Ordering::Relaxed);
+        for t in tvu.thread_hdls {
+            t.join().expect("join");
+        }
+        for t in dr_l.thread_hdls {
+            t.join().expect("join");
+        }
+        for t in dr_2.thread_hdls {
+            t.join().expect("join");
+        }
+        t_receiver.join().expect("join");
+        t_responder.join().expect("join");
+    }
+}

src/write_stage.rs

@@ -0,0 +1,77 @@
+//! The `write_stage` module implements write stage of the RPU.
+
+use bank::Bank;
+use entry::Entry;
+use entry_writer::EntryWriter;
+use packet;
+use std::io::Write;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{channel, Receiver};
+use std::sync::{Arc, Mutex};
+use std::thread::{Builder, JoinHandle};
+use streamer;
+
+pub struct WriteStage {
+    pub thread_hdl: JoinHandle<()>,
+    pub blob_receiver: streamer::BlobReceiver,
+}
+
+impl WriteStage {
+    /// Create a new Rpu that wraps the given Bank.
+    pub fn new<W: Write + Send + 'static>(
+        bank: Arc<Bank>,
+        exit: Arc<AtomicBool>,
+        blob_recycler: packet::BlobRecycler,
+        writer: Mutex<W>,
+        entry_receiver: Receiver<Entry>,
+    ) -> Self {
+        let (blob_sender, blob_receiver) = channel();
+        let thread_hdl = Builder::new()
+            .name("solana-writer".to_string())
+            .spawn(move || loop {
+                let entry_writer = EntryWriter::new(&bank);
+                let _ = entry_writer.write_and_send_entries(
+                    &blob_sender,
+                    &blob_recycler,
+                    &writer,
+                    &entry_receiver,
+                );
+                if exit.load(Ordering::Relaxed) {
+                    info!("broadcat_service exiting");
+                    break;
+                }
+            })
+            .unwrap();
+
+        WriteStage {
+            thread_hdl,
+            blob_receiver,
+        }
+    }
+
+    pub fn new_drain(
+        bank: Arc<Bank>,
+        exit: Arc<AtomicBool>,
+        entry_receiver: Receiver<Entry>,
+    ) -> Self {
+        let (_blob_sender, blob_receiver) = channel();
+        let thread_hdl = Builder::new()
+            .name("solana-drain".to_string())
+            .spawn(move || {
+                let entry_writer = EntryWriter::new(&bank);
+                loop {
+                    let _ = entry_writer.drain_entries(&entry_receiver);
+                    if exit.load(Ordering::Relaxed) {
+                        info!("drain_service exiting");
+                        break;
+                    }
+                }
+            })
+            .unwrap();
+
+        WriteStage {
+            thread_hdl,
+            blob_receiver,
+        }
+    }
+}

tests/data_replicator.rs

@@ -0,0 +1,184 @@
+#[macro_use]
+extern crate log;
+extern crate rayon;
+extern crate solana;
+
+use rayon::iter::*;
+use solana::crdt::{Crdt, TestNode};
+use solana::data_replicator::DataReplicator;
+use solana::logger;
+use solana::packet::Blob;
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, RwLock};
+use std::thread::sleep;
+use std::time::Duration;
+
+fn test_node(exit: Arc<AtomicBool>) -> (Arc<RwLock<Crdt>>, DataReplicator, UdpSocket) {
+    let tn = TestNode::new();
+    let crdt = Crdt::new(tn.data.clone());
+    let c = Arc::new(RwLock::new(crdt));
+    let w = Arc::new(RwLock::new(vec![]));
+    let d = DataReplicator::new(
+        c.clone(),
+        w,
+        tn.sockets.gossip,
+        tn.sockets.gossip_send,
+        exit,
+    ).unwrap();
+    (c, d, tn.sockets.replicate)
+}
+
+/// Test that the network converges.
+/// Run until every node in the network has a full ReplicatedData set.
+/// Check that nodes stop sending updates after all the ReplicatedData has been shared.
+/// tests that actually use this function are below
+fn run_gossip_topo<F>(topo: F)
+where
+    F: Fn(&Vec<(Arc<RwLock<Crdt>>, DataReplicator, UdpSocket)>) -> (),
+{
+    let num: usize = 5;
+    let exit = Arc::new(AtomicBool::new(false));
+    let listen: Vec<_> = (0..num).map(|_| test_node(exit.clone())).collect();
+    topo(&listen);
+    let mut done = true;
+    for i in 0..(num * 32) {
+        done = false;
+        trace!("round {}", i);
+        for &(ref c, _, _) in listen.iter() {
+            if num == c.read().unwrap().convergence() as usize {
+                done = true;
+                break;
+            }
+        }
+        //at least 1 node converged
+        if done == true {
+            break;
+        }
+        sleep(Duration::new(1, 0));
+    }
+    exit.store(true, Ordering::Relaxed);
+    for (c, dr, _) in listen.into_iter() {
+        for j in dr.thread_hdls.into_iter() {
+            j.join().unwrap();
+        }
+        // make it clear what failed
+        // protocol is to chatty, updates should stop after everyone receives `num`
+        assert!(c.read().unwrap().update_index <= num as u64);
+        // protocol is not chatty enough, everyone should get `num` entries
+        assert_eq!(c.read().unwrap().table.len(), num);
+    }
+    assert!(done);
+}
+/// ring a -> b -> c -> d -> e -> a
+#[test]
+fn gossip_ring() {
+    logger::setup();
+    run_gossip_topo(|listen| {
+        let num = listen.len();
+        for n in 0..num {
+            let y = n % listen.len();
+            let x = (n + 1) % listen.len();
+            let mut xv = listen[x].0.write().unwrap();
+            let yv = listen[y].0.read().unwrap();
+            let mut d = yv.table[&yv.me].clone();
+            d.version = 0;
+            xv.insert(&d);
+        }
+    });
+}
+
+/// star a -> (b,c,d,e)
+#[test]
+fn gossip_star() {
+    logger::setup();
+    run_gossip_topo(|listen| {
+        let num = listen.len();
+        for n in 0..(num - 1) {
+            let x = 0;
+            let y = (n + 1) % listen.len();
+            let mut xv = listen[x].0.write().unwrap();
+            let yv = listen[y].0.read().unwrap();
+            let mut yd = yv.table[&yv.me].clone();
+            yd.version = 0;
+            xv.insert(&yd);
+            trace!("star leader {:?}", &xv.me[..4]);
+        }
+    });
+}
+
+/// rstar a <- (b,c,d,e)
+#[test]
+fn gossip_rstar() {
+    logger::setup();
+    run_gossip_topo(|listen| {
+        let num = listen.len();
+        let xd = {
+            let xv = listen[0].0.read().unwrap();
+            xv.table[&xv.me].clone()
+        };
+        trace!("rstar leader {:?}", &xd.id[..4]);
+        for n in 0..(num - 1) {
+            let y = (n + 1) % listen.len();
+            let mut yv = listen[y].0.write().unwrap();
+            yv.insert(&xd);
+            trace!("rstar insert {:?} into {:?}", &xd.id[..4], &yv.me[..4]);
+        }
+    });
+}
+
+#[test]
+pub fn crdt_retransmit() {
+    logger::setup();
+    let exit = Arc::new(AtomicBool::new(false));
+    trace!("c1:");
+    let (c1, dr1, tn1) = test_node(exit.clone());
+    trace!("c2:");
+    let (c2, dr2, tn2) = test_node(exit.clone());
+    trace!("c3:");
+    let (c3, dr3, tn3) = test_node(exit.clone());
+    let c1_data = c1.read().unwrap().my_data().clone();
+    c1.write().unwrap().set_leader(c1_data.id);
+
+    c2.write().unwrap().insert(&c1_data);
+    c3.write().unwrap().insert(&c1_data);
+
+    c2.write().unwrap().set_leader(c1_data.id);
+    c3.write().unwrap().set_leader(c1_data.id);
+
+    //wait to converge
+    trace!("waiting to converge:");
+    let mut done = false;
+    for _ in 0..30 {
+        done = c1.read().unwrap().table.len() == 3 && c2.read().unwrap().table.len() == 3
+            && c3.read().unwrap().table.len() == 3;
+        if done {
+            break;
+        }
+        sleep(Duration::new(1, 0));
+    }
+    assert!(done);
+    let mut b = Blob::default();
+    b.meta.size = 10;
+    Crdt::retransmit(&c1, &Arc::new(RwLock::new(b)), &tn1).unwrap();
+    let res: Vec<_> = [tn1, tn2, tn3]
+        .into_par_iter()
+        .map(|s| {
+            let mut b = Blob::default();
+            s.set_read_timeout(Some(Duration::new(1, 0))).unwrap();
+            let res = s.recv_from(&mut b.data);
+            res.is_err() //true if failed to receive the retransmit packet
+        })
+        .collect();
+    //true if failed receive the retransmit packet, r2, and r3 should succeed
+    //r1 was the sender, so it should fail to receive the packet
+    assert_eq!(res, [true, false, false]);
+    exit.store(true, Ordering::Relaxed);
+    let mut threads = vec![];
+    threads.extend(dr1.thread_hdls.into_iter());
+    threads.extend(dr2.thread_hdls.into_iter());
+    threads.extend(dr3.thread_hdls.into_iter());
+    for t in threads.into_iter() {
+        t.join().unwrap();
+    }
+}

tests/multinode.rs

@@ -0,0 +1,174 @@
+#[macro_use]
+extern crate log;
+extern crate bincode;
+extern crate solana;
+
+use solana::bank::Bank;
+use solana::crdt::TestNode;
+use solana::crdt::{Crdt, ReplicatedData};
+use solana::data_replicator::DataReplicator;
+use solana::logger;
+use solana::mint::Mint;
+use solana::server::Server;
+use solana::signature::{KeyPair, KeyPairUtil, PublicKey};
+use solana::streamer::default_window;
+use solana::thin_client::ThinClient;
+use std::io;
+use std::io::sink;
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, RwLock};
+use std::thread::sleep;
+use std::thread::JoinHandle;
+use std::time::Duration;
+
+fn validator(
+    leader: &ReplicatedData,
+    exit: Arc<AtomicBool>,
+    alice: &Mint,
+    threads: &mut Vec<JoinHandle<()>>,
+) {
+    let validator = TestNode::new();
+    let replicant_bank = Bank::new(&alice);
+    let mut ts = Server::new_validator(
+        replicant_bank,
+        validator.data.clone(),
+        validator.sockets.requests,
+        validator.sockets.respond,
+        validator.sockets.replicate,
+        validator.sockets.gossip,
+        leader.clone(),
+        exit.clone(),
+    );
+    threads.append(&mut ts.thread_hdls);
+}
+
+fn converge(
+    leader: &ReplicatedData,
+    exit: Arc<AtomicBool>,
+    num_nodes: usize,
+    threads: &mut Vec<JoinHandle<()>>,
+) -> Vec<ReplicatedData> {
+    //lets spy on the network
+    let mut spy = TestNode::new();
+    let daddr = "0.0.0.0:0".parse().unwrap();
+    let me = spy.data.id.clone();
+    spy.data.replicate_addr = daddr;
+    spy.data.requests_addr = daddr;
+    let mut spy_crdt = Crdt::new(spy.data);
+    spy_crdt.insert(&leader);
+    spy_crdt.set_leader(leader.id);
+    let spy_ref = Arc::new(RwLock::new(spy_crdt));
+    let spy_window = default_window();
+    let dr = DataReplicator::new(
+        spy_ref.clone(),
+        spy_window,
+        spy.sockets.gossip,
+        spy.sockets.gossip_send,
+        exit,
+    ).unwrap();
+    //wait for the network to converge
+    let mut converged = false;
+    for _ in 0..30 {
+        let num = spy_ref.read().unwrap().convergence();
+        if num == num_nodes as u64 {
+            converged = true;
+            break;
+        }
+        sleep(Duration::new(1, 0));
+    }
+    assert!(converged);
+    threads.extend(dr.thread_hdls.into_iter());
+    let v: Vec<ReplicatedData> = spy_ref
+        .read()
+        .unwrap()
+        .table
+        .values()
+        .into_iter()
+        .filter(|x| x.id != me)
+        .map(|x| x.clone())
+        .collect();
+    v.clone()
+}
+
+#[test]
+fn test_multi_node() {
+    logger::setup();
+    const N: usize = 5;
+    trace!("test_multi_accountant_stub");
+    let leader = TestNode::new();
+    let alice = Mint::new(10_000);
+    let bob_pubkey = KeyPair::new().pubkey();
+    let exit = Arc::new(AtomicBool::new(false));
+
+    let leader_bank = Bank::new(&alice);
+    let server = Server::new_leader(
+        leader_bank,
+        None,
+        leader.data.clone(),
+        leader.sockets.requests,
+        leader.sockets.transaction,
+        leader.sockets.broadcast,
+        leader.sockets.respond,
+        leader.sockets.gossip,
+        exit.clone(),
+        sink(),
+    );
+
+    let mut threads = server.thread_hdls;
+    for _ in 0..N {
+        validator(&leader.data, exit.clone(), &alice, &mut threads);
+    }
+    let servers = converge(&leader.data, exit.clone(), N + 2, &mut threads);
+    //contains the leader addr as well
+    assert_eq!(servers.len(), N + 1);
+    //verify leader can do transfer
+    let leader_balance = tx_and_retry_get_balance(&leader.data, &alice, &bob_pubkey).unwrap();
+    assert_eq!(leader_balance, 500);
+    //verify validator has the same balance
+    let mut success = 0usize;
+    for server in servers.iter() {
+        let mut client = mk_client(server);
+        if let Ok(bal) = client.poll_get_balance(&bob_pubkey) {
+            trace!("validator balance {}", bal);
+            if bal == leader_balance {
+                success += 1;
+            }
+        }
+    }
+    assert_eq!(success, servers.len());
+    exit.store(true, Ordering::Relaxed);
+    for t in threads {
+        t.join().unwrap();
+    }
+}
+
+fn mk_client(leader: &ReplicatedData) -> ThinClient {
+    let requests_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+    requests_socket
+        .set_read_timeout(Some(Duration::new(1, 0)))
+        .unwrap();
+    let transactions_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+
+    ThinClient::new(
+        leader.requests_addr,
+        requests_socket,
+        leader.transactions_addr,
+        transactions_socket,
+    )
+}
+
+fn tx_and_retry_get_balance(
+    leader: &ReplicatedData,
+    alice: &Mint,
+    bob_pubkey: &PublicKey,
+) -> io::Result<i64> {
+    let mut client = mk_client(leader);
+    trace!("getting leader last_id");
+    let last_id = client.get_last_id();
+    info!("executing leader transer");
+    let _sig = client
+        .transfer(500, &alice.keypair(), *bob_pubkey, &last_id)
+        .unwrap();
+    client.poll_get_balance(bob_pubkey)
+}