Add note about installing git-lfs

Former-commit-id: 77dd1bdd4a

.codecov.yml

@@ -0,0 +1,2 @@
+ignore:
+  - "src/bin" 

.gitattributes

@@ -0,0 +1 @@
+*.a filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,4 +1,3 @@
-
+Cargo.lock
 /target/
 **/*.rs.bk
-Cargo.lock

Cargo.toml

@@ -1,19 +1,70 @@
 [package]
-name = "silk"
-description = "A silky smooth implementation of the Loom architecture"
-version = "0.1.1"
+name = "solana"
+description = "High Performance Blockchain"
+version = "0.5.0-beta"
+documentation = "https://docs.rs/solana"
+homepage = "http://solana.io/"
+repository = "https://github.com/solana-labs/solana"
 authors = [
-    "Anatoly Yakovenko <aeyakovenko@gmail.com>",
-    "Greg Fitzgerald <garious@gmail.com>",
+    "Anatoly Yakovenko <anatoly@solana.io>",
+    "Greg Fitzgerald <greg@solana.io>",
+    "Stephen Akridge <stephen@solana.io>",
 ]
 license = "Apache-2.0"
 
+[[bin]]
+name = "solana-historian-demo"
+path = "src/bin/historian-demo.rs"
+
+[[bin]]
+name = "solana-client-demo"
+path = "src/bin/client-demo.rs"
+
+[[bin]]
+name = "solana-testnode"
+path = "src/bin/testnode.rs"
+
+[[bin]]
+name = "solana-genesis"
+path = "src/bin/genesis.rs"
+
+[[bin]]
+name = "solana-genesis-demo"
+path = "src/bin/genesis-demo.rs"
+
+[[bin]]
+name = "solana-mint"
+path = "src/bin/mint.rs"
+
+[[bin]]
+name = "solana-mint-demo"
+path = "src/bin/mint-demo.rs"
+
 [badges]
-codecov = { repository = "loomprotocol/silk", branch = "master", service = "github" }
+codecov = { repository = "solana-labs/solana", branch = "master", service = "github" }
 
 [features]
 unstable = []
+ipv6 = []
+cuda = []
+erasure = []
 
 [dependencies]
 rayon = "1.0.0"
-itertools = "0.7.6"
+sha2 = "0.7.0"
+generic-array = { version = "0.9.0", default-features = false, features = ["serde"] }
+serde = "1.0.27"
+serde_derive = "1.0.27"
+serde_json = "1.0.10"
+ring = "0.12.1"
+untrusted = "0.5.1"
+bincode = "1.0.0"
+chrono = { version = "0.4.0", features = ["serde"] }
+log = "^0.4.1"
+env_logger = "^0.4.1"
+matches = "^0.1.6"
+byteorder = "^1.2.1"
+libc = "^0.2.1"
+getopts = "^0.2"
+isatty = "0.1"
+futures = "0.1"

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2018 Anatoly Yakovenko <anatoly@loomprotocol.com> and Greg Fitzgerald <garious@gmail.com>
+Copyright 2018 Anatoly Yakovenko, Greg Fitzgerald and Stephen Akridge
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

README.md

@@ -1,22 +1,99 @@
-[![Silk crate](https://img.shields.io/crates/v/silk.svg)](https://crates.io/crates/silk)
-[![Silk documentation](https://docs.rs/silk/badge.svg)](https://docs.rs/silk)
-[![Build Status](https://travis-ci.org/loomprotocol/silk.svg?branch=master)](https://travis-ci.org/loomprotocol/silk)
-[![codecov](https://codecov.io/gh/loomprotocol/silk/branch/master/graph/badge.svg)](https://codecov.io/gh/loomprotocol/silk)
+[![Solana crate](https://img.shields.io/crates/v/solana.svg)](https://crates.io/crates/solana)
+[![Solana documentation](https://docs.rs/solana/badge.svg)](https://docs.rs/solana)
+[![Build Status](https://travis-ci.org/solana-labs/solana.svg?branch=master)](https://travis-ci.org/solana-labs/solana)
+[![codecov](https://codecov.io/gh/solana-labs/solana/branch/master/graph/badge.svg)](https://codecov.io/gh/solana-labs/solana)
 
-# Silk, A Silky Smooth Implementation of the Loom Architecture
+Disclaimer
+===
 
-Loom is a new achitecture for a high performance blockchain. Its whitepaper boasts a theoretical
-throughput of 710k transactions per second on a 1 gbps network. The first implementation of the
-whitepaper is happening in the 'loomprotocol/loom' repository. That repo is aggressively moving
-forward, looking to de-risk technical claims as quickly as possible. This repo is quite a bit
-different philosophically. Here we assume the Loom architecture is sound and worthy of building
-a community around. We care a great deal about quality, clarity and short learning curve. We
-avoid the use of `unsafe` Rust and an write tests for *everything*.  Optimizations are only
-added when corresponding benchmarks are also added that demonstrate real performance boots. We
-expect the feature set here will always be a long ways behind the loom repo, but that this is
-an implementation you can take to the bank, literally.
+All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort.  It is up to the reader to check and validate their accuracy and truthfulness.  Furthermore nothing in this project constitutes a solicitation for investment.
 
-# Developing
+Solana: High Performance Blockchain
+===
+
+Solana™ is a new architecture for a high performance blockchain. It aims to support
+over 700 thousand transactions per second on a gigabit network.
+
+Introduction
+===
+
+It's possible for a centralized database to process 710,000 transactions per second on a standard gigabit network if the transactions are, on average, no more than 178 bytes. A centralized database can also replicate itself and maintain high availability without significantly compromising that transaction rate using the distributed system technique known as Optimistic Concurrency Control [H.T.Kung, J.T.Robinson (1981)]. At Solana, we're demonstrating that these same theoretical limits apply just as well to blockchain on an adversarial network. The key ingredient? Finding a way to share time when nodes can't trust one-another. Once nodes can trust time, suddenly ~40 years of distributed systems research becomes applicable to blockchain! Furthermore, and much to our surprise, it can implemented using a mechanism that has existed in Bitcoin since day one. The Bitcoin feature is called nLocktime and it can be used to postdate transactions using block height instead of a timestamp. As a Bitcoin client, you'd use block height instead of a timestamp if you don't trust the network. Block height turns out to be an instance of what's being called a Verifiable Delay Function in cryptography circles. It's a cryptographically secure way to say time has passed. In Solana, we use a far more granular verifiable delay function, a SHA 256 hash chain, to checkpoint the ledger and coordinate consensus. With it, we implement Optimistic Concurrency Control and are now well in route towards that theoretical limit of 710,000 transactions per second.
+
+Running the demo
+===
+
+First, install Rust's package manager Cargo.
+
+```bash
+$ curl https://sh.rustup.rs -sSf | sh
+$ source $HOME/.cargo/env
+```
+
+If you plan to run with GPU optimizations enabled (not recommended), you'll need a CUDA library stored in git LFS. Install git-lfs here:
+
+https://git-lfs.github.com/
+
+Now checkout the code from github:
+
+```bash
+$ git clone https://github.com/solana-labs/solana.git 
+$ cd solana
+```
+
+The testnode server is initialized with a ledger from stdin and
+generates new ledger entries on stdout. To create the input ledger, we'll need
+to create *the mint* and use it to generate a *genesis ledger*. It's done in
+two steps because the mint-demo.json file contains private keys that will be
+used later in this demo.
+
+```bash
+    $ echo 1000000000 | cargo run --release --bin solana-mint-demo > mint-demo.json
+    $ cat mint-demo.json | cargo run --release --bin solana-genesis-demo > genesis.log
+```
+
+Now you can start the server:
+
+```bash
+    $ cat genesis.log | cargo run --release --bin solana-testnode > transactions0.log
+```
+
+Wait a few seconds for the server to initialize. It will print "Ready." when it's safe
+to start sending it transactions.
+
+Then, in a separate shell, let's execute some transactions. Note we pass in
+the JSON configuration file here, not the genesis ledger.
+
+```bash
+    $ cat mint-demo.json | cargo run --release --bin solana-client-demo
+```
+
+Now kill the server with Ctrl-C, and take a look at the ledger. You should
+see something similar to:
+
+```json
+{"num_hashes":27,"id":[0, "..."],"event":"Tick"}
+{"num_hashes":3,"id":[67, "..."],"event":{"Transaction":{"tokens":42}}}
+{"num_hashes":27,"id":[0, "..."],"event":"Tick"}
+```
+
+Now restart the server from where we left off. Pass it both the genesis ledger, and
+the transaction ledger.
+
+```bash
+    $ cat genesis.log transactions0.log | cargo run --release --bin solana-testnode > transactions1.log
+```
+
+Lastly, run the client demo again, and verify that all funds were spent in the
+previous round, and so no additional transactions are added.
+
+```bash
+    $ cat mint-demo.json | cargo run --release --bin solana-client-demo
+```
+
+Stop the server again, and verify there are only Tick entries, and no Transaction entries.
+
+Developing
+===
 
 Building
 ---
@@ -32,8 +109,8 @@ $ rustup component add rustfmt-preview
 Download the source code:
 
 ```bash
-$ git clone https://github.com/loomprotocol/silk.git
-$ cd silk
+$ git clone https://github.com/solana-labs/solana.git
+$ cd solana
 ```
 
 Testing
@@ -59,3 +136,30 @@ Run the benchmarks:
 ```bash
 $ cargo +nightly bench --features="unstable"
 ```
+
+To run the benchmarks on Linux with GPU optimizations enabled:
+
+```bash
+$ cargo +nightly bench --features="unstable,cuda"
+```
+
+Code coverage
+---
+
+To generate code coverage statistics, run kcov via Docker:
+
+```bash
+$ docker run -it --rm --security-opt seccomp=unconfined --volume "$PWD:/volume" elmtai/docker-rust-kcov
+```
+
+Why coverage? While most see coverage as a code quality metric, we see it primarily as a developer
+productivity metric. When a developer makes a change to the codebase, presumably it's a *solution* to
+some problem.  Our unit-test suite is how we encode the set of *problems* the codebase solves. Running
+the test suite should indicate that your change didn't *infringe* on anyone else's solutions. Adding a
+test *protects* your solution from future changes. Say you don't understand why a line of code exists,
+try deleting it and running the unit-tests. The nearest test failure should tell you what problem
+was solved by that code. If no test fails, go ahead and submit a Pull Request that asks, "what
+problem is solved by this code?" On the other hand, if a test does fail and you can think of a
+better way to solve the same problem, a Pull Request with your solution would most certainly be
+welcome! Likewise, if rewriting a test can better communicate what code it's protecting, please
+send us that patch!

build.rs

@@ -0,0 +1,15 @@
+use std::env;
+
+fn main() {
+    println!("cargo:rustc-link-search=native=.");
+    if !env::var("CARGO_FEATURE_CUDA").is_err() {
+        println!("cargo:rustc-link-lib=static=cuda_verify_ed25519");
+        println!("cargo:rustc-link-search=native=/usr/local/cuda/lib64");
+        println!("cargo:rustc-link-lib=dylib=cudart");
+        println!("cargo:rustc-link-lib=dylib=cuda");
+        println!("cargo:rustc-link-lib=dylib=cudadevrt");
+    }
+    if !env::var("CARGO_FEATURE_ERASURE").is_err() {
+        println!("cargo:rustc-link-lib=dylib=Jerasure");
+    }
+}

doc/consensus.msc

@@ -0,0 +1,15 @@
+msc {
+  client,leader,verifier_a,verifier_b,verifier_c;
+
+  client=>leader [ label = "SUBMIT" ] ;
+  leader=>client [ label = "CONFIRMED" ] ;
+  leader=>verifier_a [ label = "CONFIRMED" ] ;
+  leader=>verifier_b [ label = "CONFIRMED" ] ;
+  leader=>verifier_c [ label = "CONFIRMED" ] ;
+  verifier_a=>leader [ label = "VERIFIED" ] ;
+  verifier_b=>leader [ label = "VERIFIED" ] ;
+  leader=>client [ label = "FINALIZED" ] ;
+  leader=>verifier_a [ label = "FINALIZED" ] ;
+  leader=>verifier_b [ label = "FINALIZED" ] ;
+  leader=>verifier_c [ label = "FINALIZED" ] ;
+}

doc/historian.md

@@ -0,0 +1,65 @@
+The Historian
+===
+
+Create a *Historian* and send it *events* to generate an *event log*, where each *entry*
+is tagged with the historian's latest *hash*. Then ensure the order of events was not tampered
+with by verifying each entry's hash can be generated from the hash in the previous entry:
+
+![historian](https://user-images.githubusercontent.com/55449/36950845-459bdb58-1fb9-11e8-850e-894586f3729b.png)
+
+```rust
+extern crate solana;
+
+use solana::historian::Historian;
+use solana::ledger::{Block, Entry, Hash};
+use solana::event::{generate_keypair, get_pubkey, sign_claim_data, Event};
+use std::thread::sleep;
+use std::time::Duration;
+use std::sync::mpsc::SendError;
+
+fn create_ledger(hist: &Historian<Hash>) -> Result<(), SendError<Event<Hash>>> {
+    sleep(Duration::from_millis(15));
+    let tokens = 42;
+    let keypair = generate_keypair();
+    let event0 = Event::new_claim(get_pubkey(&keypair), tokens, sign_claim_data(&tokens, &keypair));
+    hist.sender.send(event0)?;
+    sleep(Duration::from_millis(10));
+    Ok(())
+}
+
+fn main() {
+    let seed = Hash::default();
+    let hist = Historian::new(&seed, Some(10));
+    create_ledger(&hist).expect("send error");
+    drop(hist.sender);
+    let entries: Vec<Entry<Hash>> = hist.receiver.iter().collect();
+    for entry in &entries {
+        println!("{:?}", entry);
+    }
+    // Proof-of-History: Verify the historian learned about the events
+    // in the same order they appear in the vector.
+    assert!(entries[..].verify(&seed));
+}
+```
+
+Running the program should produce a ledger similar to:
+
+```rust
+Entry { num_hashes: 0, id: [0, ...], event: Tick }
+Entry { num_hashes: 3, id: [67, ...], event: Transaction { tokens: 42 } }
+Entry { num_hashes: 3, id: [123, ...], event: Tick }
+```
+
+Proof-of-History
+---
+
+Take note of the last line:
+
+```rust
+assert!(entries[..].verify(&seed));
+```
+
+[It's a proof!](https://en.wikipedia.org/wiki/Curry–Howard_correspondence) For each entry returned by the
+historian, we can verify that `id` is the result of applying a sha256 hash to the previous `id`
+exactly `num_hashes` times, and then hashing then event data on top of that. Because the event data is
+included in the hash, the events cannot be reordered without regenerating all the hashes.

doc/historian.msc

@@ -0,0 +1,18 @@
+msc {
+  client,historian,recorder;
+
+  recorder=>historian [ label = "e0 = Entry{id: h0, n: 0, event: Tick}" ] ;
+  recorder=>recorder [ label = "h1 = hash(h0)" ] ;
+  recorder=>recorder [ label = "h2 = hash(h1)" ] ;
+  client=>historian [ label = "Transaction(d0)" ] ;
+  historian=>recorder [ label = "Transaction(d0)" ] ;
+  recorder=>recorder [ label = "h3 = hash(h2 + d0)" ] ;
+  recorder=>historian [ label = "e1 = Entry{id: hash(h3), n: 3, event: Transaction(d0)}" ] ;
+  recorder=>recorder [ label = "h4 = hash(h3)" ] ;
+  recorder=>recorder [ label = "h5 = hash(h4)" ] ;
+  recorder=>recorder [ label = "h6 = hash(h5)" ] ;
+  recorder=>historian [ label = "e2 = Entry{id: h6, n: 3, event: Tick}" ] ;
+  client=>historian [ label = "collect()" ] ;
+  historian=>client [ label = "entries = [e0, e1, e2]" ] ;
+  client=>client [ label = "entries.verify(h0)" ] ;
+}

src/accountant.rs

@@ -0,0 +1,526 @@
+//! The `accountant` module tracks client balances, and the progress of pending
+//! transactions. It offers a high-level public API that signs transactions
+//! on behalf of the caller, and a private low-level API for when they have
+//! already been signed and verified.
+
+extern crate libc;
+
+use chrono::prelude::*;
+use event::Event;
+use hash::Hash;
+use mint::Mint;
+use plan::{Payment, Plan, Witness};
+use rayon::prelude::*;
+use signature::{KeyPair, PublicKey, Signature};
+use std::collections::hash_map::Entry::Occupied;
+use std::collections::{HashMap, HashSet, VecDeque};
+use std::result;
+use std::sync::RwLock;
+use transaction::Transaction;
+
+pub const MAX_ENTRY_IDS: usize = 1024 * 4;
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum AccountingError {
+    AccountNotFound,
+    InsufficientFunds,
+    InvalidTransferSignature,
+}
+
+pub type Result<T> = result::Result<T, AccountingError>;
+
+/// Commit funds to the 'to' party.
+fn apply_payment(balances: &RwLock<HashMap<PublicKey, RwLock<i64>>>, payment: &Payment) {
+    if balances.read().unwrap().contains_key(&payment.to) {
+        let bals = balances.read().unwrap();
+        *bals[&payment.to].write().unwrap() += payment.tokens;
+    } else {
+        let mut bals = balances.write().unwrap();
+        bals.insert(payment.to, RwLock::new(payment.tokens));
+    }
+}
+
+pub struct Accountant {
+    balances: RwLock<HashMap<PublicKey, RwLock<i64>>>,
+    pending: RwLock<HashMap<Signature, Plan>>,
+    last_ids: RwLock<VecDeque<(Hash, RwLock<HashSet<Signature>>)>>,
+    time_sources: RwLock<HashSet<PublicKey>>,
+    last_time: RwLock<DateTime<Utc>>,
+}
+
+impl Accountant {
+    /// Create an Accountant using a deposit.
+    pub fn new_from_deposit(deposit: &Payment) -> Self {
+        let balances = RwLock::new(HashMap::new());
+        apply_payment(&balances, deposit);
+        Accountant {
+            balances,
+            pending: RwLock::new(HashMap::new()),
+            last_ids: RwLock::new(VecDeque::new()),
+            time_sources: RwLock::new(HashSet::new()),
+            last_time: RwLock::new(Utc.timestamp(0, 0)),
+        }
+    }
+
+    /// Create an Accountant with only a Mint. Typically used by unit tests.
+    pub fn new(mint: &Mint) -> Self {
+        let deposit = Payment {
+            to: mint.pubkey(),
+            tokens: mint.tokens,
+        };
+        let acc = Self::new_from_deposit(&deposit);
+        acc.register_entry_id(&mint.last_id());
+        acc
+    }
+
+    fn reserve_signature(signatures: &RwLock<HashSet<Signature>>, sig: &Signature) -> bool {
+        if signatures.read().unwrap().contains(sig) {
+            return false;
+        }
+        signatures.write().unwrap().insert(*sig);
+        true
+    }
+
+    fn forget_signature(signatures: &RwLock<HashSet<Signature>>, sig: &Signature) -> bool {
+        signatures.write().unwrap().remove(sig)
+    }
+
+    fn forget_signature_with_last_id(&self, sig: &Signature, last_id: &Hash) -> bool {
+        if let Some(entry) = self.last_ids
+            .read()
+            .unwrap()
+            .iter()
+            .rev()
+            .find(|x| x.0 == *last_id)
+        {
+            return Self::forget_signature(&entry.1, sig);
+        }
+        return false;
+    }
+
+    fn reserve_signature_with_last_id(&self, sig: &Signature, last_id: &Hash) -> bool {
+        if let Some(entry) = self.last_ids
+            .read()
+            .unwrap()
+            .iter()
+            .rev()
+            .find(|x| x.0 == *last_id)
+        {
+            return Self::reserve_signature(&entry.1, sig);
+        }
+        false
+    }
+
+    /// Tell the accountant which Entry IDs exist on the ledger. This function
+    /// assumes subsequent calls correspond to later entries, and will boot
+    /// the oldest ones once its internal cache is full. Once boot, the
+    /// accountant will reject transactions using that `last_id`.
+    pub fn register_entry_id(&self, last_id: &Hash) {
+        let mut last_ids = self.last_ids.write().unwrap();
+        if last_ids.len() >= MAX_ENTRY_IDS {
+            last_ids.pop_front();
+        }
+        last_ids.push_back((*last_id, RwLock::new(HashSet::new())));
+    }
+
+    /// Deduct tokens from the 'from' address the account has sufficient
+    /// funds and isn't a duplicate.
+    pub fn process_verified_transaction_debits(&self, tr: &Transaction) -> Result<()> {
+        let bals = self.balances.read().unwrap();
+
+        // Hold a write lock before the condition check, so that a debit can't occur
+        // between checking the balance and the withdraw.
+        let option = bals.get(&tr.from);
+        if option.is_none() {
+            return Err(AccountingError::AccountNotFound);
+        }
+        let mut bal = option.unwrap().write().unwrap();
+
+        if !self.reserve_signature_with_last_id(&tr.sig, &tr.data.last_id) {
+            return Err(AccountingError::InvalidTransferSignature);
+        }
+
+        if *bal < tr.data.tokens {
+            self.forget_signature_with_last_id(&tr.sig, &tr.data.last_id);
+            return Err(AccountingError::InsufficientFunds);
+        }
+
+        *bal -= tr.data.tokens;
+
+        Ok(())
+    }
+
+    pub fn process_verified_transaction_credits(&self, tr: &Transaction) {
+        let mut plan = tr.data.plan.clone();
+        plan.apply_witness(&Witness::Timestamp(*self.last_time.read().unwrap()));
+
+        if let Some(ref payment) = plan.final_payment() {
+            apply_payment(&self.balances, payment);
+        } else {
+            let mut pending = self.pending.write().unwrap();
+            pending.insert(tr.sig, plan);
+        }
+    }
+
+    /// Process a Transaction that has already been verified.
+    pub fn process_verified_transaction(&self, tr: &Transaction) -> Result<()> {
+        self.process_verified_transaction_debits(tr)?;
+        self.process_verified_transaction_credits(tr);
+        Ok(())
+    }
+
+    /// Process a batch of verified transactions.
+    pub fn process_verified_transactions(&self, trs: Vec<Transaction>) -> Vec<Result<Transaction>> {
+        // Run all debits first to filter out any transactions that can't be processed
+        // in parallel deterministically.
+        let results: Vec<_> = trs.into_par_iter()
+            .map(|tr| self.process_verified_transaction_debits(&tr).map(|_| tr))
+            .collect(); // Calling collect() here forces all debits to complete before moving on.
+
+        results
+            .into_par_iter()
+            .map(|result| {
+                result.map(|tr| {
+                    self.process_verified_transaction_credits(&tr);
+                    tr
+                })
+            })
+            .collect()
+    }
+
+    fn partition_events(events: Vec<Event>) -> (Vec<Transaction>, Vec<Event>) {
+        let mut trs = vec![];
+        let mut rest = vec![];
+        for event in events {
+            match event {
+                Event::Transaction(tr) => trs.push(tr),
+                _ => rest.push(event),
+            }
+        }
+        (trs, rest)
+    }
+
+    pub fn process_verified_events(&self, events: Vec<Event>) -> Result<()> {
+        let (trs, rest) = Self::partition_events(events);
+        self.process_verified_transactions(trs);
+        for event in rest {
+            self.process_verified_event(&event)?;
+        }
+        Ok(())
+    }
+
+    /// Process a Witness Signature that has already been verified.
+    fn process_verified_sig(&self, from: PublicKey, tx_sig: Signature) -> Result<()> {
+        if let Occupied(mut e) = self.pending.write().unwrap().entry(tx_sig) {
+            e.get_mut().apply_witness(&Witness::Signature(from));
+            if let Some(ref payment) = e.get().final_payment() {
+                apply_payment(&self.balances, payment);
+                e.remove_entry();
+            }
+        };
+
+        Ok(())
+    }
+
+    /// Process a Witness Timestamp that has already been verified.
+    fn process_verified_timestamp(&self, from: PublicKey, dt: DateTime<Utc>) -> Result<()> {
+        // If this is the first timestamp we've seen, it probably came from the genesis block,
+        // so we'll trust it.
+        if *self.last_time.read().unwrap() == Utc.timestamp(0, 0) {
+            self.time_sources.write().unwrap().insert(from);
+        }
+
+        if self.time_sources.read().unwrap().contains(&from) {
+            if dt > *self.last_time.read().unwrap() {
+                *self.last_time.write().unwrap() = dt;
+            }
+        } else {
+            return Ok(());
+        }
+
+        // Check to see if any timelocked transactions can be completed.
+        let mut completed = vec![];
+
+        // Hold 'pending' write lock until the end of this function. Otherwise another thread can
+        // double-spend if it enters before the modified plan is removed from 'pending'.
+        let mut pending = self.pending.write().unwrap();
+        for (key, plan) in pending.iter_mut() {
+            plan.apply_witness(&Witness::Timestamp(*self.last_time.read().unwrap()));
+            if let Some(ref payment) = plan.final_payment() {
+                apply_payment(&self.balances, payment);
+                completed.push(key.clone());
+            }
+        }
+
+        for key in completed {
+            pending.remove(&key);
+        }
+
+        Ok(())
+    }
+
+    /// Process an Transaction or Witness that has already been verified.
+    pub fn process_verified_event(&self, event: &Event) -> Result<()> {
+        match *event {
+            Event::Transaction(ref tr) => self.process_verified_transaction(tr),
+            Event::Signature { from, tx_sig, .. } => self.process_verified_sig(from, tx_sig),
+            Event::Timestamp { from, dt, .. } => self.process_verified_timestamp(from, dt),
+        }
+    }
+
+    /// Create, sign, and process a Transaction from `keypair` to `to` of
+    /// `n` tokens where `last_id` is the last Entry ID observed by the client.
+    pub fn transfer(
+        &self,
+        n: i64,
+        keypair: &KeyPair,
+        to: PublicKey,
+        last_id: Hash,
+    ) -> Result<Signature> {
+        let tr = Transaction::new(keypair, to, n, last_id);
+        let sig = tr.sig;
+        self.process_verified_transaction(&tr).map(|_| sig)
+    }
+
+    /// Create, sign, and process a postdated Transaction from `keypair`
+    /// to `to` of `n` tokens on `dt` where `last_id` is the last Entry ID
+    /// observed by the client.
+    pub fn transfer_on_date(
+        &self,
+        n: i64,
+        keypair: &KeyPair,
+        to: PublicKey,
+        dt: DateTime<Utc>,
+        last_id: Hash,
+    ) -> Result<Signature> {
+        let tr = Transaction::new_on_date(keypair, to, dt, n, last_id);
+        let sig = tr.sig;
+        self.process_verified_transaction(&tr).map(|_| sig)
+    }
+
+    pub fn get_balance(&self, pubkey: &PublicKey) -> Option<i64> {
+        let bals = self.balances.read().unwrap();
+        bals.get(pubkey).map(|x| *x.read().unwrap())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bincode::serialize;
+    use hash::hash;
+    use signature::KeyPairUtil;
+
+    #[test]
+    fn test_accountant() {
+        let alice = Mint::new(10_000);
+        let bob_pubkey = KeyPair::new().pubkey();
+        let acc = Accountant::new(&alice);
+        acc.transfer(1_000, &alice.keypair(), bob_pubkey, alice.last_id())
+            .unwrap();
+        assert_eq!(acc.get_balance(&bob_pubkey).unwrap(), 1_000);
+
+        acc.transfer(500, &alice.keypair(), bob_pubkey, alice.last_id())
+            .unwrap();
+        assert_eq!(acc.get_balance(&bob_pubkey).unwrap(), 1_500);
+    }
+
+    #[test]
+    fn test_account_not_found() {
+        let mint = Mint::new(1);
+        let acc = Accountant::new(&mint);
+        assert_eq!(
+            acc.transfer(1, &KeyPair::new(), mint.pubkey(), mint.last_id()),
+            Err(AccountingError::AccountNotFound)
+        );
+    }
+
+    #[test]
+    fn test_invalid_transfer() {
+        let alice = Mint::new(11_000);
+        let acc = Accountant::new(&alice);
+        let bob_pubkey = KeyPair::new().pubkey();
+        acc.transfer(1_000, &alice.keypair(), bob_pubkey, alice.last_id())
+            .unwrap();
+        assert_eq!(
+            acc.transfer(10_001, &alice.keypair(), bob_pubkey, alice.last_id()),
+            Err(AccountingError::InsufficientFunds)
+        );
+
+        let alice_pubkey = alice.keypair().pubkey();
+        assert_eq!(acc.get_balance(&alice_pubkey).unwrap(), 10_000);
+        assert_eq!(acc.get_balance(&bob_pubkey).unwrap(), 1_000);
+    }
+
+    #[test]
+    fn test_transfer_to_newb() {
+        let alice = Mint::new(10_000);
+        let acc = Accountant::new(&alice);
+        let alice_keypair = alice.keypair();
+        let bob_pubkey = KeyPair::new().pubkey();
+        acc.transfer(500, &alice_keypair, bob_pubkey, alice.last_id())
+            .unwrap();
+        assert_eq!(acc.get_balance(&bob_pubkey).unwrap(), 500);
+    }
+
+    #[test]
+    fn test_transfer_on_date() {
+        let alice = Mint::new(1);
+        let acc = Accountant::new(&alice);
+        let alice_keypair = alice.keypair();
+        let bob_pubkey = KeyPair::new().pubkey();
+        let dt = Utc::now();
+        acc.transfer_on_date(1, &alice_keypair, bob_pubkey, dt, alice.last_id())
+            .unwrap();
+
+        // Alice's balance will be zero because all funds are locked up.
+        assert_eq!(acc.get_balance(&alice.pubkey()), Some(0));
+
+        // Bob's balance will be None because the funds have not been
+        // sent.
+        assert_eq!(acc.get_balance(&bob_pubkey), None);
+
+        // Now, acknowledge the time in the condition occurred and
+        // that bob's funds are now available.
+        acc.process_verified_timestamp(alice.pubkey(), dt).unwrap();
+        assert_eq!(acc.get_balance(&bob_pubkey), Some(1));
+
+        acc.process_verified_timestamp(alice.pubkey(), dt).unwrap(); // <-- Attack! Attempt to process completed transaction.
+        assert_ne!(acc.get_balance(&bob_pubkey), Some(2));
+    }
+
+    #[test]
+    fn test_transfer_after_date() {
+        let alice = Mint::new(1);
+        let acc = Accountant::new(&alice);
+        let alice_keypair = alice.keypair();
+        let bob_pubkey = KeyPair::new().pubkey();
+        let dt = Utc::now();
+        acc.process_verified_timestamp(alice.pubkey(), dt).unwrap();
+
+        // It's now past now, so this transfer should be processed immediately.
+        acc.transfer_on_date(1, &alice_keypair, bob_pubkey, dt, alice.last_id())
+            .unwrap();
+
+        assert_eq!(acc.get_balance(&alice.pubkey()), Some(0));
+        assert_eq!(acc.get_balance(&bob_pubkey), Some(1));
+    }
+
+    #[test]
+    fn test_cancel_transfer() {
+        let alice = Mint::new(1);
+        let acc = Accountant::new(&alice);
+        let alice_keypair = alice.keypair();
+        let bob_pubkey = KeyPair::new().pubkey();
+        let dt = Utc::now();
+        let sig = acc.transfer_on_date(1, &alice_keypair, bob_pubkey, dt, alice.last_id())
+            .unwrap();
+
+        // Alice's balance will be zero because all funds are locked up.
+        assert_eq!(acc.get_balance(&alice.pubkey()), Some(0));
+
+        // Bob's balance will be None because the funds have not been
+        // sent.
+        assert_eq!(acc.get_balance(&bob_pubkey), None);
+
+        // Now, cancel the trancaction. Alice gets her funds back, Bob never sees them.
+        acc.process_verified_sig(alice.pubkey(), sig).unwrap();
+        assert_eq!(acc.get_balance(&alice.pubkey()), Some(1));
+        assert_eq!(acc.get_balance(&bob_pubkey), None);
+
+        acc.process_verified_sig(alice.pubkey(), sig).unwrap(); // <-- Attack! Attempt to cancel completed transaction.
+        assert_ne!(acc.get_balance(&alice.pubkey()), Some(2));
+    }
+
+    #[test]
+    fn test_duplicate_event_signature() {
+        let alice = Mint::new(1);
+        let acc = Accountant::new(&alice);
+        let sig = Signature::default();
+        assert!(acc.reserve_signature_with_last_id(&sig, &alice.last_id()));
+        assert!(!acc.reserve_signature_with_last_id(&sig, &alice.last_id()));
+    }
+
+    #[test]
+    fn test_forget_signature() {
+        let alice = Mint::new(1);
+        let acc = Accountant::new(&alice);
+        let sig = Signature::default();
+        acc.reserve_signature_with_last_id(&sig, &alice.last_id());
+        assert!(acc.forget_signature_with_last_id(&sig, &alice.last_id()));
+        assert!(!acc.forget_signature_with_last_id(&sig, &alice.last_id()));
+    }
+
+    #[test]
+    fn test_max_entry_ids() {
+        let alice = Mint::new(1);
+        let acc = Accountant::new(&alice);
+        let sig = Signature::default();
+        for i in 0..MAX_ENTRY_IDS {
+            let last_id = hash(&serialize(&i).unwrap()); // Unique hash
+            acc.register_entry_id(&last_id);
+        }
+        // Assert we're no longer able to use the oldest entry ID.
+        assert!(!acc.reserve_signature_with_last_id(&sig, &alice.last_id()));
+    }
+
+    #[test]
+    fn test_debits_before_credits() {
+        let mint = Mint::new(2);
+        let acc = Accountant::new(&mint);
+        let alice = KeyPair::new();
+        let tr0 = Transaction::new(&mint.keypair(), alice.pubkey(), 2, mint.last_id());
+        let tr1 = Transaction::new(&alice, mint.pubkey(), 1, mint.last_id());
+        let trs = vec![tr0, tr1];
+        assert!(acc.process_verified_transactions(trs)[1].is_err());
+    }
+}
+
+#[cfg(all(feature = "unstable", test))]
+mod bench {
+    extern crate test;
+    use self::test::Bencher;
+    use accountant::*;
+    use bincode::serialize;
+    use hash::hash;
+    use signature::KeyPairUtil;
+
+    #[bench]
+    fn process_verified_event_bench(bencher: &mut Bencher) {
+        let mint = Mint::new(100_000_000);
+        let acc = Accountant::new(&mint);
+        // Create transactions between unrelated parties.
+        let transactions: Vec<_> = (0..4096)
+            .into_par_iter()
+            .map(|i| {
+                // Seed the 'from' account.
+                let rando0 = KeyPair::new();
+                let tr = Transaction::new(&mint.keypair(), rando0.pubkey(), 1_000, mint.last_id());
+                acc.process_verified_transaction(&tr).unwrap();
+
+                // Seed the 'to' account and a cell for its signature.
+                let last_id = hash(&serialize(&i).unwrap()); // Unique hash
+                acc.register_entry_id(&last_id);
+
+                let rando1 = KeyPair::new();
+                let tr = Transaction::new(&rando0, rando1.pubkey(), 1, last_id);
+                acc.process_verified_transaction(&tr).unwrap();
+
+                // Finally, return a transaction that's unique
+                Transaction::new(&rando0, rando1.pubkey(), 1, last_id)
+            })
+            .collect();
+        bencher.iter(|| {
+            // Since benchmarker runs this multiple times, we need to clear the signatures.
+            for sigs in acc.last_ids.read().unwrap().iter() {
+                sigs.1.write().unwrap().clear();
+            }
+
+            assert!(
+                acc.process_verified_transactions(transactions.clone())
+                    .iter()
+                    .all(|x| x.is_ok())
+            );
+        });
+    }
+}

src/accountant_skel.rs

@@ -0,0 +1,810 @@
+//! The `accountant_skel` module is a microservice that exposes the high-level
+//! Accountant API to the network. Its message encoding is currently
+//! in flux. Clients should use AccountantStub to interact with it.
+
+use accountant::Accountant;
+use bincode::{deserialize, serialize};
+use ecdsa;
+use entry::Entry;
+use event::Event;
+use hash::Hash;
+use historian::Historian;
+use packet;
+use packet::SharedPackets;
+use rayon::prelude::*;
+use recorder::Signal;
+use result::Result;
+use serde_json;
+use signature::PublicKey;
+use std::cmp::max;
+use std::collections::VecDeque;
+use std::io::Write;
+use std::net::{SocketAddr, UdpSocket};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::{channel, Receiver, Sender};
+use std::sync::{Arc, Mutex, RwLock};
+use std::thread::{spawn, JoinHandle};
+use std::time::Duration;
+use streamer;
+use transaction::Transaction;
+
+use subscribers;
+
+pub struct AccountantSkel<W: Write + Send + 'static> {
+    acc: Accountant,
+    last_id: Hash,
+    writer: W,
+    historian: Historian,
+    entry_info_subscribers: Vec<SocketAddr>,
+}
+
+#[cfg_attr(feature = "cargo-clippy", allow(large_enum_variant))]
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub enum Request {
+    Transaction(Transaction),
+    GetBalance { key: PublicKey },
+    GetLastId,
+    Subscribe { subscriptions: Vec<Subscription> },
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub enum Subscription {
+    EntryInfo,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct EntryInfo {
+    pub id: Hash,
+    pub num_hashes: u64,
+    pub num_events: u64,
+}
+
+impl Request {
+    /// Verify the request is valid.
+    pub fn verify(&self) -> bool {
+        match *self {
+            Request::Transaction(ref tr) => tr.verify_plan(),
+            _ => true,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub enum Response {
+    Balance { key: PublicKey, val: Option<i64> },
+    EntryInfo(EntryInfo),
+    LastId { id: Hash },
+}
+
+impl<W: Write + Send + 'static> AccountantSkel<W> {
+    /// Create a new AccountantSkel that wraps the given Accountant.
+    pub fn new(acc: Accountant, last_id: Hash, writer: W, historian: Historian) -> Self {
+        AccountantSkel {
+            acc,
+            last_id,
+            writer,
+            historian,
+            entry_info_subscribers: vec![],
+        }
+    }
+
+    fn notify_entry_info_subscribers(&mut self, entry: &Entry) {
+        // TODO: No need to bind().
+        let socket = UdpSocket::bind("127.0.0.1:0").expect("bind");
+
+        for addr in &self.entry_info_subscribers {
+            let entry_info = EntryInfo {
+                id: entry.id,
+                num_hashes: entry.num_hashes,
+                num_events: entry.events.len() as u64,
+            };
+            let data = serialize(&Response::EntryInfo(entry_info)).expect("serialize EntryInfo");
+            let _res = socket.send_to(&data, addr);
+        }
+    }
+
+    /// Process any Entry items that have been published by the Historian.
+    pub fn sync(&mut self) -> Hash {
+        while let Ok(entry) = self.historian.receiver.try_recv() {
+            self.last_id = entry.id;
+            self.acc.register_entry_id(&self.last_id);
+            writeln!(self.writer, "{}", serde_json::to_string(&entry).unwrap()).unwrap();
+            self.notify_entry_info_subscribers(&entry);
+        }
+        self.last_id
+    }
+
+    /// Process Request items sent by clients.
+    pub fn process_request(
+        &mut self,
+        msg: Request,
+        rsp_addr: SocketAddr,
+    ) -> Option<(Response, SocketAddr)> {
+        match msg {
+            Request::GetBalance { key } => {
+                let val = self.acc.get_balance(&key);
+                Some((Response::Balance { key, val }, rsp_addr))
+            }
+            Request::GetLastId => Some((Response::LastId { id: self.sync() }, rsp_addr)),
+            Request::Transaction(_) => unreachable!(),
+            Request::Subscribe { subscriptions } => {
+                for subscription in subscriptions {
+                    match subscription {
+                        Subscription::EntryInfo => self.entry_info_subscribers.push(rsp_addr),
+                    }
+                }
+                None
+            }
+        }
+    }
+
+    fn recv_batch(recvr: &streamer::PacketReceiver) -> Result<Vec<SharedPackets>> {
+        let timer = Duration::new(1, 0);
+        let msgs = recvr.recv_timeout(timer)?;
+        trace!("got msgs");
+        let mut batch = vec![msgs];
+        while let Ok(more) = recvr.try_recv() {
+            trace!("got more msgs");
+            batch.push(more);
+        }
+        info!("batch len {}", batch.len());
+        Ok(batch)
+    }
+
+    fn verify_batch(batch: Vec<SharedPackets>) -> Vec<Vec<(SharedPackets, Vec<u8>)>> {
+        let chunk_size = max(1, (batch.len() + 3) / 4);
+        let batches: Vec<_> = batch.chunks(chunk_size).map(|x| x.to_vec()).collect();
+        batches
+            .into_par_iter()
+            .map(|batch| {
+                let r = ecdsa::ed25519_verify(&batch);
+                batch.into_iter().zip(r).collect()
+            })
+            .collect()
+    }
+
+    fn verifier(
+        recvr: &streamer::PacketReceiver,
+        sendr: &Sender<Vec<(SharedPackets, Vec<u8>)>>,
+    ) -> Result<()> {
+        let batch = Self::recv_batch(recvr)?;
+        let verified_batches = Self::verify_batch(batch);
+        for xs in verified_batches {
+            sendr.send(xs)?;
+        }
+        Ok(())
+    }
+
+    pub fn deserialize_packets(p: &packet::Packets) -> Vec<Option<(Request, SocketAddr)>> {
+        p.packets
+            .par_iter()
+            .map(|x| {
+                deserialize(&x.data[0..x.meta.size])
+                    .map(|req| (req, x.meta.addr()))
+                    .ok()
+            })
+            .collect()
+    }
+
+    /// Split Request list into verified transactions and the rest
+    fn partition_requests(
+        req_vers: Vec<(Request, SocketAddr, u8)>,
+    ) -> (Vec<Transaction>, Vec<(Request, SocketAddr)>) {
+        let mut trs = vec![];
+        let mut reqs = vec![];
+        for (msg, rsp_addr, verify) in req_vers {
+            match msg {
+                Request::Transaction(tr) => {
+                    if verify != 0 {
+                        trs.push(tr);
+                    }
+                }
+                _ => reqs.push((msg, rsp_addr)),
+            }
+        }
+        (trs, reqs)
+    }
+
+    fn process_packets(
+        &mut self,
+        req_vers: Vec<(Request, SocketAddr, u8)>,
+    ) -> Result<Vec<(Response, SocketAddr)>> {
+        let (trs, reqs) = Self::partition_requests(req_vers);
+
+        // Process the transactions in parallel and then log the successful ones.
+        for result in self.acc.process_verified_transactions(trs) {
+            if let Ok(tr) = result {
+                self.historian
+                    .sender
+                    .send(Signal::Event(Event::Transaction(tr)))?;
+            }
+        }
+
+        // Let validators know they should not attempt to process additional
+        // transactions in parallel.
+        self.historian.sender.send(Signal::Tick)?;
+
+        // Process the remaining requests serially.
+        let rsps = reqs.into_iter()
+            .filter_map(|(req, rsp_addr)| self.process_request(req, rsp_addr))
+            .collect();
+
+        Ok(rsps)
+    }
+
+    fn serialize_response(
+        resp: Response,
+        rsp_addr: SocketAddr,
+        blob_recycler: &packet::BlobRecycler,
+    ) -> Result<packet::SharedBlob> {
+        let blob = blob_recycler.allocate();
+        {
+            let mut b = blob.write().unwrap();
+            let v = serialize(&resp)?;
+            let len = v.len();
+            b.data[..len].copy_from_slice(&v);
+            b.meta.size = len;
+            b.meta.set_addr(&rsp_addr);
+        }
+        Ok(blob)
+    }
+
+    fn serialize_responses(
+        rsps: Vec<(Response, SocketAddr)>,
+        blob_recycler: &packet::BlobRecycler,
+    ) -> Result<VecDeque<packet::SharedBlob>> {
+        let mut blobs = VecDeque::new();
+        for (resp, rsp_addr) in rsps {
+            blobs.push_back(Self::serialize_response(resp, rsp_addr, blob_recycler)?);
+        }
+        Ok(blobs)
+    }
+
+    fn process(
+        obj: &Arc<Mutex<AccountantSkel<W>>>,
+        verified_receiver: &Receiver<Vec<(SharedPackets, Vec<u8>)>>,
+        blob_sender: &streamer::BlobSender,
+        packet_recycler: &packet::PacketRecycler,
+        blob_recycler: &packet::BlobRecycler,
+    ) -> Result<()> {
+        let timer = Duration::new(1, 0);
+        let mms = verified_receiver.recv_timeout(timer)?;
+        for (msgs, vers) in mms {
+            let reqs = Self::deserialize_packets(&msgs.read().unwrap());
+            let req_vers = reqs.into_iter()
+                .zip(vers)
+                .filter_map(|(req, ver)| req.map(|(msg, addr)| (msg, addr, ver)))
+                .filter(|x| x.0.verify())
+                .collect();
+            let rsps = obj.lock().unwrap().process_packets(req_vers)?;
+            let blobs = Self::serialize_responses(rsps, blob_recycler)?;
+            if !blobs.is_empty() {
+                //don't wake up the other side if there is nothing
+                blob_sender.send(blobs)?;
+            }
+            packet_recycler.recycle(msgs);
+
+            // Write new entries to the ledger and notify subscribers.
+            obj.lock().unwrap().sync();
+        }
+
+        Ok(())
+    }
+    /// Process verified blobs, already in order
+    /// Respond with a signed hash of the state
+    fn replicate_state(
+        obj: &Arc<Mutex<AccountantSkel<W>>>,
+        verified_receiver: &streamer::BlobReceiver,
+        blob_recycler: &packet::BlobRecycler,
+    ) -> Result<()> {
+        let timer = Duration::new(1, 0);
+        let blobs = verified_receiver.recv_timeout(timer)?;
+        for msgs in &blobs {
+            let blob = msgs.read().unwrap();
+            let entries: Vec<Entry> = deserialize(&blob.data()[..blob.meta.size]).unwrap();
+            for entry in entries {
+                obj.lock().unwrap().acc.register_entry_id(&entry.id);
+
+                obj.lock()
+                    .unwrap()
+                    .acc
+                    .process_verified_events(entry.events)?;
+            }
+            //TODO respond back to leader with hash of the state
+        }
+        for blob in blobs {
+            blob_recycler.recycle(blob);
+        }
+        Ok(())
+    }
+
+    /// Create a UDP microservice that forwards messages the given AccountantSkel.
+    /// This service is the network leader
+    /// Set `exit` to shutdown its threads.
+    pub fn serve(
+        obj: &Arc<Mutex<AccountantSkel<W>>>,
+        addr: &str,
+        exit: Arc<AtomicBool>,
+    ) -> Result<Vec<JoinHandle<()>>> {
+        let read = UdpSocket::bind(addr)?;
+        // make sure we are on the same interface
+        let mut local = read.local_addr()?;
+        local.set_port(0);
+        let write = UdpSocket::bind(local)?;
+
+        let packet_recycler = packet::PacketRecycler::default();
+        let blob_recycler = packet::BlobRecycler::default();
+        let (packet_sender, packet_receiver) = channel();
+        let t_receiver =
+            streamer::receiver(read, exit.clone(), packet_recycler.clone(), packet_sender)?;
+        let (blob_sender, blob_receiver) = channel();
+        let t_responder =
+            streamer::responder(write, exit.clone(), blob_recycler.clone(), blob_receiver);
+        let (verified_sender, verified_receiver) = channel();
+
+        let exit_ = exit.clone();
+        let t_verifier = spawn(move || loop {
+            let e = Self::verifier(&packet_receiver, &verified_sender);
+            if e.is_err() && exit_.load(Ordering::Relaxed) {
+                break;
+            }
+        });
+
+        let skel = obj.clone();
+        let t_server = spawn(move || loop {
+            let e = Self::process(
+                &skel,
+                &verified_receiver,
+                &blob_sender,
+                &packet_recycler,
+                &blob_recycler,
+            );
+            if e.is_err() {
+                // Assume this was a timeout, so sync any empty entries.
+                skel.lock().unwrap().sync();
+
+                if exit.load(Ordering::Relaxed) {
+                    break;
+                }
+            }
+        });
+        Ok(vec![t_receiver, t_responder, t_server, t_verifier])
+    }
+
+    /// This service receives messages from a leader in the network and processes the transactions
+    /// on the accountant state.
+    /// # Arguments
+    /// * `obj` - The accountant state.
+    /// * `rsubs` - The subscribers.
+    /// * `exit` - The exit signal.
+    /// # Remarks
+    /// The pipeline is constructed as follows:
+    /// 1. receive blobs from the network, these are out of order
+    /// 2. verify blobs, PoH, signatures (TODO)
+    /// 3. reconstruct contiguous window
+    ///     a. order the blobs
+    ///     b. use erasure coding to reconstruct missing blobs
+    ///     c. ask the network for missing blobs, if erasure coding is insufficient
+    ///     d. make sure that the blobs PoH sequences connect (TODO)
+    /// 4. process the transaction state machine
+    /// 5. respond with the hash of the state back to the leader
+    pub fn replicate(
+        obj: &Arc<Mutex<AccountantSkel<W>>>,
+        rsubs: subscribers::Subscribers,
+        exit: Arc<AtomicBool>,
+    ) -> Result<Vec<JoinHandle<()>>> {
+        let read = UdpSocket::bind(rsubs.me.addr)?;
+        // make sure we are on the same interface
+        let mut local = read.local_addr()?;
+        local.set_port(0);
+        let write = UdpSocket::bind(local)?;
+
+        let blob_recycler = packet::BlobRecycler::default();
+        let (blob_sender, blob_receiver) = channel();
+        let t_blob_receiver = streamer::blob_receiver(
+            exit.clone(),
+            blob_recycler.clone(),
+            read,
+            blob_sender.clone(),
+        )?;
+        let (window_sender, window_receiver) = channel();
+        let (retransmit_sender, retransmit_receiver) = channel();
+
+        let subs = Arc::new(RwLock::new(rsubs));
+        let t_retransmit = streamer::retransmitter(
+            write,
+            exit.clone(),
+            subs.clone(),
+            blob_recycler.clone(),
+            retransmit_receiver,
+        );
+        //TODO
+        //the packets coming out of blob_receiver need to be sent to the GPU and verified
+        //then sent to the window, which does the erasure coding reconstruction
+        let t_window = streamer::window(
+            exit.clone(),
+            subs,
+            blob_recycler.clone(),
+            blob_receiver,
+            window_sender,
+            retransmit_sender,
+        );
+
+        let skel = obj.clone();
+        let t_server = spawn(move || loop {
+            let e = Self::replicate_state(&skel, &window_receiver, &blob_recycler);
+            if e.is_err() && exit.load(Ordering::Relaxed) {
+                break;
+            }
+        });
+        Ok(vec![t_blob_receiver, t_retransmit, t_window, t_server])
+    }
+}
+
+#[cfg(test)]
+pub fn to_packets(r: &packet::PacketRecycler, reqs: Vec<Request>) -> Vec<SharedPackets> {
+    let mut out = vec![];
+    for rrs in reqs.chunks(packet::NUM_PACKETS) {
+        let p = r.allocate();
+        p.write()
+            .unwrap()
+            .packets
+            .resize(rrs.len(), Default::default());
+        for (i, o) in rrs.iter().zip(p.write().unwrap().packets.iter_mut()) {
+            let v = serialize(&i).expect("serialize request");
+            let len = v.len();
+            o.data[..len].copy_from_slice(&v);
+            o.meta.size = len;
+        }
+        out.push(p);
+    }
+    return out;
+}
+
+#[cfg(test)]
+mod tests {
+    use accountant_skel::{to_packets, Request};
+    use bincode::serialize;
+    use ecdsa;
+    use packet::{BlobRecycler, PacketRecycler, NUM_PACKETS};
+    use transaction::{memfind, test_tx};
+
+    use accountant::Accountant;
+    use accountant_skel::AccountantSkel;
+    use accountant_stub::AccountantStub;
+    use entry::Entry;
+    use futures::Future;
+    use historian::Historian;
+    use mint::Mint;
+    use plan::Plan;
+    use recorder::Signal;
+    use signature::{KeyPair, KeyPairUtil};
+    use std::io::sink;
+    use std::net::{SocketAddr, UdpSocket};
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::{Arc, Mutex};
+    use std::thread::sleep;
+    use std::time::Duration;
+    use transaction::Transaction;
+
+    use subscribers::{Node, Subscribers};
+    use streamer;
+    use std::sync::mpsc::channel;
+    use std::collections::VecDeque;
+    use hash::{hash, Hash};
+    use event::Event;
+    use entry;
+    use chrono::prelude::*;
+
+    #[test]
+    fn test_layout() {
+        let tr = test_tx();
+        let tx = serialize(&tr).unwrap();
+        let packet = serialize(&Request::Transaction(tr)).unwrap();
+        assert_matches!(memfind(&packet, &tx), Some(ecdsa::TX_OFFSET));
+        assert_matches!(memfind(&packet, &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]), None);
+    }
+    #[test]
+    fn test_to_packets() {
+        let tr = Request::Transaction(test_tx());
+        let re = PacketRecycler::default();
+        let rv = to_packets(&re, vec![tr.clone(); 1]);
+        assert_eq!(rv.len(), 1);
+        assert_eq!(rv[0].read().unwrap().packets.len(), 1);
+
+        let rv = to_packets(&re, vec![tr.clone(); NUM_PACKETS]);
+        assert_eq!(rv.len(), 1);
+        assert_eq!(rv[0].read().unwrap().packets.len(), NUM_PACKETS);
+
+        let rv = to_packets(&re, vec![tr.clone(); NUM_PACKETS + 1]);
+        assert_eq!(rv.len(), 2);
+        assert_eq!(rv[0].read().unwrap().packets.len(), NUM_PACKETS);
+        assert_eq!(rv[1].read().unwrap().packets.len(), 1);
+    }
+
+    #[test]
+    fn test_accounting_sequential_consistency() {
+        // In this attack we'll demonstrate that a verifier can interpret the ledger
+        // differently if either the server doesn't signal the ledger to add an
+        // Entry OR if the verifier tries to parallelize across multiple Entries.
+        let mint = Mint::new(2);
+        let acc = Accountant::new(&mint);
+        let rsp_addr: SocketAddr = "0.0.0.0:0".parse().expect("socket address");
+        let historian = Historian::new(&mint.last_id(), None);
+        let mut skel = AccountantSkel::new(acc, mint.last_id(), sink(), historian);
+
+        // Process a batch that includes a transaction that receives two tokens.
+        let alice = KeyPair::new();
+        let tr = Transaction::new(&mint.keypair(), alice.pubkey(), 2, mint.last_id());
+        let req_vers = vec![(Request::Transaction(tr), rsp_addr, 1_u8)];
+        assert!(skel.process_packets(req_vers).is_ok());
+
+        // Process a second batch that spends one of those tokens.
+        let tr = Transaction::new(&alice, mint.pubkey(), 1, mint.last_id());
+        let req_vers = vec![(Request::Transaction(tr), rsp_addr, 1_u8)];
+        assert!(skel.process_packets(req_vers).is_ok());
+
+        // Collect the ledger and feed it to a new accountant.
+        skel.historian.sender.send(Signal::Tick).unwrap();
+        drop(skel.historian.sender);
+        let entries: Vec<Entry> = skel.historian.receiver.iter().collect();
+
+        // Assert the user holds one token, not two. If the server only output one
+        // entry, then the second transaction will be rejected, because it drives
+        // the account balance below zero before the credit is added.
+        let acc = Accountant::new(&mint);
+        for entry in entries {
+            acc.process_verified_events(entry.events).unwrap();
+        }
+        assert_eq!(acc.get_balance(&alice.pubkey()), Some(1));
+    }
+
+    #[test]
+    fn test_accountant_bad_sig() {
+        let serve_port = 9002;
+        let send_port = 9003;
+        let addr = format!("127.0.0.1:{}", serve_port);
+        let send_addr = format!("127.0.0.1:{}", send_port);
+        let alice = Mint::new(10_000);
+        let acc = Accountant::new(&alice);
+        let bob_pubkey = KeyPair::new().pubkey();
+        let exit = Arc::new(AtomicBool::new(false));
+        let historian = Historian::new(&alice.last_id(), Some(30));
+        let acc = Arc::new(Mutex::new(AccountantSkel::new(
+            acc,
+            alice.last_id(),
+            sink(),
+            historian,
+        )));
+        let _threads = AccountantSkel::serve(&acc, &addr, exit.clone()).unwrap();
+        sleep(Duration::from_millis(300));
+
+        let socket = UdpSocket::bind(send_addr).unwrap();
+        socket.set_read_timeout(Some(Duration::new(5, 0))).unwrap();
+
+        let mut acc = AccountantStub::new(&addr, socket);
+        let last_id = acc.get_last_id().wait().unwrap();
+
+        let tr = Transaction::new(&alice.keypair(), bob_pubkey, 500, last_id);
+
+        let _sig = acc.transfer_signed(tr).unwrap();
+
+        let last_id = acc.get_last_id().wait().unwrap();
+
+        let mut tr2 = Transaction::new(&alice.keypair(), bob_pubkey, 501, last_id);
+        tr2.data.tokens = 502;
+        tr2.data.plan = Plan::new_payment(502, bob_pubkey);
+        let _sig = acc.transfer_signed(tr2).unwrap();
+
+        assert_eq!(acc.get_balance(&bob_pubkey).wait().unwrap(), 500);
+        exit.store(true, Ordering::Relaxed);
+    }
+
+    use std::sync::{Once, ONCE_INIT};
+    extern crate env_logger;
+
+    static INIT: Once = ONCE_INIT;
+
+    /// Setup function that is only run once, even if called multiple times.
+    fn setup() {
+        INIT.call_once(|| {
+            env_logger::init().unwrap();
+        });
+    }
+
+    #[test]
+    fn test_replicate() {
+        setup();
+        let leader_sock = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let leader_addr = leader_sock.local_addr().unwrap();
+        let me_addr = "127.0.0.1:9010".parse().unwrap();
+        let target_peer_sock = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let target_peer_addr = target_peer_sock.local_addr().unwrap();
+        let source_peer_sock = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let exit = Arc::new(AtomicBool::new(false));
+
+        let node_me = Node::new([0, 0, 0, 0, 0, 0, 0, 1], 10, me_addr);
+        let node_subs = vec![Node::new([0, 0, 0, 0, 0, 0, 0, 2], 8, target_peer_addr); 1];
+        let node_leader = Node::new([0, 0, 0, 0, 0, 0, 0, 3], 20, leader_addr);
+        let subs = Subscribers::new(node_me, node_leader, &node_subs);
+
+        // setup some blob services to send blobs into the socket
+        // to simulate the source peer and get blobs out of the socket to
+        // simulate target peer
+        let recv_recycler = BlobRecycler::default();
+        let resp_recycler = BlobRecycler::default();
+        let (s_reader, r_reader) = channel();
+        let t_receiver = streamer::blob_receiver(
+            exit.clone(),
+            recv_recycler.clone(),
+            target_peer_sock,
+            s_reader,
+        ).unwrap();
+        let (s_responder, r_responder) = channel();
+        let t_responder = streamer::responder(
+            source_peer_sock,
+            exit.clone(),
+            resp_recycler.clone(),
+            r_responder,
+        );
+
+        let starting_balance = 10_000;
+        let alice = Mint::new(starting_balance);
+        let acc = Accountant::new(&alice);
+        let historian = Historian::new(&alice.last_id(), Some(30));
+        let acc = Arc::new(Mutex::new(AccountantSkel::new(
+            acc,
+            alice.last_id(),
+            sink(),
+            historian,
+        )));
+
+        let _threads = AccountantSkel::replicate(&acc, subs, exit.clone()).unwrap();
+
+        let mut alice_ref_balance = starting_balance;
+        let mut msgs = VecDeque::new();
+        let mut cur_hash = Hash::default();
+        let num_blobs = 10;
+        let transfer_amount = 501;
+        let bob_keypair = KeyPair::new();
+        for i in 0..num_blobs {
+            let b = resp_recycler.allocate();
+            let b_ = b.clone();
+            let mut w = b.write().unwrap();
+            w.set_index(i).unwrap();
+
+            let tr0 = Event::new_timestamp(&bob_keypair, Utc::now());
+            let entry0 = entry::create_entry(&cur_hash, i, vec![tr0]);
+            acc.lock().unwrap().acc.register_entry_id(&cur_hash);
+            cur_hash = hash(&cur_hash);
+
+            let tr1 = Transaction::new(
+                &alice.keypair(),
+                bob_keypair.pubkey(),
+                transfer_amount,
+                cur_hash,
+            );
+            acc.lock().unwrap().acc.register_entry_id(&cur_hash);
+            cur_hash = hash(&cur_hash);
+            let entry1 =
+                entry::create_entry(&cur_hash, i + num_blobs, vec![Event::Transaction(tr1)]);
+            acc.lock().unwrap().acc.register_entry_id(&cur_hash);
+            cur_hash = hash(&cur_hash);
+
+            alice_ref_balance -= transfer_amount;
+
+            let serialized_entry = serialize(&vec![entry0, entry1]).unwrap();
+
+            w.data_mut()[..serialized_entry.len()].copy_from_slice(&serialized_entry);
+            w.set_size(serialized_entry.len());
+            w.meta.set_addr(&me_addr);
+            drop(w);
+            msgs.push_back(b_);
+        }
+
+        // send the blobs into the socket
+        s_responder.send(msgs).expect("send");
+
+        // receive retransmitted messages
+        let timer = Duration::new(1, 0);
+        let mut msgs: Vec<_> = Vec::new();
+        while let Ok(msg) = r_reader.recv_timeout(timer) {
+            trace!("msg: {:?}", msg);
+            msgs.push(msg);
+        }
+
+        let alice_balance = acc.lock()
+            .unwrap()
+            .acc
+            .get_balance(&alice.keypair().pubkey())
+            .unwrap();
+        assert_eq!(alice_balance, alice_ref_balance);
+
+        let bob_balance = acc.lock()
+            .unwrap()
+            .acc
+            .get_balance(&bob_keypair.pubkey())
+            .unwrap();
+        assert_eq!(bob_balance, starting_balance - alice_ref_balance);
+
+        exit.store(true, Ordering::Relaxed);
+        t_receiver.join().expect("join");
+        t_responder.join().expect("join");
+    }
+
+}
+
+#[cfg(all(feature = "unstable", test))]
+mod bench {
+    extern crate test;
+    use self::test::Bencher;
+    use accountant::{Accountant, MAX_ENTRY_IDS};
+    use accountant_skel::*;
+    use bincode::serialize;
+    use hash::hash;
+    use mint::Mint;
+    use signature::{KeyPair, KeyPairUtil};
+    use std::collections::HashSet;
+    use std::io::sink;
+    use std::time::Instant;
+    use transaction::Transaction;
+
+    #[bench]
+    fn process_packets_bench(_bencher: &mut Bencher) {
+        let mint = Mint::new(100_000_000);
+        let acc = Accountant::new(&mint);
+        let rsp_addr: SocketAddr = "0.0.0.0:0".parse().expect("socket address");
+        // Create transactions between unrelated parties.
+        let txs = 100_000;
+        let last_ids: Mutex<HashSet<Hash>> = Mutex::new(HashSet::new());
+        let transactions: Vec<_> = (0..txs)
+            .into_par_iter()
+            .map(|i| {
+                // Seed the 'to' account and a cell for its signature.
+                let dummy_id = i % (MAX_ENTRY_IDS as i32);
+                let last_id = hash(&serialize(&dummy_id).unwrap()); // Semi-unique hash
+                {
+                    let mut last_ids = last_ids.lock().unwrap();
+                    if !last_ids.contains(&last_id) {
+                        last_ids.insert(last_id);
+                        acc.register_entry_id(&last_id);
+                    }
+                }
+
+                // Seed the 'from' account.
+                let rando0 = KeyPair::new();
+                let tr = Transaction::new(&mint.keypair(), rando0.pubkey(), 1_000, last_id);
+                acc.process_verified_transaction(&tr).unwrap();
+
+                let rando1 = KeyPair::new();
+                let tr = Transaction::new(&rando0, rando1.pubkey(), 2, last_id);
+                acc.process_verified_transaction(&tr).unwrap();
+
+                // Finally, return a transaction that's unique
+                Transaction::new(&rando0, rando1.pubkey(), 1, last_id)
+            })
+            .collect();
+
+        let req_vers = transactions
+            .into_iter()
+            .map(|tr| (Request::Transaction(tr), rsp_addr, 1_u8))
+            .collect();
+
+        let historian = Historian::new(&mint.last_id(), None);
+        let mut skel = AccountantSkel::new(acc, mint.last_id(), sink(), historian);
+
+        let now = Instant::now();
+        assert!(skel.process_packets(req_vers).is_ok());
+        let duration = now.elapsed();
+        let sec = duration.as_secs() as f64 + duration.subsec_nanos() as f64 / 1_000_000_000.0;
+        let tps = txs as f64 / sec;
+
+        // Ensure that all transactions were successfully logged.
+        drop(skel.historian.sender);
+        let entries: Vec<Entry> = skel.historian.receiver.iter().collect();
+        assert_eq!(entries.len(), 1);
+        assert_eq!(entries[0].events.len(), txs as usize);
+
+        println!("{} tps", tps);
+    }
+}

src/accountant_stub.rs

@@ -0,0 +1,201 @@
+//! The `accountant_stub` module is a client-side object that interfaces with a server-side Accountant
+//! object via the network interface exposed by AccountantSkel. Client code should use
+//! this object instead of writing messages to the network directly. The binary
+//! encoding of its messages are unstable and may change in future releases.
+
+use accountant_skel::{Request, Response, Subscription};
+use bincode::{deserialize, serialize};
+use futures::future::{ok, FutureResult};
+use hash::Hash;
+use signature::{KeyPair, PublicKey, Signature};
+use std::collections::HashMap;
+use std::io;
+use std::net::UdpSocket;
+use transaction::Transaction;
+
+pub struct AccountantStub {
+    pub addr: String,
+    pub socket: UdpSocket,
+    last_id: Option<Hash>,
+    num_events: u64,
+    balances: HashMap<PublicKey, Option<i64>>,
+}
+
+impl AccountantStub {
+    /// Create a new AccountantStub that will interface with AccountantSkel
+    /// over `socket`. To receive responses, the caller must bind `socket`
+    /// to a public address before invoking AccountantStub methods.
+    pub fn new(addr: &str, socket: UdpSocket) -> Self {
+        let stub = AccountantStub {
+            addr: addr.to_string(),
+            socket,
+            last_id: None,
+            num_events: 0,
+            balances: HashMap::new(),
+        };
+        stub.init();
+        stub
+    }
+
+    pub fn init(&self) {
+        let subscriptions = vec![Subscription::EntryInfo];
+        let req = Request::Subscribe { subscriptions };
+        let data = serialize(&req).expect("serialize Subscribe");
+        let _res = self.socket.send_to(&data, &self.addr);
+    }
+
+    pub fn recv_response(&self) -> io::Result<Response> {
+        let mut buf = vec![0u8; 1024];
+        self.socket.recv_from(&mut buf)?;
+        let resp = deserialize(&buf).expect("deserialize balance");
+        Ok(resp)
+    }
+
+    pub fn process_response(&mut self, resp: Response) {
+        match resp {
+            Response::Balance { key, val } => {
+                self.balances.insert(key, val);
+            }
+            Response::LastId { id } => {
+                self.last_id = Some(id);
+            }
+            Response::EntryInfo(entry_info) => {
+                self.last_id = Some(entry_info.id);
+                self.num_events += entry_info.num_events;
+            }
+        }
+    }
+
+    /// Send a signed Transaction to the server for processing. This method
+    /// does not wait for a response.
+    pub fn transfer_signed(&self, tr: Transaction) -> io::Result<usize> {
+        let req = Request::Transaction(tr);
+        let data = serialize(&req).unwrap();
+        self.socket.send_to(&data, &self.addr)
+    }
+
+    /// Creates, signs, and processes a Transaction. Useful for writing unit-tests.
+    pub fn transfer(
+        &self,
+        n: i64,
+        keypair: &KeyPair,
+        to: PublicKey,
+        last_id: &Hash,
+    ) -> io::Result<Signature> {
+        let tr = Transaction::new(keypair, to, n, *last_id);
+        let sig = tr.sig;
+        self.transfer_signed(tr).map(|_| sig)
+    }
+
+    /// Request the balance of the user holding `pubkey`. This method blocks
+    /// until the server sends a response. If the response packet is dropped
+    /// by the network, this method will hang indefinitely.
+    pub fn get_balance(&mut self, pubkey: &PublicKey) -> FutureResult<i64, i64> {
+        let req = Request::GetBalance { key: *pubkey };
+        let data = serialize(&req).expect("serialize GetBalance");
+        self.socket
+            .send_to(&data, &self.addr)
+            .expect("buffer error");
+        let mut done = false;
+        while !done {
+            let resp = self.recv_response().expect("recv response");
+            if let &Response::Balance { ref key, .. } = &resp {
+                done = key == pubkey;
+            }
+            self.process_response(resp);
+        }
+        ok(self.balances[pubkey].unwrap())
+    }
+
+    /// Request the last Entry ID from the server. This method blocks
+    /// until the server sends a response. At the time of this writing,
+    /// it also has the side-effect of causing the server to log any
+    /// entries that have been published by the Historian.
+    pub fn get_last_id(&mut self) -> FutureResult<Hash, ()> {
+        let req = Request::GetLastId;
+        let data = serialize(&req).expect("serialize GetId");
+        self.socket
+            .send_to(&data, &self.addr)
+            .expect("buffer error");
+        let mut done = false;
+        while !done {
+            let resp = self.recv_response().expect("recv response");
+            if let &Response::LastId { .. } = &resp {
+                done = true;
+            }
+            self.process_response(resp);
+        }
+        ok(self.last_id.unwrap_or(Hash::default()))
+    }
+
+    /// Return the number of transactions the server processed since creating
+    /// this stub instance.
+    pub fn transaction_count(&mut self) -> u64 {
+        // Wait for at least one EntryInfo.
+        let mut done = false;
+        while !done {
+            let resp = self.recv_response().expect("recv response");
+            if let &Response::EntryInfo(_) = &resp {
+                done = true;
+            }
+            self.process_response(resp);
+        }
+
+        // Then take the rest.
+        self.socket.set_nonblocking(true).expect("set nonblocking");
+        loop {
+            match self.recv_response() {
+                Err(_) => break,
+                Ok(resp) => self.process_response(resp),
+            }
+        }
+        self.socket.set_nonblocking(false).expect("set blocking");
+        self.num_events
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use accountant::Accountant;
+    use accountant_skel::AccountantSkel;
+    use futures::Future;
+    use historian::Historian;
+    use mint::Mint;
+    use signature::{KeyPair, KeyPairUtil};
+    use std::io::sink;
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::{Arc, Mutex};
+    use std::thread::sleep;
+    use std::time::Duration;
+
+    // TODO: Figure out why this test sometimes hangs on TravisCI.
+    #[test]
+    fn test_accountant_stub() {
+        let addr = "127.0.0.1:9000";
+        let send_addr = "127.0.0.1:9001";
+        let alice = Mint::new(10_000);
+        let acc = Accountant::new(&alice);
+        let bob_pubkey = KeyPair::new().pubkey();
+        let exit = Arc::new(AtomicBool::new(false));
+        let historian = Historian::new(&alice.last_id(), Some(30));
+        let acc = Arc::new(Mutex::new(AccountantSkel::new(
+            acc,
+            alice.last_id(),
+            sink(),
+            historian,
+        )));
+        let _threads = AccountantSkel::serve(&acc, addr, exit.clone()).unwrap();
+        sleep(Duration::from_millis(300));
+
+        let socket = UdpSocket::bind(send_addr).unwrap();
+        socket.set_read_timeout(Some(Duration::new(5, 0))).unwrap();
+
+        let mut acc = AccountantStub::new(addr, socket);
+        let last_id = acc.get_last_id().wait().unwrap();
+        let _sig = acc.transfer(500, &alice.keypair(), bob_pubkey, &last_id)
+            .unwrap();
+        assert_eq!(acc.get_balance(&bob_pubkey).wait().unwrap(), 500);
+        exit.store(true, Ordering::Relaxed);
+    }
+}

src/bin/client-demo.rs

@@ -0,0 +1,143 @@
+extern crate futures;
+extern crate getopts;
+extern crate isatty;
+extern crate rayon;
+extern crate serde_json;
+extern crate solana;
+extern crate untrusted;
+
+use futures::Future;
+use getopts::Options;
+use isatty::stdin_isatty;
+use rayon::prelude::*;
+use solana::accountant_stub::AccountantStub;
+use solana::mint::MintDemo;
+use solana::signature::{KeyPair, KeyPairUtil};
+use solana::transaction::Transaction;
+use std::env;
+use std::io::{stdin, Read};
+use std::net::UdpSocket;
+use std::process::exit;
+use std::time::Instant;
+use untrusted::Input;
+
+fn print_usage(program: &str, opts: Options) {
+    let mut brief = format!("Usage: cat <mint.json> | {} [options]\n\n", program);
+    brief += "  Solana client demo creates a number of transactions and\n";
+    brief += "  sends them to a target node.";
+    brief += "  Takes json formatted mint file to stdin.";
+
+    print!("{}", opts.usage(&brief));
+}
+
+fn main() {
+    let mut threads = 4usize;
+    let mut addr: String = "127.0.0.1:8000".to_string();
+    let mut send_addr: String = "127.0.0.1:8001".to_string();
+
+    let mut opts = Options::new();
+    opts.optopt("s", "", "server address", "host:port");
+    opts.optopt("c", "", "client address", "host:port");
+    opts.optopt("t", "", "number of threads", "4");
+    opts.optflag("h", "help", "print help");
+    let args: Vec<String> = env::args().collect();
+    let matches = match opts.parse(&args[1..]) {
+        Ok(m) => m,
+        Err(e) => {
+            eprintln!("{}", e);
+            exit(1);
+        }
+    };
+
+    if matches.opt_present("h") {
+        let program = args[0].clone();
+        print_usage(&program, opts);
+        return;
+    }
+    if matches.opt_present("s") {
+        addr = matches.opt_str("s").unwrap();
+    }
+    if matches.opt_present("c") {
+        send_addr = matches.opt_str("c").unwrap();
+    }
+    if matches.opt_present("t") {
+        threads = matches.opt_str("t").unwrap().parse().expect("integer");
+    }
+
+    if stdin_isatty() {
+        eprintln!("nothing found on stdin, expected a json file");
+        exit(1);
+    }
+
+    let mut buffer = String::new();
+    let num_bytes = stdin().read_to_string(&mut buffer).unwrap();
+    if num_bytes == 0 {
+        eprintln!("empty file on stdin, expected a json file");
+        exit(1);
+    }
+
+    println!("Parsing stdin...");
+    let demo: MintDemo = serde_json::from_str(&buffer).unwrap_or_else(|e| {
+        eprintln!("failed to parse json: {}", e);
+        exit(1);
+    });
+
+    let socket = UdpSocket::bind(&send_addr).unwrap();
+    let mut acc = AccountantStub::new(&addr, socket);
+
+    println!("Get last ID...");
+    let last_id = acc.get_last_id().wait().unwrap();
+
+    println!("Creating keypairs...");
+    let txs = demo.users.len() / 2;
+    let keypairs: Vec<_> = demo.users
+        .into_par_iter()
+        .map(|(pkcs8, _)| KeyPair::from_pkcs8(Input::from(&pkcs8)).unwrap())
+        .collect();
+    let keypair_pairs: Vec<_> = keypairs.chunks(2).collect();
+
+    println!("Signing transactions...");
+    let now = Instant::now();
+    let transactions: Vec<_> = keypair_pairs
+        .into_par_iter()
+        .map(|chunk| Transaction::new(&chunk[0], chunk[1].pubkey(), 1, last_id))
+        .collect();
+    let duration = now.elapsed();
+    let ns = duration.as_secs() * 1_000_000_000 + u64::from(duration.subsec_nanos());
+    let bsps = txs as f64 / ns as f64;
+    let nsps = ns as f64 / txs as f64;
+    println!(
+        "Done. {} thousand signatures per second, {}us per signature",
+        bsps * 1_000_000_f64,
+        nsps / 1_000_f64
+    );
+
+    let initial_tx_count = acc.transaction_count();
+
+    println!("Transfering {} transactions in {} batches", txs, threads);
+    let now = Instant::now();
+    let sz = transactions.len() / threads;
+    let chunks: Vec<_> = transactions.chunks(sz).collect();
+    chunks.into_par_iter().for_each(|trs| {
+        println!("Transferring 1 unit {} times...", trs.len());
+        let send_addr = "0.0.0.0:0";
+        let socket = UdpSocket::bind(send_addr).unwrap();
+        let acc = AccountantStub::new(&addr, socket);
+        for tr in trs {
+            acc.transfer_signed(tr.clone()).unwrap();
+        }
+    });
+
+    println!("Waiting for half the transactions to complete...",);
+    let mut tx_count = acc.transaction_count();
+    while tx_count < transactions.len() as u64 / 2 {
+        tx_count = acc.transaction_count();
+    }
+    let txs = tx_count - initial_tx_count;
+    println!("Transactions processed {}", txs);
+
+    let duration = now.elapsed();
+    let ns = duration.as_secs() * 1_000_000_000 + u64::from(duration.subsec_nanos());
+    let tps = (txs * 1_000_000_000) as f64 / ns as f64;
+    println!("Done. {} tps", tps);
+}

src/bin/genesis-demo.rs

@@ -0,0 +1,73 @@
+extern crate isatty;
+extern crate rayon;
+extern crate ring;
+extern crate serde_json;
+extern crate solana;
+extern crate untrusted;
+
+use isatty::stdin_isatty;
+use rayon::prelude::*;
+use solana::accountant::MAX_ENTRY_IDS;
+use solana::entry::{create_entry, next_tick};
+use solana::event::Event;
+use solana::mint::MintDemo;
+use solana::signature::{KeyPair, KeyPairUtil};
+use solana::transaction::Transaction;
+use std::io::{stdin, Read};
+use std::process::exit;
+use untrusted::Input;
+
+// Generate a ledger with lots and lots of accounts.
+fn main() {
+    if stdin_isatty() {
+        eprintln!("nothing found on stdin, expected a json file");
+        exit(1);
+    }
+
+    let mut buffer = String::new();
+    let num_bytes = stdin().read_to_string(&mut buffer).unwrap();
+    if num_bytes == 0 {
+        eprintln!("empty file on stdin, expected a json file");
+        exit(1);
+    }
+
+    let demo: MintDemo = serde_json::from_str(&buffer).unwrap_or_else(|e| {
+        eprintln!("failed to parse json: {}", e);
+        exit(1);
+    });
+
+    let num_accounts = demo.users.len();
+    let last_id = demo.mint.last_id();
+    let mint_keypair = demo.mint.keypair();
+
+    eprintln!("Signing {} transactions...", num_accounts);
+    let events: Vec<_> = demo.users
+        .into_par_iter()
+        .map(|(pkcs8, tokens)| {
+            let rando = KeyPair::from_pkcs8(Input::from(&pkcs8)).unwrap();
+            let tr = Transaction::new(&mint_keypair, rando.pubkey(), tokens, last_id);
+            Event::Transaction(tr)
+        })
+        .collect();
+
+    for entry in demo.mint.create_entries() {
+        println!("{}", serde_json::to_string(&entry).unwrap());
+    }
+
+    eprintln!("Logging the creation of {} accounts...", num_accounts);
+    let entry = create_entry(&last_id, 0, events);
+    println!("{}", serde_json::to_string(&entry).unwrap());
+
+    eprintln!("Creating {} empty entries...", MAX_ENTRY_IDS);
+    // Offer client lots of entry IDs to use for each transaction's last_id.
+    let mut last_id = last_id;
+    for _ in 0..MAX_ENTRY_IDS {
+        let entry = next_tick(&last_id, 1);
+        last_id = entry.id;
+        let serialized = serde_json::to_string(&entry).unwrap_or_else(|e| {
+            eprintln!("failed to serialize: {}", e);
+            exit(1);
+        });
+        println!("{}", serialized);
+    }
+}

src/bin/genesis.rs

@@ -0,0 +1,36 @@
+//! A command-line executable for generating the chain's genesis block.
+
+extern crate isatty;
+extern crate serde_json;
+extern crate solana;
+
+use isatty::stdin_isatty;
+use solana::mint::Mint;
+use std::io::{stdin, Read};
+use std::process::exit;
+
+fn main() {
+    if stdin_isatty() {
+        eprintln!("nothing found on stdin, expected a json file");
+        exit(1);
+    }
+
+    let mut buffer = String::new();
+    let num_bytes = stdin().read_to_string(&mut buffer).unwrap();
+    if num_bytes == 0 {
+        eprintln!("empty file on stdin, expected a json file");
+        exit(1);
+    }
+
+    let mint: Mint = serde_json::from_str(&buffer).unwrap_or_else(|e| {
+        eprintln!("failed to parse json: {}", e);
+        exit(1);
+    });
+    for x in mint.create_entries() {
+        let serialized = serde_json::to_string(&x).unwrap_or_else(|e| {
+            eprintln!("failed to serialize: {}", e);
+            exit(1);
+        });
+        println!("{}", serialized);
+    }
+}

src/bin/historian-demo.rs

@@ -0,0 +1,37 @@
+extern crate solana;
+
+use solana::entry::Entry;
+use solana::event::Event;
+use solana::hash::Hash;
+use solana::historian::Historian;
+use solana::ledger::Block;
+use solana::recorder::Signal;
+use solana::signature::{KeyPair, KeyPairUtil};
+use solana::transaction::Transaction;
+use std::sync::mpsc::SendError;
+use std::thread::sleep;
+use std::time::Duration;
+
+fn create_ledger(hist: &Historian, seed: &Hash) -> Result<(), SendError<Signal>> {
+    sleep(Duration::from_millis(15));
+    let keypair = KeyPair::new();
+    let tr = Transaction::new(&keypair, keypair.pubkey(), 42, *seed);
+    let signal0 = Signal::Event(Event::Transaction(tr));
+    hist.sender.send(signal0)?;
+    sleep(Duration::from_millis(10));
+    Ok(())
+}
+
+fn main() {
+    let seed = Hash::default();
+    let hist = Historian::new(&seed, Some(10));
+    create_ledger(&hist, &seed).expect("send error");
+    drop(hist.sender);
+    let entries: Vec<Entry> = hist.receiver.iter().collect();
+    for entry in &entries {
+        println!("{:?}", entry);
+    }
+    // Proof-of-History: Verify the historian learned about the events
+    // in the same order they appear in the vector.
+    assert!(entries[..].verify(&seed));
+}

src/bin/mint-demo.rs

@@ -0,0 +1,33 @@
+extern crate rayon;
+extern crate ring;
+extern crate serde_json;
+extern crate solana;
+
+use rayon::prelude::*;
+use ring::rand::SystemRandom;
+use solana::mint::{Mint, MintDemo};
+use solana::signature::KeyPair;
+use std::io;
+
+fn main() {
+    let mut input_text = String::new();
+    io::stdin().read_line(&mut input_text).unwrap();
+    let trimmed = input_text.trim();
+    let tokens = trimmed.parse::<i64>().unwrap();
+
+    let mint = Mint::new(tokens);
+    let tokens_per_user = 1_000;
+    let num_accounts = tokens / tokens_per_user;
+    let rnd = SystemRandom::new();
+
+    let users: Vec<_> = (0..num_accounts)
+        .into_par_iter()
+        .map(|_| {
+            let pkcs8 = KeyPair::generate_pkcs8(&rnd).unwrap().to_vec();
+            (pkcs8, tokens_per_user)
+        })
+        .collect();
+
+    let demo = MintDemo { mint, users };
+    println!("{}", serde_json::to_string(&demo).unwrap());
+}

src/bin/mint.rs

@@ -0,0 +1,29 @@
+extern crate isatty;
+extern crate serde_json;
+extern crate solana;
+
+use isatty::stdin_isatty;
+use solana::mint::Mint;
+use std::io;
+use std::process::exit;
+
+fn main() {
+    let mut input_text = String::new();
+    if stdin_isatty() {
+        eprintln!("nothing found on stdin, expected a token number");
+        exit(1);
+    }
+
+    io::stdin().read_line(&mut input_text).unwrap();
+    let trimmed = input_text.trim();
+    let tokens = trimmed.parse::<i64>().unwrap_or_else(|e| {
+        eprintln!("{}", e);
+        exit(1);
+    });
+    let mint = Mint::new(tokens);
+    let serialized = serde_json::to_string(&mint).unwrap_or_else(|e| {
+        eprintln!("failed to serialize: {}", e);
+        exit(1);
+    });
+    println!("{}", serialized);
+}

src/bin/testnode.rs

@@ -0,0 +1,111 @@
+extern crate env_logger;
+extern crate getopts;
+extern crate isatty;
+extern crate serde_json;
+extern crate solana;
+
+use getopts::Options;
+use isatty::stdin_isatty;
+use solana::accountant::Accountant;
+use solana::accountant_skel::AccountantSkel;
+use solana::entry::Entry;
+use solana::event::Event;
+use solana::historian::Historian;
+use std::env;
+use std::io::{stdin, stdout, Read};
+use std::process::exit;
+use std::sync::atomic::AtomicBool;
+use std::sync::{Arc, Mutex};
+
+fn print_usage(program: &str, opts: Options) {
+    let mut brief = format!("Usage: cat <transaction.log> | {} [options]\n\n", program);
+    brief += "  Run a Solana node to handle transactions and\n";
+    brief += "  write a new transaction log to stdout.\n";
+    brief += "  Takes existing transaction log from stdin.";
+
+    print!("{}", opts.usage(&brief));
+}
+
+fn main() {
+    env_logger::init().unwrap();
+    let mut port = 8000u16;
+    let mut opts = Options::new();
+    opts.optopt("p", "", "port", "port");
+    opts.optflag("h", "help", "print help");
+    let args: Vec<String> = env::args().collect();
+    let matches = match opts.parse(&args[1..]) {
+        Ok(m) => m,
+        Err(e) => {
+            eprintln!("{}", e);
+            exit(1);
+        }
+    };
+    if matches.opt_present("h") {
+        let program = args[0].clone();
+        print_usage(&program, opts);
+        return;
+    }
+    if matches.opt_present("p") {
+        port = matches.opt_str("p").unwrap().parse().expect("port");
+    }
+    let addr = format!("0.0.0.0:{}", port);
+
+    if stdin_isatty() {
+        eprintln!("nothing found on stdin, expected a log file");
+        exit(1);
+    }
+
+    let mut buffer = String::new();
+    let num_bytes = stdin().read_to_string(&mut buffer).unwrap();
+    if num_bytes == 0 {
+        eprintln!("empty file on stdin, expected a log file");
+        exit(1);
+    }
+
+    eprintln!("Initializing...");
+    let mut entries = buffer.lines().map(|line| {
+        serde_json::from_str(&line).unwrap_or_else(|e| {
+            eprintln!("failed to parse json: {}", e);
+            exit(1);
+        })
+    });
+
+    // The first item in the ledger is required to be an entry with zero num_hashes,
+    // which implies its id can be used as the ledger's seed.
+    let entry0 = entries.next().unwrap();
+
+    // The second item in the ledger is a special transaction where the to and from
+    // fields are the same. That entry should be treated as a deposit, not a
+    // transfer to oneself.
+    let entry1: Entry = entries.next().unwrap();
+    let deposit = if let Event::Transaction(ref tr) = entry1.events[0] {
+        tr.data.plan.final_payment()
+    } else {
+        None
+    };
+
+    let acc = Accountant::new_from_deposit(&deposit.unwrap());
+    acc.register_entry_id(&entry0.id);
+    acc.register_entry_id(&entry1.id);
+
+    let mut last_id = entry1.id;
+    for entry in entries {
+        last_id = entry.id;
+        acc.process_verified_events(entry.events).unwrap();
+        acc.register_entry_id(&last_id);
+    }
+
+    let historian = Historian::new(&last_id, Some(1000));
+    let exit = Arc::new(AtomicBool::new(false));
+    let skel = Arc::new(Mutex::new(AccountantSkel::new(
+        acc,
+        last_id,
+        stdout(),
+        historian,
+    )));
+    let threads = AccountantSkel::serve(&skel, &addr, exit.clone()).unwrap();
+    eprintln!("Ready. Listening on {}", addr);
+    for t in threads {
+        t.join().expect("join");
+    }
+}

src/crdt.rs

@@ -0,0 +1,372 @@
+//! The `crdt` module defines a data structure that is shared by all the nodes in the network over
+//! a gossip control plane.  The goal is to share small bits of of-chain information and detect and
+//! repair partitions.
+//!
+//! This CRDT only supports a very limited set of types.  A map of PublicKey -> Versioned Struct.
+//! The last version is always picked durring an update.
+
+use bincode::{deserialize, serialize};
+use byteorder::{LittleEndian, ReadBytesExt};
+use hash::Hash;
+use result::Result;
+use ring::rand::{SecureRandom, SystemRandom};
+use signature::{PublicKey, Signature};
+use std::collections::HashMap;
+use std::io::Cursor;
+use std::net::{SocketAddr, UdpSocket};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, RwLock};
+use std::thread::{sleep, spawn, JoinHandle};
+use std::time::Duration;
+
+/// Structure to be replicated by the network
+#[derive(Serialize, Deserialize, Clone)]
+pub struct ReplicatedData {
+    id: PublicKey,
+    sig: Signature,
+    /// should always be increasing
+    version: u64,
+    /// address to connect to for gossip
+    gossip_addr: SocketAddr,
+    /// address to connect to for replication
+    replicate_addr: SocketAddr,
+    /// address to connect to when this node is leader
+    lead_addr: SocketAddr,
+    /// current leader identity
+    current_leader_id: PublicKey,
+    /// last verified hash that was submitted to the leader
+    last_verified_hash: Hash,
+    /// last verified count, always increasing
+    last_verified_count: u64,
+}
+
+impl ReplicatedData {
+    pub fn new(id: PublicKey, gossip_addr: SocketAddr) -> ReplicatedData {
+        let daddr = "0.0.0.0:0".parse().unwrap();
+        ReplicatedData {
+            id,
+            sig: Signature::default(),
+            version: 0,
+            gossip_addr,
+            replicate_addr: daddr,
+            lead_addr: daddr,
+            current_leader_id: PublicKey::default(),
+            last_verified_hash: Hash::default(),
+            last_verified_count: 0,
+        }
+    }
+}
+
+/// `Crdt` structure keeps a table of `ReplicatedData` structs
+/// # Properties
+/// * `table` - map of public id's to versioned and signed ReplicatedData structs
+/// * `local` - map of public id's to what `self.update_index` `self.table` was updated
+/// * `remote` - map of public id's to the `remote.update_index` was sent
+/// * `update_index` - my update index
+/// # Remarks
+/// This implements two services, `gossip` and `listen`.
+/// * `gossip` - asynchronously ask nodes to send updates
+/// * `listen` - listen for requests and responses
+/// No attempt to keep track of timeouts or dropped requests is made, or should be.
+pub struct Crdt {
+    table: HashMap<PublicKey, ReplicatedData>,
+    /// Value of my update index when entry in table was updated.
+    /// Nodes will ask for updates since `update_index`, and this node
+    /// should respond with all the identities that are greater then the
+    /// request's `update_index` in this list
+    local: HashMap<PublicKey, u64>,
+    /// The value of the remote update index that i have last seen
+    /// This Node will ask external nodes for updates since the value in this list
+    remote: HashMap<PublicKey, u64>,
+    update_index: u64,
+    me: PublicKey,
+    timeout: Duration,
+}
+// TODO These messages should be signed, and go through the gpu pipeline for spam filtering
+#[derive(Serialize, Deserialize)]
+enum Protocol {
+    /// forward your own latest data structure when requesting an update
+    /// this doesn't update the `remote` update index, but it allows the
+    /// recepient of this request to add knowledge of this node to the network
+    RequestUpdates(u64, ReplicatedData),
+    //TODO might need a since?
+    /// from id, form's last update index, ReplicatedData
+    ReceiveUpdates(PublicKey, u64, Vec<ReplicatedData>),
+}
+
+impl Crdt {
+    pub fn new(me: ReplicatedData) -> Crdt {
+        assert_eq!(me.version, 0);
+        let mut g = Crdt {
+            table: HashMap::new(),
+            local: HashMap::new(),
+            remote: HashMap::new(),
+            me: me.id,
+            update_index: 1,
+            timeout: Duration::new(0, 100_000),
+        };
+        g.local.insert(me.id, g.update_index);
+        g.table.insert(me.id, me);
+        g
+    }
+    pub fn import(&mut self, v: &ReplicatedData) {
+        // TODO check that last_verified types are always increasing
+        // TODO probably an error or attack
+        if self.me != v.id {
+            self.insert(v);
+        }
+    }
+    pub fn insert(&mut self, v: &ReplicatedData) {
+        if self.table.get(&v.id).is_none() || (v.version > self.table[&v.id].version) {
+            trace!("insert! {}", v.version);
+            self.update_index += 1;
+            let _ = self.table.insert(v.id, v.clone());
+            let _ = self.local.insert(v.id, self.update_index);
+        } else {
+            trace!("INSERT FAILED {}", v.version);
+        }
+    }
+    fn random() -> u64 {
+        let rnd = SystemRandom::new();
+        let mut buf = [0u8; 8];
+        rnd.fill(&mut buf).unwrap();
+        let mut rdr = Cursor::new(&buf);
+        rdr.read_u64::<LittleEndian>().unwrap()
+    }
+    fn get_updates_since(&self, v: u64) -> (PublicKey, u64, Vec<ReplicatedData>) {
+        trace!("get updates since {}", v);
+        let data = self.table
+            .values()
+            .filter(|x| self.local[&x.id] > v)
+            .cloned()
+            .collect();
+        let id = self.me;
+        let ups = self.update_index;
+        (id, ups, data)
+    }
+
+    /// Create a random gossip request
+    /// # Returns
+    /// (A,B,C)
+    /// * A - Remote gossip address
+    /// * B - My gossip address
+    /// * C - Remote update index to request updates since
+    fn gossip_request(&self) -> (SocketAddr, Protocol) {
+        let n = (Self::random() as usize) % self.table.len();
+        trace!("random {:?} {}", &self.me[0..1], n);
+        let v = self.table.values().nth(n).unwrap().clone();
+        let remote_update_index = *self.remote.get(&v.id).unwrap_or(&0);
+        let req = Protocol::RequestUpdates(remote_update_index, self.table[&self.me].clone());
+        (v.gossip_addr, req)
+    }
+
+    /// At random pick a node and try to get updated changes from them
+    fn run_gossip(obj: &Arc<RwLock<Self>>) -> Result<()> {
+        //TODO we need to keep track of stakes and weight the selection by stake size
+        //TODO cache sockets
+
+        // Lock the object only to do this operation and not for any longer
+        // especially not when doing the `sock.send_to`
+        let (remote_gossip_addr, req) = obj.read().unwrap().gossip_request();
+        let sock = UdpSocket::bind("0.0.0.0:0")?;
+        // TODO this will get chatty, so we need to first ask for number of updates since
+        // then only ask for specific data that we dont have
+        let r = serialize(&req)?;
+        sock.send_to(&r, remote_gossip_addr)?;
+        Ok(())
+    }
+
+    /// Apply updates that we received from the identity `from`
+    /// # Arguments
+    /// * `from` - identity of the sender of the updates
+    /// * `update_index` - the number of updates that `from` has completed and this set of `data` represents
+    /// * `data` - the update data
+    fn apply_updates(&mut self, from: PublicKey, update_index: u64, data: &[ReplicatedData]) {
+        trace!("got updates {}", data.len());
+        // TODO we need to punish/spam resist here
+        // sig verify the whole update and slash anyone who sends a bad update
+        for v in data {
+            self.import(&v);
+        }
+        *self.remote.entry(from).or_insert(update_index) = update_index;
+    }
+
+    /// randomly pick a node and ask them for updates asynchronously
+    pub fn gossip(obj: Arc<RwLock<Self>>, exit: Arc<AtomicBool>) -> JoinHandle<()> {
+        spawn(move || loop {
+            let _ = Self::run_gossip(&obj);
+            if exit.load(Ordering::Relaxed) {
+                return;
+            }
+            //TODO this should be a tuned parameter
+            sleep(obj.read().unwrap().timeout);
+        })
+    }
+
+    /// Process messages from the network
+    fn run_listen(obj: &Arc<RwLock<Self>>, sock: &UdpSocket) -> Result<()> {
+        //TODO cache connections
+        let mut buf = vec![0u8; 1024 * 64];
+        let (amt, src) = sock.recv_from(&mut buf)?;
+        trace!("got request from {}", src);
+        buf.resize(amt, 0);
+        let r = deserialize(&buf)?;
+        match r {
+            // TODO sigverify these
+            Protocol::RequestUpdates(v, reqdata) => {
+                trace!("RequestUpdates {}", v);
+                let addr = reqdata.gossip_addr;
+                // only lock for this call, dont lock durring IO `sock.send_to` or `sock.recv_from`
+                let (from, ups, data) = obj.read().unwrap().get_updates_since(v);
+                trace!("get updates since response {} {}", v, data.len());
+                let rsp = serialize(&Protocol::ReceiveUpdates(from, ups, data))?;
+                trace!("send_to {}", addr);
+                //TODO verify reqdata belongs to sender
+                obj.write().unwrap().import(&reqdata);
+                sock.send_to(&rsp, addr).unwrap();
+                trace!("send_to done!");
+            }
+            Protocol::ReceiveUpdates(from, ups, data) => {
+                trace!("ReceivedUpdates");
+                obj.write().unwrap().apply_updates(from, ups, &data);
+            }
+        }
+        Ok(())
+    }
+    pub fn listen(
+        obj: Arc<RwLock<Self>>,
+        sock: UdpSocket,
+        exit: Arc<AtomicBool>,
+    ) -> JoinHandle<()> {
+        sock.set_read_timeout(Some(Duration::new(2, 0))).unwrap();
+        spawn(move || loop {
+            let _ = Self::run_listen(&obj, &sock);
+            if exit.load(Ordering::Relaxed) {
+                return;
+            }
+        })
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crdt::{Crdt, ReplicatedData};
+    use signature::KeyPair;
+    use signature::KeyPairUtil;
+    use std::net::UdpSocket;
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::{Arc, RwLock};
+    use std::thread::{sleep, JoinHandle};
+    use std::time::Duration;
+
+    /// Test that the network converges.
+    /// Run until every node in the network has a full ReplicatedData set.
+    /// Check that nodes stop sending updates after all the ReplicatedData has been shared.
+    /// tests that actually use this function are below
+    fn run_gossip_topo<F>(topo: F)
+    where
+        F: Fn(&Vec<(Arc<RwLock<Crdt>>, JoinHandle<()>)>) -> (),
+    {
+        let num: usize = 5;
+        let exit = Arc::new(AtomicBool::new(false));
+        let listen: Vec<_> = (0..num)
+            .map(|_| {
+                let listener = UdpSocket::bind("0.0.0.0:0").unwrap();
+                let pubkey = KeyPair::new().pubkey();
+                let d = ReplicatedData::new(pubkey, listener.local_addr().unwrap());
+                let crdt = Crdt::new(d);
+                let c = Arc::new(RwLock::new(crdt));
+                let l = Crdt::listen(c.clone(), listener, exit.clone());
+                (c, l)
+            })
+            .collect();
+        topo(&listen);
+        let gossip: Vec<_> = listen
+            .iter()
+            .map(|&(ref c, _)| Crdt::gossip(c.clone(), exit.clone()))
+            .collect();
+        let mut done = true;
+        for _ in 0..(num * 16) {
+            done = true;
+            for &(ref c, _) in listen.iter() {
+                trace!(
+                    "done updates {} {}",
+                    c.read().unwrap().table.len(),
+                    c.read().unwrap().update_index
+                );
+                //make sure the number of updates doesn't grow unbounded
+                assert!(c.read().unwrap().update_index <= num as u64);
+                //make sure we got all the updates
+                if c.read().unwrap().table.len() != num {
+                    done = false;
+                }
+            }
+            if done == true {
+                break;
+            }
+            sleep(Duration::new(1, 0));
+        }
+        exit.store(true, Ordering::Relaxed);
+        for j in gossip {
+            j.join().unwrap();
+        }
+        for (c, j) in listen.into_iter() {
+            j.join().unwrap();
+            // make it clear what failed
+            // protocol is to chatty, updates should stop after everyone receives `num`
+            assert!(c.read().unwrap().update_index <= num as u64);
+            // protocol is not chatty enough, everyone should get `num` entries
+            assert_eq!(c.read().unwrap().table.len(), num);
+        }
+        assert!(done);
+    }
+    /// ring a -> b -> c -> d -> e -> a
+    #[test]
+    fn gossip_ring_test() {
+        run_gossip_topo(|listen| {
+            let num = listen.len();
+            for n in 0..num {
+                let y = n % listen.len();
+                let x = (n + 1) % listen.len();
+                let mut xv = listen[x].0.write().unwrap();
+                let yv = listen[y].0.read().unwrap();
+                let mut d = yv.table[&yv.me].clone();
+                d.version = 0;
+                xv.insert(&d);
+            }
+        });
+    }
+
+    /// star (b,c,d,e) -> a
+    #[test]
+    fn gossip_star_test() {
+        run_gossip_topo(|listen| {
+            let num = listen.len();
+            for n in 0..(num - 1) {
+                let x = 0;
+                let y = (n + 1) % listen.len();
+                let mut xv = listen[x].0.write().unwrap();
+                let yv = listen[y].0.read().unwrap();
+                let mut d = yv.table[&yv.me].clone();
+                d.version = 0;
+                xv.insert(&d);
+            }
+        });
+    }
+
+    /// Test that insert drops messages that are older
+    #[test]
+    fn insert_test() {
+        let mut d = ReplicatedData::new(KeyPair::new().pubkey(), "127.0.0.1:1234".parse().unwrap());
+        assert_eq!(d.version, 0);
+        let mut crdt = Crdt::new(d.clone());
+        assert_eq!(crdt.table[&d.id].version, 0);
+        d.version = 2;
+        crdt.insert(&d);
+        assert_eq!(crdt.table[&d.id].version, 2);
+        d.version = 1;
+        crdt.insert(&d);
+        assert_eq!(crdt.table[&d.id].version, 2);
+    }
+
+}

src/ecdsa.rs

@@ -0,0 +1,194 @@
+use packet::{Packet, SharedPackets};
+use std::mem::size_of;
+use transaction::{PUB_KEY_OFFSET, SIGNED_DATA_OFFSET, SIG_OFFSET};
+
+pub const TX_OFFSET: usize = 4;
+
+#[cfg(feature = "cuda")]
+#[repr(C)]
+struct Elems {
+    elems: *const Packet,
+    num: u32,
+}
+
+#[cfg(feature = "cuda")]
+#[link(name = "cuda_verify_ed25519")]
+extern "C" {
+    fn ed25519_verify_many(
+        vecs: *const Elems,
+        num: u32,          //number of vecs
+        message_size: u32, //size of each element inside the elems field of the vec
+        public_key_offset: u32,
+        signature_offset: u32,
+        signed_message_offset: u32,
+        signed_message_len_offset: u32,
+        out: *mut u8, //combined length of all the items in vecs
+    ) -> u32;
+}
+
+#[cfg(not(feature = "cuda"))]
+fn verify_packet(packet: &Packet) -> u8 {
+    use ring::signature;
+    use signature::{PublicKey, Signature};
+    use untrusted;
+
+    let msg_start = TX_OFFSET + SIGNED_DATA_OFFSET;
+    let sig_start = TX_OFFSET + SIG_OFFSET;
+    let sig_end = sig_start + size_of::<Signature>();
+    let pub_key_start = TX_OFFSET + PUB_KEY_OFFSET;
+    let pub_key_end = pub_key_start + size_of::<PublicKey>();
+
+    if packet.meta.size <= msg_start {
+        return 0;
+    }
+
+    let msg_end = packet.meta.size;
+    signature::verify(
+        &signature::ED25519,
+        untrusted::Input::from(&packet.data[pub_key_start..pub_key_end]),
+        untrusted::Input::from(&packet.data[msg_start..msg_end]),
+        untrusted::Input::from(&packet.data[sig_start..sig_end]),
+    ).is_ok() as u8
+}
+
+#[cfg(not(feature = "cuda"))]
+pub fn ed25519_verify(batches: &Vec<SharedPackets>) -> Vec<Vec<u8>> {
+    use rayon::prelude::*;
+
+    batches
+        .into_par_iter()
+        .map(|p| {
+            p.read()
+                .unwrap()
+                .packets
+                .par_iter()
+                .map(verify_packet)
+                .collect()
+        })
+        .collect()
+}
+
+#[cfg(feature = "cuda")]
+pub fn ed25519_verify(batches: &Vec<SharedPackets>) -> Vec<Vec<u8>> {
+    use packet::PACKET_DATA_SIZE;
+
+    let mut out = Vec::new();
+    let mut elems = Vec::new();
+    let mut locks = Vec::new();
+    let mut rvs = Vec::new();
+
+    for packets in batches {
+        locks.push(packets.read().unwrap());
+    }
+    let mut num = 0;
+    for p in locks {
+        elems.push(Elems {
+            elems: p.packets.as_ptr(),
+            num: p.packets.len() as u32,
+        });
+        let mut v = Vec::new();
+        v.resize(p.packets.len(), 0);
+        rvs.push(v);
+        num += p.packets.len();
+    }
+    out.resize(num, 0);
+    trace!("Starting verify num packets: {}", num);
+    trace!("elem len: {}", elems.len() as u32);
+    trace!("packet sizeof: {}", size_of::<Packet>() as u32);
+    trace!("pub key: {}", (TX_OFFSET + PUB_KEY_OFFSET) as u32);
+    trace!("sig offset: {}", (TX_OFFSET + SIG_OFFSET) as u32);
+    trace!("sign data: {}", (TX_OFFSET + SIGNED_DATA_OFFSET) as u32);
+    trace!("len offset: {}", PACKET_DATA_SIZE as u32);
+    unsafe {
+        let res = ed25519_verify_many(
+            elems.as_ptr(),
+            elems.len() as u32,
+            size_of::<Packet>() as u32,
+            (TX_OFFSET + PUB_KEY_OFFSET) as u32,
+            (TX_OFFSET + SIG_OFFSET) as u32,
+            (TX_OFFSET + SIGNED_DATA_OFFSET) as u32,
+            PACKET_DATA_SIZE as u32,
+            out.as_mut_ptr(),
+        );
+        if res != 0 {
+            trace!("RETURN!!!: {}", res);
+        }
+    }
+    trace!("done verify");
+    let mut num = 0;
+    for vs in rvs.iter_mut() {
+        for mut v in vs.iter_mut() {
+            *v = out[num];
+            if *v != 0 {
+                trace!("VERIFIED PACKET!!!!!");
+            }
+            num += 1;
+        }
+    }
+    rvs
+}
+
+#[cfg(test)]
+mod tests {
+    use accountant_skel::Request;
+    use bincode::serialize;
+    use ecdsa;
+    use packet::{Packet, Packets, SharedPackets};
+    use std::sync::RwLock;
+    use transaction::test_tx;
+    use transaction::Transaction;
+
+    fn make_packet_from_transaction(tr: Transaction) -> Packet {
+        let tx = serialize(&Request::Transaction(tr)).unwrap();
+        let mut packet = Packet::default();
+        packet.meta.size = tx.len();
+        packet.data[..packet.meta.size].copy_from_slice(&tx);
+        return packet;
+    }
+
+    fn test_verify_n(n: usize, modify_data: bool) {
+        let tr = test_tx();
+        let mut packet = make_packet_from_transaction(tr);
+
+        // jumble some data to test failure
+        if modify_data {
+            packet.data[20] = 10;
+        }
+
+        // generate packet vector
+        let mut packets = Packets::default();
+        packets.packets = Vec::new();
+        for _ in 0..n {
+            packets.packets.push(packet.clone());
+        }
+        let shared_packets = SharedPackets::new(RwLock::new(packets));
+        let batches = vec![shared_packets.clone(), shared_packets.clone()];
+
+        // verify packets
+        let ans = ecdsa::ed25519_verify(&batches);
+
+        // check result
+        let ref_ans = if modify_data { 0u8 } else { 1u8 };
+        assert_eq!(ans, vec![vec![ref_ans; n], vec![ref_ans; n]]);
+    }
+
+    #[test]
+    fn test_verify_zero() {
+        test_verify_n(0, false);
+    }
+
+    #[test]
+    fn test_verify_one() {
+        test_verify_n(1, false);
+    }
+
+    #[test]
+    fn test_verify_seventy_one() {
+        test_verify_n(71, false);
+    }
+
+    #[test]
+    fn test_verify_fail() {
+        test_verify_n(5, true);
+    }
+}

src/entry.rs

@@ -0,0 +1,176 @@
+//! The `entry` module is a fundamental building block of Proof of History. It contains a
+//! unique ID that is the hash of the Entry before it, plus the hash of the
+//! transactions within it. Entries cannot be reordered, and its field `num_hashes`
+//! represents an approximate amount of time since the last Entry was created.
+use event::Event;
+use hash::{extend_and_hash, hash, Hash};
+use rayon::prelude::*;
+
+/// Each Entry contains three pieces of data. The `num_hashes` field is the number
+/// of hashes performed since the previous entry.  The `id` field is the result
+/// of hashing `id` from the previous entry `num_hashes` times.  The `events`
+/// field points to Events that took place shortly after `id` was generated.
+///
+/// If you divide `num_hashes` by the amount of time it takes to generate a new hash, you
+/// get a duration estimate since the last Entry. Since processing power increases
+/// over time, one should expect the duration `num_hashes` represents to decrease proportionally.
+/// Though processing power varies across nodes, the network gives priority to the
+/// fastest processor. Duration should therefore be estimated by assuming that the hash
+/// was generated by the fastest processor at the time the entry was recorded.
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub struct Entry {
+    pub num_hashes: u64,
+    pub id: Hash,
+    pub events: Vec<Event>,
+}
+
+impl Entry {
+    /// Creates a Entry from the number of hashes `num_hashes` since the previous event
+    /// and that resulting `id`.
+    pub fn new_tick(num_hashes: u64, id: &Hash) -> Self {
+        Entry {
+            num_hashes,
+            id: *id,
+            events: vec![],
+        }
+    }
+
+    /// Verifies self.id is the result of hashing a `start_hash` `self.num_hashes` times.
+    /// If the event is not a Tick, then hash that as well.
+    pub fn verify(&self, start_hash: &Hash) -> bool {
+        self.events.par_iter().all(|event| event.verify())
+            && self.id == next_hash(start_hash, self.num_hashes, &self.events)
+    }
+}
+
+fn add_event_data(hash_data: &mut Vec<u8>, event: &Event) {
+    match *event {
+        Event::Transaction(ref tr) => {
+            hash_data.push(0u8);
+            hash_data.extend_from_slice(&tr.sig);
+        }
+        Event::Signature { ref sig, .. } => {
+            hash_data.push(1u8);
+            hash_data.extend_from_slice(sig);
+        }
+        Event::Timestamp { ref sig, .. } => {
+            hash_data.push(2u8);
+            hash_data.extend_from_slice(sig);
+        }
+    }
+}
+
+/// Creates the hash `num_hashes` after `start_hash`. If the event contains
+/// a signature, the final hash will be a hash of both the previous ID and
+/// the signature.
+pub fn next_hash(start_hash: &Hash, num_hashes: u64, events: &[Event]) -> Hash {
+    let mut id = *start_hash;
+    for _ in 1..num_hashes {
+        id = hash(&id);
+    }
+
+    // Hash all the event data
+    let mut hash_data = vec![];
+    for event in events {
+        add_event_data(&mut hash_data, event);
+    }
+
+    if !hash_data.is_empty() {
+        extend_and_hash(&id, &hash_data)
+    } else if num_hashes != 0 {
+        hash(&id)
+    } else {
+        id
+    }
+}
+
+/// Creates the next Entry `num_hashes` after `start_hash`.
+pub fn create_entry(start_hash: &Hash, cur_hashes: u64, events: Vec<Event>) -> Entry {
+    let num_hashes = cur_hashes + if events.is_empty() { 0 } else { 1 };
+    let id = next_hash(start_hash, 0, &events);
+    Entry {
+        num_hashes,
+        id,
+        events,
+    }
+}
+
+/// Creates the next Tick Entry `num_hashes` after `start_hash`.
+pub fn create_entry_mut(start_hash: &mut Hash, cur_hashes: &mut u64, events: Vec<Event>) -> Entry {
+    let entry = create_entry(start_hash, *cur_hashes, events);
+    *start_hash = entry.id;
+    *cur_hashes = 0;
+    entry
+}
+
+/// Creates the next Tick Entry `num_hashes` after `start_hash`.
+pub fn next_tick(start_hash: &Hash, num_hashes: u64) -> Entry {
+    Entry {
+        num_hashes,
+        id: next_hash(start_hash, num_hashes, &[]),
+        events: vec![],
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use chrono::prelude::*;
+    use entry::create_entry;
+    use event::Event;
+    use hash::hash;
+    use signature::{KeyPair, KeyPairUtil};
+    use transaction::Transaction;
+
+    #[test]
+    fn test_entry_verify() {
+        let zero = Hash::default();
+        let one = hash(&zero);
+        assert!(Entry::new_tick(0, &zero).verify(&zero)); // base case
+        assert!(!Entry::new_tick(0, &zero).verify(&one)); // base case, bad
+        assert!(next_tick(&zero, 1).verify(&zero)); // inductive step
+        assert!(!next_tick(&zero, 1).verify(&one)); // inductive step, bad
+    }
+
+    #[test]
+    fn test_event_reorder_attack() {
+        let zero = Hash::default();
+
+        // First, verify entries
+        let keypair = KeyPair::new();
+        let tr0 = Event::Transaction(Transaction::new(&keypair, keypair.pubkey(), 0, zero));
+        let tr1 = Event::Transaction(Transaction::new(&keypair, keypair.pubkey(), 1, zero));
+        let mut e0 = create_entry(&zero, 0, vec![tr0.clone(), tr1.clone()]);
+        assert!(e0.verify(&zero));
+
+        // Next, swap two events and ensure verification fails.
+        e0.events[0] = tr1; // <-- attack
+        e0.events[1] = tr0;
+        assert!(!e0.verify(&zero));
+    }
+
+    #[test]
+    fn test_witness_reorder_attack() {
+        let zero = Hash::default();
+
+        // First, verify entries
+        let keypair = KeyPair::new();
+        let tr0 = Event::new_timestamp(&keypair, Utc::now());
+        let tr1 = Event::new_signature(&keypair, Default::default());
+        let mut e0 = create_entry(&zero, 0, vec![tr0.clone(), tr1.clone()]);
+        assert!(e0.verify(&zero));
+
+        // Next, swap two witness events and ensure verification fails.
+        e0.events[0] = tr1; // <-- attack
+        e0.events[1] = tr0;
+        assert!(!e0.verify(&zero));
+    }
+
+    #[test]
+    fn test_next_tick() {
+        let zero = Hash::default();
+        let tick = next_tick(&zero, 1);
+        assert_eq!(tick.num_hashes, 1);
+        assert_ne!(tick.id, zero);
+    }
+}

src/erasure.rs

@@ -0,0 +1,420 @@
+// Support erasure coding
+
+use packet::{BlobRecycler, SharedBlob};
+use std::result;
+
+//TODO(sakridge) pick these values
+const NUM_CODED: usize = 10;
+const MAX_MISSING: usize = 2;
+const NUM_DATA: usize = NUM_CODED - MAX_MISSING;
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum ErasureError {
+    NotEnoughBlocksToDecode,
+    DecodeError,
+    InvalidBlockSize,
+}
+
+pub type Result<T> = result::Result<T, ErasureError>;
+
+// k = number of data devices
+// m = number of coding devices
+// w = word size
+
+extern "C" {
+    fn jerasure_matrix_encode(
+        k: i32,
+        m: i32,
+        w: i32,
+        matrix: *const i32,
+        data_ptrs: *const *const u8,
+        coding_ptrs: *const *mut u8,
+        size: i32,
+    );
+    fn jerasure_matrix_decode(
+        k: i32,
+        m: i32,
+        w: i32,
+        matrix: *const i32,
+        row_k_ones: i32,
+        erasures: *const i32,
+        data_ptrs: *const *mut u8,
+        coding_ptrs: *const *const u8,
+        size: i32,
+    ) -> i32;
+    fn galois_single_divide(a: i32, b: i32, w: i32) -> i32;
+}
+
+fn get_matrix(m: i32, k: i32, w: i32) -> Vec<i32> {
+    let mut matrix = vec![0; (m * k) as usize];
+    for i in 0..m {
+        for j in 0..k {
+            unsafe {
+                matrix[(i * k + j) as usize] = galois_single_divide(1, i ^ (m + j), w);
+            }
+        }
+    }
+    matrix
+}
+
+pub const ERASURE_W: i32 = 32;
+
+// Generate coding blocks into coding
+//   There are some alignment restrictions, blocks should be aligned by 16 bytes
+//   which means their size should be >= 16 bytes
+pub fn generate_coding_blocks(coding: &mut [&mut [u8]], data: &[&[u8]]) -> Result<()> {
+    if data.len() == 0 {
+        return Ok(());
+    }
+    let m = coding.len() as i32;
+    let block_len = data[0].len();
+    let matrix: Vec<i32> = get_matrix(m, data.len() as i32, ERASURE_W);
+    let mut coding_arg = Vec::new();
+    let mut data_arg = Vec::new();
+    for block in data {
+        if block_len != block.len() {
+            return Err(ErasureError::InvalidBlockSize);
+        }
+        data_arg.push(block.as_ptr());
+    }
+    for mut block in coding {
+        if block_len != block.len() {
+            return Err(ErasureError::InvalidBlockSize);
+        }
+        coding_arg.push(block.as_mut_ptr());
+    }
+
+    unsafe {
+        jerasure_matrix_encode(
+            data.len() as i32,
+            m,
+            ERASURE_W,
+            matrix.as_ptr(),
+            data_arg.as_ptr(),
+            coding_arg.as_ptr(),
+            data[0].len() as i32,
+        );
+    }
+    Ok(())
+}
+
+// Recover data + coding blocks into data blocks
+//   data: array of blocks to recover into
+//   coding: arry of coding blocks
+//   erasures: list of indices in data where blocks should be recovered
+pub fn decode_blocks(data: &mut [&mut [u8]], coding: &[&[u8]], erasures: &[i32]) -> Result<()> {
+    if data.len() == 0 {
+        return Ok(());
+    }
+    let block_len = data[0].len();
+    let matrix: Vec<i32> = get_matrix(coding.len() as i32, data.len() as i32, ERASURE_W);
+
+    // generate coding pointers, blocks should be the same size
+    let mut coding_arg: Vec<*const u8> = Vec::new();
+    for x in coding.iter() {
+        if x.len() != block_len {
+            return Err(ErasureError::InvalidBlockSize);
+        }
+        coding_arg.push(x.as_ptr());
+    }
+
+    // generate data pointers, blocks should be the same size
+    let mut data_arg: Vec<*mut u8> = Vec::new();
+    for x in data.iter_mut() {
+        if x.len() != block_len {
+            return Err(ErasureError::InvalidBlockSize);
+        }
+        data_arg.push(x.as_mut_ptr());
+    }
+    unsafe {
+        let ret = jerasure_matrix_decode(
+            data.len() as i32,
+            coding.len() as i32,
+            ERASURE_W,
+            matrix.as_ptr(),
+            0,
+            erasures.as_ptr(),
+            data_arg.as_ptr(),
+            coding_arg.as_ptr(),
+            data[0].len() as i32,
+        );
+        trace!("jerasure_matrix_decode ret: {}", ret);
+        for x in data[erasures[0] as usize][0..8].iter() {
+            trace!("{} ", x)
+        }
+        trace!("");
+        if ret < 0 {
+            return Err(ErasureError::DecodeError);
+        }
+    }
+    Ok(())
+}
+
+// Generate coding blocks in window from consumed to consumed+NUM_DATA
+pub fn generate_coding(
+    re: &BlobRecycler,
+    window: &mut Vec<Option<SharedBlob>>,
+    consumed: usize,
+) -> Result<()> {
+    let mut data_blobs = Vec::new();
+    let mut coding_blobs = Vec::new();
+    let mut data_locks = Vec::new();
+    let mut data_ptrs: Vec<&[u8]> = Vec::new();
+    let mut coding_locks = Vec::new();
+    let mut coding_ptrs: Vec<&mut [u8]> = Vec::new();
+    for i in consumed..consumed + NUM_DATA {
+        let n = i % window.len();
+        data_blobs.push(window[n].clone().unwrap());
+    }
+    for b in &data_blobs {
+        data_locks.push(b.write().unwrap());
+    }
+    for (i, l) in data_locks.iter_mut().enumerate() {
+        trace!("i: {} data: {}", i, l.data[0]);
+        data_ptrs.push(&l.data);
+    }
+
+    // generate coding ptr array
+    let coding_start = consumed + NUM_DATA;
+    let coding_end = consumed + NUM_CODED;
+    for i in coding_start..coding_end {
+        let n = i % window.len();
+        window[n] = Some(re.allocate());
+        coding_blobs.push(window[n].clone().unwrap());
+    }
+    for b in &coding_blobs {
+        coding_locks.push(b.write().unwrap());
+    }
+    for (i, l) in coding_locks.iter_mut().enumerate() {
+        trace!("i: {} data: {}", i, l.data[0]);
+        coding_ptrs.push(&mut l.data);
+    }
+
+    generate_coding_blocks(coding_ptrs.as_mut_slice(), &data_ptrs)?;
+    trace!("consumed: {}", consumed);
+    Ok(())
+}
+
+// Recover missing blocks into window
+//   missing blocks should be None, will use re
+//   to allocate new ones. Returns err if not enough
+//   coding blocks are present to restore
+pub fn recover(
+    re: &BlobRecycler,
+    window: &mut Vec<Option<SharedBlob>>,
+    consumed: usize,
+) -> Result<()> {
+    //recover with erasure coding
+    let mut data_missing = 0;
+    let mut coded_missing = 0;
+    let coding_start = consumed + NUM_DATA;
+    let coding_end = consumed + NUM_CODED;
+    for i in consumed..coding_end {
+        let n = i % window.len();
+        if window[n].is_none() {
+            if i >= coding_start {
+                coded_missing += 1;
+            } else {
+                data_missing += 1;
+            }
+        }
+    }
+    trace!("missing: data: {} coding: {}", data_missing, coded_missing);
+    if data_missing > 0 {
+        if (data_missing + coded_missing) <= MAX_MISSING {
+            let mut blobs: Vec<SharedBlob> = Vec::new();
+            let mut locks = Vec::new();
+            let mut data_ptrs: Vec<&mut [u8]> = Vec::new();
+            let mut coding_ptrs: Vec<&[u8]> = Vec::new();
+            let mut erasures: Vec<i32> = Vec::new();
+            for i in consumed..coding_end {
+                let j = i % window.len();
+                let mut b = &mut window[j];
+                if b.is_some() {
+                    blobs.push(b.clone().unwrap());
+                    continue;
+                }
+                let n = re.allocate();
+                *b = Some(n.clone());
+                //mark the missing memory
+                blobs.push(n);
+                erasures.push((i - consumed) as i32);
+            }
+            erasures.push(-1);
+            trace!("erasures: {:?}", erasures);
+            //lock everything
+            for b in &blobs {
+                locks.push(b.write().unwrap());
+            }
+            for (i, l) in locks.iter_mut().enumerate() {
+                if i >= NUM_DATA {
+                    trace!("pushing coding: {}", i);
+                    coding_ptrs.push(&l.data);
+                } else {
+                    trace!("pushing data: {}", i);
+                    data_ptrs.push(&mut l.data);
+                }
+            }
+            trace!(
+                "coding_ptrs.len: {} data_ptrs.len {}",
+                coding_ptrs.len(),
+                data_ptrs.len()
+            );
+            decode_blocks(data_ptrs.as_mut_slice(), &coding_ptrs, &erasures)?;
+        } else {
+            return Err(ErasureError::NotEnoughBlocksToDecode);
+        }
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod test {
+    use erasure;
+    use packet::{BlobRecycler, SharedBlob, PACKET_DATA_SIZE};
+    extern crate env_logger;
+
+    #[test]
+    pub fn test_coding() {
+        let zero_vec = vec![0; 16];
+        let mut vs: Vec<Vec<u8>> = (0..4).map(|i| (i..(16 + i)).collect()).collect();
+        let v_orig: Vec<u8> = vs[0].clone();
+
+        let m = 2;
+        let mut coding_blocks: Vec<_> = (0..m).map(|_| vec![0u8; 16]).collect();
+
+        {
+            let mut coding_blocks_slices: Vec<_> =
+                coding_blocks.iter_mut().map(|x| x.as_mut_slice()).collect();
+            let v_slices: Vec<_> = vs.iter().map(|x| x.as_slice()).collect();
+
+            assert!(
+                erasure::generate_coding_blocks(
+                    coding_blocks_slices.as_mut_slice(),
+                    v_slices.as_slice()
+                ).is_ok()
+            );
+        }
+        trace!("coding blocks:");
+        for b in &coding_blocks {
+            trace!("{:?}", b);
+        }
+        let erasure: i32 = 1;
+        let erasures = vec![erasure, -1];
+        // clear an entry
+        vs[erasure as usize].copy_from_slice(zero_vec.as_slice());
+
+        {
+            let coding_blocks_slices: Vec<_> = coding_blocks.iter().map(|x| x.as_slice()).collect();
+            let mut v_slices: Vec<_> = vs.iter_mut().map(|x| x.as_mut_slice()).collect();
+
+            assert!(
+                erasure::decode_blocks(
+                    v_slices.as_mut_slice(),
+                    coding_blocks_slices.as_slice(),
+                    erasures.as_slice(),
+                ).is_ok()
+            );
+        }
+
+        trace!("vs:");
+        for v in &vs {
+            trace!("{:?}", v);
+        }
+        assert_eq!(v_orig, vs[0]);
+    }
+
+    fn print_window(window: &Vec<Option<SharedBlob>>) {
+        for (i, w) in window.iter().enumerate() {
+            print!("window({}): ", i);
+            if w.is_some() {
+                let window_lock = w.clone().unwrap();
+                let window_data = window_lock.read().unwrap().data;
+                for i in 0..8 {
+                    print!("{} ", window_data[i]);
+                }
+            } else {
+                print!("null");
+            }
+            println!("");
+        }
+    }
+
+    #[test]
+    pub fn test_window_recover() {
+        let mut window = Vec::new();
+        let blob_recycler = BlobRecycler::default();
+        let offset = 4;
+        for i in 0..(4 * erasure::NUM_CODED + 1) {
+            let b = blob_recycler.allocate();
+            let b_ = b.clone();
+            let data_len = b.read().unwrap().data.len();
+            let mut w = b.write().unwrap();
+            w.set_index(i as u64).unwrap();
+            assert_eq!(i as u64, w.get_index().unwrap());
+            w.meta.size = PACKET_DATA_SIZE;
+            for k in 0..data_len {
+                w.data[k] = (k + i) as u8;
+            }
+            window.push(Some(b_));
+        }
+        println!("** after-gen:");
+        print_window(&window);
+        assert!(erasure::generate_coding(&blob_recycler, &mut window, offset).is_ok());
+        assert!(
+            erasure::generate_coding(&blob_recycler, &mut window, offset + erasure::NUM_CODED)
+                .is_ok()
+        );
+        assert!(
+            erasure::generate_coding(
+                &blob_recycler,
+                &mut window,
+                offset + (2 * erasure::NUM_CODED)
+            ).is_ok()
+        );
+        assert!(
+            erasure::generate_coding(
+                &blob_recycler,
+                &mut window,
+                offset + (3 * erasure::NUM_CODED)
+            ).is_ok()
+        );
+        println!("** after-coding:");
+        print_window(&window);
+        let refwindow = window[offset + 1].clone();
+        window[offset + 1] = None;
+        window[offset + 2] = None;
+        window[offset + erasure::NUM_CODED + 3] = None;
+        window[offset + (2 * erasure::NUM_CODED) + 0] = None;
+        window[offset + (2 * erasure::NUM_CODED) + 1] = None;
+        window[offset + (2 * erasure::NUM_CODED) + 2] = None;
+        let window_l0 = &(window[offset + (3 * erasure::NUM_CODED)]).clone().unwrap();
+        window_l0.write().unwrap().data[0] = 55;
+        println!("** after-nulling:");
+        print_window(&window);
+        assert!(erasure::recover(&blob_recycler, &mut window, offset).is_ok());
+        assert!(erasure::recover(&blob_recycler, &mut window, offset + erasure::NUM_CODED).is_ok());
+        assert!(
+            erasure::recover(
+                &blob_recycler,
+                &mut window,
+                offset + (2 * erasure::NUM_CODED)
+            ).is_err()
+        );
+        assert!(
+            erasure::recover(
+                &blob_recycler,
+                &mut window,
+                offset + (3 * erasure::NUM_CODED)
+            ).is_ok()
+        );
+        println!("** after-restore:");
+        print_window(&window);
+        let window_l = window[offset + 1].clone().unwrap();
+        let ref_l = refwindow.clone().unwrap();
+        assert_eq!(
+            window_l.read().unwrap().data.to_vec(),
+            ref_l.read().unwrap().data.to_vec()
+        );
+    }
+}

src/event.rs

@@ -1,151 +1,67 @@
-//! The `event` crate provides the foundational data structures for Proof-of-History
+//! The `event` module handles events, which may be a `Transaction`, or a `Witness` used to process a pending
+//! Transaction.
 
-/// A Proof-of-History is an ordered log of events in time. Each entry contains three
-/// pieces of data. The 'num_hashes' field is the number of hashes performed since the previous
-/// entry.  The 'end_hash' field is the result of hashing 'end_hash' from the previous entry
-/// 'num_hashes' times.  The 'data' field is an optional foreign key (a hash) pointing to some
-/// arbitrary data that a client is looking to associate with the entry.
-///
-/// If you divide 'num_hashes' by the amount of time it takes to generate a new hash, you
-/// get a duration estimate since the last event. Since processing power increases
-/// over time, one should expect the duration 'num_hashes' represents to decrease proportionally.
-/// Though processing power varies across nodes, the network gives priority to the
-/// fastest processor. Duration should therefore be estimated by assuming that the hash
-/// was generated by the fastest processor at the time the entry was logged.
-pub struct Event {
-    pub num_hashes: u64,
-    pub end_hash: u64,
-    pub data: EventData,
-}
+use bincode::serialize;
+use chrono::prelude::*;
+use signature::{KeyPair, KeyPairUtil, PublicKey, Signature, SignatureUtil};
+use transaction::Transaction;
 
-/// When 'data' is Tick, the event represents a simple clock tick, and exists for the
-/// sole purpose of improving the performance of event log verification. A tick can
-/// be generated in 'num_hashes' hashes and verified in 'num_hashes' hashes.  By logging
-/// a hash alongside the tick, each tick and be verified in parallel using the 'end_hash'
-/// of the preceding tick to seed its hashing.
-pub enum EventData {
-    Tick,
-    UserDataKey(u64),
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub enum Event {
+    Transaction(Transaction),
+    Signature {
+        from: PublicKey,
+        tx_sig: Signature,
+        sig: Signature,
+    },
+    Timestamp {
+        from: PublicKey,
+        dt: DateTime<Utc>,
+        sig: Signature,
+    },
 }
 
 impl Event {
-    /// Creates an Event from the number of hashes 'num_hashes' since the previous event
-    /// and that resulting 'end_hash'.
-    pub fn new_tick(num_hashes: u64, end_hash: u64) -> Self {
-        let data = EventData::Tick;
-        Event {
-            num_hashes,
-            end_hash,
-            data,
+    /// Create and sign a new Witness Timestamp. Used for unit-testing.
+    pub fn new_timestamp(from: &KeyPair, dt: DateTime<Utc>) -> Self {
+        let sign_data = serialize(&dt).unwrap();
+        let sig = Signature::clone_from_slice(from.sign(&sign_data).as_ref());
+        Event::Timestamp {
+            from: from.pubkey(),
+            dt,
+            sig,
         }
     }
 
-    /// Verifies self.end_hash is the result of hashing a 'start_hash' 'self.num_hashes' times.
-    pub fn verify(self: &Self, start_hash: u64) -> bool {
-        self.end_hash == next_tick(start_hash, self.num_hashes).end_hash
+    /// Create and sign a new Witness Signature. Used for unit-testing.
+    pub fn new_signature(from: &KeyPair, tx_sig: Signature) -> Self {
+        let sig = Signature::clone_from_slice(from.sign(&tx_sig).as_ref());
+        Event::Signature {
+            from: from.pubkey(),
+            tx_sig,
+            sig,
+        }
     }
-}
 
-/// Creates the next Tick Event 'num_hashes' after 'start_hash'.
-pub fn next_tick(start_hash: u64, num_hashes: u64) -> Event {
-    use std::collections::hash_map::DefaultHasher;
-    use std::hash::{Hash, Hasher};
-    let mut end_hash = start_hash;
-    let mut hasher = DefaultHasher::new();
-    for _ in 0..num_hashes {
-        end_hash.hash(&mut hasher);
-        end_hash = hasher.finish();
+    /// Verify the Event's signature's are valid and if a transaction, that its
+    /// spending plan is valid.
+    pub fn verify(&self) -> bool {
+        match *self {
+            Event::Transaction(ref tr) => tr.verify_sig(),
+            Event::Signature { from, tx_sig, sig } => sig.verify(&from, &tx_sig),
+            Event::Timestamp { from, dt, sig } => sig.verify(&from, &serialize(&dt).unwrap()),
+        }
     }
-    Event::new_tick(num_hashes, end_hash)
-}
-
-/// Verifies the hashes and counts of a slice of events are all consistent.
-pub fn verify_slice(events: &[Event], start_hash: u64) -> bool {
-    use rayon::prelude::*;
-    let genesis = [Event::new_tick(0, start_hash)];
-    let event_pairs = genesis.par_iter().chain(events).zip(events);
-    event_pairs.all(|(x0, x1)| x1.verify(x0.end_hash))
-}
-
-/// Verifies the hashes and events serially. Exists only for reference.
-pub fn verify_slice_seq(events: &[Event], start_hash: u64) -> bool {
-    let genesis = [Event::new_tick(0, start_hash)];
-    let mut event_pairs = genesis.iter().chain(events).zip(events);
-    event_pairs.all(|(x0, x1)| x1.verify(x0.end_hash))
-}
-
-/// Create a vector of Ticks of length 'len' from 'start_hash' hash and 'num_hashes'.
-pub fn create_ticks(start_hash: u64, num_hashes: u64, len: usize) -> Vec<Event> {
-    use itertools::unfold;
-    let mut events = unfold(start_hash, |state| {
-        let event = next_tick(*state, num_hashes);
-        *state = event.end_hash;
-        return Some(event);
-    });
-    events.by_ref().take(len).collect()
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use signature::{KeyPair, KeyPairUtil};
 
     #[test]
     fn test_event_verify() {
-        assert!(Event::new_tick(0, 0).verify(0)); // base case
-        assert!(!Event::new_tick(0, 0).verify(1)); // base case, bad
-        assert!(next_tick(0, 1).verify(0)); // inductive step
-        assert!(!next_tick(0, 1).verify(1)); // inductive step, bad
-    }
-
-    #[test]
-    fn test_next_tick() {
-        assert_eq!(next_tick(0, 1).num_hashes, 1)
-    }
-
-    fn verify_slice_generic(verify_slice: fn(&[Event], u64) -> bool) {
-        assert!(verify_slice(&vec![], 0)); // base case
-        assert!(verify_slice(&vec![Event::new_tick(0, 0)], 0)); // singleton case 1
-        assert!(!verify_slice(&vec![Event::new_tick(0, 0)], 1)); // singleton case 2, bad
-        assert!(verify_slice(&create_ticks(0, 0, 2), 0)); // inductive step
-
-        let mut bad_ticks = create_ticks(0, 0, 2);
-        bad_ticks[1].end_hash = 1;
-        assert!(!verify_slice(&bad_ticks, 0)); // inductive step, bad
-    }
-
-    #[test]
-    fn test_verify_slice() {
-        verify_slice_generic(verify_slice);
-    }
-
-    #[test]
-    fn test_verify_slice_seq() {
-        verify_slice_generic(verify_slice_seq);
-    }
-
-}
-
-#[cfg(all(feature = "unstable", test))]
-mod bench {
-    extern crate test;
-    use self::test::Bencher;
-    use event;
-
-    #[bench]
-    fn event_bench(bencher: &mut Bencher) {
-        let start_hash = 0;
-        let events = event::create_ticks(start_hash, 100_000, 8);
-        bencher.iter(|| {
-            assert!(event::verify_slice(&events, start_hash));
-        });
-    }
-
-    #[bench]
-    fn event_bench_seq(bencher: &mut Bencher) {
-        let start_hash = 0;
-        let events = event::create_ticks(start_hash, 100_000, 8);
-        bencher.iter(|| {
-            assert!(event::verify_slice_seq(&events, start_hash));
-        });
+        assert!(Event::new_timestamp(&KeyPair::new(), Utc::now()).verify());
+        assert!(Event::new_signature(&KeyPair::new(), Signature::default()).verify());
     }
 }

src/hash.rs

@@ -0,0 +1,21 @@
+//! The `hash` module provides functions for creating SHA-256 hashes.
+
+use generic_array::typenum::U32;
+use generic_array::GenericArray;
+use sha2::{Digest, Sha256};
+
+pub type Hash = GenericArray<u8, U32>;
+
+/// Return a Sha256 hash for the given data.
+pub fn hash(val: &[u8]) -> Hash {
+    let mut hasher = Sha256::default();
+    hasher.input(val);
+    hasher.result()
+}
+
+/// Return the hash of the given hash extended with the given value.
+pub fn extend_and_hash(id: &Hash, val: &[u8]) -> Hash {
+    let mut hash_data = id.to_vec();
+    hash_data.extend_from_slice(val);
+    hash(&hash_data)
+}

src/historian.rs

@@ -0,0 +1,113 @@
+//! The `historian` module provides a microservice for generating a Proof of History.
+//! It manages a thread containing a Proof of History Recorder.
+
+use entry::Entry;
+use hash::Hash;
+use recorder::{ExitReason, Recorder, Signal};
+use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
+use std::thread::{spawn, JoinHandle};
+use std::time::Instant;
+
+pub struct Historian {
+    pub sender: SyncSender<Signal>,
+    pub receiver: Receiver<Entry>,
+    pub thread_hdl: JoinHandle<ExitReason>,
+}
+
+impl Historian {
+    pub fn new(start_hash: &Hash, ms_per_tick: Option<u64>) -> Self {
+        let (sender, event_receiver) = sync_channel(10_000);
+        let (entry_sender, receiver) = sync_channel(10_000);
+        let thread_hdl =
+            Historian::create_recorder(*start_hash, ms_per_tick, event_receiver, entry_sender);
+        Historian {
+            sender,
+            receiver,
+            thread_hdl,
+        }
+    }
+
+    /// A background thread that will continue tagging received Event messages and
+    /// sending back Entry messages until either the receiver or sender channel is closed.
+    fn create_recorder(
+        start_hash: Hash,
+        ms_per_tick: Option<u64>,
+        receiver: Receiver<Signal>,
+        sender: SyncSender<Entry>,
+    ) -> JoinHandle<ExitReason> {
+        spawn(move || {
+            let mut recorder = Recorder::new(receiver, sender, start_hash);
+            let now = Instant::now();
+            loop {
+                if let Err(err) = recorder.process_events(now, ms_per_tick) {
+                    return err;
+                }
+                if ms_per_tick.is_some() {
+                    recorder.hash();
+                }
+            }
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use ledger::Block;
+    use std::thread::sleep;
+    use std::time::Duration;
+
+    #[test]
+    fn test_historian() {
+        let zero = Hash::default();
+        let hist = Historian::new(&zero, None);
+
+        hist.sender.send(Signal::Tick).unwrap();
+        sleep(Duration::new(0, 1_000_000));
+        hist.sender.send(Signal::Tick).unwrap();
+        sleep(Duration::new(0, 1_000_000));
+        hist.sender.send(Signal::Tick).unwrap();
+
+        let entry0 = hist.receiver.recv().unwrap();
+        let entry1 = hist.receiver.recv().unwrap();
+        let entry2 = hist.receiver.recv().unwrap();
+
+        assert_eq!(entry0.num_hashes, 0);
+        assert_eq!(entry1.num_hashes, 0);
+        assert_eq!(entry2.num_hashes, 0);
+
+        drop(hist.sender);
+        assert_eq!(
+            hist.thread_hdl.join().unwrap(),
+            ExitReason::RecvDisconnected
+        );
+
+        assert!([entry0, entry1, entry2].verify(&zero));
+    }
+
+    #[test]
+    fn test_historian_closed_sender() {
+        let zero = Hash::default();
+        let hist = Historian::new(&zero, None);
+        drop(hist.receiver);
+        hist.sender.send(Signal::Tick).unwrap();
+        assert_eq!(
+            hist.thread_hdl.join().unwrap(),
+            ExitReason::SendDisconnected
+        );
+    }
+
+    #[test]
+    fn test_ticking_historian() {
+        let zero = Hash::default();
+        let hist = Historian::new(&zero, Some(20));
+        sleep(Duration::from_millis(300));
+        hist.sender.send(Signal::Tick).unwrap();
+        drop(hist.sender);
+        let entries: Vec<Entry> = hist.receiver.iter().collect();
+        assert!(entries.len() > 1);
+
+        // Ensure the ID is not the seed.
+        assert_ne!(entries[0].id, zero);
+    }
+}

src/ledger.rs

@@ -0,0 +1,67 @@
+//! The `ledger` module provides functions for parallel verification of the
+//! Proof of History ledger.
+
+use entry::{next_tick, Entry};
+use hash::Hash;
+use rayon::prelude::*;
+
+pub trait Block {
+    /// Verifies the hashes and counts of a slice of events are all consistent.
+    fn verify(&self, start_hash: &Hash) -> bool;
+}
+
+impl Block for [Entry] {
+    fn verify(&self, start_hash: &Hash) -> bool {
+        let genesis = [Entry::new_tick(0, start_hash)];
+        let entry_pairs = genesis.par_iter().chain(self).zip(self);
+        entry_pairs.all(|(x0, x1)| x1.verify(&x0.id))
+    }
+}
+
+/// Create a vector of Ticks of length `len` from `start_hash` hash and `num_hashes`.
+pub fn next_ticks(start_hash: &Hash, num_hashes: u64, len: usize) -> Vec<Entry> {
+    let mut id = *start_hash;
+    let mut ticks = vec![];
+    for _ in 0..len {
+        let entry = next_tick(&id, num_hashes);
+        id = entry.id;
+        ticks.push(entry);
+    }
+    ticks
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use hash::hash;
+
+    #[test]
+    fn test_verify_slice() {
+        let zero = Hash::default();
+        let one = hash(&zero);
+        assert!(vec![][..].verify(&zero)); // base case
+        assert!(vec![Entry::new_tick(0, &zero)][..].verify(&zero)); // singleton case 1
+        assert!(!vec![Entry::new_tick(0, &zero)][..].verify(&one)); // singleton case 2, bad
+        assert!(next_ticks(&zero, 0, 2)[..].verify(&zero)); // inductive step
+
+        let mut bad_ticks = next_ticks(&zero, 0, 2);
+        bad_ticks[1].id = one;
+        assert!(!bad_ticks.verify(&zero)); // inductive step, bad
+    }
+}
+
+#[cfg(all(feature = "unstable", test))]
+mod bench {
+    extern crate test;
+    use self::test::Bencher;
+    use ledger::*;
+
+    #[bench]
+    fn event_bench(bencher: &mut Bencher) {
+        let start_hash = Hash::default();
+        let entries = next_ticks(&start_hash, 10_000, 8);
+        bencher.iter(|| {
+            assert!(entries.verify(&start_hash));
+        });
+    }
+}

src/lib.rs

@@ -1,4 +1,43 @@
 #![cfg_attr(feature = "unstable", feature(test))]
+pub mod accountant;
+pub mod accountant_skel;
+pub mod accountant_stub;
+pub mod crdt;
+pub mod ecdsa;
+pub mod entry;
+#[cfg(feature = "erasure")]
+pub mod erasure;
 pub mod event;
-extern crate itertools;
+pub mod hash;
+pub mod historian;
+pub mod ledger;
+pub mod mint;
+pub mod packet;
+pub mod plan;
+pub mod recorder;
+pub mod result;
+pub mod signature;
+pub mod streamer;
+pub mod subscribers;
+pub mod transaction;
+extern crate bincode;
+extern crate byteorder;
+extern crate chrono;
+extern crate generic_array;
+extern crate libc;
+#[macro_use]
+extern crate log;
 extern crate rayon;
+extern crate ring;
+extern crate serde;
+#[macro_use]
+extern crate serde_derive;
+extern crate serde_json;
+extern crate sha2;
+extern crate untrusted;
+
+extern crate futures;
+
+#[cfg(test)]
+#[macro_use]
+extern crate matches;

src/mint.rs

@@ -0,0 +1,89 @@
+//! The `mint` module is a library for generating the chain's genesis block.
+
+use entry::create_entry;
+use entry::Entry;
+use event::Event;
+use hash::{hash, Hash};
+use ring::rand::SystemRandom;
+use signature::{KeyPair, KeyPairUtil, PublicKey};
+use transaction::Transaction;
+use untrusted::Input;
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct Mint {
+    pub pkcs8: Vec<u8>,
+    pubkey: PublicKey,
+    pub tokens: i64,
+}
+
+impl Mint {
+    pub fn new(tokens: i64) -> Self {
+        let rnd = SystemRandom::new();
+        let pkcs8 = KeyPair::generate_pkcs8(&rnd).unwrap().to_vec();
+        let keypair = KeyPair::from_pkcs8(Input::from(&pkcs8)).unwrap();
+        let pubkey = keypair.pubkey();
+        Mint {
+            pkcs8,
+            pubkey,
+            tokens,
+        }
+    }
+
+    pub fn seed(&self) -> Hash {
+        hash(&self.pkcs8)
+    }
+
+    pub fn last_id(&self) -> Hash {
+        self.create_entries()[1].id
+    }
+
+    pub fn keypair(&self) -> KeyPair {
+        KeyPair::from_pkcs8(Input::from(&self.pkcs8)).unwrap()
+    }
+
+    pub fn pubkey(&self) -> PublicKey {
+        self.pubkey
+    }
+
+    pub fn create_events(&self) -> Vec<Event> {
+        let keypair = self.keypair();
+        let tr = Transaction::new(&keypair, self.pubkey(), self.tokens, self.seed());
+        vec![Event::Transaction(tr)]
+    }
+
+    pub fn create_entries(&self) -> Vec<Entry> {
+        let e0 = create_entry(&self.seed(), 0, vec![]);
+        let e1 = create_entry(&e0.id, 0, self.create_events());
+        vec![e0, e1]
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct MintDemo {
+    pub mint: Mint,
+    pub users: Vec<(Vec<u8>, i64)>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use ledger::Block;
+    use plan::Plan;
+
+    #[test]
+    fn test_create_events() {
+        let mut events = Mint::new(100).create_events().into_iter();
+        if let Event::Transaction(tr) = events.next().unwrap() {
+            if let Plan::Pay(payment) = tr.data.plan {
+                assert_eq!(tr.from, payment.to);
+            }
+        }
+        assert_eq!(events.next(), None);
+    }
+
+    #[test]
+    fn test_verify_entries() {
+        let entries = Mint::new(100).create_entries();
+        assert!(entries[..].verify(&entries[0].id));
+    }
+}

src/packet.rs

@@ -0,0 +1,396 @@
+//! The `packet` module defines data structures and methods to pull data from the network.
+use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
+use result::{Error, Result};
+use std::collections::VecDeque;
+use std::fmt;
+use std::io;
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, UdpSocket};
+use std::sync::{Arc, Mutex, RwLock};
+use std::mem::size_of;
+
+pub type SharedPackets = Arc<RwLock<Packets>>;
+pub type SharedBlob = Arc<RwLock<Blob>>;
+pub type PacketRecycler = Recycler<Packets>;
+pub type BlobRecycler = Recycler<Blob>;
+
+pub const NUM_PACKETS: usize = 1024 * 8;
+const BLOB_SIZE: usize = 64 * 1024;
+pub const PACKET_DATA_SIZE: usize = 256;
+pub const NUM_BLOBS: usize = (NUM_PACKETS * PACKET_DATA_SIZE) / BLOB_SIZE;
+
+#[derive(Clone, Default)]
+#[repr(C)]
+pub struct Meta {
+    pub size: usize,
+    pub addr: [u16; 8],
+    pub port: u16,
+    pub v6: bool,
+}
+
+#[derive(Clone)]
+#[repr(C)]
+pub struct Packet {
+    pub data: [u8; PACKET_DATA_SIZE],
+    pub meta: Meta,
+}
+
+impl fmt::Debug for Packet {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "Packet {{ size: {:?}, addr: {:?} }}",
+            self.meta.size,
+            self.meta.addr()
+        )
+    }
+}
+
+impl Default for Packet {
+    fn default() -> Packet {
+        Packet {
+            data: [0u8; PACKET_DATA_SIZE],
+            meta: Meta::default(),
+        }
+    }
+}
+
+impl Meta {
+    pub fn addr(&self) -> SocketAddr {
+        if !self.v6 {
+            let addr = [
+                self.addr[0] as u8,
+                self.addr[1] as u8,
+                self.addr[2] as u8,
+                self.addr[3] as u8,
+            ];
+            let ipv4: Ipv4Addr = From::<[u8; 4]>::from(addr);
+            SocketAddr::new(IpAddr::V4(ipv4), self.port)
+        } else {
+            let ipv6: Ipv6Addr = From::<[u16; 8]>::from(self.addr);
+            SocketAddr::new(IpAddr::V6(ipv6), self.port)
+        }
+    }
+
+    pub fn set_addr(&mut self, a: &SocketAddr) {
+        match *a {
+            SocketAddr::V4(v4) => {
+                let ip = v4.ip().octets();
+                self.addr[0] = u16::from(ip[0]);
+                self.addr[1] = u16::from(ip[1]);
+                self.addr[2] = u16::from(ip[2]);
+                self.addr[3] = u16::from(ip[3]);
+                self.port = a.port();
+            }
+            SocketAddr::V6(v6) => {
+                self.addr = v6.ip().segments();
+                self.port = a.port();
+                self.v6 = true;
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct Packets {
+    pub packets: Vec<Packet>,
+}
+
+//auto derive doesn't support large arrays
+impl Default for Packets {
+    fn default() -> Packets {
+        Packets {
+            packets: vec![Packet::default(); NUM_PACKETS],
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct Blob {
+    pub data: [u8; BLOB_SIZE],
+    pub meta: Meta,
+}
+
+impl fmt::Debug for Blob {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "Blob {{ size: {:?}, addr: {:?} }}",
+            self.meta.size,
+            self.meta.addr()
+        )
+    }
+}
+
+//auto derive doesn't support large arrays
+impl Default for Blob {
+    fn default() -> Blob {
+        Blob {
+            data: [0u8; BLOB_SIZE],
+            meta: Meta::default(),
+        }
+    }
+}
+
+pub struct Recycler<T> {
+    gc: Arc<Mutex<Vec<Arc<RwLock<T>>>>>,
+}
+
+impl<T: Default> Default for Recycler<T> {
+    fn default() -> Recycler<T> {
+        Recycler {
+            gc: Arc::new(Mutex::new(vec![])),
+        }
+    }
+}
+
+impl<T: Default> Clone for Recycler<T> {
+    fn clone(&self) -> Recycler<T> {
+        Recycler {
+            gc: self.gc.clone(),
+        }
+    }
+}
+
+impl<T: Default> Recycler<T> {
+    pub fn allocate(&self) -> Arc<RwLock<T>> {
+        let mut gc = self.gc.lock().expect("recycler lock");
+        gc.pop()
+            .unwrap_or_else(|| Arc::new(RwLock::new(Default::default())))
+    }
+    pub fn recycle(&self, msgs: Arc<RwLock<T>>) {
+        let mut gc = self.gc.lock().expect("recycler lock");
+        gc.push(msgs);
+    }
+}
+
+impl Packets {
+    fn run_read_from(&mut self, socket: &UdpSocket) -> Result<usize> {
+        self.packets.resize(NUM_PACKETS, Packet::default());
+        let mut i = 0;
+        //DOCUMENTED SIDE-EFFECT
+        //Performance out of the IO without poll
+        //  * block on the socket until its readable
+        //  * set the socket to non blocking
+        //  * read until it fails
+        //  * set it back to blocking before returning
+        socket.set_nonblocking(false)?;
+        for p in &mut self.packets {
+            p.meta.size = 0;
+            match socket.recv_from(&mut p.data) {
+                Err(_) if i > 0 => {
+                    trace!("got {:?} messages", i);
+                    break;
+                }
+                Err(e) => {
+                    info!("recv_from err {:?}", e);
+                    return Err(Error::IO(e));
+                }
+                Ok((nrecv, from)) => {
+                    p.meta.size = nrecv;
+                    p.meta.set_addr(&from);
+                    if i == 0 {
+                        socket.set_nonblocking(true)?;
+                    }
+                }
+            }
+            i += 1;
+        }
+        Ok(i)
+    }
+    pub fn recv_from(&mut self, socket: &UdpSocket) -> Result<()> {
+        let sz = self.run_read_from(socket)?;
+        self.packets.resize(sz, Packet::default());
+        Ok(())
+    }
+    pub fn send_to(&self, socket: &UdpSocket) -> Result<()> {
+        for p in &self.packets {
+            let a = p.meta.addr();
+            socket.send_to(&p.data[..p.meta.size], &a)?;
+        }
+        Ok(())
+    }
+}
+
+const BLOB_INDEX_SIZE: usize = size_of::<u64>();
+
+impl Blob {
+    pub fn get_index(&self) -> Result<u64> {
+        let mut rdr = io::Cursor::new(&self.data[0..BLOB_INDEX_SIZE]);
+        let r = rdr.read_u64::<LittleEndian>()?;
+        Ok(r)
+    }
+    pub fn set_index(&mut self, ix: u64) -> Result<()> {
+        let mut wtr = vec![];
+        wtr.write_u64::<LittleEndian>(ix)?;
+        self.data[..BLOB_INDEX_SIZE].clone_from_slice(&wtr);
+        Ok(())
+    }
+    pub fn data(&self) -> &[u8] {
+        &self.data[BLOB_INDEX_SIZE..]
+    }
+    pub fn data_mut(&mut self) -> &mut [u8] {
+        &mut self.data[BLOB_INDEX_SIZE..]
+    }
+    pub fn set_size(&mut self, size: usize) {
+        self.meta.size = size + BLOB_INDEX_SIZE;
+    }
+    pub fn recv_from(re: &BlobRecycler, socket: &UdpSocket) -> Result<VecDeque<SharedBlob>> {
+        let mut v = VecDeque::new();
+        //DOCUMENTED SIDE-EFFECT
+        //Performance out of the IO without poll
+        //  * block on the socket until its readable
+        //  * set the socket to non blocking
+        //  * read until it fails
+        //  * set it back to blocking before returning
+        socket.set_nonblocking(false)?;
+        for i in 0..NUM_BLOBS {
+            let r = re.allocate();
+            {
+                let mut p = r.write().unwrap();
+                match socket.recv_from(&mut p.data) {
+                    Err(_) if i > 0 => {
+                        trace!("got {:?} messages", i);
+                        break;
+                    }
+                    Err(e) => {
+                        info!("recv_from err {:?}", e);
+                        return Err(Error::IO(e));
+                    }
+                    Ok((nrecv, from)) => {
+                        p.meta.size = nrecv;
+                        p.meta.set_addr(&from);
+                        if i == 0 {
+                            socket.set_nonblocking(true)?;
+                        }
+                    }
+                }
+            }
+            v.push_back(r);
+        }
+        Ok(v)
+    }
+    pub fn send_to(
+        re: &BlobRecycler,
+        socket: &UdpSocket,
+        v: &mut VecDeque<SharedBlob>,
+    ) -> Result<()> {
+        while let Some(r) = v.pop_front() {
+            {
+                let p = r.read().unwrap();
+                let a = p.meta.addr();
+                socket.send_to(&p.data[..p.meta.size], &a)?;
+            }
+            re.recycle(r);
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use packet::{Blob, BlobRecycler, Packet, PacketRecycler, Packets};
+    use std::collections::VecDeque;
+    use std::io;
+    use std::io::Write;
+    use std::net::UdpSocket;
+    #[test]
+    pub fn packet_recycler_test() {
+        let r = PacketRecycler::default();
+        let p = r.allocate();
+        r.recycle(p);
+        assert_eq!(r.gc.lock().unwrap().len(), 1);
+        let _ = r.allocate();
+        assert_eq!(r.gc.lock().unwrap().len(), 0);
+    }
+    #[test]
+    pub fn blob_recycler_test() {
+        let r = BlobRecycler::default();
+        let p = r.allocate();
+        r.recycle(p);
+        assert_eq!(r.gc.lock().unwrap().len(), 1);
+        let _ = r.allocate();
+        assert_eq!(r.gc.lock().unwrap().len(), 0);
+    }
+    #[test]
+    pub fn packet_send_recv() {
+        let reader = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let addr = reader.local_addr().unwrap();
+        let sender = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let saddr = sender.local_addr().unwrap();
+        let r = PacketRecycler::default();
+        let p = r.allocate();
+        p.write().unwrap().packets.resize(10, Packet::default());
+        for m in p.write().unwrap().packets.iter_mut() {
+            m.meta.set_addr(&addr);
+            m.meta.size = 256;
+        }
+        p.read().unwrap().send_to(&sender).unwrap();
+        p.write().unwrap().recv_from(&reader).unwrap();
+        for m in p.write().unwrap().packets.iter_mut() {
+            assert_eq!(m.meta.size, 256);
+            assert_eq!(m.meta.addr(), saddr);
+        }
+
+        r.recycle(p);
+    }
+
+    #[test]
+    pub fn blob_send_recv() {
+        trace!("start");
+        let reader = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let addr = reader.local_addr().unwrap();
+        let sender = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let r = BlobRecycler::default();
+        let p = r.allocate();
+        p.write().unwrap().meta.set_addr(&addr);
+        p.write().unwrap().meta.size = 1024;
+        let mut v = VecDeque::new();
+        v.push_back(p);
+        assert_eq!(v.len(), 1);
+        Blob::send_to(&r, &sender, &mut v).unwrap();
+        trace!("send_to");
+        assert_eq!(v.len(), 0);
+        let mut rv = Blob::recv_from(&r, &reader).unwrap();
+        trace!("recv_from");
+        assert_eq!(rv.len(), 1);
+        let rp = rv.pop_front().unwrap();
+        assert_eq!(rp.write().unwrap().meta.size, 1024);
+        r.recycle(rp);
+    }
+
+    #[cfg(all(feature = "ipv6", test))]
+    #[test]
+    pub fn blob_ipv6_send_recv() {
+        let reader = UdpSocket::bind("[::1]:0").expect("bind");
+        let addr = reader.local_addr().unwrap();
+        let sender = UdpSocket::bind("[::1]:0").expect("bind");
+        let r = BlobRecycler::default();
+        let p = r.allocate();
+        p.write().unwrap().meta.set_addr(&addr);
+        p.write().unwrap().meta.size = 1024;
+        let mut v = VecDeque::default();
+        v.push_back(p);
+        Blob::send_to(&r, &sender, &mut v).unwrap();
+        let mut rv = Blob::recv_from(&r, &reader).unwrap();
+        let rp = rv.pop_front().unwrap();
+        assert_eq!(rp.write().unwrap().meta.size, 1024);
+        r.recycle(rp);
+    }
+
+    #[test]
+    pub fn debug_trait() {
+        write!(io::sink(), "{:?}", Packet::default()).unwrap();
+        write!(io::sink(), "{:?}", Packets::default()).unwrap();
+        write!(io::sink(), "{:?}", Blob::default()).unwrap();
+    }
+    #[test]
+    pub fn blob_test() {
+        let mut b = Blob::default();
+        b.set_index(<u64>::max_value()).unwrap();
+        assert_eq!(b.get_index().unwrap(), <u64>::max_value());
+        b.data_mut()[0] = 1;
+        assert_eq!(b.data()[0], 1);
+        assert_eq!(b.get_index().unwrap(), <u64>::max_value());
+    }
+
+}

src/plan.rs

@@ -0,0 +1,176 @@
+//! The `plan` module provides a domain-specific language for payment plans. Users create Plan objects that
+//! are given to an interpreter. The interpreter listens for `Witness` events,
+//! which it uses to reduce the payment plan. When the plan is reduced to a
+//! `Payment`, the payment is executed.
+
+use chrono::prelude::*;
+use signature::PublicKey;
+use std::mem;
+
+pub enum Witness {
+    Timestamp(DateTime<Utc>),
+    Signature(PublicKey),
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub enum Condition {
+    Timestamp(DateTime<Utc>),
+    Signature(PublicKey),
+}
+
+impl Condition {
+    /// Return true if the given Witness satisfies this Condition.
+    pub fn is_satisfied(&self, witness: &Witness) -> bool {
+        match (self, witness) {
+            (&Condition::Signature(ref pubkey), &Witness::Signature(ref from)) => pubkey == from,
+            (&Condition::Timestamp(ref dt), &Witness::Timestamp(ref last_time)) => dt <= last_time,
+            _ => false,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub struct Payment {
+    pub tokens: i64,
+    pub to: PublicKey,
+}
+
+#[repr(C)]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub enum Plan {
+    Pay(Payment),
+    After(Condition, Payment),
+    Race((Condition, Payment), (Condition, Payment)),
+}
+
+impl Plan {
+    /// Create the simplest spending plan - one that pays `tokens` to PublicKey.
+    pub fn new_payment(tokens: i64, to: PublicKey) -> Self {
+        Plan::Pay(Payment { tokens, to })
+    }
+
+    /// Create a spending plan that pays `tokens` to `to` after being witnessed by `from`.
+    pub fn new_authorized_payment(from: PublicKey, tokens: i64, to: PublicKey) -> Self {
+        Plan::After(Condition::Signature(from), Payment { tokens, to })
+    }
+
+    /// Create a spending plan that pays `tokens` to `to` after the given DateTime.
+    pub fn new_future_payment(dt: DateTime<Utc>, tokens: i64, to: PublicKey) -> Self {
+        Plan::After(Condition::Timestamp(dt), Payment { tokens, to })
+    }
+
+    /// Create a spending plan that pays `tokens` to `to` after the given DateTime
+    /// unless cancelled by `from`.
+    pub fn new_cancelable_future_payment(
+        dt: DateTime<Utc>,
+        from: PublicKey,
+        tokens: i64,
+        to: PublicKey,
+    ) -> Self {
+        Plan::Race(
+            (Condition::Timestamp(dt), Payment { tokens, to }),
+            (Condition::Signature(from), Payment { tokens, to: from }),
+        )
+    }
+
+    /// Return Payment if the spending plan requires no additional Witnesses.
+    pub fn final_payment(&self) -> Option<Payment> {
+        match *self {
+            Plan::Pay(ref payment) => Some(payment.clone()),
+            _ => None,
+        }
+    }
+
+    /// Return true if the plan spends exactly `spendable_tokens`.
+    pub fn verify(&self, spendable_tokens: i64) -> bool {
+        match *self {
+            Plan::Pay(ref payment) | Plan::After(_, ref payment) => {
+                payment.tokens == spendable_tokens
+            }
+            Plan::Race(ref a, ref b) => {
+                a.1.tokens == spendable_tokens && b.1.tokens == spendable_tokens
+            }
+        }
+    }
+
+    /// Apply a witness to the spending plan to see if the plan can be reduced.
+    /// If so, modify the plan in-place.
+    pub fn apply_witness(&mut self, witness: &Witness) {
+        let new_payment = match *self {
+            Plan::After(ref cond, ref payment) if cond.is_satisfied(witness) => Some(payment),
+            Plan::Race((ref cond, ref payment), _) if cond.is_satisfied(witness) => Some(payment),
+            Plan::Race(_, (ref cond, ref payment)) if cond.is_satisfied(witness) => Some(payment),
+            _ => None,
+        }.cloned();
+
+        if let Some(payment) = new_payment {
+            mem::replace(self, Plan::Pay(payment));
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_signature_satisfied() {
+        let sig = PublicKey::default();
+        assert!(Condition::Signature(sig).is_satisfied(&Witness::Signature(sig)));
+    }
+
+    #[test]
+    fn test_timestamp_satisfied() {
+        let dt1 = Utc.ymd(2014, 11, 14).and_hms(8, 9, 10);
+        let dt2 = Utc.ymd(2014, 11, 14).and_hms(10, 9, 8);
+        assert!(Condition::Timestamp(dt1).is_satisfied(&Witness::Timestamp(dt1)));
+        assert!(Condition::Timestamp(dt1).is_satisfied(&Witness::Timestamp(dt2)));
+        assert!(!Condition::Timestamp(dt2).is_satisfied(&Witness::Timestamp(dt1)));
+    }
+
+    #[test]
+    fn test_verify_plan() {
+        let dt = Utc.ymd(2014, 11, 14).and_hms(8, 9, 10);
+        let from = PublicKey::default();
+        let to = PublicKey::default();
+        assert!(Plan::new_payment(42, to).verify(42));
+        assert!(Plan::new_authorized_payment(from, 42, to).verify(42));
+        assert!(Plan::new_future_payment(dt, 42, to).verify(42));
+        assert!(Plan::new_cancelable_future_payment(dt, from, 42, to).verify(42));
+    }
+
+    #[test]
+    fn test_authorized_payment() {
+        let from = PublicKey::default();
+        let to = PublicKey::default();
+
+        let mut plan = Plan::new_authorized_payment(from, 42, to);
+        plan.apply_witness(&Witness::Signature(from));
+        assert_eq!(plan, Plan::new_payment(42, to));
+    }
+
+    #[test]
+    fn test_future_payment() {
+        let dt = Utc.ymd(2014, 11, 14).and_hms(8, 9, 10);
+        let to = PublicKey::default();
+
+        let mut plan = Plan::new_future_payment(dt, 42, to);
+        plan.apply_witness(&Witness::Timestamp(dt));
+        assert_eq!(plan, Plan::new_payment(42, to));
+    }
+
+    #[test]
+    fn test_cancelable_future_payment() {
+        let dt = Utc.ymd(2014, 11, 14).and_hms(8, 9, 10);
+        let from = PublicKey::default();
+        let to = PublicKey::default();
+
+        let mut plan = Plan::new_cancelable_future_payment(dt, from, 42, to);
+        plan.apply_witness(&Witness::Timestamp(dt));
+        assert_eq!(plan, Plan::new_payment(42, to));
+
+        let mut plan = Plan::new_cancelable_future_payment(dt, from, 42, to);
+        plan.apply_witness(&Witness::Signature(from));
+        assert_eq!(plan, Plan::new_payment(42, from));
+    }
+}

src/recorder.rs

@@ -0,0 +1,89 @@
+//! The `recorder` module provides an object for generating a Proof of History.
+//! It records Event items on behalf of its users. It continuously generates
+//! new hashes, only stopping to check if it has been sent an Event item. It
+//! tags each Event with an Entry, and sends it back. The Entry includes the
+//! Event, the latest hash, and the number of hashes since the last event.
+//! The resulting stream of entries represents ordered events in time.
+
+use entry::{create_entry_mut, Entry};
+use event::Event;
+use hash::{hash, Hash};
+use std::mem;
+use std::sync::mpsc::{Receiver, SyncSender, TryRecvError};
+use std::time::{Duration, Instant};
+
+#[cfg_attr(feature = "cargo-clippy", allow(large_enum_variant))]
+pub enum Signal {
+    Tick,
+    Event(Event),
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum ExitReason {
+    RecvDisconnected,
+    SendDisconnected,
+}
+
+pub struct Recorder {
+    sender: SyncSender<Entry>,
+    receiver: Receiver<Signal>,
+    last_hash: Hash,
+    events: Vec<Event>,
+    num_hashes: u64,
+    num_ticks: u64,
+}
+
+impl Recorder {
+    pub fn new(receiver: Receiver<Signal>, sender: SyncSender<Entry>, last_hash: Hash) -> Self {
+        Recorder {
+            receiver,
+            sender,
+            last_hash,
+            events: vec![],
+            num_hashes: 0,
+            num_ticks: 0,
+        }
+    }
+
+    pub fn hash(&mut self) {
+        self.last_hash = hash(&self.last_hash);
+        self.num_hashes += 1;
+    }
+
+    pub fn record_entry(&mut self) -> Result<(), ExitReason> {
+        let events = mem::replace(&mut self.events, vec![]);
+        let entry = create_entry_mut(&mut self.last_hash, &mut self.num_hashes, events);
+        self.sender
+            .send(entry)
+            .or(Err(ExitReason::SendDisconnected))?;
+        Ok(())
+    }
+
+    pub fn process_events(
+        &mut self,
+        epoch: Instant,
+        ms_per_tick: Option<u64>,
+    ) -> Result<(), ExitReason> {
+        loop {
+            if let Some(ms) = ms_per_tick {
+                if epoch.elapsed() > Duration::from_millis((self.num_ticks + 1) * ms) {
+                    self.record_entry()?;
+                    self.num_ticks += 1;
+                }
+            }
+
+            match self.receiver.try_recv() {
+                Ok(signal) => match signal {
+                    Signal::Tick => {
+                        self.record_entry()?;
+                    }
+                    Signal::Event(event) => {
+                        self.events.push(event);
+                    }
+                },
+                Err(TryRecvError::Empty) => return Ok(()),
+                Err(TryRecvError::Disconnected) => return Err(ExitReason::RecvDisconnected),
+            };
+        }
+    }
+}

src/result.rs

@@ -0,0 +1,132 @@
+//! The `result` module exposes a Result type that propagates one of many different Error types.
+
+use bincode;
+use serde_json;
+use std;
+use std::any::Any;
+use accountant;
+
+#[derive(Debug)]
+pub enum Error {
+    IO(std::io::Error),
+    JSON(serde_json::Error),
+    AddrParse(std::net::AddrParseError),
+    JoinError(Box<Any + Send + 'static>),
+    RecvError(std::sync::mpsc::RecvError),
+    RecvTimeoutError(std::sync::mpsc::RecvTimeoutError),
+    Serialize(std::boxed::Box<bincode::ErrorKind>),
+    AccountingError(accountant::AccountingError),
+    SendError,
+    Services,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl std::convert::From<std::sync::mpsc::RecvError> for Error {
+    fn from(e: std::sync::mpsc::RecvError) -> Error {
+        Error::RecvError(e)
+    }
+}
+impl std::convert::From<std::sync::mpsc::RecvTimeoutError> for Error {
+    fn from(e: std::sync::mpsc::RecvTimeoutError) -> Error {
+        Error::RecvTimeoutError(e)
+    }
+}
+impl std::convert::From<accountant::AccountingError> for Error {
+    fn from(e: accountant::AccountingError) -> Error {
+        Error::AccountingError(e)
+    }
+}
+impl<T> std::convert::From<std::sync::mpsc::SendError<T>> for Error {
+    fn from(_e: std::sync::mpsc::SendError<T>) -> Error {
+        Error::SendError
+    }
+}
+impl std::convert::From<Box<Any + Send + 'static>> for Error {
+    fn from(e: Box<Any + Send + 'static>) -> Error {
+        Error::JoinError(e)
+    }
+}
+
+impl std::convert::From<std::io::Error> for Error {
+    fn from(e: std::io::Error) -> Error {
+        Error::IO(e)
+    }
+}
+impl std::convert::From<serde_json::Error> for Error {
+    fn from(e: serde_json::Error) -> Error {
+        Error::JSON(e)
+    }
+}
+impl std::convert::From<std::net::AddrParseError> for Error {
+    fn from(e: std::net::AddrParseError) -> Error {
+        Error::AddrParse(e)
+    }
+}
+impl std::convert::From<std::boxed::Box<bincode::ErrorKind>> for Error {
+    fn from(e: std::boxed::Box<bincode::ErrorKind>) -> Error {
+        Error::Serialize(e)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use result::Error;
+    use result::Result;
+    use serde_json;
+    use std::io;
+    use std::io::Write;
+    use std::net::SocketAddr;
+    use std::sync::mpsc::channel;
+    use std::sync::mpsc::RecvError;
+    use std::sync::mpsc::RecvTimeoutError;
+    use std::thread;
+
+    fn addr_parse_error() -> Result<SocketAddr> {
+        let r = "12fdfasfsafsadfs".parse()?;
+        Ok(r)
+    }
+
+    fn join_error() -> Result<()> {
+        let r = thread::spawn(|| panic!("hi")).join()?;
+        Ok(r)
+    }
+    fn json_error() -> Result<()> {
+        let r = serde_json::from_slice("=342{;;;;:}".as_bytes())?;
+        Ok(r)
+    }
+    fn send_error() -> Result<()> {
+        let (s, r) = channel();
+        drop(r);
+        s.send(())?;
+        Ok(())
+    }
+
+    #[test]
+    fn from_test() {
+        assert_matches!(addr_parse_error(), Err(Error::AddrParse(_)));
+        assert_matches!(Error::from(RecvError {}), Error::RecvError(_));
+        assert_matches!(
+            Error::from(RecvTimeoutError::Timeout),
+            Error::RecvTimeoutError(_)
+        );
+        assert_matches!(send_error(), Err(Error::SendError));
+        assert_matches!(join_error(), Err(Error::JoinError(_)));
+        let ioe = io::Error::new(io::ErrorKind::NotFound, "hi");
+        assert_matches!(Error::from(ioe), Error::IO(_));
+    }
+    #[test]
+    fn fmt_test() {
+        write!(io::sink(), "{:?}", addr_parse_error()).unwrap();
+        write!(io::sink(), "{:?}", Error::from(RecvError {})).unwrap();
+        write!(io::sink(), "{:?}", Error::from(RecvTimeoutError::Timeout)).unwrap();
+        write!(io::sink(), "{:?}", send_error()).unwrap();
+        write!(io::sink(), "{:?}", join_error()).unwrap();
+        write!(io::sink(), "{:?}", json_error()).unwrap();
+        write!(
+            io::sink(),
+            "{:?}",
+            Error::from(io::Error::new(io::ErrorKind::NotFound, "hi"))
+        ).unwrap();
+    }
+}

src/signature.rs

@@ -0,0 +1,43 @@
+//! The `signature` module provides functionality for public, and private keys.
+
+use generic_array::typenum::{U32, U64};
+use generic_array::GenericArray;
+use ring::signature::Ed25519KeyPair;
+use ring::{rand, signature};
+use untrusted;
+
+pub type KeyPair = Ed25519KeyPair;
+pub type PublicKey = GenericArray<u8, U32>;
+pub type Signature = GenericArray<u8, U64>;
+
+pub trait KeyPairUtil {
+    fn new() -> Self;
+    fn pubkey(&self) -> PublicKey;
+}
+
+impl KeyPairUtil for Ed25519KeyPair {
+    /// Return a new ED25519 keypair
+    fn new() -> Self {
+        let rng = rand::SystemRandom::new();
+        let pkcs8_bytes = signature::Ed25519KeyPair::generate_pkcs8(&rng).unwrap();
+        signature::Ed25519KeyPair::from_pkcs8(untrusted::Input::from(&pkcs8_bytes)).unwrap()
+    }
+
+    /// Return the public key for the given keypair
+    fn pubkey(&self) -> PublicKey {
+        GenericArray::clone_from_slice(self.public_key_bytes())
+    }
+}
+
+pub trait SignatureUtil {
+    fn verify(&self, peer_public_key_bytes: &[u8], msg_bytes: &[u8]) -> bool;
+}
+
+impl SignatureUtil for GenericArray<u8, U64> {
+    fn verify(&self, peer_public_key_bytes: &[u8], msg_bytes: &[u8]) -> bool {
+        let peer_public_key = untrusted::Input::from(peer_public_key_bytes);
+        let msg = untrusted::Input::from(msg_bytes);
+        let sig = untrusted::Input::from(self);
+        signature::verify(&signature::ED25519, peer_public_key, msg, sig).is_ok()
+    }
+}

src/streamer.rs

@@ -0,0 +1,549 @@
+//! The `streamer` module defines a set of services for effecently pulling data from udp sockets.
+use packet::{Blob, BlobRecycler, PacketRecycler, SharedBlob, SharedPackets, NUM_BLOBS};
+use result::Result;
+use std::collections::VecDeque;
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc;
+use std::sync::{Arc, RwLock};
+use std::thread::{spawn, JoinHandle};
+use std::time::Duration;
+use subscribers::Subscribers;
+
+pub type PacketReceiver = mpsc::Receiver<SharedPackets>;
+pub type PacketSender = mpsc::Sender<SharedPackets>;
+pub type BlobSender = mpsc::Sender<VecDeque<SharedBlob>>;
+pub type BlobReceiver = mpsc::Receiver<VecDeque<SharedBlob>>;
+
+fn recv_loop(
+    sock: &UdpSocket,
+    exit: &Arc<AtomicBool>,
+    re: &PacketRecycler,
+    channel: &PacketSender,
+) -> Result<()> {
+    loop {
+        let msgs = re.allocate();
+        let msgs_ = msgs.clone();
+        loop {
+            match msgs.write().unwrap().recv_from(sock) {
+                Ok(()) => {
+                    channel.send(msgs_)?;
+                    break;
+                }
+                Err(_) => {
+                    if exit.load(Ordering::Relaxed) {
+                        re.recycle(msgs_);
+                        return Ok(());
+                    }
+                }
+            }
+        }
+    }
+}
+
+pub fn receiver(
+    sock: UdpSocket,
+    exit: Arc<AtomicBool>,
+    recycler: PacketRecycler,
+    channel: PacketSender,
+) -> Result<JoinHandle<()>> {
+    let timer = Duration::new(1, 0);
+    sock.set_read_timeout(Some(timer))?;
+    Ok(spawn(move || {
+        let _ = recv_loop(&sock, &exit, &recycler, &channel);
+        ()
+    }))
+}
+
+fn recv_send(sock: &UdpSocket, recycler: &BlobRecycler, r: &BlobReceiver) -> Result<()> {
+    let timer = Duration::new(1, 0);
+    let mut msgs = r.recv_timeout(timer)?;
+    Blob::send_to(recycler, sock, &mut msgs)?;
+    Ok(())
+}
+
+pub fn responder(
+    sock: UdpSocket,
+    exit: Arc<AtomicBool>,
+    recycler: BlobRecycler,
+    r: BlobReceiver,
+) -> JoinHandle<()> {
+    spawn(move || loop {
+        if recv_send(&sock, &recycler, &r).is_err() && exit.load(Ordering::Relaxed) {
+            break;
+        }
+    })
+}
+
+//TODO, we would need to stick block authentication before we create the
+//window.
+fn recv_blobs(recycler: &BlobRecycler, sock: &UdpSocket, s: &BlobSender) -> Result<()> {
+    let dq = Blob::recv_from(recycler, sock)?;
+    if !dq.is_empty() {
+        s.send(dq)?;
+    }
+    Ok(())
+}
+
+pub fn blob_receiver(
+    exit: Arc<AtomicBool>,
+    recycler: BlobRecycler,
+    sock: UdpSocket,
+    s: BlobSender,
+) -> Result<JoinHandle<()>> {
+    //DOCUMENTED SIDE-EFFECT
+    //1 second timeout on socket read
+    let timer = Duration::new(1, 0);
+    sock.set_read_timeout(Some(timer))?;
+    let t = spawn(move || loop {
+        if exit.load(Ordering::Relaxed) {
+            break;
+        }
+        let ret = recv_blobs(&recycler, &sock, &s);
+        if ret.is_err() {
+            break;
+        }
+    });
+    Ok(t)
+}
+
+fn recv_window(
+    window: &mut Vec<Option<SharedBlob>>,
+    subs: &Arc<RwLock<Subscribers>>,
+    recycler: &BlobRecycler,
+    consumed: &mut usize,
+    r: &BlobReceiver,
+    s: &BlobSender,
+    retransmit: &BlobSender,
+) -> Result<()> {
+    let timer = Duration::new(1, 0);
+    let mut dq = r.recv_timeout(timer)?;
+    while let Ok(mut nq) = r.try_recv() {
+        dq.append(&mut nq)
+    }
+    {
+        //retransmit all leader blocks
+        let mut retransmitq = VecDeque::new();
+        let rsubs = subs.read().unwrap();
+        for b in &dq {
+            let p = b.read().unwrap();
+            //TODO this check isn't safe against adverserial packets
+            //we need to maintain a sequence window
+            trace!(
+                "idx: {} addr: {:?} leader: {:?}",
+                p.get_index().unwrap(),
+                p.meta.addr(),
+                rsubs.leader.addr
+            );
+            if p.meta.addr() == rsubs.leader.addr {
+                //TODO
+                //need to copy the retransmited blob
+                //otherwise we get into races with which thread
+                //should do the recycling
+                //
+                //a better absraction would be to recycle when the blob
+                //is dropped via a weakref to the recycler
+                let nv = recycler.allocate();
+                {
+                    let mut mnv = nv.write().unwrap();
+                    let sz = p.meta.size;
+                    mnv.meta.size = sz;
+                    mnv.data[..sz].copy_from_slice(&p.data[..sz]);
+                }
+                retransmitq.push_back(nv);
+            }
+        }
+        if !retransmitq.is_empty() {
+            retransmit.send(retransmitq)?;
+        }
+    }
+    //send a contiguous set of blocks
+    let mut contq = VecDeque::new();
+    while let Some(b) = dq.pop_front() {
+        let b_ = b.clone();
+        let p = b.write().unwrap();
+        let pix = p.get_index()? as usize;
+        let w = pix % NUM_BLOBS;
+        //TODO, after the block are authenticated
+        //if we get different blocks at the same index
+        //that is a network failure/attack
+        trace!("window w: {} size: {}", w, p.meta.size);
+        {
+            if window[w].is_none() {
+                window[w] = Some(b_);
+            } else {
+                debug!("duplicate blob at index {:}", w);
+            }
+            loop {
+                let k = *consumed % NUM_BLOBS;
+                trace!("k: {} consumed: {}", k, *consumed);
+                if window[k].is_none() {
+                    break;
+                }
+                contq.push_back(window[k].clone().unwrap());
+                window[k] = None;
+                *consumed += 1;
+            }
+        }
+    }
+    trace!("sending contq.len: {}", contq.len());
+    if !contq.is_empty() {
+        s.send(contq)?;
+    }
+    Ok(())
+}
+
+pub fn window(
+    exit: Arc<AtomicBool>,
+    subs: Arc<RwLock<Subscribers>>,
+    recycler: BlobRecycler,
+    r: BlobReceiver,
+    s: BlobSender,
+    retransmit: BlobSender,
+) -> JoinHandle<()> {
+    spawn(move || {
+        let mut window = vec![None; NUM_BLOBS];
+        let mut consumed = 0;
+        loop {
+            if exit.load(Ordering::Relaxed) {
+                break;
+            }
+            let _ = recv_window(
+                &mut window,
+                &subs,
+                &recycler,
+                &mut consumed,
+                &r,
+                &s,
+                &retransmit,
+            );
+        }
+    })
+}
+
+fn retransmit(
+    subs: &Arc<RwLock<Subscribers>>,
+    recycler: &BlobRecycler,
+    r: &BlobReceiver,
+    sock: &UdpSocket,
+) -> Result<()> {
+    let timer = Duration::new(1, 0);
+    let mut dq = r.recv_timeout(timer)?;
+    while let Ok(mut nq) = r.try_recv() {
+        dq.append(&mut nq);
+    }
+    {
+        let wsubs = subs.read().unwrap();
+        for b in &dq {
+            let mut mb = b.write().unwrap();
+            wsubs.retransmit(&mut mb, sock)?;
+        }
+    }
+    while let Some(b) = dq.pop_front() {
+        recycler.recycle(b);
+    }
+    Ok(())
+}
+
+/// Service to retransmit messages from the leader to layer 1 nodes.
+/// See `subscribers` for network layer definitions.
+/// # Arguments
+/// * `sock` - Socket to read from.  Read timeout is set to 1.
+/// * `exit` - Boolean to signal system exit.
+/// * `subs` - Shared Subscriber structure.  This structure needs to be updated and popualted by
+/// the accountant.
+/// * `recycler` - Blob recycler.
+/// * `r` - Receive channel for blobs to be retransmitted to all the layer 1 nodes.
+pub fn retransmitter(
+    sock: UdpSocket,
+    exit: Arc<AtomicBool>,
+    subs: Arc<RwLock<Subscribers>>,
+    recycler: BlobRecycler,
+    r: BlobReceiver,
+) -> JoinHandle<()> {
+    spawn(move || loop {
+        if exit.load(Ordering::Relaxed) {
+            break;
+        }
+        let _ = retransmit(&subs, &recycler, &r, &sock);
+    })
+}
+
+#[cfg(all(feature = "unstable", test))]
+mod bench {
+    extern crate test;
+    use self::test::Bencher;
+    use packet::{Packet, PacketRecycler, PACKET_DATA_SIZE};
+    use result::Result;
+    use std::net::{SocketAddr, UdpSocket};
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::mpsc::channel;
+    use std::sync::{Arc, Mutex};
+    use std::thread::sleep;
+    use std::thread::{spawn, JoinHandle};
+    use std::time::Duration;
+    use std::time::SystemTime;
+    use streamer::{receiver, PacketReceiver};
+
+    fn producer(
+        addr: &SocketAddr,
+        recycler: PacketRecycler,
+        exit: Arc<AtomicBool>,
+    ) -> JoinHandle<()> {
+        let send = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let msgs = recycler.allocate();
+        let msgs_ = msgs.clone();
+        msgs.write().unwrap().packets.resize(10, Packet::default());
+        for w in msgs.write().unwrap().packets.iter_mut() {
+            w.meta.size = PACKET_DATA_SIZE;
+            w.meta.set_addr(&addr);
+        }
+        spawn(move || loop {
+            if exit.load(Ordering::Relaxed) {
+                return;
+            }
+            let mut num = 0;
+            for p in msgs_.read().unwrap().packets.iter() {
+                let a = p.meta.addr();
+                send.send_to(&p.data[..p.meta.size], &a).unwrap();
+                num += 1;
+            }
+            assert_eq!(num, 10);
+        })
+    }
+
+    fn sink(
+        recycler: PacketRecycler,
+        exit: Arc<AtomicBool>,
+        rvs: Arc<Mutex<usize>>,
+        r: PacketReceiver,
+    ) -> JoinHandle<()> {
+        spawn(move || loop {
+            if exit.load(Ordering::Relaxed) {
+                return;
+            }
+            let timer = Duration::new(1, 0);
+            match r.recv_timeout(timer) {
+                Ok(msgs) => {
+                    let msgs_ = msgs.clone();
+                    *rvs.lock().unwrap() += msgs.read().unwrap().packets.len();
+                    recycler.recycle(msgs_);
+                }
+                _ => (),
+            }
+        })
+    }
+    fn run_streamer_bench() -> Result<()> {
+        let read = UdpSocket::bind("127.0.0.1:0")?;
+        let addr = read.local_addr()?;
+        let exit = Arc::new(AtomicBool::new(false));
+        let pack_recycler = PacketRecycler::default();
+
+        let (s_reader, r_reader) = channel();
+        let t_reader = receiver(read, exit.clone(), pack_recycler.clone(), s_reader)?;
+        let t_producer1 = producer(&addr, pack_recycler.clone(), exit.clone());
+        let t_producer2 = producer(&addr, pack_recycler.clone(), exit.clone());
+        let t_producer3 = producer(&addr, pack_recycler.clone(), exit.clone());
+
+        let rvs = Arc::new(Mutex::new(0));
+        let t_sink = sink(pack_recycler.clone(), exit.clone(), rvs.clone(), r_reader);
+
+        let start = SystemTime::now();
+        let start_val = *rvs.lock().unwrap();
+        sleep(Duration::new(5, 0));
+        let elapsed = start.elapsed().unwrap();
+        let end_val = *rvs.lock().unwrap();
+        let time = elapsed.as_secs() * 10000000000 + elapsed.subsec_nanos() as u64;
+        let ftime = (time as f64) / 10000000000f64;
+        let fcount = (end_val - start_val) as f64;
+        println!("performance: {:?}", fcount / ftime);
+        exit.store(true, Ordering::Relaxed);
+        t_reader.join()?;
+        t_producer1.join()?;
+        t_producer2.join()?;
+        t_producer3.join()?;
+        t_sink.join()?;
+        Ok(())
+    }
+    #[bench]
+    pub fn streamer_bench(_bench: &mut Bencher) {
+        run_streamer_bench().unwrap();
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use packet::{Blob, BlobRecycler, Packet, PacketRecycler, Packets, PACKET_DATA_SIZE};
+    use std::collections::VecDeque;
+    use std::io;
+    use std::io::Write;
+    use std::net::UdpSocket;
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::mpsc::channel;
+    use std::sync::{Arc, RwLock};
+    use std::time::Duration;
+    use streamer::{blob_receiver, receiver, responder, retransmitter, window, BlobReceiver,
+                   PacketReceiver};
+    use subscribers::{Node, Subscribers};
+
+    fn get_msgs(r: PacketReceiver, num: &mut usize) {
+        for _t in 0..5 {
+            let timer = Duration::new(1, 0);
+            match r.recv_timeout(timer) {
+                Ok(m) => *num += m.read().unwrap().packets.len(),
+                e => println!("error {:?}", e),
+            }
+            if *num == 10 {
+                break;
+            }
+        }
+    }
+    #[test]
+    pub fn streamer_debug() {
+        write!(io::sink(), "{:?}", Packet::default()).unwrap();
+        write!(io::sink(), "{:?}", Packets::default()).unwrap();
+        write!(io::sink(), "{:?}", Blob::default()).unwrap();
+    }
+    #[test]
+    pub fn streamer_send_test() {
+        let read = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let addr = read.local_addr().unwrap();
+        let send = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let exit = Arc::new(AtomicBool::new(false));
+        let pack_recycler = PacketRecycler::default();
+        let resp_recycler = BlobRecycler::default();
+        let (s_reader, r_reader) = channel();
+        let t_receiver = receiver(read, exit.clone(), pack_recycler.clone(), s_reader).unwrap();
+        let (s_responder, r_responder) = channel();
+        let t_responder = responder(send, exit.clone(), resp_recycler.clone(), r_responder);
+        let mut msgs = VecDeque::new();
+        for i in 0..10 {
+            let b = resp_recycler.allocate();
+            let b_ = b.clone();
+            let mut w = b.write().unwrap();
+            w.data[0] = i as u8;
+            w.meta.size = PACKET_DATA_SIZE;
+            w.meta.set_addr(&addr);
+            msgs.push_back(b_);
+        }
+        s_responder.send(msgs).expect("send");
+        let mut num = 0;
+        get_msgs(r_reader, &mut num);
+        assert_eq!(num, 10);
+        exit.store(true, Ordering::Relaxed);
+        t_receiver.join().expect("join");
+        t_responder.join().expect("join");
+    }
+
+    fn get_blobs(r: BlobReceiver, num: &mut usize) {
+        for _t in 0..5 {
+            let timer = Duration::new(1, 0);
+            match r.recv_timeout(timer) {
+                Ok(m) => {
+                    for (i, v) in m.iter().enumerate() {
+                        assert_eq!(v.read().unwrap().get_index().unwrap() as usize, *num + i);
+                    }
+                    *num += m.len();
+                }
+                e => println!("error {:?}", e),
+            }
+            if *num == 10 {
+                break;
+            }
+        }
+    }
+
+    #[test]
+    pub fn window_send_test() {
+        let read = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let addr = read.local_addr().unwrap();
+        let send = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let exit = Arc::new(AtomicBool::new(false));
+        let subs = Arc::new(RwLock::new(Subscribers::new(
+            Node::default(),
+            Node::new([0; 8], 0, send.local_addr().unwrap()),
+            &[],
+        )));
+        let resp_recycler = BlobRecycler::default();
+        let (s_reader, r_reader) = channel();
+        let t_receiver =
+            blob_receiver(exit.clone(), resp_recycler.clone(), read, s_reader).unwrap();
+        let (s_window, r_window) = channel();
+        let (s_retransmit, r_retransmit) = channel();
+        let t_window = window(
+            exit.clone(),
+            subs,
+            resp_recycler.clone(),
+            r_reader,
+            s_window,
+            s_retransmit,
+        );
+        let (s_responder, r_responder) = channel();
+        let t_responder = responder(send, exit.clone(), resp_recycler.clone(), r_responder);
+        let mut msgs = VecDeque::new();
+        for v in 0..10 {
+            let i = 9 - v;
+            let b = resp_recycler.allocate();
+            let b_ = b.clone();
+            let mut w = b.write().unwrap();
+            w.set_index(i).unwrap();
+            assert_eq!(i, w.get_index().unwrap());
+            w.meta.size = PACKET_DATA_SIZE;
+            w.meta.set_addr(&addr);
+            msgs.push_back(b_);
+        }
+        s_responder.send(msgs).expect("send");
+        let mut num = 0;
+        get_blobs(r_window, &mut num);
+        assert_eq!(num, 10);
+        let mut q = r_retransmit.recv().unwrap();
+        while let Ok(mut nq) = r_retransmit.try_recv() {
+            q.append(&mut nq);
+        }
+        assert_eq!(q.len(), 10);
+        exit.store(true, Ordering::Relaxed);
+        t_receiver.join().expect("join");
+        t_responder.join().expect("join");
+        t_window.join().expect("join");
+    }
+
+    #[test]
+    pub fn retransmit() {
+        let read = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let send = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let exit = Arc::new(AtomicBool::new(false));
+        let subs = Arc::new(RwLock::new(Subscribers::new(
+            Node::default(),
+            Node::default(),
+            &[Node::new([0; 8], 1, read.local_addr().unwrap())],
+        )));
+        let (s_retransmit, r_retransmit) = channel();
+        let blob_recycler = BlobRecycler::default();
+        let saddr = send.local_addr().unwrap();
+        let t_retransmit = retransmitter(
+            send,
+            exit.clone(),
+            subs,
+            blob_recycler.clone(),
+            r_retransmit,
+        );
+        let mut bq = VecDeque::new();
+        let b = blob_recycler.allocate();
+        b.write().unwrap().meta.size = 10;
+        bq.push_back(b);
+        s_retransmit.send(bq).unwrap();
+        let (s_blob_receiver, r_blob_receiver) = channel();
+        let t_receiver =
+            blob_receiver(exit.clone(), blob_recycler.clone(), read, s_blob_receiver).unwrap();
+        let mut oq = r_blob_receiver.recv().unwrap();
+        assert_eq!(oq.len(), 1);
+        let o = oq.pop_front().unwrap();
+        let ro = o.read().unwrap();
+        assert_eq!(ro.meta.size, 10);
+        assert_eq!(ro.meta.addr(), saddr);
+        exit.store(true, Ordering::Relaxed);
+        t_receiver.join().expect("join");
+        t_retransmit.join().expect("join");
+    }
+
+}

src/subscribers.rs

@@ -0,0 +1,149 @@
+//! The `subscribers` module defines data structures to keep track of nodes on the network.
+//! The network is arranged in layers:
+//!
+//! * layer 0 - Leader.
+//! * layer 1 - As many nodes as we can fit to quickly get reliable `2/3+1` finality
+//! * layer 2 - Everyone else, if layer 1 is `2^10`, layer 2 should be able to fit `2^20` number of nodes.
+//!
+//! It's up to the external state machine to keep this updated.
+use packet::Blob;
+use rayon::prelude::*;
+use result::{Error, Result};
+use std::net::{SocketAddr, UdpSocket};
+
+use std::fmt;
+
+#[derive(Clone, PartialEq)]
+pub struct Node {
+    pub id: [u64; 8],
+    pub weight: u64,
+    pub addr: SocketAddr,
+}
+
+//sockaddr doesn't implement default
+impl Default for Node {
+    fn default() -> Node {
+        Node {
+            id: [0; 8],
+            weight: 0,
+            addr: "0.0.0.0:0".parse().unwrap(),
+        }
+    }
+}
+
+impl Node {
+    pub fn new(id: [u64; 8], weight: u64, addr: SocketAddr) -> Node {
+        Node { id, weight, addr }
+    }
+    fn key(&self) -> i64 {
+        (self.weight as i64).checked_neg().unwrap()
+    }
+}
+
+impl fmt::Debug for Node {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Node {{ weight: {} addr: {} }}", self.weight, self.addr)
+    }
+}
+
+pub struct Subscribers {
+    data: Vec<Node>,
+    pub me: Node,
+    pub leader: Node,
+}
+
+impl Subscribers {
+    pub fn new(me: Node, leader: Node, network: &[Node]) -> Subscribers {
+        let mut h = Subscribers {
+            data: vec![],
+            me: me.clone(),
+            leader: leader.clone(),
+        };
+        h.insert(&[me, leader]);
+        h.insert(network);
+        h
+    }
+
+    /// retransmit messages from the leader to layer 1 nodes
+    pub fn retransmit(&self, blob: &mut Blob, s: &UdpSocket) -> Result<()> {
+        let errs: Vec<_> = self.data
+            .par_iter()
+            .map(|i| {
+                if self.me == *i {
+                    return Ok(0);
+                }
+                if self.leader == *i {
+                    return Ok(0);
+                }
+                trace!("retransmit blob to {}", i.addr);
+                s.send_to(&blob.data[..blob.meta.size], &i.addr)
+            })
+            .collect();
+        for e in errs {
+            trace!("retransmit result {:?}", e);
+            match e {
+                Err(e) => return Err(Error::IO(e)),
+                _ => (),
+            }
+        }
+        Ok(())
+    }
+    pub fn insert(&mut self, ns: &[Node]) {
+        self.data.extend_from_slice(ns);
+        self.data.sort_by_key(Node::key);
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use packet::Blob;
+    use rayon::prelude::*;
+    use std::net::UdpSocket;
+    use std::time::Duration;
+    use subscribers::{Node, Subscribers};
+
+    #[test]
+    pub fn subscriber() {
+        let mut me = Node::default();
+        me.weight = 10;
+        let mut leader = Node::default();
+        leader.weight = 11;
+        let mut s = Subscribers::new(me, leader, &[]);
+        assert_eq!(s.data.len(), 2);
+        assert_eq!(s.data[0].weight, 11);
+        assert_eq!(s.data[1].weight, 10);
+        let mut n = Node::default();
+        n.weight = 12;
+        s.insert(&[n]);
+        assert_eq!(s.data.len(), 3);
+        assert_eq!(s.data[0].weight, 12);
+    }
+    #[test]
+    pub fn retransmit() {
+        let s1 = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let s2 = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let s3 = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        let n1 = Node::new([0; 8], 0, s1.local_addr().unwrap());
+        let n2 = Node::new([0; 8], 0, s2.local_addr().unwrap());
+        let mut s = Subscribers::new(n1.clone(), n2.clone(), &[]);
+        let n3 = Node::new([0; 8], 0, s3.local_addr().unwrap());
+        s.insert(&[n3]);
+        let mut b = Blob::default();
+        b.meta.size = 10;
+        let s4 = UdpSocket::bind("127.0.0.1:0").expect("bind");
+        s.retransmit(&mut b, &s4).unwrap();
+        let res: Vec<_> = [s1, s2, s3]
+            .into_par_iter()
+            .map(|s| {
+                let mut b = Blob::default();
+                s.set_read_timeout(Some(Duration::new(1, 0))).unwrap();
+                s.recv_from(&mut b.data).is_err()
+            })
+            .collect();
+        assert_eq!(res, [true, true, false]);
+        let mut n4 = Node::default();
+        n4.addr = "255.255.255.255:1".parse().unwrap();
+        s.insert(&[n4]);
+        assert!(s.retransmit(&mut b, &s4).is_err());
+    }
+}

src/transaction.rs

@@ -0,0 +1,259 @@
+//! The `transaction` module provides functionality for creating log transactions.
+
+use bincode::serialize;
+use chrono::prelude::*;
+use hash::Hash;
+use plan::{Condition, Payment, Plan};
+use rayon::prelude::*;
+use signature::{KeyPair, KeyPairUtil, PublicKey, Signature, SignatureUtil};
+
+pub const SIGNED_DATA_OFFSET: usize = 112;
+pub const SIG_OFFSET: usize = 8;
+pub const PUB_KEY_OFFSET: usize = 80;
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub struct TransactionData {
+    pub tokens: i64,
+    pub last_id: Hash,
+    pub plan: Plan,
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub struct Transaction {
+    pub sig: Signature,
+    pub from: PublicKey,
+    pub data: TransactionData,
+}
+
+impl Transaction {
+    /// Create and sign a new Transaction. Used for unit-testing.
+    pub fn new(from_keypair: &KeyPair, to: PublicKey, tokens: i64, last_id: Hash) -> Self {
+        let from = from_keypair.pubkey();
+        let plan = Plan::Pay(Payment { tokens, to });
+        let mut tr = Transaction {
+            sig: Signature::default(),
+            data: TransactionData {
+                plan,
+                tokens,
+                last_id,
+            },
+            from: from,
+        };
+        tr.sign(from_keypair);
+        tr
+    }
+
+    /// Create and sign a postdated Transaction. Used for unit-testing.
+    pub fn new_on_date(
+        from_keypair: &KeyPair,
+        to: PublicKey,
+        dt: DateTime<Utc>,
+        tokens: i64,
+        last_id: Hash,
+    ) -> Self {
+        let from = from_keypair.pubkey();
+        let plan = Plan::Race(
+            (Condition::Timestamp(dt), Payment { tokens, to }),
+            (Condition::Signature(from), Payment { tokens, to: from }),
+        );
+        let mut tr = Transaction {
+            data: TransactionData {
+                plan,
+                tokens,
+                last_id,
+            },
+            from: from,
+            sig: Signature::default(),
+        };
+        tr.sign(from_keypair);
+        tr
+    }
+
+    fn get_sign_data(&self) -> Vec<u8> {
+        serialize(&(&self.data)).unwrap()
+    }
+
+    /// Sign this transaction.
+    pub fn sign(&mut self, keypair: &KeyPair) {
+        let sign_data = self.get_sign_data();
+        self.sig = Signature::clone_from_slice(keypair.sign(&sign_data).as_ref());
+    }
+
+    pub fn verify_sig(&self) -> bool {
+        self.sig.verify(&self.from, &self.get_sign_data())
+    }
+
+    pub fn verify_plan(&self) -> bool {
+        self.data.plan.verify(self.data.tokens)
+    }
+}
+
+#[cfg(test)]
+pub fn test_tx() -> Transaction {
+    let keypair1 = KeyPair::new();
+    let pubkey1 = keypair1.pubkey();
+    let zero = Hash::default();
+    Transaction::new(&keypair1, pubkey1, 42, zero)
+}
+
+#[cfg(test)]
+pub fn memfind<A: Eq>(a: &[A], b: &[A]) -> Option<usize> {
+    assert!(a.len() >= b.len());
+    let end = a.len() - b.len() + 1;
+    for i in 0..end {
+        if a[i..i + b.len()] == b[..] {
+            return Some(i);
+        }
+    }
+    None
+}
+
+/// Verify a batch of signatures.
+pub fn verify_signatures(transactions: &[Transaction]) -> bool {
+    transactions.par_iter().all(|tr| tr.verify_sig())
+}
+
+/// Verify a batch of spending plans.
+pub fn verify_plans(transactions: &[Transaction]) -> bool {
+    transactions.par_iter().all(|tr| tr.verify_plan())
+}
+
+/// Verify a batch of transactions.
+pub fn verify_transactions(transactions: &[Transaction]) -> bool {
+    verify_signatures(transactions) && verify_plans(transactions)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bincode::{deserialize, serialize};
+
+    #[test]
+    fn test_claim() {
+        let keypair = KeyPair::new();
+        let zero = Hash::default();
+        let tr0 = Transaction::new(&keypair, keypair.pubkey(), 42, zero);
+        assert!(tr0.verify_plan());
+    }
+
+    #[test]
+    fn test_transfer() {
+        let zero = Hash::default();
+        let keypair0 = KeyPair::new();
+        let keypair1 = KeyPair::new();
+        let pubkey1 = keypair1.pubkey();
+        let tr0 = Transaction::new(&keypair0, pubkey1, 42, zero);
+        assert!(tr0.verify_plan());
+    }
+
+    #[test]
+    fn test_serialize_claim() {
+        let plan = Plan::Pay(Payment {
+            tokens: 0,
+            to: Default::default(),
+        });
+        let claim0 = Transaction {
+            data: TransactionData {
+                plan,
+                tokens: 0,
+                last_id: Default::default(),
+            },
+            from: Default::default(),
+            sig: Default::default(),
+        };
+        let buf = serialize(&claim0).unwrap();
+        let claim1: Transaction = deserialize(&buf).unwrap();
+        assert_eq!(claim1, claim0);
+    }
+
+    #[test]
+    fn test_token_attack() {
+        let zero = Hash::default();
+        let keypair = KeyPair::new();
+        let pubkey = keypair.pubkey();
+        let mut tr = Transaction::new(&keypair, pubkey, 42, zero);
+        tr.data.tokens = 1_000_000; // <-- attack, part 1!
+        if let Plan::Pay(ref mut payment) = tr.data.plan {
+            payment.tokens = tr.data.tokens; // <-- attack, part 2!
+        };
+        assert!(tr.verify_plan());
+        assert!(!tr.verify_sig());
+    }
+
+    #[test]
+    fn test_hijack_attack() {
+        let keypair0 = KeyPair::new();
+        let keypair1 = KeyPair::new();
+        let thief_keypair = KeyPair::new();
+        let pubkey1 = keypair1.pubkey();
+        let zero = Hash::default();
+        let mut tr = Transaction::new(&keypair0, pubkey1, 42, zero);
+        if let Plan::Pay(ref mut payment) = tr.data.plan {
+            payment.to = thief_keypair.pubkey(); // <-- attack!
+        };
+        assert!(tr.verify_plan());
+        assert!(!tr.verify_sig());
+    }
+    #[test]
+    fn test_layout() {
+        let tr = test_tx();
+        let sign_data = tr.get_sign_data();
+        let tx = serialize(&tr).unwrap();
+        assert_matches!(memfind(&tx, &sign_data), Some(SIGNED_DATA_OFFSET));
+        assert_matches!(memfind(&tx, &tr.sig), Some(SIG_OFFSET));
+        assert_matches!(memfind(&tx, &tr.from), Some(PUB_KEY_OFFSET));
+    }
+
+    #[test]
+    fn test_overspend_attack() {
+        let keypair0 = KeyPair::new();
+        let keypair1 = KeyPair::new();
+        let zero = Hash::default();
+        let mut tr = Transaction::new(&keypair0, keypair1.pubkey(), 1, zero);
+        if let Plan::Pay(ref mut payment) = tr.data.plan {
+            payment.tokens = 2; // <-- attack!
+        }
+        assert!(!tr.verify_plan());
+
+        // Also, ensure all branchs of the plan spend all tokens
+        if let Plan::Pay(ref mut payment) = tr.data.plan {
+            payment.tokens = 0; // <-- whoops!
+        }
+        assert!(!tr.verify_plan());
+    }
+
+    #[test]
+    fn test_verify_transactions() {
+        let alice_keypair = KeyPair::new();
+        let bob_pubkey = KeyPair::new().pubkey();
+        let carol_pubkey = KeyPair::new().pubkey();
+        let last_id = Hash::default();
+        let tr0 = Transaction::new(&alice_keypair, bob_pubkey, 1, last_id);
+        let tr1 = Transaction::new(&alice_keypair, carol_pubkey, 1, last_id);
+        let transactions = vec![tr0, tr1];
+        assert!(verify_transactions(&transactions));
+    }
+}
+
+#[cfg(all(feature = "unstable", test))]
+mod bench {
+    extern crate test;
+    use self::test::Bencher;
+    use transaction::*;
+
+    #[bench]
+    fn verify_signatures_bench(bencher: &mut Bencher) {
+        let alice_keypair = KeyPair::new();
+        let last_id = Hash::default();
+        let transactions: Vec<_> = (0..64)
+            .into_par_iter()
+            .map(|_| {
+                let rando_pubkey = KeyPair::new().pubkey();
+                Transaction::new(&alice_keypair, rando_pubkey, 1, last_id)
+            })
+            .collect();
+        bencher.iter(|| {
+            assert!(verify_signatures(&transactions));
+        });
+    }
+}