Update Cargo.toml

And authors update

.buildkite/hooks/post-checkout

@@ -0,0 +1,2 @@
+CI_BUILD_START=$(date +%s)
+export CI_BUILD_START

.buildkite/hooks/post-checkout.sh

@@ -0,0 +1 @@
+post-checkout

.buildkite/hooks/post-command

@@ -0,0 +1,50 @@
+#!/bin/bash -e
+
+#
+# Save target/ for the next CI build on this machine
+#
+if [[ -n $CARGO_TARGET_CACHE_NAME ]]; then
+  (
+    d=$HOME/cargo-target-cache/"$CARGO_TARGET_CACHE_NAME"
+    mkdir -p "$d"
+    set -x
+    rsync -a --delete --link-dest="$PWD" target "$d"
+    du -hs "$d"
+  )
+fi
+
+#
+# Add job_stats data point
+#
+if [[ -z $CI_BUILD_START ]]; then
+  echo Error: CI_BUILD_START empty
+else
+  CI_BUILD_DURATION=$(( $(date +%s) - CI_BUILD_START + 1 ))
+
+  CI_LABEL=${BUILDKITE_LABEL:-build label missing}
+
+  PR=false
+  if [[ $BUILDKITE_BRANCH =~ pull/* ]]; then
+    PR=true
+  fi
+
+  SUCCESS=true
+  if [[ $BUILDKITE_COMMAND_EXIT_STATUS != 0 ]]; then
+    SUCCESS=false
+  fi
+
+  point_tags="pipeline=$BUILDKITE_PIPELINE_SLUG,job=$CI_LABEL,pr=$PR,success=$SUCCESS"
+  point_tags="${point_tags// /\\ }"  # Escape spaces
+
+  point_fields="duration=$CI_BUILD_DURATION"
+  point_fields="${point_fields// /\\ }"  # Escape spaces
+
+  point="job_stats,$point_tags $point_fields"
+  echo "Influx data point: $point"
+  if [[ -n $INFLUX_USERNAME && -n $INFLUX_PASSWORD ]]; then
+    echo "https://metrics.solana.com:8086/write?db=ci&u=${INFLUX_USERNAME}&p=${INFLUX_PASSWORD}" \
+      | xargs curl -XPOST --data-binary "$point"
+  else
+    echo Influx user credentials not found
+  fi
+fi

.buildkite/hooks/post-command.sh

@@ -0,0 +1 @@
+post-command

.buildkite/hooks/pre-command

@@ -0,0 +1,13 @@
+#!/bin/bash -e
+
+[[ -n "$CARGO_TARGET_CACHE_NAME" ]] || exit 0
+
+#
+# Restore target/ from the previous CI build on this machine
+#
+(
+  d=$HOME/cargo-target-cache/"$CARGO_TARGET_CACHE_NAME"
+  mkdir -p "$d"/target
+  set -x
+  rsync -a --delete --link-dest="$d" "$d"/target .
+)

.buildkite/hooks/pre-command.sh

@@ -0,0 +1 @@
+pre-command

.clippy.toml

@@ -0,0 +1 @@
+too-many-arguments-threshold = 9

.codecov.yml

@@ -1,2 +1,5 @@
 ignore:
   - "src/bin" 
+coverage:
+  status:
+    patch: off

.gitignore

@@ -1,3 +1,11 @@
-
+Cargo.lock
 /target/
 **/*.rs.bk
+.cargo
+
+# node configuration files
+/config/
+/config-private/
+/config-drone/
+/config-validator/
+/config-client/

.travis.yml

@@ -1,22 +0,0 @@
-language: rust
-required: sudo
-services:
-- docker
-matrix:
-  allow_failures:
-  - rust: nightly
-  include:
-  - rust: stable
-  - rust: nightly
-    env:
-    - FEATURES='asm,unstable'
-before_script: |
-  export PATH="$PATH:$HOME/.cargo/bin"
-  rustup component add rustfmt-preview
-script:
-- cargo fmt -- --write-mode=diff
-- cargo build --verbose --features "$FEATURES"
-- cargo test --verbose --features "$FEATURES"
-after_success: |
-  docker run -it --rm --security-opt seccomp=unconfined --volume "$PWD:/volume" elmtai/docker-rust-kcov
-  bash <(curl -s https://codecov.io/bash) -s target/cov

Cargo.lock

@@ -1,482 +0,0 @@
-[[package]]
-name = "arrayref"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "arrayvec"
-version = "0.4.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "bincode"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "bitflags"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "block-buffer"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "byte-tools"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "byteorder"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "cfg-if"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "chrono"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
- "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "crossbeam-deque"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "crossbeam-epoch 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "crossbeam-utils"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "digest"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "dtoa"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "either"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "fake-simd"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "fuchsia-zircon"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "fuchsia-zircon-sys"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "gcc"
-version = "0.3.54"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "generic-array"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
- "typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "generic-array"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "serde 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
- "typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "itoa"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "lazy_static"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "lazy_static"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "libc"
-version = "0.2.39"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "memoffset"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "nodrop"
-version = "0.1.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "num"
-version = "0.1.42"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)",
- "num-iter 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)",
- "num-traits 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "num-integer"
-version = "0.1.36"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "num-traits 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "num-iter"
-version = "0.1.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)",
- "num-traits 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "num-traits"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "num_cpus"
-version = "1.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "proc-macro2"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "quote"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rand"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rayon"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rayon"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "either 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "rayon-core"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.1.37"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "ring"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)",
- "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "scopeguard"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "serde"
-version = "1.0.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "serde_derive"
-version = "1.0.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "quote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_derive_internals 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "syn 0.12.13 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "serde_derive_internals"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "syn 0.12.13 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "serde_json"
-version = "1.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "num-traits 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "sha2"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "sha2-asm"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)",
- "generic-array 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "silk"
-version = "0.3.3"
-dependencies = [
- "bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "rayon 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "ring 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_derive 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_json 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "sha2 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "sha2-asm 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "syn"
-version = "0.12.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "quote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "time"
-version = "0.1.39"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)",
- "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "typenum"
-version = "1.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "unicode-xid"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "untrusted"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "winapi"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
-[metadata]
-"checksum arrayref 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0fd1479b7c29641adbd35ff3b5c293922d696a92f25c8c975da3e0acbc87258f"
-"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"
-"checksum bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bda13183df33055cbb84b847becce220d392df502ebe7a4a78d7021771ed94d0"
-"checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf"
-"checksum block-buffer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab"
-"checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40"
-"checksum byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "652805b7e73fada9d85e9a6682a4abd490cb52d96aeecc12e33a0de34dfd0d23"
-"checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de"
-"checksum chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7c20ebe0b2b08b0aeddba49c609fe7957ba2e33449882cb186a180bc60682fa9"
-"checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3"
-"checksum crossbeam-epoch 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "59796cc6cbbdc6bb319161349db0c3250ec73ec7fcb763a51065ec4e2e158552"
-"checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9"
-"checksum digest 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "00a49051fef47a72c9623101b19bd71924a45cca838826caae3eaa4d00772603"
-"checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab"
-"checksum either 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "740178ddf48b1a9e878e6d6509a1442a2d42fd2928aae8e7a6f8a36fb01981b3"
-"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
-"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
-"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
-"checksum gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)" = "5e33ec290da0d127825013597dbdfc28bee4964690c7ce1166cbc2a7bd08b1bb"
-"checksum generic-array 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fceb69994e330afed50c93524be68c42fa898c2d9fd4ee8da03bd7363acd26f2"
-"checksum generic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d"
-"checksum itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8324a32baf01e2ae060e9de58ed0bc2320c9a2833491ee36cd3b4c414de4db8c"
-"checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73"
-"checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d"
-"checksum libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)" = "f54263ad99207254cf58b5f701ecb432c717445ea2ee8af387334bdd1a03fdff"
-"checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3"
-"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2"
-"checksum num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e"
-"checksum num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f8d26da319fb45674985c78f1d1caf99aa4941f785d384a2ae36d0740bc3e2fe"
-"checksum num-iter 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "4b226df12c5a59b63569dd57fafb926d91b385dfce33d8074a412411b689d593"
-"checksum num-traits 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3c2bd9b9d21e48e956b763c9f37134dc62d9e95da6edb3f672cacb6caf3cd3"
-"checksum num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c51a3322e4bca9d212ad9a158a02abc6934d005490c054a2778df73a70aa0a30"
-"checksum proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cd07deb3c6d1d9ff827999c7f9b04cdfd66b1b17ae508e14fe47b620f2282ae0"
-"checksum quote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1eca14c727ad12702eb4b6bfb5a232287dcf8385cb8ca83a3eeaf6519c44c408"
-"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5"
-"checksum rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b614fe08b6665cb9a231d07ac1364b0ef3cb3698f1239ee0c4c3a88a524f54c8"
-"checksum rayon 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "485541959c8ecc49865526fe6c4de9653dd6e60d829d6edf0be228167b60372d"
-"checksum rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9d24ad214285a7729b174ed6d3bcfcb80177807f959d95fafd5bfc5c4f201ac8"
-"checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd"
-"checksum ring 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6f7d28b30a72c01b458428e0ae988d4149c20d902346902be881e3edc4bb325c"
-"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
-"checksum serde 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)" = "e928fecdb00fe608c96f83a012633383564e730962fc7a0b79225a6acf056798"
-"checksum serde_derive 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)" = "95f666a2356d87ce4780ea15b14b13532785579a5cad2dcba5292acc75f6efe2"
-"checksum serde_derive_internals 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1fc848d073be32cd982380c06587ea1d433bc1a4c4a111de07ec2286a3ddade8"
-"checksum serde_json 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)" = "57781ed845b8e742fc2bf306aba8e3b408fe8c366b900e3769fbc39f49eb8b39"
-"checksum sha2 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7daca11f2fdb8559c4f6c588386bed5e2ad4b6605c1442935a7f08144a918688"
-"checksum sha2-asm 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e319010fd740857efd4428b8ee1b74f311aeb0fda1ece174a9bad6741182d26"
-"checksum syn 0.12.13 (registry+https://github.com/rust-lang/crates.io-index)" = "517f6da31bc53bf080b9a77b29fbd0ff8da2f5a2ebd24c73c2238274a94ac7cb"
-"checksum time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "a15375f1df02096fb3317256ce2cee6a1f42fc84ea5ad5fc8c421cfe40c73098"
-"checksum typenum 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "13a99dc6780ef33c78780b826cf9d2a78840b72cae9474de4bcaf9051e60ebbd"
-"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
-"checksum untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f392d7819dbe58833e26872f5f6f0d68b7bbbe90fc3667e98731c4a15ad9a7ae"
-"checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3"
-"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

Cargo.toml

@@ -1,52 +1,63 @@
 [package]
-name = "silk"
-description = "A silky smooth implementation of the Loom architecture"
-version = "0.3.3"
-documentation = "https://docs.rs/silk"
-homepage = "http://loomprotocol.com/"
-repository = "https://github.com/loomprotocol/silk"
+name = "solana"
+description = "Blockchain, Rebuilt for Scale"
+version = "0.7.0"
+documentation = "https://docs.rs/solana"
+homepage = "http://solana.com/"
+readme = "README.md"
+repository = "https://github.com/solana-labs/solana"
 authors = [
-    "Anatoly Yakovenko <aeyakovenko@gmail.com>",
-    "Greg Fitzgerald <garious@gmail.com>",
+    "Anatoly Yakovenko <anatoly@solana.com>",
+    "Greg Fitzgerald <greg@solana.com>",
+    "Stephen Akridge <stephen@solana.com>",
+    "Michael Vines <mvines@solana.com>",
+    "Rob Walker <rob@solana.com>",
+    "Pankaj Garg <pankaj@solana.com>",
+    "Tyera Eulberg <tyera@solana.com>",
 ]
 license = "Apache-2.0"
 
 [[bin]]
-name = "silk-historian-demo"
-path = "src/bin/historian-demo.rs"
-
-[[bin]]
-name = "silk-client-demo"
+name = "solana-client-demo"
 path = "src/bin/client-demo.rs"
 
 [[bin]]
-name = "silk-testnode"
-path = "src/bin/testnode.rs"
+name = "solana-wallet"
+path = "src/bin/wallet.rs"
 
 [[bin]]
-name = "silk-genesis"
+name = "solana-fullnode"
+path = "src/bin/fullnode.rs"
+
+[[bin]]
+name = "solana-keygen"
+path = "src/bin/keygen.rs"
+
+[[bin]]
+name = "solana-fullnode-config"
+path = "src/bin/fullnode-config.rs"
+
+[[bin]]
+name = "solana-genesis"
 path = "src/bin/genesis.rs"
 
 [[bin]]
-name = "silk-genesis-demo"
-path = "src/bin/genesis-demo.rs"
-
-[[bin]]
-name = "silk-mint"
-path = "src/bin/mint.rs"
+name = "solana-drone"
+path = "src/bin/drone.rs"
 
 [badges]
-codecov = { repository = "loomprotocol/silk", branch = "master", service = "github" }
+codecov = { repository = "solana-labs/solana", branch = "master", service = "github" }
 
 [features]
 unstable = []
-asm = ["sha2-asm"]
+ipv6 = []
+cuda = []
+erasure = []
 
 [dependencies]
 rayon = "1.0.0"
 sha2 = "0.7.0"
-sha2-asm = {version="0.3", optional=true}
-generic-array = { version = "0.9.0", default-features = false, features = ["serde"] }
+generic-array = { version = "0.11.1", default-features = false, features = ["serde"] }
 serde = "1.0.27"
 serde_derive = "1.0.27"
 serde_json = "1.0.10"
@@ -54,3 +65,47 @@ ring = "0.12.1"
 untrusted = "0.5.1"
 bincode = "1.0.0"
 chrono = { version = "0.4.0", features = ["serde"] }
+log = "0.4.2"
+env_logger = "0.5.10"
+matches = "0.1.6"
+byteorder = "1.2.1"
+libc = "0.2.1"
+getopts = "0.2"
+atty = "0.2"
+rand = "0.5.1"
+pnet_datalink = "0.21.0"
+tokio = "0.1"
+tokio-codec = "0.1"
+tokio-core = "0.1.17"
+tokio-io = "0.1"
+itertools = "0.7.8"
+bs58 = "0.2.0"
+p2p = "0.5.2"
+futures = "0.1.21"
+clap = "2.31"
+reqwest = "0.8.6"
+influx_db_client = "0.3.4"
+dirs = "1.0.2"
+
+[dev-dependencies]
+criterion = "0.2"
+
+[[bench]]
+name = "bank"
+harness = false
+
+[[bench]]
+name = "banking_stage"
+harness = false
+
+[[bench]]
+name = "ledger"
+harness = false
+
+[[bench]]
+name = "signature"
+harness = false
+
+[[bench]]
+name = "streamer"
+harness = false

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2018 Anatoly Yakovenko <anatoly@loomprotocol.com> and Greg Fitzgerald <garious@gmail.com>
+Copyright 2018 Solana Labs, Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

README.md

@@ -1,29 +1,42 @@
-[![Silk crate](https://img.shields.io/crates/v/silk.svg)](https://crates.io/crates/silk)
-[![Silk documentation](https://docs.rs/silk/badge.svg)](https://docs.rs/silk)
-[![Build Status](https://travis-ci.org/loomprotocol/silk.svg?branch=master)](https://travis-ci.org/loomprotocol/silk)
-[![codecov](https://codecov.io/gh/loomprotocol/silk/branch/master/graph/badge.svg)](https://codecov.io/gh/loomprotocol/silk)
+[![Solana crate](https://img.shields.io/crates/v/solana.svg)](https://crates.io/crates/solana)
+[![Solana documentation](https://docs.rs/solana/badge.svg)](https://docs.rs/solana)
+[![Build status](https://badge.buildkite.com/d4c4d7da9154e3a8fb7199325f430ccdb05be5fc1e92777e51.svg?branch=master)](https://solana-ci-gate.herokuapp.com/buildkite_public_log?https://buildkite.com/solana-labs/solana/builds/latest/master)
+[![codecov](https://codecov.io/gh/solana-labs/solana/branch/master/graph/badge.svg)](https://codecov.io/gh/solana-labs/solana)
+
+Blockchain, Rebuilt for Scale
+===
+
+Solana™ is a new blockchain architecture built from the ground up for scale. The architecture supports
+up to 710 thousand transactions per second on a gigabit network.
 
 Disclaimer
 ===
 
 All claims, content, designs, algorithms, estimates, roadmaps, specifications, and performance measurements described in this project are done with the author's best effort.  It is up to the reader to check and validate their accuracy and truthfulness.  Furthermore nothing in this project constitutes a solicitation for investment.
 
-Silk, a silky smooth implementation of the Loom specification
+Introduction
 ===
 
-Loom™ is a new architecture for a high performance blockchain. Its white paper boasts a theoretical
-throughput of 710k transactions per second on a 1 gbps network. The specification is implemented
-in two git repositories. Research is performed in the loom repository. That work drives the
-Loom specification forward. This repository, on the other hand, aims to implement the specification
-as-is.  We care a great deal about quality, clarity and short learning curve. We avoid the use
-of `unsafe` Rust and write tests for *everything*.  Optimizations are only added when
-corresponding benchmarks are also added that demonstrate real performance boosts. We expect the
-feature set here will always be a ways behind the loom repo, but that this is an implementation
-you can take to the bank, literally.
+It's possible for a centralized database to process 710,000 transactions per second on a standard gigabit network if the transactions are, on average, no more than 176 bytes. A centralized database can also replicate itself and maintain high availability without significantly compromising that transaction rate using the distributed system technique known as Optimistic Concurrency Control [H.T.Kung, J.T.Robinson (1981)]. At Solana, we're demonstrating that these same theoretical limits apply just as well to blockchain on an adversarial network. The key ingredient? Finding a way to share time when nodes can't trust one-another. Once nodes can trust time, suddenly ~40 years of distributed systems research becomes applicable to blockchain! Furthermore, and much to our surprise, it can implemented using a mechanism that has existed in Bitcoin since day one. The Bitcoin feature is called nLocktime and it can be used to postdate transactions using block height instead of a timestamp. As a Bitcoin client, you'd use block height instead of a timestamp if you don't trust the network. Block height turns out to be an instance of what's being called a Verifiable Delay Function in cryptography circles. It's a cryptographically secure way to say time has passed. In Solana, we use a far more granular verifiable delay function, a SHA 256 hash chain, to checkpoint the ledger and coordinate consensus. With it, we implement Optimistic Concurrency Control and are now well in route towards that theoretical limit of 710,000 transactions per second.
 
-Running the demo
+
+Testnet Demos
 ===
 
+The Solana repo contains all the scripts you might need to spin up your own
+local testnet. Depending on what you're looking to achieve, you may want to
+run a different variation, as the full-fledged, performance-enhanced
+multinode testnet is considerably more complex to set up than a Rust-only,
+singlenode testnode.  If you are looking to develop high-level features, such
+as experimenting with smart contracts, save yourself some setup headaches and
+stick to the Rust-only singlenode demo.  If you're doing performance optimization
+of the transaction pipeline, consider the enhanced singlenode demo. If you're
+doing consensus work, you'll need at least a Rust-only multinode demo. If you want
+to reproduce our TPS metrics, run the enhanced multinode demo.
+
+For all four variations, you'd need the latest Rust toolchain and the Solana
+source code:
+
 First, install Rust's package manager Cargo.
 
 ```bash
@@ -31,60 +44,189 @@ $ curl https://sh.rustup.rs -sSf | sh
 $ source $HOME/.cargo/env
 ```
 
-Install the silk executables:
+Now checkout the code from github:
 
 ```bash
-    $ cargo install silk
+$ git clone https://github.com/solana-labs/solana.git
+$ cd solana
 ```
 
-The testnode server is initialized with a transaction log from stdin and
-generates new log entries on stdout. To create the input log, we'll need
-to create *the mint* and use it to generate a *genesis log*. It's done in
-two steps because the mint.json file contains a private key that will be
-used later in this demo.
+The demo code is sometimes broken between releases as we add new low-level
+features, so if this is your first time running the demo, you'll improve
+your odds of success if you check out the
+[latest release](https://github.com/solana-labs/solana/releases)
+before proceeding:
 
 ```bash
-    $ echo 500 | silk-mint > mint.json
-    $ cat mint.json | silk-genesis > genesis.log
+$ git checkout v0.7.0-beta
 ```
 
-Now you can start the server:
+Configuration Setup
+---
+
+The network is initialized with a genesis ledger and leader/validator configuration files.
+These files can be generated by running the following script.
 
 ```bash
-    $ cat genesis.log | silk-testnode > transactions0.log
+$ ./multinode-demo/setup.sh
 ```
 
-Then, in a separate shell, let's execute some transactions. Note we pass in
-the JSON configuration file here, not the genesis log.
+Singlenode Testnet
+---
+
+Before you start a fullnode, make sure you know the IP address of the machine you
+want to be the leader for the demo, and make sure that udp ports 8000-10000 are
+open on all the machines you want to test with.
+
+Now start the server:
 
 ```bash
-    $ cat mint.json | silk-client-demo
+$ ./multinode-demo/leader.sh
 ```
 
-Now kill the server with Ctrl-C, and take a look at the transaction log. You should
-see something similar to:
+Wait a few seconds for the server to initialize. It will print "Ready." when it's ready to
+receive transactions.
 
-```json
-{"num_hashes":27,"id":[0, "..."],"event":"Tick"}
-{"num_hashes":3,"id":[67, "..."],"event":{"Transaction":{"asset":42}}}
-{"num_hashes":27,"id":[0, "..."],"event":"Tick"}
-```
+Drone
+---
 
-Now restart the server from where we left off. Pass it both the genesis log, and
-the transaction log.
+In order for the below test client and validators to work, we'll also need to
+spin up a drone to give out some test tokens.  The drone delivers Milton
+Friedman-style "air drops" (free tokens to requesting clients) to be used in
+test transactions.
+
+Start the drone on the leader node with:
 
 ```bash
-    $ cat genesis.log transactions0.log | silk-testnode > transactions1.log
+$ ./multinode-demo/drone.sh
 ```
 
-Lastly, run the client demo again, and verify that all funds were spent in the
-previous round, and so no additional transactions are added.
+
+Multinode Testnet
+---
+
+To run a multinode testnet, after starting a leader node, spin up some validator nodes:
 
 ```bash
-    $ cat mint.json | silk-client-demo
+$ ./multinode-demo/validator.sh ubuntu@10.0.1.51:~/solana 10.0.1.51
 ```
 
-Stop the server again, and verify there are only Tick entries, and no Transaction entries.
+To run a performance-enhanced leader or validator (on Linux),
+[CUDA 9.2](https://developer.nvidia.com/cuda-downloads) must be installed on
+your system:
+```bash
+$ ./fetch-perf-libs.sh
+$ SOLANA_CUDA=1 ./multinode-demo/leader.sh
+$ SOLANA_CUDA=1 ./multinode-demo/validator.sh ubuntu@10.0.1.51:~/solana 10.0.1.51
+
+```
+
+
+
+Testnet Client Demo
+---
+
+Now that your singlenode or multinode testnet is up and running, in a separate shell, let's send it some transactions! Note we pass in
+the JSON configuration file here, not the genesis ledger.
+
+```bash
+$ ./multinode-demo/client.sh ubuntu@10.0.1.51:~/solana 2 #The leader machine and the total number of nodes in the network
+```
+
+What just happened? The client demo spins up several threads to send 500,000 transactions
+to the testnet as quickly as it can. The client then pings the testnet periodically to see
+how many transactions it processed in that time. Take note that the demo intentionally
+floods the network with UDP packets, such that the network will almost certainly drop a
+bunch of them. This ensures the testnet has an opportunity to reach 710k TPS. The client
+demo completes after it has convinced itself the testnet won't process any additional
+transactions. You should see several TPS measurements printed to the screen. In the
+multinode variation, you'll see TPS measurements for each validator node as well.
+
+Linux Snap
+---
+A Linux [Snap](https://snapcraft.io/) is available, which can be used to
+easily get Solana running on supported Linux systems without building anything
+from source.  The `edge` Snap channel is updated daily with the latest
+development from the `master` branch.  To install:
+```bash
+$ sudo snap install solana --edge --devmode
+```
+(`--devmode` flag is required only for `solana.fullnode-cuda`)
+
+Once installed the usual Solana programs will be available as `solona.*` instead
+of `solana-*`.  For example, `solana.fullnode` instead of `solana-fullnode`.
+
+Update to the latest version at any time with
+```bash
+$ snap info solana
+$ sudo snap refresh solana --devmode
+```
+
+### Daemon support
+The snap supports running a leader, validator or leader+drone node as a system
+daemon.
+
+Run `sudo snap get solana` to view the current daemon configuration, and
+`sudo snap logs -f solana` to view the daemon logs.
+
+Disable the daemon at any time by running:
+```bash
+$ sudo snap set solana mode=
+```
+
+Runtime configuration files for the daemon can be found in
+`/var/snap/solana/current/config`.
+
+#### Leader daemon
+```bash
+$ sudo snap set solana mode=leader
+```
+
+If CUDA is available:
+```bash
+$ sudo snap set solana mode=leader enable-cuda=1
+```
+
+`rsync` must be configured and running on the leader.
+
+1. Ensure rsync is installed with `sudo apt-get -y install rsync`
+2. Edit `/etc/rsyncd.conf` to include the following
+```
+[config]
+path = /var/snap/solana/current/config
+hosts allow = *
+read only = true
+```
+3. Run `sudo systemctl enable rsync; sudo systemctl start rsync`
+4. Test by running `rsync -Pzravv rsync://<ip-address-of-leader>/config
+solana-config` from another machine.  **If the leader is running on a cloud
+provider it may be necessary to configure the Firewall rules to permit ingress
+to port tcp:873, tcp:9900 and the port range udp:8000-udp:10000**
+
+
+To run both the Leader and Drone:
+```bash
+$ sudo snap set solana mode=leader+drone
+
+```
+
+#### Validator daemon
+```bash
+$ sudo snap set solana mode=validator
+
+```
+If CUDA is available:
+```bash
+$ sudo snap set solana mode=validator enable-cuda=1
+```
+
+By default the validator will connect to **testnet.solana.com**, override
+the leader IP address by running:
+```bash
+$ sudo snap set solana mode=validator leader-address=127.0.0.1 #<-- change IP address
+```
+It's assumed that the leader will be running `rsync` configured as described in
+the previous **Leader daemon** section.
 
 Developing
 ===
@@ -100,11 +242,22 @@ $ source $HOME/.cargo/env
 $ rustup component add rustfmt-preview
 ```
 
+If your rustc version is lower than 1.26.1, please update it:
+
+```bash
+$ rustup update
+```
+
+On Linux systems you may need to install libssl-dev and pkg-config.  On Ubuntu:
+```bash
+$ sudo apt-get install libssl-dev pkg-config
+```
+
 Download the source code:
 
 ```bash
-$ git clone https://github.com/loomprotocol/silk.git
-$ cd silk
+$ git clone https://github.com/solana-labs/solana.git
+$ cd solana
 ```
 
 Testing
@@ -113,9 +266,37 @@ Testing
 Run the test suite:
 
 ```bash
-cargo test
+$ cargo test
 ```
 
+To emulate all the tests that will run on a Pull Request, run:
+```bash
+$ ./ci/run-local.sh
+```
+
+Debugging
+---
+
+There are some useful debug messages in the code, you can enable them on a per-module and per-level
+basis with the normal RUST\_LOG environment variable. Run the fullnode with this syntax:
+```bash
+$ RUST_LOG=solana::streamer=debug,solana::server=info cat genesis.log | ./target/release/solana-fullnode > transactions0.log
+```
+to see the debug and info sections for streamer and server respectively. Generally
+we are using debug for infrequent debug messages, trace for potentially frequent messages and
+info for performance-related logging.
+
+Attaching to a running process with gdb
+
+```
+$ sudo gdb
+attach <PID>
+set logging on
+thread apply all bt
+```
+
+This will dump all the threads stack traces into gdb.txt
+
 Benchmarking
 ---
 
@@ -128,5 +309,37 @@ $ rustup install nightly
 Run the benchmarks:
 
 ```bash
-$ cargo +nightly bench --features="asm,unstable"
+$ cargo +nightly bench --features="unstable"
 ```
+
+Code coverage
+---
+
+To generate code coverage statistics, install cargo-cov. Note: the tool currently only works
+in Rust nightly.
+
+```bash
+$ cargo +nightly install cargo-cov
+```
+
+Run cargo-cov and generate a report:
+
+```bash
+$ cargo +nightly cov test
+$ cargo +nightly cov report --open
+```
+
+The coverage report will be written to `./target/cov/report/index.html`
+
+
+Why coverage? While most see coverage as a code quality metric, we see it primarily as a developer
+productivity metric. When a developer makes a change to the codebase, presumably it's a *solution* to
+some problem.  Our unit-test suite is how we encode the set of *problems* the codebase solves. Running
+the test suite should indicate that your change didn't *infringe* on anyone else's solutions. Adding a
+test *protects* your solution from future changes. Say you don't understand why a line of code exists,
+try deleting it and running the unit-tests. The nearest test failure should tell you what problem
+was solved by that code. If no test fails, go ahead and submit a Pull Request that asks, "what
+problem is solved by this code?" On the other hand, if a test does fail and you can think of a
+better way to solve the same problem, a Pull Request with your solution would most certainly be
+welcome! Likewise, if rewriting a test can better communicate what code it's protecting, please
+send us that patch!

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-slate

benches/bank.rs

@@ -0,0 +1,66 @@
+#[macro_use]
+extern crate criterion;
+extern crate bincode;
+extern crate rayon;
+extern crate solana;
+
+use bincode::serialize;
+use criterion::{Bencher, Criterion};
+use rayon::prelude::*;
+use solana::bank::*;
+use solana::hash::hash;
+use solana::mint::Mint;
+use solana::signature::{KeyPair, KeyPairUtil};
+use solana::transaction::Transaction;
+
+fn bench_process_transaction(bencher: &mut Bencher) {
+    let mint = Mint::new(100_000_000);
+    let bank = Bank::new(&mint);
+
+    // Create transactions between unrelated parties.
+    let transactions: Vec<_> = (0..4096)
+        .into_par_iter()
+        .map(|i| {
+            // Seed the 'from' account.
+            let rando0 = KeyPair::new();
+            let tx = Transaction::new(&mint.keypair(), rando0.pubkey(), 10_000, mint.last_id());
+            assert!(bank.process_transaction(&tx).is_ok());
+
+            // Seed the 'to' account and a cell for its signature.
+            let last_id = hash(&serialize(&i).unwrap()); // Unique hash
+            bank.register_entry_id(&last_id);
+
+            let rando1 = KeyPair::new();
+            let tx = Transaction::new(&rando0, rando1.pubkey(), 1, last_id);
+            assert!(bank.process_transaction(&tx).is_ok());
+
+            // Finally, return the transaction to the benchmark.
+            tx
+        })
+        .collect();
+
+    bencher.iter_with_setup(
+        || {
+            // Since benchmarker runs this multiple times, we need to clear the signatures.
+            bank.clear_signatures();
+            transactions.clone()
+        },
+        |transactions| {
+            let results = bank.process_transactions(transactions);
+            assert!(results.iter().all(Result::is_ok));
+        },
+    )
+}
+
+fn bench(criterion: &mut Criterion) {
+    criterion.bench_function("bench_process_transaction", |bencher| {
+        bench_process_transaction(bencher);
+    });
+}
+
+criterion_group!(
+    name = benches;
+    config = Criterion::default().sample_size(2);
+    targets = bench
+);
+criterion_main!(benches);

benches/banking_stage.rs

@@ -0,0 +1,226 @@
+extern crate bincode;
+#[macro_use]
+extern crate criterion;
+extern crate rayon;
+extern crate solana;
+
+use criterion::{Bencher, Criterion};
+use rayon::prelude::*;
+use solana::bank::Bank;
+use solana::banking_stage::BankingStage;
+use solana::mint::Mint;
+use solana::packet::{to_packets_chunked, PacketRecycler};
+use solana::record_stage::Signal;
+use solana::signature::{KeyPair, KeyPairUtil};
+use solana::transaction::Transaction;
+use std::iter;
+use std::sync::mpsc::{channel, Receiver};
+use std::sync::Arc;
+
+// use self::test::Bencher;
+// use bank::{Bank, MAX_ENTRY_IDS};
+// use bincode::serialize;
+// use hash::hash;
+// use mint::Mint;
+// use rayon::prelude::*;
+// use signature::{KeyPair, KeyPairUtil};
+// use std::collections::HashSet;
+// use std::time::Instant;
+// use transaction::Transaction;
+//
+// fn bench_process_transactions(_bencher: &mut Bencher) {
+//     let mint = Mint::new(100_000_000);
+//     let bank = Bank::new(&mint);
+//     // Create transactions between unrelated parties.
+//     let txs = 100_000;
+//     let last_ids: Mutex<HashSet<Hash>> = Mutex::new(HashSet::new());
+//     let transactions: Vec<_> = (0..txs)
+//         .into_par_iter()
+//         .map(|i| {
+//             // Seed the 'to' account and a cell for its signature.
+//             let dummy_id = i % (MAX_ENTRY_IDS as i32);
+//             let last_id = hash(&serialize(&dummy_id).unwrap()); // Semi-unique hash
+//             {
+//                 let mut last_ids = last_ids.lock().unwrap();
+//                 if !last_ids.contains(&last_id) {
+//                     last_ids.insert(last_id);
+//                     bank.register_entry_id(&last_id);
+//                 }
+//             }
+//
+//             // Seed the 'from' account.
+//             let rando0 = KeyPair::new();
+//             let tx = Transaction::new(&mint.keypair(), rando0.pubkey(), 1_000, last_id);
+//             bank.process_transaction(&tx).unwrap();
+//
+//             let rando1 = KeyPair::new();
+//             let tx = Transaction::new(&rando0, rando1.pubkey(), 2, last_id);
+//             bank.process_transaction(&tx).unwrap();
+//
+//             // Finally, return a transaction that's unique
+//             Transaction::new(&rando0, rando1.pubkey(), 1, last_id)
+//         })
+//         .collect();
+//
+//     let banking_stage = EventProcessor::new(bank, &mint.last_id(), None);
+//
+//     let now = Instant::now();
+//     assert!(banking_stage.process_transactions(transactions).is_ok());
+//     let duration = now.elapsed();
+//     let sec = duration.as_secs() as f64 + duration.subsec_nanos() as f64 / 1_000_000_000.0;
+//     let tps = txs as f64 / sec;
+//
+//     // Ensure that all transactions were successfully logged.
+//     drop(banking_stage.historian_input);
+//     let entries: Vec<Entry> = banking_stage.output.lock().unwrap().iter().collect();
+//     assert_eq!(entries.len(), 1);
+//     assert_eq!(entries[0].transactions.len(), txs as usize);
+//
+//     println!("{} tps", tps);
+// }
+
+fn check_txs(batches: usize, receiver: &Receiver<Signal>, ref_tx_count: usize) {
+    let mut total = 0;
+    for _ in 0..batches {
+        let signal = receiver.recv().unwrap();
+        if let Signal::Transactions(transactions) = signal {
+            total += transactions.len();
+        } else {
+            assert!(false);
+        }
+    }
+    assert_eq!(total, ref_tx_count);
+}
+
+fn bench_banking_stage_multi_accounts(bencher: &mut Bencher) {
+    let tx = 10_000_usize;
+    let mint_total = 1_000_000_000_000;
+    let mint = Mint::new(mint_total);
+    let num_dst_accounts = 8 * 1024;
+    let num_src_accounts = 8 * 1024;
+
+    let srckeys: Vec<_> = (0..num_src_accounts).map(|_| KeyPair::new()).collect();
+    let dstkeys: Vec<_> = (0..num_dst_accounts)
+        .map(|_| KeyPair::new().pubkey())
+        .collect();
+
+    let transactions: Vec<_> = (0..tx)
+        .map(|i| {
+            Transaction::new(
+                &srckeys[i % num_src_accounts],
+                dstkeys[i % num_dst_accounts],
+                i as i64,
+                mint.last_id(),
+            )
+        })
+        .collect();
+
+    let (verified_sender, verified_receiver) = channel();
+    let (signal_sender, signal_receiver) = channel();
+    let packet_recycler = PacketRecycler::default();
+
+    let setup_transactions: Vec<_> = (0..num_src_accounts)
+        .map(|i| {
+            Transaction::new(
+                &mint.keypair(),
+                srckeys[i].pubkey(),
+                mint_total / num_src_accounts as i64,
+                mint.last_id(),
+            )
+        })
+        .collect();
+
+    bencher.iter(move || {
+        let bank = Arc::new(Bank::new(&mint));
+
+        let verified_setup: Vec<_> =
+            to_packets_chunked(&packet_recycler, &setup_transactions.clone(), tx)
+                .into_iter()
+                .map(|x| {
+                    let len = (*x).read().unwrap().packets.len();
+                    (x, iter::repeat(1).take(len).collect())
+                })
+                .collect();
+
+        let verified_setup_len = verified_setup.len();
+        verified_sender.send(verified_setup).unwrap();
+        BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler)
+            .unwrap();
+
+        check_txs(verified_setup_len, &signal_receiver, num_src_accounts);
+
+        let verified: Vec<_> = to_packets_chunked(&packet_recycler, &transactions.clone(), 192)
+            .into_iter()
+            .map(|x| {
+                let len = (*x).read().unwrap().packets.len();
+                (x, iter::repeat(1).take(len).collect())
+            })
+            .collect();
+
+        let verified_len = verified.len();
+        verified_sender.send(verified).unwrap();
+        BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler)
+            .unwrap();
+
+        check_txs(verified_len, &signal_receiver, tx);
+    });
+}
+
+fn bench_banking_stage_single_from(bencher: &mut Bencher) {
+    let tx = 10_000_usize;
+    let mint = Mint::new(1_000_000_000_000);
+    let mut pubkeys = Vec::new();
+    let num_keys = 8;
+    for _ in 0..num_keys {
+        pubkeys.push(KeyPair::new().pubkey());
+    }
+
+    let transactions: Vec<_> = (0..tx)
+        .into_par_iter()
+        .map(|i| {
+            Transaction::new(
+                &mint.keypair(),
+                pubkeys[i % num_keys],
+                i as i64,
+                mint.last_id(),
+            )
+        })
+        .collect();
+
+    let (verified_sender, verified_receiver) = channel();
+    let (signal_sender, signal_receiver) = channel();
+    let packet_recycler = PacketRecycler::default();
+
+    bencher.iter(move || {
+        let bank = Arc::new(Bank::new(&mint));
+        let verified: Vec<_> = to_packets_chunked(&packet_recycler, &transactions.clone(), tx)
+            .into_iter()
+            .map(|x| {
+                let len = (*x).read().unwrap().packets.len();
+                (x, iter::repeat(1).take(len).collect())
+            })
+            .collect();
+        let verified_len = verified.len();
+        verified_sender.send(verified).unwrap();
+        BankingStage::process_packets(&bank, &verified_receiver, &signal_sender, &packet_recycler)
+            .unwrap();
+
+        check_txs(verified_len, &signal_receiver, tx);
+    });
+}
+
+fn bench(criterion: &mut Criterion) {
+    criterion.bench_function("bench_banking_stage_multi_accounts", |bencher| {
+        bench_banking_stage_multi_accounts(bencher);
+    });
+    criterion.bench_function("bench_process_stage_single_from", |bencher| {
+        bench_banking_stage_single_from(bencher);
+    });
+}
+
+criterion_group!(
+    name = benches;
+    config = Criterion::default().sample_size(2);
+    targets = bench
+);
+criterion_main!(benches);

benches/ledger.rs

@@ -0,0 +1,40 @@
+#[macro_use]
+extern crate criterion;
+extern crate solana;
+
+use criterion::{Bencher, Criterion};
+use solana::hash::{hash, Hash};
+use solana::ledger::{next_entries, reconstruct_entries_from_blobs, Block};
+use solana::packet::BlobRecycler;
+use solana::signature::{KeyPair, KeyPairUtil};
+use solana::transaction::Transaction;
+use std::collections::VecDeque;
+
+fn bench_block_to_blobs_to_block(bencher: &mut Bencher) {
+    let zero = Hash::default();
+    let one = hash(&zero);
+    let keypair = KeyPair::new();
+    let tx0 = Transaction::new(&keypair, keypair.pubkey(), 1, one);
+    let transactions = vec![tx0; 10];
+    let entries = next_entries(&zero, 1, transactions);
+
+    let blob_recycler = BlobRecycler::default();
+    bencher.iter(|| {
+        let mut blob_q = VecDeque::new();
+        entries.to_blobs(&blob_recycler, &mut blob_q);
+        assert_eq!(reconstruct_entries_from_blobs(blob_q).unwrap(), entries);
+    });
+}
+
+fn bench(criterion: &mut Criterion) {
+    criterion.bench_function("bench_block_to_blobs_to_block", |bencher| {
+        bench_block_to_blobs_to_block(bencher);
+    });
+}
+
+criterion_group!(
+    name = benches;
+    config = Criterion::default().sample_size(2);
+    targets = bench
+);
+criterion_main!(benches);

benches/signature.rs

@@ -0,0 +1,24 @@
+#[macro_use]
+extern crate criterion;
+extern crate solana;
+
+use criterion::{Bencher, Criterion};
+use solana::signature::GenKeys;
+
+fn bench_gen_keys(b: &mut Bencher) {
+    let rnd = GenKeys::new([0u8; 32]);
+    b.iter(|| rnd.gen_n_keypairs(1000));
+}
+
+fn bench(criterion: &mut Criterion) {
+    criterion.bench_function("bench_gen_keys", |bencher| {
+        bench_gen_keys(bencher);
+    });
+}
+
+criterion_group!(
+    name = benches;
+    config = Criterion::default().sample_size(2);
+    targets = bench
+);
+criterion_main!(benches);

benches/streamer.rs

@@ -0,0 +1,117 @@
+#[macro_use]
+extern crate log;
+extern crate solana;
+#[macro_use]
+extern crate criterion;
+
+use criterion::{Bencher, Criterion};
+use solana::packet::{Packet, PacketRecycler, BLOB_SIZE, PACKET_DATA_SIZE};
+use solana::result::Result;
+use solana::streamer::{receiver, PacketReceiver};
+use std::net::{SocketAddr, UdpSocket};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::channel;
+use std::sync::{Arc, Mutex};
+use std::thread::sleep;
+use std::thread::{spawn, JoinHandle};
+use std::time::Duration;
+use std::time::SystemTime;
+
+fn producer(addr: &SocketAddr, recycler: PacketRecycler, exit: Arc<AtomicBool>) -> JoinHandle<()> {
+    let send = UdpSocket::bind("0.0.0.0:0").unwrap();
+    let msgs = recycler.allocate();
+    let msgs_ = msgs.clone();
+    msgs.write().unwrap().packets.resize(10, Packet::default());
+    for w in msgs.write().unwrap().packets.iter_mut() {
+        w.meta.size = PACKET_DATA_SIZE;
+        w.meta.set_addr(&addr);
+    }
+    spawn(move || loop {
+        if exit.load(Ordering::Relaxed) {
+            return;
+        }
+        let mut num = 0;
+        for p in msgs_.read().unwrap().packets.iter() {
+            let a = p.meta.addr();
+            assert!(p.meta.size < BLOB_SIZE);
+            send.send_to(&p.data[..p.meta.size], &a).unwrap();
+            num += 1;
+        }
+        assert_eq!(num, 10);
+    })
+}
+
+fn sink(
+    recycler: PacketRecycler,
+    exit: Arc<AtomicBool>,
+    rvs: Arc<Mutex<usize>>,
+    r: PacketReceiver,
+) -> JoinHandle<()> {
+    spawn(move || loop {
+        if exit.load(Ordering::Relaxed) {
+            return;
+        }
+        let timer = Duration::new(1, 0);
+        match r.recv_timeout(timer) {
+            Ok(msgs) => {
+                *rvs.lock().unwrap() += msgs.read().unwrap().packets.len();
+                recycler.recycle(msgs);
+            }
+            _ => (),
+        }
+    })
+}
+
+fn bench_streamer_with_result() -> Result<()> {
+    let read = UdpSocket::bind("127.0.0.1:0")?;
+    read.set_read_timeout(Some(Duration::new(1, 0)))?;
+
+    let addr = read.local_addr()?;
+    let exit = Arc::new(AtomicBool::new(false));
+    let pack_recycler = PacketRecycler::default();
+
+    let (s_reader, r_reader) = channel();
+    let t_reader = receiver(read, exit.clone(), pack_recycler.clone(), s_reader);
+    let t_producer1 = producer(&addr, pack_recycler.clone(), exit.clone());
+    let t_producer2 = producer(&addr, pack_recycler.clone(), exit.clone());
+    let t_producer3 = producer(&addr, pack_recycler.clone(), exit.clone());
+
+    let rvs = Arc::new(Mutex::new(0));
+    let t_sink = sink(pack_recycler.clone(), exit.clone(), rvs.clone(), r_reader);
+
+    let start = SystemTime::now();
+    let start_val = *rvs.lock().unwrap();
+    sleep(Duration::new(5, 0));
+    let elapsed = start.elapsed().unwrap();
+    let end_val = *rvs.lock().unwrap();
+    let time = elapsed.as_secs() * 10000000000 + elapsed.subsec_nanos() as u64;
+    let ftime = (time as f64) / 10000000000f64;
+    let fcount = (end_val - start_val) as f64;
+    trace!("performance: {:?}", fcount / ftime);
+    exit.store(true, Ordering::Relaxed);
+    t_reader.join()?;
+    t_producer1.join()?;
+    t_producer2.join()?;
+    t_producer3.join()?;
+    t_sink.join()?;
+    Ok(())
+}
+
+fn bench_streamer(bencher: &mut Bencher) {
+    bencher.iter(|| {
+        bench_streamer_with_result().unwrap();
+    });
+}
+
+fn bench(criterion: &mut Criterion) {
+    criterion.bench_function("bench_streamer", |bencher| {
+        bench_streamer(bencher);
+    });
+}
+
+criterion_group!(
+    name = benches;
+    config = Criterion::default().sample_size(2);
+    targets = bench
+);
+criterion_main!(benches);

build.rs

@@ -0,0 +1,16 @@
+use std::env;
+
+fn main() {
+    println!("cargo:rustc-link-search=native=.");
+    if !env::var("CARGO_FEATURE_CUDA").is_err() {
+        println!("cargo:rustc-link-lib=static=cuda_verify_ed25519");
+        println!("cargo:rustc-link-search=native=/usr/local/cuda/lib64");
+        println!("cargo:rustc-link-lib=dylib=cudart");
+        println!("cargo:rustc-link-lib=dylib=cuda");
+        println!("cargo:rustc-link-lib=dylib=cudadevrt");
+    }
+    if !env::var("CARGO_FEATURE_ERASURE").is_err() {
+        println!("cargo:rustc-link-lib=dylib=Jerasure");
+        println!("cargo:rustc-link-lib=dylib=gf_complete");
+    }
+}

ci/.gitignore

@@ -0,0 +1,3 @@
+/node_modules/
+/package-lock.json
+/snapcraft.credentials

ci/README.md

@@ -0,0 +1,89 @@
+
+Our CI infrastructure is built around [BuildKite](https://buildkite.com) with some
+additional GitHub integration provided by https://github.com/mvines/ci-gate
+
+## Agent Queues
+
+We define two [Agent Queues](https://buildkite.com/docs/agent/v3/queues):
+`queue=default` and `queue=cuda`.  The `default` queue should be favored and
+runs on lower-cost CPU instances.  The `cuda` queue is only necessary for
+running **tests** that depend on GPU (via CUDA) access -- CUDA builds may still
+be run on the `default` queue, and the [buildkite artifact
+system](https://buildkite.com/docs/builds/artifacts) used to transfer build
+products over to a GPU instance for testing.
+
+## Buildkite Agent Management
+
+### Buildkite GCP Setup
+
+CI runs on Google Cloud Platform via two Compute Engine Instance groups:
+`ci-default` and `ci-cuda`.  Autoscaling is currently disabled and the number of
+VM Instances in each group is manually adjusted.
+
+#### Updating a CI Disk Image
+
+Each Instance group has its own disk image, `ci-default-vX` and
+`ci-cuda-vY`, where *X* and *Y* are incremented each time the image is changed.
+
+The process to update a disk image is as follows (TODO: make this less manual):
+
+1. Create a new VM Instance using the disk image to modify.
+2. Once the VM boots, ssh to it and modify the disk as desired.
+3. Stop the VM Instance running the modified disk.  Remember the name of the VM disk
+4. From another machine, `gcloud auth login`, then create a new Disk Image based
+off the modified VM Instance:
+```
+ $ gcloud compute images create ci-default-$(date +%Y%m%d%H%M) --source-disk xxx --source-disk-zone us-east1-b --family ci-default
+
+```
+or
+```
+  $ gcloud compute images create ci-cuda-$(date +%Y%m%d%H%M) --source-disk xxx --source-disk-zone us-east1-b --family ci-cuda
+```
+5. Delete the new VM instance.
+6. Go to the Instance templates tab, find the existing template named
+`ci-default-vX` or `ci-cuda-vY` and select it.  Use the "Copy" button to create
+a new Instance template called `ci-default-vX+1` or `ci-cuda-vY+1` with the
+newly created Disk image.
+7. Go to the Instance Groups tag and find the applicable group, `ci-default` or
+`ci-cuda`.  Edit the Instance Group in two steps: (a) Set the number of
+instances to 0 and wait for them all to terminate, (b) Update the Instance
+template and restore the number of instances to the original value.
+8. Clean up the previous version by deleting it from Instance Templates and
+Images. 
+
+
+## Reference
+
+### Buildkite AWS CloudFormation Setup
+
+**AWS CloudFormation is currently inactive, although it may be restored in the
+future**
+
+AWS CloudFormation can be used to scale machines up and down based on the
+current CI load.  If no machine is currently running it can take up to 60
+seconds to spin up a new instance, please remain calm during this time.
+
+#### AMI
+We use a custom AWS AMI built via https://github.com/solana-labs/elastic-ci-stack-for-aws/tree/solana/cuda.
+
+Use the following process to update this AMI as dependencies change:
+```bash
+$ export AWS_ACCESS_KEY_ID=my_access_key
+$ export AWS_SECRET_ACCESS_KEY=my_secret_access_key
+$ git clone https://github.com/solana-labs/elastic-ci-stack-for-aws.git -b solana/cuda
+$ cd elastic-ci-stack-for-aws/
+$ make build
+$ make build-ami
+```
+
+Watch for the *"amazon-ebs: AMI:"* log message to extract the name of the new
+AMI.  For example:
+```
+amazon-ebs: AMI: ami-07118545e8b4ce6dc
+```
+The new AMI should also now be visible in your EC2 Dashboard.  Go to the desired
+AWS CloudFormation stack, update the **ImageId** field to the new AMI id, and
+*apply* the stack changes.
+
+

ci/buildkite-snap.yml

@@ -0,0 +1,4 @@
+steps:
+  - command: "ci/snap.sh"
+    timeout_in_minutes: 40
+    name: "snap [public]"

ci/buildkite.yml

@@ -0,0 +1,44 @@
+steps:
+  - command: "ci/docker-run.sh rust ci/test-stable.sh"
+    name: "stable [public]"
+    env:
+      CARGO_TARGET_CACHE_NAME: "stable"
+    timeout_in_minutes: 30
+  - command: "ci/shellcheck.sh"
+    name: "shellcheck [public]"
+    timeout_in_minutes: 20
+  - command: "ci/docker-run.sh solanalabs/rust-nightly ci/test-nightly.sh"
+    name: "nightly [public]"
+    env:
+      CARGO_TARGET_CACHE_NAME: "nightly"
+    timeout_in_minutes: 30
+  - command: "ci/test-stable-perf.sh"
+    name: "stable-perf [public]"
+    env:
+      CARGO_TARGET_CACHE_NAME: "stable-perf"
+    timeout_in_minutes: 20
+    retry:
+      automatic:
+      - exit_status: "*"
+        limit: 2
+    agents:
+      - "queue=cuda"
+  - command: "ci/pr-snap.sh"
+    timeout_in_minutes: 20
+    name: "snap [public]"
+  - wait
+  - command: "ci/publish-crate.sh"
+    timeout_in_minutes: 20
+    name: "publish crate [public]"
+  - command: "ci/hoover.sh"
+    timeout_in_minutes: 20
+    name: "clean agent [public]"
+  - trigger: "solana-snap"
+    branches: "!pull/*"
+    async: true
+    build:
+      message: "${BUILDKITE_MESSAGE}"
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"
+      env:
+        TRIGGERED_BUILDKITE_TAG: "${BUILDKITE_TAG}"

ci/docker-run.sh

@@ -0,0 +1,49 @@
+#!/bin/bash -e
+
+usage() {
+  echo "Usage: $0 [docker image name] [command]"
+  echo
+  echo Runs command in the specified docker image with
+  echo a CI-appropriate environment
+  echo
+}
+
+cd "$(dirname "$0")/.."
+
+IMAGE="$1"
+if [[ -z "$IMAGE" ]]; then
+  echo Error: image not defined
+  exit 1
+fi
+
+docker pull "$IMAGE"
+shift
+
+ARGS=(
+  --workdir /solana
+  --volume "$PWD:/solana"
+  --volume "$HOME:/home"
+  --env "CARGO_HOME=/home/.cargo"
+  --rm
+)
+
+# kcov tries to set the personality of the binary which docker
+# doesn't allow by default.
+ARGS+=(--security-opt "seccomp=unconfined")
+
+# Ensure files are created with the current host uid/gid
+if [[ -z "$SOLANA_DOCKER_RUN_NOSETUID" ]]; then
+  ARGS+=(--user "$(id -u):$(id -g)")
+fi
+
+# Environment variables to propagate into the container
+ARGS+=(
+  --env BUILDKITE_BRANCH
+  --env BUILDKITE_TAG
+  --env CODECOV_TOKEN
+  --env CRATES_IO_TOKEN
+  --env SNAPCRAFT_CREDENTIALS_KEY
+)
+
+set -x
+exec docker run "${ARGS[@]}" "$IMAGE" "$@"

ci/docker-rust-nightly/Dockerfile

@@ -0,0 +1,6 @@
+FROM rustlang/rust:nightly
+
+RUN cargo install --force clippy cargo-cov && \
+    echo deb http://ftp.debian.org/debian stretch-backports main >> /etc/apt/sources.list && \
+    apt update && \
+    apt install -y llvm-6.0

ci/docker-rust-nightly/README.md

@@ -0,0 +1,6 @@
+Docker image containing rust nightly and some preinstalled crates used in CI.
+
+This image may be manually updated by running `./build.sh` if you are a member
+of the [Solana Labs](https://hub.docker.com/u/solanalabs/) Docker Hub
+organization, but it is also automatically updated periodically by
+[this automation](https://buildkite.com/solana-labs/solana-ci-docker-rust-nightly).

ci/docker-rust-nightly/build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -ex
+
+cd "$(dirname "$0")"
+
+docker build -t solanalabs/rust-nightly .
+docker push solanalabs/rust-nightly

ci/docker-snapcraft/Dockerfile

@@ -0,0 +1,7 @@
+FROM snapcraft/xenial-amd64
+
+# Update snapcraft to latest version
+RUN apt-get update -qq \
+ && apt-get install -y snapcraft daemontools \
+ && rm -rf /var/lib/apt/lists/* \
+ && snapcraft --version

ci/docker-snapcraft/build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -ex
+
+cd "$(dirname "$0")"
+
+docker build -t solanalabs/snapcraft .
+docker push solanalabs/snapcraft

ci/hoover.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Regular maintenance performed on a buildkite agent to control disk usage
+#
+
+echo --- Delete all exited containers first
+(
+  set -x
+  exited=$(docker ps -aq --no-trunc --filter "status=exited")
+  if [[ -n "$exited" ]]; then
+    # shellcheck disable=SC2086 # Don't want to double quote "$exited"
+    docker rm $exited
+  fi
+)
+
+echo --- Delete untagged images
+(
+  set -x
+  untagged=$(docker images | grep '<none>'| awk '{ print $3 }')
+  if [[ -n "$untagged" ]]; then
+    # shellcheck disable=SC2086 # Don't want to double quote "$untagged"
+    docker rmi $untagged
+  fi
+)
+
+echo --- Delete all dangling images
+(
+  set -x
+  dangling=$(docker images --filter 'dangling=true' -q --no-trunc | sort | uniq)
+  if [[ -n "$dangling" ]]; then
+    # shellcheck disable=SC2086 # Don't want to double quote "$dangling"
+    docker rmi $dangling
+  fi
+)
+
+echo --- Remove unused docker networks
+(
+  set -x
+  docker network prune -f
+)
+
+echo "--- Delete /tmp files older than 1 day owned by $(whoami)"
+(
+  set -x
+  find /tmp -maxdepth 1 -user "$(whoami)" -mtime +1 -print0 | xargs -0 rm -rf
+)
+
+echo --- System Status
+(
+  set -x
+  docker images
+  docker ps
+  docker network ls
+  df -h
+)
+
+exit 0

ci/is-pr.sh

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+#
+# The standard BUILDKITE_PULL_REQUEST environment variable is always "false" due
+# to how solana-ci-gate is used to trigger PR builds rather than using the
+# standard Buildkite PR trigger.
+#
+
+[[ $BUILDKITE_BRANCH =~ pull/* ]]

ci/pr-snap.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+#
+# Only run snap.sh for pull requests that modify files under /snap
+#
+
+cd "$(dirname "$0")"
+
+if ./is-pr.sh; then
+  affected_files="$(buildkite-agent meta-data get affected_files)"
+  echo "Affected files in this PR: $affected_files"
+  if [[ ! ":$affected_files:" =~ :snap/ ]]; then
+    echo "Skipping snap build as no files under /snap were modified"
+    exit 0
+  fi
+  exec ./snap.sh
+else
+  echo "Skipping snap build as this is not a pull request"
+fi

ci/publish-crate.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+if [[ -z "$BUILDKITE_TAG" ]]; then
+  # Skip publish if this is not a tagged release
+  exit 0
+fi
+
+if [[ -z "$CRATES_IO_TOKEN" ]]; then
+  echo CRATES_IO_TOKEN undefined
+  exit 1
+fi
+
+# TODO: Ensure the published version matches the contents of BUILDKITE_TAG
+ci/docker-run.sh rust \
+  bash -exc "cargo package; cargo publish --token $CRATES_IO_TOKEN"
+
+exit 0

ci/run-local.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+#
+# Run the entire buildkite CI pipeline locally for pre-testing before sending a
+# Github pull request
+#
+
+cd "$(dirname "$0")/.."
+BKRUN=ci/node_modules/.bin/bkrun
+
+if [[ ! -x $BKRUN ]]; then
+  (
+    set -x
+    cd ci/
+    npm install bkrun
+  )
+fi
+
+set -x
+exec ./ci/node_modules/.bin/bkrun ci/buildkite.yml

ci/shellcheck.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -e
+#
+# Reference: https://github.com/koalaman/shellcheck/wiki/Directive
+
+cd "$(dirname "$0")/.."
+
+set -x
+find . -name "*.sh" -not -regex ".*/.cargo/.*" -not -regex ".*/node_modules/.*" -print0 \
+  | xargs -0 \
+      ci/docker-run.sh koalaman/shellcheck --color=always --external-sources --shell=bash
+exit 0

ci/snap.sh

@@ -0,0 +1,53 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+DRYRUN=
+if [[ -z $BUILDKITE_BRANCH ]] || ./ci/is-pr.sh; then
+  DRYRUN="echo"
+fi
+
+# BUILDKITE_TAG is the normal environment variable set by Buildkite.  However
+# when this script is run from a triggered pipeline, TRIGGERED_BUILDKITE_TAG is
+# used instead of BUILDKITE_TAG (due to Buildkite limitations that prevents
+# BUILDKITE_TAG from propagating through to triggered pipelines)
+if [[ -z "$BUILDKITE_TAG" && -z "$TRIGGERED_BUILDKITE_TAG" ]]; then
+  SNAP_CHANNEL=edge
+else
+  SNAP_CHANNEL=beta
+fi
+
+if [[ -z $DRYRUN ]]; then
+  [[ -n $SNAPCRAFT_CREDENTIALS_KEY ]] || {
+    echo SNAPCRAFT_CREDENTIALS_KEY not defined
+    exit 1;
+  }
+  (
+    openssl aes-256-cbc -d \
+      -in ci/snapcraft.credentials.enc \
+      -out ci/snapcraft.credentials \
+      -k "$SNAPCRAFT_CREDENTIALS_KEY"
+
+    snapcraft login --with ci/snapcraft.credentials
+  ) || {
+    rm -f ci/snapcraft.credentials;
+    exit 1
+  }
+fi
+
+set -x
+
+echo --- checking for multilog
+if [[ ! -x /usr/bin/multilog ]]; then
+  echo "multilog not found, install with: sudo apt-get install -y daemontools"
+  exit 1
+fi
+
+echo --- build
+snapcraft
+
+source ci/upload_ci_artifact.sh
+upload_ci_artifact solana_*.snap
+
+echo --- publish
+$DRYRUN snapcraft push solana_*.snap --release $SNAP_CHANNEL

ci/test-nightly.sh

@@ -0,0 +1,32 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+export RUST_BACKTRACE=1
+rustc --version
+cargo --version
+
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+_ cargo build --verbose --features unstable
+_ cargo test --verbose --features unstable
+_ cargo clippy -- --deny=warnings
+
+exit 0
+
+# Coverage disabled (see issue #433)
+_ cargo cov test
+_ cargo cov report
+
+echo --- Coverage report:
+ls -l target/cov/report/index.html
+
+if [[ -z "$CODECOV_TOKEN" ]]; then
+  echo CODECOV_TOKEN undefined
+else
+  bash <(curl -s https://codecov.io/bash) -x 'llvm-cov-6.0 gcov'
+fi
+

ci/test-stable-perf.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+./fetch-perf-libs.sh
+
+export LD_LIBRARY_PATH=$PWD:/usr/local/cuda/lib64
+export PATH=$PATH:/usr/local/cuda/bin
+export RUST_BACKTRACE=1
+
+set -x
+exec cargo test --features=cuda,erasure

ci/test-stable.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+
+cd "$(dirname "$0")/.."
+
+export RUST_BACKTRACE=1
+rustc --version
+cargo --version
+
+_() {
+  echo "--- $*"
+  "$@"
+}
+
+_ rustup component add rustfmt-preview
+_ cargo fmt -- --write-mode=check
+_ cargo build --verbose
+_ cargo test --verbose
+_ cargo bench --verbose

ci/testnet-deploy.sh

@@ -0,0 +1,165 @@
+#!/bin/bash -e
+#
+# Deploys the Solana software running on the testnet full nodes
+#
+# This script must be run by a user/machine that has successfully authenticated
+# with GCP and has sufficient permission.
+#
+cd "$(dirname "$0")/.."
+
+# TODO: Switch over to rolling updates
+ROLLING_UPDATE=false
+#ROLLING_UPDATE=true
+
+if [[ -z $SOLANA_METRICS_CONFIG ]]; then
+  echo Error: SOLANA_METRICS_CONFIG environment variable is unset
+  exit 1
+fi
+
+# Default to edge channel.  To select the beta channel:
+#   export SOLANA_SNAP_CHANNEL=beta
+if [[ -z $SOLANA_SNAP_CHANNEL ]]; then
+  SOLANA_SNAP_CHANNEL=edge
+fi
+
+case $SOLANA_SNAP_CHANNEL in
+edge)
+  publicUrl=master.testnet.solana.com
+  publicIp=$(dig +short $publicUrl | head -n1)
+  ;;
+beta)
+  publicUrl=testnet.solana.com
+  publicIp="" # Use default value
+  ;;
+*)
+  echo Error: Unknown SOLANA_SNAP_CHANNEL=$SOLANA_SNAP_CHANNEL
+  exit 1
+  ;;
+esac
+
+resourcePrefix=${publicUrl//./-}
+vmlist=("$resourcePrefix":us-west1-b) # Leader is hard coded as the first entry
+validatorNamePrefix=$resourcePrefix-validator-
+
+echo "--- Available validators for $publicUrl"
+filter="name~^$validatorNamePrefix"
+gcloud compute instances list --filter="$filter"
+while read -r vmName vmZone status; do
+  if [[ $status != RUNNING ]]; then
+    echo "Warning: $vmName is not RUNNING, ignoring it."
+    continue
+  fi
+  vmlist+=("$vmName:$vmZone")
+done < <(gcloud compute instances list --filter="$filter" --format 'value(name,zone,status)')
+
+wait_for_node() {
+  declare pid=$1
+
+  declare ok=true
+  wait "$pid" || ok=false
+  cat "log-$pid.txt"
+  if ! $ok; then
+    echo ^^^ +++
+    exit 1
+  fi
+}
+
+if ! $ROLLING_UPDATE; then
+  count=1
+  for info in "${vmlist[@]}"; do
+    nodePosition="($count/${#vmlist[*]})"
+    vmName=${info%:*}
+    vmZone=${info#*:}
+    echo "--- Shutting down $vmName in zone $vmZone $nodePosition"
+    gcloud compute ssh "$vmName" --zone "$vmZone" \
+      --ssh-flag="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+      --command="echo sudo snap remove solana" &
+
+    if [[ $((count % 10)) = 0 ]]; then
+      #  Slow down deployment to avoid triggering GCP login
+      #  quota limits (each |ssh| counts as a login)
+      sleep 3
+    fi
+
+    count=$((count + 1))
+  done
+
+  wait
+fi
+
+echo "--- Refreshing leader for $publicUrl"
+leader=true
+pids=()
+count=1
+for info in "${vmlist[@]}"; do
+  nodePosition="($count/${#vmlist[*]})"
+
+  vmName=${info%:*}
+  vmZone=${info#*:}
+  echo "Starting refresh for $vmName $nodePosition"
+
+  (
+    SECONDS=0
+    echo "--- $vmName in zone $vmZone $nodePosition"
+    commonNodeConfig="\
+      rust-log=$RUST_LOG \
+      default-metrics-rate=$SOLANA_DEFAULT_METRICS_RATE \
+      metrics-config=$SOLANA_METRICS_CONFIG \
+    "
+    if $leader; then
+      nodeConfig="mode=leader+drone $commonNodeConfig"
+      if [[ -n $SOLANA_CUDA ]]; then
+        nodeConfig="$nodeConfig enable-cuda=1"
+      fi
+    else
+      nodeConfig="mode=validator leader-address=$publicIp $commonNodeConfig"
+    fi
+
+    set -x
+    gcloud compute ssh "$vmName" --zone "$vmZone" \
+      --ssh-flag="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -t" \
+      --command="\
+        set -ex; \
+        logmarker='solana deploy $(date)/$RANDOM'; \
+        sudo snap remove solana; \
+        logger \$logmarker; \
+        sudo snap install solana --$SOLANA_SNAP_CHANNEL --devmode; \
+        sudo snap set solana $nodeConfig; \
+        snap info solana; \
+        echo Slight delay to get more syslog output; \
+        sleep 2; \
+        sudo grep -Pzo \"\$logmarker(.|\\n)*\" /var/log/syslog \
+      "
+    echo "Succeeded in ${SECONDS} seconds"
+  ) > "log-$vmName.txt" 2>&1 &
+  pid=$!
+  # Rename log file so it can be discovered later by $pid
+  mv "log-$vmName.txt" "log-$pid.txt"
+
+  if $leader; then
+    echo Waiting for leader...
+    # Wait for the leader to initialize before starting the validators
+    # TODO: Remove this limitation eventually.
+    wait_for_node "$pid"
+
+    echo "--- Refreshing validators"
+  else
+    #  Slow down deployment to ~20 machines a minute to avoid triggering GCP login
+    #  quota limits (each |ssh| counts as a login)
+    sleep 3
+
+    pids+=("$pid")
+  fi
+  leader=false
+  count=$((count + 1))
+done
+
+echo --- Waiting for validators
+for pid in "${pids[@]}"; do
+  wait_for_node "$pid"
+done
+
+echo "--- $publicUrl sanity test"
+USE_SNAP=1 ci/testnet-sanity.sh $publicUrl
+
+exit 0

ci/testnet-sanity.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -e
+#
+# Perform a quick sanity test on the specific testnet
+#
+
+cd "$(dirname "$0")/.."
+
+TESTNET=$1
+if [[ -z $TESTNET ]]; then
+  TESTNET=testnet.solana.com
+fi
+
+echo "--- $TESTNET: wallet sanity"
+multinode-demo/test/wallet-sanity.sh $TESTNET
+
+echo --- fin
+exit 0

ci/upload_ci_artifact.sh

@@ -0,0 +1,18 @@
+# |source| me
+
+upload_ci_artifact() {
+  echo "--- artifact: $1"
+  if [[ -r "$1" ]]; then
+    ls -l "$1"
+    if ${BUILDKITE:-false}; then
+      (
+        set -x
+        buildkite-agent artifact upload "$1"
+      )
+    fi
+  else
+    echo ^^^ +++
+    echo "$1 not found"
+  fi
+}
+

doc/consensus.msc

@@ -0,0 +1,15 @@
+msc {
+  client,leader,verifier_a,verifier_b,verifier_c;
+
+  client=>leader [ label = "SUBMIT" ] ;
+  leader=>client [ label = "CONFIRMED" ] ;
+  leader=>verifier_a [ label = "CONFIRMED" ] ;
+  leader=>verifier_b [ label = "CONFIRMED" ] ;
+  leader=>verifier_c [ label = "CONFIRMED" ] ;
+  verifier_a=>leader [ label = "VERIFIED" ] ;
+  verifier_b=>leader [ label = "VERIFIED" ] ;
+  leader=>client [ label = "FINALIZED" ] ;
+  leader=>verifier_a [ label = "FINALIZED" ] ;
+  leader=>verifier_b [ label = "FINALIZED" ] ;
+  leader=>verifier_c [ label = "FINALIZED" ] ;
+}

doc/historian.md

@@ -1,65 +0,0 @@
-The Historian
-===
-
-Create a *Historian* and send it *events* to generate an *event log*, where each log *entry*
-is tagged with the historian's latest *hash*. Then ensure the order of events was not tampered
-with by verifying each entry's hash can be generated from the hash in the previous entry:
-
-![historian](https://user-images.githubusercontent.com/55449/36950845-459bdb58-1fb9-11e8-850e-894586f3729b.png)
-
-```rust
-extern crate silk;
-
-use silk::historian::Historian;
-use silk::log::{verify_slice, Entry, Hash};
-use silk::event::{generate_keypair, get_pubkey, sign_claim_data, Event};
-use std::thread::sleep;
-use std::time::Duration;
-use std::sync::mpsc::SendError;
-
-fn create_log(hist: &Historian<Hash>) -> Result<(), SendError<Event<Hash>>> {
-    sleep(Duration::from_millis(15));
-    let asset = Hash::default();
-    let keypair = generate_keypair();
-    let event0 = Event::new_claim(get_pubkey(&keypair), asset, sign_claim_data(&asset, &keypair));
-    hist.sender.send(event0)?;
-    sleep(Duration::from_millis(10));
-    Ok(())
-}
-
-fn main() {
-    let seed = Hash::default();
-    let hist = Historian::new(&seed, Some(10));
-    create_log(&hist).expect("send error");
-    drop(hist.sender);
-    let entries: Vec<Entry<Hash>> = hist.receiver.iter().collect();
-    for entry in &entries {
-        println!("{:?}", entry);
-    }
-    // Proof-of-History: Verify the historian learned about the events
-    // in the same order they appear in the vector.
-    assert!(verify_slice(&entries, &seed));
-}
-```
-
-Running the program should produce a log similar to:
-
-```rust
-Entry { num_hashes: 0, id: [0, ...], event: Tick }
-Entry { num_hashes: 3, id: [67, ...], event: Transaction { asset: [37, ...] } }
-Entry { num_hashes: 3, id: [123, ...], event: Tick }
-```
-
-Proof-of-History
----
-
-Take note of the last line:
-
-```rust
-assert!(verify_slice(&entries, &seed));
-```
-
-[It's a proof!](https://en.wikipedia.org/wiki/Curry–Howard_correspondence) For each entry returned by the
-historian, we can verify that `id` is the result of applying a sha256 hash to the previous `id`
-exactly `num_hashes` times, and then hashing then event data on top of that. Because the event data is
-included in the hash, the events cannot be reordered without regenerating all the hashes.

doc/historian.msc

@@ -1,18 +0,0 @@
-msc {
-  client,historian,logger;
-
-  logger=>historian [ label = "e0 = Entry{id: h0, n: 0, event: Tick}" ] ;
-  logger=>logger [ label = "h1 = hash(h0)" ] ;
-  logger=>logger [ label = "h2 = hash(h1)" ] ;
-  client=>historian [ label = "Transaction(d0)" ] ;
-  historian=>logger [ label = "Transaction(d0)" ] ;
-  logger=>logger [ label = "h3 = hash(h2 + d0)" ] ;
-  logger=>historian [ label = "e1 = Entry{id: hash(h3), n: 3, event: Transaction(d0)}" ] ;
-  logger=>logger [ label = "h4 = hash(h3)" ] ;
-  logger=>logger [ label = "h5 = hash(h4)" ] ;
-  logger=>logger [ label = "h6 = hash(h5)" ] ;
-  logger=>historian [ label = "e2 = Entry{id: h6, n: 3, event: Tick}" ] ;
-  client=>historian [ label = "collect()" ] ;
-  historian=>client [ label = "entries = [e0, e1, e2]" ] ;
-  client=>client [ label = "verify_slice(entries, h0)" ] ;
-}

fetch-perf-libs.sh

@@ -0,0 +1,37 @@
+#!/bin/bash -e
+
+if [[ $(uname) != Linux ]]; then
+  echo Performance libraries are only available for Linux
+  exit 1
+fi
+
+if [[ $(uname -m) != x86_64 ]]; then
+  echo Performance libraries are only available for x86_64 architecture
+  exit 1
+fi
+
+(
+  set -x
+  curl -o solana-perf.tgz \
+    https://solana-perf.s3.amazonaws.com/master/x86_64-unknown-linux-gnu/solana-perf.tgz
+  tar zxvf solana-perf.tgz
+)
+
+if [[ -r /usr/local/cuda/version.txt && -r cuda-version.txt ]]; then
+  if ! diff /usr/local/cuda/version.txt cuda-version.txt > /dev/null; then
+    echo ==============================================
+    echo Warning: possible CUDA version mismatch
+    echo
+    echo "Expected version: $(cat cuda-version.txt)"
+    echo "Detected version: $(cat /usr/local/cuda/version.txt)"
+    echo ==============================================
+  fi
+else
+  echo ==============================================
+  echo Warning: unable to validate CUDA version
+  echo ==============================================
+fi
+
+echo "Downloaded solana-perf version: $(cat solana-perf-HEAD.txt)"
+
+exit 0

multinode-demo/client.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# usage: $0 <rsync network path to solana repo on leader machine> <number of nodes in the network>"
+#
+
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+leader=$1
+if [[ -z $leader ]]; then
+  if [[ -d "$SNAP" ]]; then
+    leader=testnet.solana.com # Default to testnet when running as a Snap
+  else
+    leader=$here/.. # Default to local solana repo
+  fi
+fi
+count=${2:-1}
+
+rsync_leader_url=$(rsync_url "$leader")
+
+set -ex
+mkdir -p "$SOLANA_CONFIG_CLIENT_DIR"
+$rsync -vPz "$rsync_leader_url"/config/leader.json "$SOLANA_CONFIG_CLIENT_DIR"/
+
+client_json="$SOLANA_CONFIG_CLIENT_DIR"/client.json
+[[ -r $client_json ]] || $solana_keygen -o "$client_json"
+
+$solana_client_demo \
+  -n "$count" \
+  -l "$SOLANA_CONFIG_CLIENT_DIR"/leader.json \
+  -k "$SOLANA_CONFIG_CLIENT_DIR"/client.json \
+

multinode-demo/common.sh

@@ -0,0 +1,168 @@
+# |source| this file
+#
+# Disable complaints about unused variables in this file:
+# shellcheck disable=2034
+
+# shellcheck disable=2154 # 'here' is referenced but not assigned
+if [[ -z $here ]]; then
+  echo "|here| is not defined"
+  exit 1
+fi
+
+rsync=rsync
+leader_logger="cat"
+validator_logger="cat"
+drone_logger="cat"
+
+if [[ -d "$SNAP" ]]; then # Running inside a Linux Snap?
+  solana_program() {
+    declare program="$1"
+    if [[ "$program" = wallet || "$program" = client-demo ]]; then
+      # TODO: Merge wallet.sh/client.sh functionality into
+      #       solana-wallet/solana-demo-client proper and remove this special case
+      printf "%s/bin/solana-%s" "$SNAP" "$program"
+    else
+      printf "%s/command-%s.wrapper" "$SNAP" "$program"
+    fi
+  }
+  rsync="$SNAP"/bin/rsync
+  multilog="$SNAP/bin/multilog t s16777215"
+  leader_logger="$multilog $SNAP_DATA/leader"
+  validator_logger="$multilog t $SNAP_DATA/validator"
+  drone_logger="$multilog $SNAP_DATA/drone"
+  # Create log directories manually to prevent multilog from creating them as
+  # 0700
+  mkdir -p "$SNAP_DATA"/{drone,leader,validator}
+
+  SOLANA_METRICS_CONFIG="$(snapctl get metrics-config)"
+  SOLANA_DEFAULT_METRICS_RATE="$(snapctl get default-metrics-rate)"
+  SOLANA_CUDA="$(snapctl get enable-cuda)"
+  RUST_LOG="$(snapctl get rust-log)"
+
+elif [[ -n "$USE_SNAP" ]]; then # Use the Linux Snap binaries
+  solana_program() {
+    declare program="$1"
+    printf "solana.%s" "$program"
+  }
+elif [[ -n "$USE_INSTALL" ]]; then # Assume |cargo install| was run
+  solana_program() {
+    declare program="$1"
+    printf "solana-%s" "$program"
+  }
+  # CUDA was/wasn't selected at build time, can't affect CUDA state here
+  unset SOLANA_CUDA
+else
+  solana_program() {
+    declare program="$1"
+    declare features=""
+    if [[ "$program" =~ ^(.*)-cuda$ ]]; then
+      program=${BASH_REMATCH[1]}
+      features="--features=cuda"
+    fi
+    if [[ -z "$DEBUG" ]]; then
+      maybe_release=--release
+    fi
+    printf "cargo run $maybe_release --bin solana-%s %s -- " "$program" "$features"
+  }
+  if [[ -n $SOLANA_CUDA ]]; then
+    # Locate perf libs downloaded by |./fetch-perf-libs.sh|
+    LD_LIBRARY_PATH=$(cd "$here" && dirname "$PWD"):$LD_LIBRARY_PATH
+    export LD_LIBRARY_PATH
+  fi
+fi
+
+solana_client_demo=$(solana_program client-demo)
+solana_wallet=$(solana_program wallet)
+solana_drone=$(solana_program drone)
+solana_fullnode=$(solana_program fullnode)
+solana_fullnode_config=$(solana_program fullnode-config)
+solana_fullnode_cuda=$(solana_program fullnode-cuda)
+solana_genesis=$(solana_program genesis)
+solana_keygen=$(solana_program keygen)
+
+export RUST_LOG=${RUST_LOG:-solana=info} # if RUST_LOG is unset, default to info
+export RUST_BACKTRACE=1
+
+
+# The SOLANA_METRICS_CONFIG environment variable is formatted as a
+# comma-delimited list of parameters. All parameters are optional.
+#
+# Example:
+#   export SOLANA_METRICS_CONFIG="host=<metrics host>,db=<database name>,u=<username>,p=<password>"
+#
+configure_metrics() {
+  [[ -n $SOLANA_METRICS_CONFIG ]] || return
+
+  declare metrics_params
+  IFS=',' read -r -a metrics_params <<< "$SOLANA_METRICS_CONFIG"
+  for param in "${metrics_params[@]}"; do
+    IFS='=' read -r -a pair <<< "$param"
+    if [[ "${#pair[@]}" != 2 ]]; then
+      echo Error: invalid metrics parameter: "$param" >&2
+    else
+      declare name="${pair[0]}"
+      declare value="${pair[1]}"
+      case "$name" in
+      host)
+        export INFLUX_HOST="$value"
+        echo INFLUX_HOST="$INFLUX_HOST" >&2
+        ;;
+      db)
+        export INFLUX_DATABASE="$value"
+        echo INFLUX_DATABASE="$INFLUX_DATABASE" >&2
+        ;;
+      u)
+        export INFLUX_USERNAME="$value"
+        echo INFLUX_USERNAME="$INFLUX_USERNAME" >&2
+        ;;
+      p)
+        export INFLUX_PASSWORD="$value"
+        echo INFLUX_PASSWORD="********" >&2
+        ;;
+      *)
+        echo Error: Unknown metrics parameter name: "$name" >&2
+        ;;
+      esac
+    fi
+  done
+}
+configure_metrics
+
+tune_networking() {
+  # Reference: https://medium.com/@CameronSparr/increase-os-udp-buffers-to-improve-performance-51d167bb1360
+  if [[ $(uname) = Linux ]]; then
+    (
+      set -x +e
+      # test the existence of the sysctls before trying to set them
+      # go ahead and return true and don't exit if these calls fail
+      sysctl net.core.rmem_max 2>/dev/null 1>/dev/null &&
+          sudo sysctl -w net.core.rmem_max=26214400 1>/dev/null 2>/dev/null
+
+      sysctl net.core.rmem_default 2>/dev/null 1>/dev/null &&
+          sudo sysctl -w net.core.rmem_default=26214400 1>/dev/null 2>/dev/null
+    ) || true
+  fi
+}
+
+SOLANA_CONFIG_DIR=${SNAP_DATA:-$PWD}/config
+SOLANA_CONFIG_PRIVATE_DIR=${SNAP_DATA:-$PWD}/config-private
+SOLANA_CONFIG_CLIENT_DIR=${SNAP_USER_DATA:-$PWD}/config-client
+
+rsync_url() { # adds the 'rsync://` prefix to URLs that need it
+  declare url="$1"
+
+  if [[ "$url" =~ ^.*:.*$ ]]; then
+    # assume remote-shell transport when colon is present, use $url unmodified
+    echo "$url"
+    return
+  fi
+
+  if [[ -d "$url" ]]; then
+    # assume local directory if $url is a valid directory, use $url unmodified
+    echo "$url"
+    return
+  fi
+
+  # Default to rsync:// URL
+  echo "rsync://$url"
+}

multinode-demo/drone.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# usage: $0 <rsync network path to solana repo on leader machine>
+#
+
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+SOLANA_CONFIG_DIR="$SOLANA_CONFIG_DIR"-drone
+
+if [[ -d "$SNAP" ]]; then
+  # Exit if mode is not yet configured
+  # (typically the case after the Snap is first installed)
+  [[ -n "$(snapctl get mode)" ]] || exit 0
+
+  # Select leader from the Snap configuration
+  leader_address="$(snapctl get leader-address)"
+  if [[ -z "$leader_address" ]]; then
+    # Assume drone is running on the same node as the leader by default
+    leader_address="localhost"
+  fi
+  leader="$leader_address"
+else
+  leader=${1:-${here}/..}  # Default to local solana repo
+fi
+
+[[ -f "$SOLANA_CONFIG_PRIVATE_DIR"/mint.json ]] || {
+  echo "$SOLANA_CONFIG_PRIVATE_DIR/mint.json not found, create it by running:"
+  echo
+  echo "  ${here}/setup.sh -t leader"
+  exit 1
+}
+
+rsync_leader_url=$(rsync_url "$leader")
+set -ex
+mkdir -p "$SOLANA_CONFIG_DIR"
+$rsync -vPz "$rsync_leader_url"/config/leader.json "$SOLANA_CONFIG_DIR"/
+
+set -o pipefail
+$solana_drone \
+  -l "$SOLANA_CONFIG_DIR"/leader.json -k "$SOLANA_CONFIG_PRIVATE_DIR"/mint.json \
+2>&1 | $drone_logger

multinode-demo/gce_multinode.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+command=$1
+prefix=
+num_nodes=
+out_file=
+image_name="ubuntu-16-04-cuda-9-2-new"
+
+shift
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 <create|delete> <-p prefix> <-n num_nodes> <-o file> [-i image-name]
+
+Manage a GCE multinode network
+
+ create|delete    - Create or delete the network
+ -p prefix        - A common prefix for node names, to avoid collision
+ -n num_nodes     - Number of nodes
+ -o out_file      - Used for create option. Outputs an array of IP addresses
+                    of new nodes to the file
+ -i image_name    - Existing image on GCE (default $image_name)
+
+EOF
+  exit $exitcode
+}
+
+while getopts "h?p:i:n:o:" opt; do
+  case $opt in
+  h | \?)
+    usage
+    ;;
+  p)
+    prefix=$OPTARG
+    ;;
+  i)
+    image_name=$OPTARG
+    ;;
+  o)
+    out_file=$OPTARG
+    ;;
+  n)
+    num_nodes=$OPTARG
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+
+set -e
+
+[[ -n $command ]] || usage "Need a command (create|delete)"
+
+[[ -n $prefix ]] || usage "Need a prefix for GCE instance names"
+
+[[ -n $num_nodes ]] || usage "Need number of nodes"
+
+nodes=()
+for i in $(seq 1 "$num_nodes"); do
+  nodes+=("$prefix$i")
+done
+
+if [[ $command == "create" ]]; then
+  [[ -n $out_file ]] || usage "Need an outfile to store IP Addresses"
+
+  ip_addr_list=$(gcloud beta compute instances create "${nodes[@]}" --zone=us-west1-b --tags=testnet \
+    --image="$image_name" | awk '/RUNNING/ {print $5}')
+
+  echo "ip_addr_array=($ip_addr_list)" >"$out_file"
+elif [[ $command == "delete" ]]; then
+  gcloud beta compute instances delete "${nodes[@]}"
+else
+  usage "Unknown command: $command"
+fi

multinode-demo/leader.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+if [[ -d "$SNAP" ]]; then
+  # Exit if mode is not yet configured
+  # (typically the case after the Snap is first installed)
+  [[ -n "$(snapctl get mode)" ]] || exit 0
+fi
+
+[[ -f "$SOLANA_CONFIG_DIR"/leader.json ]] || {
+  echo "$SOLANA_CONFIG_DIR/leader.json not found, create it by running:"
+  echo
+  echo "  ${here}/setup.sh -t leader"
+  exit 1
+}
+
+if [[ -n "$SOLANA_CUDA" ]]; then
+  program="$solana_fullnode_cuda"
+else
+  program="$solana_fullnode"
+fi
+
+tune_networking
+
+set -xo pipefail
+$program \
+  --identity "$SOLANA_CONFIG_DIR"/leader.json \
+  --ledger "$SOLANA_CONFIG_DIR"/ledger.log \
+2>&1 | $leader_logger

multinode-demo/remote_leader.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+[[ -n $FORCE ]] || exit
+
+chmod 600 ~/.ssh/authorized_keys ~/.ssh/id_rsa
+
+PATH="$HOME"/.cargo/bin:"$PATH"
+
+./fetch-perf-libs.sh
+
+# Run setup
+USE_INSTALL=1 ./multinode-demo/setup.sh -p
+USE_INSTALL=1 SOLANA_CUDA=1 ./multinode-demo/leader.sh >leader.log 2>&1 &
+USE_INSTALL=1 ./multinode-demo/drone.sh >drone.log 2>&1 &

multinode-demo/remote_nodes.sh

@@ -0,0 +1,185 @@
+#!/bin/bash
+
+command=$1
+ip_addr_file=
+remote_user=
+ssh_keys=
+
+shift
+
+usage() {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 <start|stop> <-f IP Addr Array file> <-u username> [-k ssh-keys]
+
+Manage a GCE multinode network
+
+ start|stop    - Create or delete the network
+ -f file       - A bash script that exports an array of IP addresses, ip_addr_array.
+                 Elements of the array are public IP address of remote nodes.
+ -u username   - The username for logging into remote nodes.
+ -k ssh-keys   - Path to public/private key pair that remote nodes can use to perform
+                 rsync and ssh among themselves. Must contain pub, and priv keys.
+
+EOF
+  exit $exitcode
+}
+
+while getopts "h?f:u:k:" opt; do
+  case $opt in
+  h | \?)
+    usage
+    ;;
+  f)
+    ip_addr_file=$OPTARG
+    ;;
+  u)
+    remote_user=$OPTARG
+    ;;
+  k)
+    ssh_keys=$OPTARG
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+
+set -e
+
+# Sample IP Address array file contents
+# ip_addr_array=(192.168.1.1 192.168.1.5 192.168.2.2)
+
+[[ -n $command ]] || usage "Need a command (start|stop)"
+[[ -n $ip_addr_file ]] || usage "Need a file with IP address array"
+[[ -n $remote_user ]] || usage "Need the username for remote nodes"
+
+ip_addr_array=()
+# Get IP address array
+# shellcheck source=/dev/null
+source "$ip_addr_file"
+
+build_project() {
+  echo "Build started at $(date)"
+  SECONDS=0
+
+  # Build and install locally
+  PATH="$HOME"/.cargo/bin:"$PATH"
+  cargo install --force
+
+  echo "Build took $SECONDS seconds"
+}
+
+common_start_setup() {
+  ip_addr=$1
+
+  # Killing sshguard for now. TODO: Find a better solution
+  # sshguard is blacklisting IP address after ssh-keyscan and ssh login attempts
+  ssh "$remote_user@$ip_addr" " \
+    set -ex; \
+    sudo service sshguard stop; \
+    sudo apt-get --assume-yes install rsync libssl-dev; \
+    mkdir -p ~/.ssh ~/solana ~/.cargo/bin; \
+  " >log/"$ip_addr".log
+
+  # If provided, deploy SSH keys
+  if [[ -n $ssh_keys ]]; then
+    {
+      rsync -vPrz "$ssh_keys"/id_rsa "$remote_user@$ip_addr":~/.ssh/
+      rsync -vPrz "$ssh_keys"/id_rsa.pub "$remote_user@$ip_addr":~/.ssh/
+      rsync -vPrz "$ssh_keys"/id_rsa.pub "$remote_user@$ip_addr":~/.ssh/authorized_keys
+      rsync -vPrz ./multinode-demo "$remote_user@$ip_addr":~/solana/
+    } >>log/"$ip_addr".log
+  fi
+}
+
+start_leader() {
+  common_start_setup "$1"
+
+  {
+    rsync -vPrz ~/.cargo/bin/solana* "$remote_user@$ip_addr":~/.cargo/bin/
+    rsync -vPrz ./fetch-perf-libs.sh "$remote_user@$ip_addr":~/solana/
+    ssh -n -f "$remote_user@$ip_addr" 'cd solana; FORCE=1 ./multinode-demo/remote_leader.sh'
+  } >>log/"$1".log
+
+  leader_ip=$1
+  leader_time=$SECONDS
+  SECONDS=0
+}
+
+start_validator() {
+  common_start_setup "$1"
+
+  ssh -n -f "$remote_user@$ip_addr" "cd solana; FORCE=1 ./multinode-demo/remote_validator.sh $leader_ip" >>log/"$1".log
+}
+
+start_all_nodes() {
+  echo "Deployment started at $(date)"
+  SECONDS=0
+  count=0
+  leader_ip=
+  leader_time=
+
+  mkdir -p log
+
+  for ip_addr in "${ip_addr_array[@]}"; do
+    if ((!count)); then
+      # Start the leader on the first node
+      echo "Leader node $ip_addr, killing previous instance and restarting"
+      start_leader "$ip_addr"
+    else
+      # Start validator on all other nodes
+      echo "Validator[$count] node $ip_addr, killing previous instance and restarting"
+      start_validator "$ip_addr" &
+      # TBD: Remove the sleep or reduce time once GCP login quota is increased
+      sleep 2
+    fi
+
+    ((count = count + 1))
+  done
+
+  wait
+
+  ((validator_count = count - 1))
+
+  echo "Deployment finished at $(date)"
+  echo "Leader deployment too $leader_time seconds"
+  echo "$validator_count Validator deployment took $SECONDS seconds"
+}
+
+stop_all_nodes() {
+  SECONDS=0
+  local count=0
+  for ip_addr in "${ip_addr_array[@]}"; do
+    ssh-keygen -R "$ip_addr" >log/local.log
+    ssh-keyscan "$ip_addr" >>~/.ssh/known_hosts 2>/dev/null
+
+    echo "Stopping node[$count] $ip_addr. Remote user $remote_user"
+
+    ssh -n -f "$remote_user@$ip_addr" " \
+    set -ex; \
+    sudo service sshguard stop; \
+    pkill -9 solana-; \
+    pkill -9 validator; \
+    pkill -9 leader; \
+    "
+    sleep 2
+    ((count = count + 1))
+    echo "Stopped node[$count] $ip_addr"
+  done
+  echo "Stopping $count nodes took $SECONDS seconds"
+}
+
+if [[ $command == "start" ]]; then
+  build_project
+  stop_all_nodes
+  start_all_nodes
+elif [[ $command == "stop" ]]; then
+  stop_all_nodes
+else
+  usage "Unknown command: $command"
+fi

multinode-demo/remote_validator.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -e
+
+[[ -n $FORCE ]] || exit
+
+chmod 600 ~/.ssh/authorized_keys ~/.ssh/id_rsa
+
+PATH="$HOME"/.cargo/bin:"$PATH"
+
+touch ~/.ssh/known_hosts
+ssh-keygen -R "$1" 2>/dev/null
+ssh-keyscan "$1" >>~/.ssh/known_hosts 2>/dev/null
+
+rsync -vPrz "$1":~/.cargo/bin/solana* ~/.cargo/bin/
+
+# Run setup
+USE_INSTALL=1 ./multinode-demo/setup.sh -p
+USE_INSTALL=1 ./multinode-demo/validator.sh "$1":~/solana "$1" >validator.log 2>&1

multinode-demo/setup.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+usage () {
+  exitcode=0
+  if [[ -n "$1" ]]; then
+    exitcode=1
+    echo "Error: $*"
+  fi
+  cat <<EOF
+usage: $0 [-n num_tokens] [-l] [-p] [-t node_type]
+
+Creates a fullnode configuration
+
+ -n num_tokens  - Number of tokens to create
+ -l             - Detect network address from local machine configuration, which
+                  may be a private IP address unaccessible on the Intenet (default)
+ -p             - Detect public address using public Internet servers
+ -t node_type   - Create configuration files only for this kind of node.  Valid
+                  options are validator or leader.  Creates configuration files
+                  for both by default
+
+EOF
+  exit $exitcode
+}
+
+ip_address_arg=-l
+num_tokens=1000000000
+node_type_leader=true
+node_type_validator=true
+while getopts "h?n:lpt:" opt; do
+  case $opt in
+  h|\?)
+    usage
+    exit 0
+    ;;
+  l)
+    ip_address_arg=-l
+    ;;
+  p)
+    ip_address_arg=-p
+    ;;
+  n)
+    num_tokens="$OPTARG"
+    ;;
+  t)
+    node_type="$OPTARG"
+    case $OPTARG in
+    leader)
+      node_type_leader=true
+      node_type_validator=false
+      ;;
+    validator)
+      node_type_leader=false
+      node_type_validator=true
+      ;;
+    *)
+      usage "Error: unknown node type: $node_type"
+      ;;
+    esac
+    ;;
+  *)
+    usage "Error: unhandled option: $opt"
+    ;;
+  esac
+done
+
+
+leader_address_args=("$ip_address_arg")
+validator_address_args=("$ip_address_arg" -b 9000)
+leader_id_path="$SOLANA_CONFIG_PRIVATE_DIR"/leader-id.json
+validator_id_path="$SOLANA_CONFIG_PRIVATE_DIR"/validator-id.json
+mint_path="$SOLANA_CONFIG_PRIVATE_DIR"/mint.json
+
+set -e
+
+echo "Cleaning $SOLANA_CONFIG_DIR"
+rm -rvf "$SOLANA_CONFIG_DIR"
+mkdir -p "$SOLANA_CONFIG_DIR"
+
+rm -rvf "$SOLANA_CONFIG_PRIVATE_DIR"
+mkdir -p "$SOLANA_CONFIG_PRIVATE_DIR"
+
+$solana_keygen -o "$leader_id_path"
+$solana_keygen -o "$validator_id_path"
+
+if $node_type_leader; then
+  echo "Creating $mint_path with $num_tokens tokens"
+  $solana_keygen -o "$mint_path"
+
+  echo "Creating $SOLANA_CONFIG_DIR/ledger.log"
+  $solana_genesis --tokens="$num_tokens" < "$mint_path" > "$SOLANA_CONFIG_DIR"/ledger.log
+
+  echo "Creating $SOLANA_CONFIG_DIR/leader.json"
+  $solana_fullnode_config --keypair="$leader_id_path" "${leader_address_args[@]}" > "$SOLANA_CONFIG_DIR"/leader.json
+fi
+
+
+if $node_type_validator; then
+  echo "Creating $SOLANA_CONFIG_DIR/validator.json"
+  $solana_fullnode_config --keypair="$validator_id_path" "${validator_address_args[@]}" > "$SOLANA_CONFIG_DIR"/validator.json
+fi
+
+ls -lh "$SOLANA_CONFIG_DIR"/
+if $node_type_leader; then
+  ls -lh "$SOLANA_CONFIG_PRIVATE_DIR"
+fi

multinode-demo/test/wallet-sanity.sh

@@ -0,0 +1,47 @@
+#!/bin/bash -e
+#
+# Wallet sanity test
+#
+
+here=$(dirname "$0")
+cd "$here"
+
+if [[ -n "$USE_SNAP" ]]; then
+  # TODO: Merge wallet.sh functionality into solana-wallet proper and
+  #       remove this USE_SNAP case
+  wallet="solana.wallet $1"
+else
+  wallet="../wallet.sh $1"
+fi
+
+# Tokens transferred to this address are lost forever...
+garbage_address=vS3ngn1TfQmpsW1Z4NkLuqNAQFF3dYQw8UZ6TCx9bmq
+
+check_balance_output() {
+  declare expected_output="$1"
+  exec 42>&1
+  output=$($wallet balance | tee >(cat - >&42))
+  if [[ ! "$output" =~ $expected_output ]]; then
+    echo "Balance is incorrect.  Expected: $expected_output"
+    exit 1
+  fi
+}
+
+pay_and_confirm() {
+  exec 42>&1
+  signature=$($wallet pay "$@" | tee >(cat - >&42))
+  $wallet confirm "$signature"
+}
+
+$wallet reset
+$wallet address
+check_balance_output "Your balance is: 0"
+$wallet airdrop --tokens 60
+check_balance_output "Your balance is: 60"
+$wallet airdrop --tokens 40
+check_balance_output "Your balance is: 100"
+pay_and_confirm --to $garbage_address --tokens 99
+check_balance_output "Your balance is: 1"
+
+echo PASS
+exit 0

multinode-demo/validator.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+usage() {
+  if [[ -n "$1" ]]; then
+    echo "$*"
+    echo
+  fi
+  echo "usage: $0 [rsync network path to solana repo on leader machine] [network ip address of leader]"
+  exit 1
+}
+
+if [[ "$1" = "-h" || -n "$3" ]]; then
+  usage
+fi
+
+if [[ -d "$SNAP" ]]; then
+  # Exit if mode is not yet configured
+  # (typically the case after the Snap is first installed)
+  [[ -n "$(snapctl get mode)" ]] || exit 0
+
+  # Select leader from the Snap configuration
+  leader_address="$(snapctl get leader-address)"
+  if [[ -z "$leader_address" ]]; then
+    # Assume public testnet by default
+    leader_address=35.230.65.68  # testnet.solana.com
+  fi
+  leader="$leader_address"
+else
+  if [[ -n "$3" ]]; then
+    usage
+  fi
+
+  if [[ -z "$1" ]]; then
+    leader=${1:-${here}/..}    # Default to local solana repo
+    leader_address=${2:-127.0.0.1}  # Default to local leader
+  elif [[ -z "$2" ]]; then
+    leader="$1"
+    leader_address=$(dig +short "$1" | head -n1)
+    if [[ -z "$leader_address" ]]; then
+      usage "Error: unable to resolve IP address for $leader"
+    fi
+  else
+    leader="$1"
+    leader_address="$2"
+  fi
+fi
+leader_port=8001
+
+if [[ -n "$SOLANA_CUDA" ]]; then
+  program="$solana_fullnode_cuda"
+else
+  program="$solana_fullnode"
+fi
+
+
+[[ -f "$SOLANA_CONFIG_DIR"/validator.json ]] || {
+  echo "$SOLANA_CONFIG_DIR/validator.json not found, create it by running:"
+  echo
+  echo "  ${here}/setup.sh -t validator"
+  exit 1
+}
+
+rsync_leader_url=$(rsync_url "$leader")
+
+tune_networking
+
+SOLANA_LEADER_CONFIG_DIR="$SOLANA_CONFIG_DIR"/leader-config
+rm -rf "$SOLANA_LEADER_CONFIG_DIR"
+set -ex
+$rsync -vPrz "$rsync_leader_url"/config/ "$SOLANA_LEADER_CONFIG_DIR"
+
+# migrate from old ledger format?  why not...
+if [[ ! -f "$SOLANA_LEADER_CONFIG_DIR"/ledger.log &&
+          -f "$SOLANA_LEADER_CONFIG_DIR"/genesis.log ]]; then
+  (shopt -s nullglob &&
+     cat "$SOLANA_LEADER_CONFIG_DIR"/genesis.log \
+         "$SOLANA_LEADER_CONFIG_DIR"/tx-*.log) > "$SOLANA_LEADER_CONFIG_DIR"/ledger.log
+fi
+
+# Ensure the validator has at least 1 token before connecting to the network
+# TODO: Remove this workaround
+while ! $solana_wallet \
+          -l "$SOLANA_LEADER_CONFIG_DIR"/leader.json \
+          -k "$SOLANA_CONFIG_PRIVATE_DIR"/validator-id.json airdrop --tokens 1; do
+  sleep 1
+done
+
+set -o pipefail
+$program \
+  --identity "$SOLANA_CONFIG_DIR"/validator.json \
+  --testnet "$leader_address:$leader_port" \
+  --ledger "$SOLANA_LEADER_CONFIG_DIR"/ledger.log \
+2>&1 | $validator_logger

multinode-demo/wallet.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# usage: $0 <rsync network path to solana repo on leader machine>"
+#
+
+here=$(dirname "$0")
+# shellcheck source=multinode-demo/common.sh
+source "$here"/common.sh
+
+# if $1 isn't host:path, something.com, or a valid local path
+if [[ ${1%:} != "$1" || "$1" =~ [^.]\.[^.] || -d $1 ]]; then
+  leader=$1 # interpret
+  shift
+else
+  if [[ -d "$SNAP" ]]; then
+    leader=testnet.solana.com # Default to testnet when running as a Snap
+  else
+    leader=$here/.. # Default to local solana repo
+  fi
+fi
+
+if [[ "$1" = "reset" ]]; then
+  echo Wallet resetting
+  rm -rf "$SOLANA_CONFIG_CLIENT_DIR"
+  exit 0
+fi
+
+rsync_leader_url=$(rsync_url "$leader")
+
+set -e
+mkdir -p "$SOLANA_CONFIG_CLIENT_DIR"
+if [[ ! -r "$SOLANA_CONFIG_CLIENT_DIR"/leader.json ]]; then
+  echo "Fetching leader configuration from $rsync_leader_url"
+  $rsync -Pz "$rsync_leader_url"/config/leader.json "$SOLANA_CONFIG_CLIENT_DIR"/
+fi
+
+client_id_path="$SOLANA_CONFIG_CLIENT_DIR"/id.json
+if [[ ! -r $client_id_path ]]; then
+  echo "Generating client identity: $client_id_path"
+  $solana_keygen -o "$client_id_path"
+fi
+
+# shellcheck disable=SC2086 # $solana_wallet should not be quoted
+exec $solana_wallet \
+  -l "$SOLANA_CONFIG_CLIENT_DIR"/leader.json -k "$client_id_path" "$@"

rfcs/rfc-001-smart-contracts-engine.md

@@ -0,0 +1,190 @@
+# Smart Contracts Engine
+ 
+The goal of this RFC is to define a set of constraints for APIs and runtime such that we can execute our smart contracts safely on massively parallel hardware such as a GPU.  Our runtime is built around an OS *syscall* primitive.  The difference in blockchain is that now the OS does a cryptographic check of memory region ownership before accessing the memory in the Solana kernel.
+
+## Version
+
+version 0.1 
+
+## Toolchain Stack
+
+     +---------------------+       +---------------------+
+     |                     |       |                     |
+     |   +------------+    |       |   +------------+    |
+     |   |            |    |       |   |            |    |
+     |   |  frontend  |    |       |   |  verifier  |    |
+     |   |            |    |       |   |            |    |
+     |   +-----+------+    |       |   +-----+------+    |
+     |         |           |       |         |           |
+     |         |           |       |         |           |
+     |   +-----+------+    |       |   +-----+------+    |
+     |   |            |    |       |   |            |    |
+     |   |    llvm    |    |       |   |   loader   |    |
+     |   |            |    +------>+   |            |    |
+     |   +-----+------+    |       |   +-----+------+    |
+     |         |           |       |         |           |
+     |         |           |       |         |           |
+     |   +-----+------+    |       |   +-----+------+    |
+     |   |            |    |       |   |            |    |
+     |   |    ELF     |    |       |   |   runtime  |    |
+     |   |            |    |       |   |            |    |
+     |   +------------+    |       |   +------------+    |
+     |                     |       |                     |
+     |        client       |       |       solana        |
+     +---------------------+       +---------------------+
+
+                [Figure 1. Smart Contracts Stack]
+
+In Figure 1 an untrusted client, creates a program in the front-end language of her choice, (like C/C++/Rust/Lua), and compiles it with LLVM to a position independent shared object ELF, targeting BPF bytecode. Solana will safely load and execute the ELF.
+
+## Bytecode
+
+Our bytecode is based on Berkley Packet Filter. The requirements for BPF overlap almost exactly with the requirements we have:
+
+1. Deterministic amount of time to execute the code
+2. Bytecode that is portable between machine instruction sets
+3. Verified memory accesses
+4. Fast to load the object, verify the bytecode and JIT to local machine instruction set
+
+For 1, that means that loops are unrolled, and for any jumps back we can guard them with a check against the number of instruction that have been executed at this point.  If the limit is reached, the program yields its execution.  This involves saving the stack and current instruction index.
+
+For 2, the BPF bytecode already easily maps to x86–64, arm64 and other instruction sets. 
+
+For 3, every load and store that is relative can be checked to be within the expected memory that is passed into the ELF.  Dynamic load and stores can do a runtime check against available memory, these will be slow and should be avoided.
+
+For 4, Fully linked PIC ELF with just a single RX segment. Effectively we are linking a shared object with `-fpic -target bpf` and with a linker script to collect everything into a single RX segment. Writable globals are not supported.
+
+### Address Checks
+
+The interface to the module takes a `&mut Vec<Vec<u8>>` in rust, or a `int sz, void* data[sz], int szs[sz]` in `C`.  Given the module's bytecode, for each method, we need to analyze the bounds on load and stores into each buffer the module uses.  This check needs to be done `on chain`, and after those bounds are computed we can verify that the user supplied array of buffers will not cause a memory fault.  For load and stores that we cannot analyze, we can replace with a `safe_load` and `safe_store` instruction that will check the table for access.
+
+## Loader
+The loader is our first smart contract. The job of this contract is to load the actual program with its own instance data.  The loader will verify the bytecode and that the object implements the expected entry points.
+
+Since there is only one RX segment, the context for the contract instance is passed into each entry point as well as the event data for that entry point.
+
+A client will create a transaction to create a new loader instance:
+
+`Solana_NewLoader(Loader Instance PubKey, proof of key ownership, space I need for my elf)`
+
+A client will then do a bunch of transactions to load its elf into the loader instance they created:
+
+`Loader_UploadElf(Loader Instance PubKey, proof of key ownership, pos start, pos end, data)`
+
+At this point the client can create a new instance of the module with its own instance address:
+
+`Loader_NewInstance(Loader Instance PubKey, proof of key ownership, Instance PubKey, proof of key ownership)`
+
+Once the instance has been created, the client may need to upload more user data to solana to configure this instance:
+
+`Instance_UploadModuleData(Instance PubKey, proof of key ownership, pos start, pos end, data)`
+
+Now clients can `start` the instance:
+
+`Instance_Start(Instance PubKey, proof of key ownership)`
+
+## Runtime
+
+Our goal with the runtime is to have a general purpose execution environment that is highly parallelizable and doesn't require dynamic resource management. We want to execute as many contracts as we can in parallel, and have them pass or fail without a destructive state change.
+
+### State and Entry Point
+
+State is addressed by an account which is at the moment simply the PubKey.  Our goal is to eliminate dynamic memory allocation in the smart contract itself, so the contract is a function that takes a mapping of [(PubKey,State)] and returns [(PubKey, State')].  The output of keys is a subset of the input.  Three basic kinds of state exist:
+
+* Instance State
+* Participant State
+* Caller State
+
+There isn't any difference in how each is implemented, but conceptually Participant State is memory that is allocated for each participant in the contract.  Instance State is memory that is allocated for the contract itself, and Caller State is memory that the transactions caller has allocated.
+
+
+### Call
+
+```
+void call(
+    const struct instance_data *data,
+    const uint8_t kind[],  //instance|participant|caller|read|write
+    const uint8_t *keys[],
+    uint8_t *data[],
+    int num,
+    uint8_t dirty[],        //dirty memory bits
+    uint8_t *userdata,      //current transaction data
+);
+```
+
+To call this operation, the transaction that is destined to the contract instance specifies what keyed state it should present to the `call` function.  To allocate the state memory or a call context, the client has to first call a function on the contract with the designed address that will own the state.
+
+At its core, this is a system call that requires cryptographic proof of ownership of memory regions instead of an OS that checks page tables for access rights.
+
+* `Instance_AllocateContext(Instance PubKey, My PubKey, Proof of key ownership)`
+
+Any transaction can then call `call` on the contract with a set of keys.  It's up to the contract itself to manage ownership:
+
+* `Instance_Call(Instance PubKey, [Context PubKeys], proofs of ownership, userdata...)`
+
+Contracts should be able to read any state that is part of solana, but only write to state that the contract allocated.
+
+#### Caller State
+
+Caller `state` is memory allocated for the `call` that belongs to the public key that is issuing the `call`.  This is the caller's context.
+
+#### Instance State
+
+Instance `state` is memory that belongs to this contract instance.  We may also need module-wide `state` as well.
+
+#### Participant State
+
+Participant `state` is any other memory.  In some cases it may make sense to have these allocated as part of the call by the caller.
+
+### Reduce
+
+Some operations on the contract will require iteration over all the keys.  To make this parallelizable the iteration is broken up into reduce calls which are combined.
+
+```
+void reduce_m(
+    const struct instance_data *data,
+    const uint8_t *keys[],
+    const uint8_t *data[],
+    int num,
+    uint8_t *reduce_data,
+);
+
+void reduce_r(
+    const struct instance_data *data,
+    const uint8_t *reduce_data[],
+    int num,
+    uint8_t *reduce_data,
+);
+```
+
+### Execution
+
+Transactions are batched and processed in parallel at each stage.
+```
++-----------+    +--------------+      +-----------+    +---------------+
+| sigverify |-+->| debit commit |---+->| execution |-+->| memory commit |
++-----------+ |  +--------------+   |  +-----------+ |  +---------------+
+              |                     |                |
+              |  +---------------+  |                |  +--------------+
+              |->| memory verify |->+                +->| debit undo   |
+                 +---------------+                   |  +--------------+
+                                                     |
+                                                     |  +---------------+
+                                                     +->| credit commit |
+                                                        +---------------+
+
+
+```
+The `debit verify` stage is very similar to `memory verify`.  Proof of key ownership is used to check if the callers key has some state allocated with the contract, then the memory is loaded and executed.  After execution stage, the dirty pages are written back by the contract.  Because know all the memory accesses during execution, we can batch transactions that do not interfere with each other.  We can also apply the `debit undo` and `credit commit` stages of the transaction.  `debit undo` is run in case of an exception during contract execution, only transfers may be reversed, fees are commited to solana.
+
+### GPU execution
+
+A single contract can read and write to separate key pairs without interference.  These separate calls to the same contract can execute on the same GPU thread over different memory using different SIMD lanes.
+
+## Notes
+
+1. There is no dynamic memory allocation.
+2. Persistant Memory is allocated to a Key with ownership
+3. Contracts can `call` to update key owned state
+4. Contracts can `reduce` over the memory to aggregate state
+5. `call` is just a *syscall* that does a cryptographic check of memory owndershp

rfcs/rfc-002-consensus.md

@@ -0,0 +1,122 @@
+# Consensus
+
+VERY WIP
+
+The goal of this RFC is to define the consensus algorithm used in solana.  This proposal covers a Proof of Stake algorithm that leverages Proof of History.  PoH is a permissionless clock for blockchain that is available before consensus.  This PoS approach leverages PoH to make strong assumptions about time between partitions.
+
+## Version
+
+version 0.1
+
+## Message Flow
+
+1. Transactions are ingested at the leader.
+2. Leader filters for valid transactions
+3. Leader executes valid transactions on its state
+4. Leader packages transactions into blobs
+5. Leader transmits blobs to validator nodes.
+    a. The set of supermajority + `M` by stake weight of nodes is rotated in round robin fashion.
+6. Validators retransmit blobs to peers in their set and to further downstream nodes.
+7. Validators validate the transactions and execute them on their state.
+8. Validators compute the hash of the state.
+9. Validators transmit votes to the leader.
+    a. Votes are signatures of the hash of the computed state.
+10. Leader executes the votes as any other transaction and broadcasts them out to the network
+11. Validators observe their votes, and all the votes from the network.
+12. Validators continue voting if the supermajority of stake is observed in the vote for the same hash.
+
+Supermajority is defined as `2/3rds + 1` vote of the PoS stakes.
+
+## Staking
+
+Validators `stake` some of their spendable sol into a staking account.  The stakes are not spendable and can only be used for voting.
+
+```
+CreateStake(
+    PoH count,
+    PoH hash,
+    source public key,
+    amount,
+    destination public key,
+    proof of ownership of destination public key,
+    signature of the message with the source keypair
+)
+```
+
+Creating the stake has a warmup period of TBD.  Unstaking requires the node to miss a certain amount of validation votes.
+
+## Validation Votes
+
+```
+Validate(
+    PoH count,
+    PoH hash,
+    stake public key,
+    signature of the state,
+    signature of the message with the stake keypair
+)
+```
+
+## Validator Slashing
+
+Validators `stake` some of their spendable sol into a staking account.  The stakes are not spendable and can only be used for voting.
+
+```
+Slash(Validate(
+    PoH count,
+    PoH hash,
+    stake public key,
+    ...
+    signature of the message with the stake keypair
+))
+```
+
+When the `Slash` vote is processed, validators should lookup `PoH hash` at `PoH count` and compare it with the message.  If they do not match, the stake at `stake public key` should be set to `0`.
+
+## Leader Slashing
+
+TBD.  The goal of this is to discourage leaders from generating multiple PoH streams.
+
+## Validation Vote Contract
+
+The goal of this contract is to simulate economic cost of mining on a shorter branch.
+
+1. With my signature I am certifying that I computed `state hash` at `PoH count` and `PoH hash`.
+2. I will not vote on a branch that doesn't contain this message for at least `N` counts, or until `PoH count` + `N` is reached by the PoH stream.
+3. I will not vote for any other branch below `PoH count`.
+    a. if there are other votes not present in this PoH history the validator may need to `cancel` them before creating this vote.
+
+## Leader Seed Generation
+
+Leader selection is decided via a random seed.  The process is as follows:
+
+1. Periodically at a specific `PoH count` select the first vote signatures that create a supermajority from the previous round.
+2. append them together
+3. hash the string for `N` counts via a similar process as PoH itself.
+4. The resulting hash is the random seed for `M` counts, where M > N
+
+## Leader Ranking and Rotation
+
+Leader's transmit for a count of `T`.  When `T` is reached all the validators should switch to the next ranked leader.  To rank leaders, the supermajority + `M` nodes are shuffled with the using the above calculated random seed.
+
+TBD: define a ranking for critical partitions without a node from supermajority + `M` set.
+
+## Partition selection
+
+Validators should select the first branch to reach finality, or the highest ranking leader.
+
+## Examples
+
+### Small Partition
+1. Network partition M occurs for 10% of the nodes
+2. The larger partition K, with 90% of the stake weight continues to operate as normal
+3. M cycles through the ranks until one of them is leader.
+4. M validators observe 10% of the vote pool, finality is not reached
+5. M and K re-connect.
+6. M validators cancel their votes on K which are below K's `PoH count`
+
+### Leader Timeout
+1. Next rank node observes a timeout.
+2. Nodes receiving both PoH streams pick the higher rank node.
+3. 2, causes a partition, since nodes can only vote for 1 leader.
+4. Partition is resolved just like in the [Small Partition](#small-parition)

rfcs/rfc-003-storage.md

@@ -0,0 +1,54 @@
+# Storage
+
+The goal of this RFC is to define a protocol for storing a very large ledger over a p2p network that is verified by solana validators.  At full capacity on a 1gbps network solana will generate 4 petabytes of data per year.  To prevent the network from centralizing around full nodes that have to store the full data set this protocol proposes a way for mining nodes to provide storage capacity for pieces of the network.
+
+# Version 
+
+version 0.1
+
+# Background
+
+The basic idea to Proof of Replication is encrypting a dataset with a public symmetric key using CBC encryption, then hash the encrypted dataset. The main problem with the naive approach is that a dishonest storage node can stream the encryption and delete the data as its hashed. The simple solution is to force the hash to be done on the reverse of the encryption, or perhaps with a random order. This ensures that all the data is present during the generation of the proof and it also requires the validator to have the entirety of the encrypted data present for verification of every proof of every identity. So the space required to validate is `(Number of Proofs)*(data size)`
+
+# Optimization with PoH 
+
+Our improvement on this approach is to randomly sample the encrypted blocks faster than it takes to encrypt, and record the hash of those samples into the PoH ledger. Thus the blocks stay in the exact same order for every PoRep and verification can stream the data and verify all the proofs in a single batch. This way we can verify multiple proofs concurrently, each one on its own CUDA core. With the current generation of graphics cards our network can support up to 14k replication identities or symmetric keys. The total space required for verification is `(2 CBC blocks) * (Number of Identities)`, with core count of equal to (Number of Identities). A CBC block is expected to be 1MB in size.
+
+# Network
+
+Validators for PoRep are the same validators that are verifying transactions. They have some stake that they have put up as collateral that ensures that their work is honest. If you can prove that a validator verified a fake PoRep, then the validators stake can be slashed.
+
+Replicators are specialized thin clients. They download a part of the ledger and store it, and provide PoReps of storing the ledger. For each verified PoRep replicators earn a reward of sol from the mining pool.
+
+# Constraints
+
+We have the following constraints:
+* At most 14k  replication identities can be used, because thats how many CUDA cores we can fit in a $5k box at the moment.
+* Verification requires generating the CBC blocks. That requires space of 2 blocks per identity, and 1 CUDA core per identity for the same dataset. So as many identities at once should be batched with as many proofs for those identities verified concurrently for the same dataset.
+
+# Validation and Replication Protocol
+
+1. Network sets the replication target number, lets say 1k. 1k PoRep identities are created from signatures of a PoH hash. So they are tied to a specific PoH hash. It doesn't matter who creates them, or simply the last 1k validation signatures we saw for the ledger at that count. This maybe just the initial batch of identities, because we want to stagger identity rotation.
+2. Any client can use any of these identities to create PoRep proofs. Replicator identities are the CBC encryption keys.
+3. Periodically at a specific PoH count, replicator that want to create PoRep proofs sign the PoH hash at that count. That signature is the seed used to pick the block and identity to replicate. A block is 1TB of ledger.
+4. Periodically at a specific PoH count, replicator submits PoRep proofs for their selected block. A signature of the PoH hash at that count is the seed used to sample the 1TB encrypted block, and hash it. This is done faster than it takes to encrypt the 1TB block with the original identity.
+5. Replicators must submit some number of fake proofs, which they can prove to be fake by providing the seed for the hash result.
+6. Periodically at a specific PoH count, validators sign the hash and use the signature to select the 1TB block that they need to validate. They batch all the identities and proofs and submit approval for all the verified ones.
+7. After #6, replicator client submit the proofs of fake proofs.
+
+For any random seed, we force everyone to use a signature that is derived from a PoH hash. Everyone must use the same count, so the same PoH hash is signed by every participant. The signatures are then each cryptographically tied to the keypair, which prevents a leader from grinding on the resulting value for more than 1 identity.
+
+We need to stagger the rotation of the identity keys. Once this gets going, the next identity could be generated by hashing itself with a PoH hash, or via some other process based on the validation signatures.
+
+Since there are many more client identities then encryption identities, we need to split the reward for multiple clients, and prevent Sybil attacks from generating many clients to acquire the same block of data. To remain BFT we want to avoid a single human entity from storing all the replications of a single chunk of the ledger.
+
+Our solution to this is to force the clients to continue using the same identity. If the first round is used to acquire the same block for many client identities, the second round for the same client identities will force a redistribution of the signatures, and therefore PoRep identities and blocks. Thus to get a reward for storage clients need to store the first block for free and the network can reward long lived client identities more than new ones.
+
+# Notes
+
+* We can reduce the costs of verification of PoRep by using PoH, and actually make it feasible to verify a large number of proofs for a global dataset.
+* We can eliminate grinding by forcing everyone to sign the same PoH hash and use the signatures as the seed
+* The game between validators and replicators is over random blocks and random encryption identities and random data samples. The goal of randomization is to prevent colluding groups from having overlap on data or validation.
+* Replicator clients fish for lazy validators by submitting fake proofs that they can prove are fake.
+* Replication identities are just symmetric encryption keys, the number of them on the network is our storage replication target. Many more client identities can exist than replicator identities, so unlimited number of clients can provide proofs of the same replicator identity.
+* To defend against Sybil client identities that try to store the same block we force the clients to store for multiple rounds before receiving a reward.

scripts/perf-plot.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+import matplotlib
+matplotlib.use('Agg')
+
+import matplotlib.pyplot as plt
+import json
+import sys
+
+stages_to_counters = {}
+stages_to_time = {}
+
+if len(sys.argv) != 2:
+    print("USAGE: {} <input file>".format(sys.argv[0]))
+    sys.exit(1)
+
+with open(sys.argv[1]) as fh:
+    for line in fh.readlines():
+        if "COUNTER" in line:
+            json_part = line[line.find("{"):]
+            x = json.loads(json_part)
+            counter = x['name']
+            if not (counter in stages_to_counters):
+                stages_to_counters[counter] = []
+                stages_to_time[counter] = []
+            stages_to_counters[counter].append(x['counts'])
+            stages_to_time[counter].append(x['now'])
+
+fig, ax = plt.subplots()
+
+for stage in stages_to_counters.keys():
+    plt.plot(stages_to_time[stage], stages_to_counters[stage], label=stage)
+
+plt.xlabel('ms')
+plt.ylabel('count')
+
+plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3,
+           ncol=2, mode="expand", borderaxespad=0.)
+
+plt.locator_params(axis='x', nbins=10)
+plt.grid(True)
+
+plt.savefig("perf.pdf")

snap/README.md

@@ -0,0 +1,17 @@
+## Development
+
+If you're running Ubuntu 16.04 and already have `snapcraft` installed, simply
+run:
+```
+$ snapcraft
+```
+
+For other systems we provide a docker image that can be used for snap
+development:
+```
+$ ./ci/docker-run.sh solanalabs/snapcraft snapcraft -d
+```
+
+## Reference
+* https://docs.snapcraft.io/
+

snap/hooks/configure

@@ -0,0 +1,34 @@
+#!/bin/bash -e
+
+echo Stopping daemons
+snapctl stop --disable solana.daemon-drone
+snapctl stop --disable solana.daemon-leader
+snapctl stop --disable solana.daemon-validator
+
+mode="$(snapctl get mode)"
+if [[ -z "$mode" ]]; then
+  exit 0
+fi
+
+ip_address_arg=-p # Use public IP address (TODO: make this configurable?)
+num_tokens="$(snapctl get num-tokens)"
+
+case $mode in
+leader+drone)
+  $SNAP/bin/setup.sh ${num_tokens:+-n $num_tokens} ${ip_address_arg} -t leader
+  snapctl start --enable solana.daemon-leader
+  snapctl start --enable solana.daemon-drone
+  ;;
+leader)
+  $SNAP/bin/setup.sh ${num_tokens:+-n $num_tokens} ${ip_address_arg} -t leader
+  snapctl start --enable solana.daemon-leader
+  ;;
+validator)
+  $SNAP/bin/setup.sh ${ip_address_arg} -t validator
+  snapctl start --enable solana.daemon-validator
+  ;;
+*)
+  echo "Error: Unknown mode: $mode"
+  exit 1
+  ;;
+esac

snap/snapcraft.yaml

@@ -0,0 +1,118 @@
+name: solana
+version: git
+summary: Blockchain, Rebuilt for Scale
+description: |
+  710,000 tx/s with off-the-shelf hardware and no sharding.
+  Scales with Moore's Law.
+grade: devel
+
+# TODO: solana-perf-fullnode does not yet run with 'strict' confinement due to the
+# CUDA dependency, so use 'devmode' confinement for now
+confinement: devmode
+
+hooks:
+  configure:
+    plugs: [network]
+
+apps:
+  drone:
+    command: solana-drone
+    plugs:
+      - network
+      - network-bind
+  fullnode:
+    command: solana-fullnode
+    plugs:
+      - network
+      - network-bind
+      - home
+  fullnode-cuda:
+    command: solana-fullnode-cuda
+    plugs:
+      - network
+      - network-bind
+      - home
+  fullnode-config:
+    command: solana-fullnode-config
+    plugs:
+      - network
+      - network-bind
+      - home
+  genesis:
+    command: solana-genesis
+  keygen:
+    command: solana-keygen
+    plugs:
+      - home
+  client-demo:
+    # TODO: Merge client.sh functionality into solana-client-demo proper
+    command: client.sh
+    #command: solana-client-demo
+    plugs:
+      - network
+      - network-bind
+      - home
+  wallet:
+    # TODO: Merge wallet.sh functionality into solana-wallet proper
+    command: wallet.sh
+    #command: solana-wallet
+    plugs:
+      - network
+      - home
+  daemon-validator:
+    daemon: simple
+    command: validator.sh
+    plugs:
+      - network
+      - network-bind
+  daemon-leader:
+    daemon: simple
+    command: leader.sh
+    plugs:
+      - network
+      - network-bind
+  daemon-drone:
+    daemon: simple
+    command: drone.sh
+    plugs:
+      - network
+      - network-bind
+
+parts:
+  solana:
+    plugin: nil
+    prime:
+      - bin
+      - usr/lib
+    override-build: |
+      # Install CUDA 9.2 runtime
+      mkdir -p $SNAPCRAFT_PART_INSTALL/usr/
+      cp -rav /usr/local/cuda-9.2/targets/x86_64-linux/lib/ $SNAPCRAFT_PART_INSTALL/usr/lib
+      mkdir -p $SNAPCRAFT_PART_INSTALL/usr/lib/x86_64-linux-gnu/
+      cp -rav /usr/lib/x86_64-linux-gnu/libcuda.* $SNAPCRAFT_PART_INSTALL/usr/lib/x86_64-linux-gnu/
+      mkdir -p $SNAPCRAFT_PART_INSTALL/usr/lib/nvidia-396/
+      cp -v /usr/lib/nvidia-396/libnvidia-fatbinaryloader.so* $SNAPCRAFT_PART_INSTALL/usr/lib/nvidia-396/
+
+      # Build/install solana-fullnode-cuda
+      ./fetch-perf-libs.sh
+      cargo install --features=cuda --root $SNAPCRAFT_PART_INSTALL --bin solana-fullnode
+      mv $SNAPCRAFT_PART_INSTALL/bin/solana-fullnode $SNAPCRAFT_PART_INSTALL
+      rm -rf $SNAPCRAFT_PART_INSTALL/bin/*
+      mv $SNAPCRAFT_PART_INSTALL/solana-fullnode $SNAPCRAFT_PART_INSTALL/bin/solana-fullnode-cuda
+      mkdir -p $SNAPCRAFT_PART_INSTALL/usr/lib/
+      cp -f libJerasure.so $SNAPCRAFT_PART_INSTALL/usr/lib/libJerasure.so.2
+      cp -f libgf_complete.so $SNAPCRAFT_PART_INSTALL/usr/lib/libgf_complete.so.1
+
+      # Build/install all other programs
+      cargo install --root $SNAPCRAFT_PART_INSTALL --bins
+
+      # Install multinode scripts
+      mkdir -p $SNAPCRAFT_PART_INSTALL/bin
+      cp -av multinode-demo/* $SNAPCRAFT_PART_INSTALL/bin/
+
+      # TODO: build rsync/multilog from source instead of sneaking it in from the host
+      # system...
+      set -x
+      mkdir -p $SNAPCRAFT_PART_INSTALL/bin
+      cp -av /usr/bin/rsync $SNAPCRAFT_PART_INSTALL/bin/
+      cp -av /usr/bin/multilog $SNAPCRAFT_PART_INSTALL/bin/

src/accountant.rs

@@ -1,409 +0,0 @@
-//! The `accountant` is a client of the `historian`. It uses the historian's
-//! event log to record transactions. Its users can deposit funds and
-//! transfer funds to other users.
-
-use hash::Hash;
-use entry::Entry;
-use event::Event;
-use transaction::{Condition, Transaction};
-use signature::{KeyPair, PublicKey, Signature};
-use mint::Mint;
-use historian::{reserve_signature, Historian};
-use std::sync::mpsc::SendError;
-use std::collections::{HashMap, HashSet};
-use std::result;
-use chrono::prelude::*;
-
-#[derive(Debug, PartialEq, Eq)]
-pub enum AccountingError {
-    InsufficientFunds,
-    InvalidTransfer,
-    InvalidTransferSignature,
-    SendError,
-}
-
-pub type Result<T> = result::Result<T, AccountingError>;
-
-pub struct Accountant {
-    pub historian: Historian,
-    pub balances: HashMap<PublicKey, i64>,
-    pub first_id: Hash,
-    pub last_id: Hash,
-    pending: HashMap<Signature, Transaction<i64>>,
-    time_sources: HashSet<PublicKey>,
-    last_time: DateTime<Utc>,
-}
-
-impl Accountant {
-    pub fn new_from_entries<I>(entries: I, ms_per_tick: Option<u64>) -> Self
-    where
-        I: IntoIterator<Item = Entry>,
-    {
-        let mut entries = entries.into_iter();
-
-        // The first item in the log is required to be an entry with zero num_hashes,
-        // which implies its id can be used as the log's seed.
-        let entry0 = entries.next().unwrap();
-        let start_hash = entry0.id;
-
-        let hist = Historian::new(&start_hash, ms_per_tick);
-        let mut acc = Accountant {
-            historian: hist,
-            balances: HashMap::new(),
-            first_id: start_hash,
-            last_id: start_hash,
-            pending: HashMap::new(),
-            time_sources: HashSet::new(),
-            last_time: Utc.timestamp(0, 0),
-        };
-
-        // The second item in the log is a special transaction where the to and from
-        // fields are the same. That entry should be treated as a deposit, not a
-        // transfer to oneself.
-        let entry1 = entries.next().unwrap();
-        acc.process_verified_event(&entry1.event, true).unwrap();
-
-        for entry in entries {
-            acc.process_verified_event(&entry.event, false).unwrap();
-        }
-        acc
-    }
-
-    pub fn new(mint: &Mint, ms_per_tick: Option<u64>) -> Self {
-        Self::new_from_entries(mint.create_entries(), ms_per_tick)
-    }
-
-    pub fn sync(self: &mut Self) -> Hash {
-        while let Ok(entry) = self.historian.receiver.try_recv() {
-            self.last_id = entry.id;
-        }
-        self.last_id
-    }
-
-    fn is_deposit(allow_deposits: bool, from: &PublicKey, to: &PublicKey) -> bool {
-        allow_deposits && from == to
-    }
-
-    pub fn process_transaction(self: &mut Self, tr: Transaction<i64>) -> Result<()> {
-        if !tr.verify() {
-            return Err(AccountingError::InvalidTransfer);
-        }
-
-        if self.get_balance(&tr.from).unwrap_or(0) < tr.asset {
-            return Err(AccountingError::InsufficientFunds);
-        }
-
-        self.process_verified_transaction(&tr, false)?;
-        if let Err(SendError(_)) = self.historian.sender.send(Event::Transaction(tr)) {
-            return Err(AccountingError::SendError);
-        }
-
-        Ok(())
-    }
-
-    /// Commit funds to the 'to' party.
-    fn complete_transaction(self: &mut Self, tr: &Transaction<i64>) {
-        if self.balances.contains_key(&tr.to) {
-            if let Some(x) = self.balances.get_mut(&tr.to) {
-                *x += tr.asset;
-            }
-        } else {
-            self.balances.insert(tr.to, tr.asset);
-        }
-    }
-
-    /// Return funds to the 'from' party.
-    fn cancel_transaction(self: &mut Self, tr: &Transaction<i64>) {
-        if let Some(x) = self.balances.get_mut(&tr.from) {
-            *x += tr.asset;
-        }
-    }
-
-    // TODO: Move this to transaction.rs
-    fn all_satisfied(&self, conds: &[Condition]) -> bool {
-        let mut satisfied = true;
-        for cond in conds {
-            if let &Condition::Timestamp(dt) = cond {
-                if dt > self.last_time {
-                    satisfied = false;
-                }
-            } else {
-                satisfied = false;
-            }
-        }
-        satisfied
-    }
-
-    fn process_verified_transaction(
-        self: &mut Self,
-        tr: &Transaction<i64>,
-        allow_deposits: bool,
-    ) -> Result<()> {
-        if !reserve_signature(&mut self.historian.signatures, &tr.sig) {
-            return Err(AccountingError::InvalidTransferSignature);
-        }
-
-        if !tr.unless_any.is_empty() {
-            // TODO: Check to see if the transaction is expired.
-        }
-
-        if !Self::is_deposit(allow_deposits, &tr.from, &tr.to) {
-            if let Some(x) = self.balances.get_mut(&tr.from) {
-                *x -= tr.asset;
-            }
-        }
-
-        if !self.all_satisfied(&tr.if_all) {
-            self.pending.insert(tr.sig, tr.clone());
-            return Ok(());
-        }
-
-        self.complete_transaction(tr);
-        Ok(())
-    }
-
-    fn process_verified_sig(&mut self, from: PublicKey, tx_sig: Signature) -> Result<()> {
-        let mut cancel = false;
-        if let Some(tr) = self.pending.get(&tx_sig) {
-            // Cancel:
-            // if Signature(from) is in unless_any, return funds to tx.from, and remove the tx from this map.
-
-            // TODO: Use find().
-            for cond in &tr.unless_any {
-                if let Condition::Signature(pubkey) = *cond {
-                    if from == pubkey {
-                        cancel = true;
-                        break;
-                    }
-                }
-            }
-        }
-
-        if cancel {
-            if let Some(tr) = self.pending.remove(&tx_sig) {
-                self.cancel_transaction(&tr);
-            }
-        }
-
-        // Process Multisig:
-        // otherwise, if "Signature(from) is in if_all, remove it. If that causes that list
-        // to be empty, add the asset to to, and remove the tx from this map.
-        Ok(())
-    }
-
-    fn process_verified_timestamp(&mut self, from: PublicKey, dt: DateTime<Utc>) -> Result<()> {
-        // If this is the first timestamp we've seen, it probably came from the genesis block,
-        // so we'll trust it.
-        if self.last_time == Utc.timestamp(0, 0) {
-            self.time_sources.insert(from);
-        }
-
-        if self.time_sources.contains(&from) {
-            if dt > self.last_time {
-                self.last_time = dt;
-            }
-        } else {
-            return Ok(());
-        }
-        // TODO: Lookup pending Transaction waiting on time, signed by a whitelisted PublicKey.
-
-        // Expire:
-        // if a Timestamp after this DateTime is in unless_any, return funds to tx.from,
-        // and remove the tx from this map.
-
-        // Check to see if any timelocked transactions can be completed.
-        let mut completed = vec![];
-        for (key, tr) in &self.pending {
-            for cond in &tr.if_all {
-                if let Condition::Timestamp(dt) = *cond {
-                    if self.last_time >= dt {
-                        if tr.if_all.len() == 1 {
-                            completed.push(*key);
-                        }
-                    }
-                }
-            }
-            // TODO: Add this in once we start removing constraints
-            //if tr.if_all.is_empty() {
-            //    // TODO: Remove tr from pending
-            //    self.complete_transaction(tr);
-            //}
-        }
-
-        for key in completed {
-            if let Some(tr) = self.pending.remove(&key) {
-                self.complete_transaction(&tr);
-            }
-        }
-
-        Ok(())
-    }
-
-    fn process_verified_event(self: &mut Self, event: &Event, allow_deposits: bool) -> Result<()> {
-        match *event {
-            Event::Tick => Ok(()),
-            Event::Transaction(ref tr) => self.process_verified_transaction(tr, allow_deposits),
-            Event::Signature { from, tx_sig, .. } => self.process_verified_sig(from, tx_sig),
-            Event::Timestamp { from, dt, .. } => self.process_verified_timestamp(from, dt),
-        }
-    }
-
-    pub fn transfer(
-        self: &mut Self,
-        n: i64,
-        keypair: &KeyPair,
-        to: PublicKey,
-    ) -> Result<Signature> {
-        let tr = Transaction::new(keypair, to, n, self.last_id);
-        let sig = tr.sig;
-        self.process_transaction(tr).map(|_| sig)
-    }
-
-    pub fn transfer_on_date(
-        self: &mut Self,
-        n: i64,
-        keypair: &KeyPair,
-        to: PublicKey,
-        dt: DateTime<Utc>,
-    ) -> Result<Signature> {
-        let tr = Transaction::new_on_date(keypair, to, dt, n, self.last_id);
-        let sig = tr.sig;
-        self.process_transaction(tr).map(|_| sig)
-    }
-
-    pub fn get_balance(self: &Self, pubkey: &PublicKey) -> Option<i64> {
-        self.balances.get(pubkey).map(|x| *x)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use signature::KeyPairUtil;
-    use logger::ExitReason;
-
-    #[test]
-    fn test_accountant() {
-        let alice = Mint::new(10_000);
-        let bob_pubkey = KeyPair::new().pubkey();
-        let mut acc = Accountant::new(&alice, Some(2));
-        acc.transfer(1_000, &alice.keypair(), bob_pubkey).unwrap();
-        assert_eq!(acc.get_balance(&bob_pubkey).unwrap(), 1_000);
-
-        acc.transfer(500, &alice.keypair(), bob_pubkey).unwrap();
-        assert_eq!(acc.get_balance(&bob_pubkey).unwrap(), 1_500);
-
-        drop(acc.historian.sender);
-        assert_eq!(
-            acc.historian.thread_hdl.join().unwrap(),
-            ExitReason::RecvDisconnected
-        );
-    }
-
-    #[test]
-    fn test_invalid_transfer() {
-        let alice = Mint::new(11_000);
-        let mut acc = Accountant::new(&alice, Some(2));
-        let bob_pubkey = KeyPair::new().pubkey();
-        acc.transfer(1_000, &alice.keypair(), bob_pubkey).unwrap();
-        assert_eq!(
-            acc.transfer(10_001, &alice.keypair(), bob_pubkey),
-            Err(AccountingError::InsufficientFunds)
-        );
-
-        let alice_pubkey = alice.keypair().pubkey();
-        assert_eq!(acc.get_balance(&alice_pubkey).unwrap(), 10_000);
-        assert_eq!(acc.get_balance(&bob_pubkey).unwrap(), 1_000);
-
-        drop(acc.historian.sender);
-        assert_eq!(
-            acc.historian.thread_hdl.join().unwrap(),
-            ExitReason::RecvDisconnected
-        );
-    }
-
-    #[test]
-    fn test_transfer_to_newb() {
-        let alice = Mint::new(10_000);
-        let mut acc = Accountant::new(&alice, Some(2));
-        let alice_keypair = alice.keypair();
-        let bob_pubkey = KeyPair::new().pubkey();
-        acc.transfer(500, &alice_keypair, bob_pubkey).unwrap();
-        assert_eq!(acc.get_balance(&bob_pubkey).unwrap(), 500);
-
-        drop(acc.historian.sender);
-        assert_eq!(
-            acc.historian.thread_hdl.join().unwrap(),
-            ExitReason::RecvDisconnected
-        );
-    }
-
-    #[test]
-    fn test_transfer_on_date() {
-        let alice = Mint::new(1);
-        let mut acc = Accountant::new(&alice, Some(2));
-        let alice_keypair = alice.keypair();
-        let bob_pubkey = KeyPair::new().pubkey();
-        let dt = Utc::now();
-        acc.transfer_on_date(1, &alice_keypair, bob_pubkey, dt)
-            .unwrap();
-
-        // Alice's balance will be zero because all funds are locked up.
-        assert_eq!(acc.get_balance(&alice.pubkey()), Some(0));
-
-        // Bob's balance will be None because the funds have not been
-        // sent.
-        assert_eq!(acc.get_balance(&bob_pubkey), None);
-
-        // Now, acknowledge the time in the condition occurred and
-        // that bob's funds are now available.
-        acc.process_verified_timestamp(alice.pubkey(), dt).unwrap();
-        assert_eq!(acc.get_balance(&bob_pubkey), Some(1));
-
-        acc.process_verified_timestamp(alice.pubkey(), dt).unwrap(); // <-- Attack! Attempt to process completed transaction.
-        assert_ne!(acc.get_balance(&bob_pubkey), Some(2));
-    }
-
-    #[test]
-    fn test_transfer_after_date() {
-        let alice = Mint::new(1);
-        let mut acc = Accountant::new(&alice, Some(2));
-        let alice_keypair = alice.keypair();
-        let bob_pubkey = KeyPair::new().pubkey();
-        let dt = Utc::now();
-        acc.process_verified_timestamp(alice.pubkey(), dt).unwrap();
-
-        // It's now past now, so this transfer should be processed immediately.
-        acc.transfer_on_date(1, &alice_keypair, bob_pubkey, dt)
-            .unwrap();
-
-        assert_eq!(acc.get_balance(&alice.pubkey()), Some(0));
-        assert_eq!(acc.get_balance(&bob_pubkey), Some(1));
-    }
-
-    #[test]
-    fn test_cancel_transfer() {
-        let alice = Mint::new(1);
-        let mut acc = Accountant::new(&alice, Some(2));
-        let alice_keypair = alice.keypair();
-        let bob_pubkey = KeyPair::new().pubkey();
-        let dt = Utc::now();
-        let sig = acc.transfer_on_date(1, &alice_keypair, bob_pubkey, dt)
-            .unwrap();
-
-        // Alice's balance will be zero because all funds are locked up.
-        assert_eq!(acc.get_balance(&alice.pubkey()), Some(0));
-
-        // Bob's balance will be None because the funds have not been
-        // sent.
-        assert_eq!(acc.get_balance(&bob_pubkey), None);
-
-        // Now, cancel the trancaction. Alice gets her funds back, Bob never sees them.
-        acc.process_verified_sig(alice.pubkey(), sig).unwrap();
-        assert_eq!(acc.get_balance(&alice.pubkey()), Some(1));
-        assert_eq!(acc.get_balance(&bob_pubkey), None);
-
-        acc.process_verified_sig(alice.pubkey(), sig).unwrap(); // <-- Attack! Attempt to cancel completed transaction.
-        assert_ne!(acc.get_balance(&alice.pubkey()), Some(2));
-    }
-}

src/accountant_skel.rs

@@ -1,75 +0,0 @@
-use std::io;
-use accountant::Accountant;
-use transaction::Transaction;
-use signature::PublicKey;
-use hash::Hash;
-use entry::Entry;
-use std::net::UdpSocket;
-use bincode::{deserialize, serialize};
-
-pub struct AccountantSkel {
-    pub acc: Accountant,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub enum Request {
-    Transaction(Transaction<i64>),
-    GetBalance { key: PublicKey },
-    GetEntries { last_id: Hash },
-    GetId { is_last: bool },
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub enum Response {
-    Balance { key: PublicKey, val: Option<i64> },
-    Entries { entries: Vec<Entry> },
-    Id { id: Hash, is_last: bool },
-}
-
-impl AccountantSkel {
-    pub fn new(acc: Accountant) -> Self {
-        AccountantSkel { acc }
-    }
-
-    pub fn process_request(self: &mut Self, msg: Request) -> Option<Response> {
-        match msg {
-            Request::Transaction(tr) => {
-                if let Err(err) = self.acc.process_transaction(tr) {
-                    eprintln!("Transaction error: {:?}", err);
-                }
-                None
-            }
-            Request::GetBalance { key } => {
-                let val = self.acc.get_balance(&key);
-                Some(Response::Balance { key, val })
-            }
-            Request::GetEntries { .. } => Some(Response::Entries { entries: vec![] }),
-            Request::GetId { is_last } => Some(Response::Id {
-                id: if is_last {
-                    self.acc.sync();
-                    self.acc.last_id
-                } else {
-                    self.acc.first_id
-                },
-                is_last,
-            }),
-        }
-    }
-
-    /// UDP Server that forwards messages to Accountant methods.
-    pub fn serve(self: &mut Self, addr: &str) -> io::Result<()> {
-        let socket = UdpSocket::bind(addr)?;
-        let mut buf = vec![0u8; 1024];
-        loop {
-            //println!("skel: Waiting for incoming packets...");
-            let (_sz, src) = socket.recv_from(&mut buf)?;
-
-            // TODO: Return a descriptive error message if deserialization fails.
-            let req = deserialize(&buf).expect("deserialize request");
-
-            if let Some(resp) = self.process_request(req) {
-                socket.send_to(&serialize(&resp).expect("serialize response"), &src)?;
-            }
-        }
-    }
-}

src/accountant_stub.rs

@@ -1,139 +0,0 @@
-//! The `accountant` is a client of the `historian`. It uses the historian's
-//! event log to record transactions. Its users can deposit funds and
-//! transfer funds to other users.
-
-use std::net::UdpSocket;
-use std::io;
-use bincode::{deserialize, serialize};
-use transaction::Transaction;
-use signature::{KeyPair, PublicKey, Signature};
-use hash::Hash;
-use entry::Entry;
-use accountant_skel::{Request, Response};
-
-pub struct AccountantStub {
-    pub addr: String,
-    pub socket: UdpSocket,
-    pub last_id: Option<Hash>,
-}
-
-impl AccountantStub {
-    pub fn new(addr: &str, socket: UdpSocket) -> Self {
-        AccountantStub {
-            addr: addr.to_string(),
-            socket,
-            last_id: None,
-        }
-    }
-
-    pub fn transfer_signed(&self, tr: Transaction<i64>) -> io::Result<usize> {
-        let req = Request::Transaction(tr);
-        let data = serialize(&req).unwrap();
-        self.socket.send_to(&data, &self.addr)
-    }
-
-    pub fn transfer(
-        &self,
-        n: i64,
-        keypair: &KeyPair,
-        to: PublicKey,
-        last_id: &Hash,
-    ) -> io::Result<Signature> {
-        let tr = Transaction::new(keypair, to, n, *last_id);
-        let sig = tr.sig;
-        self.transfer_signed(tr).map(|_| sig)
-    }
-
-    pub fn get_balance(&self, pubkey: &PublicKey) -> io::Result<Option<i64>> {
-        let req = Request::GetBalance { key: *pubkey };
-        let data = serialize(&req).expect("serialize GetBalance");
-        self.socket.send_to(&data, &self.addr)?;
-        let mut buf = vec![0u8; 1024];
-        self.socket.recv_from(&mut buf)?;
-        let resp = deserialize(&buf).expect("deserialize balance");
-        if let Response::Balance { key, val } = resp {
-            assert_eq!(key, *pubkey);
-            return Ok(val);
-        }
-        Ok(None)
-    }
-
-    fn get_id(&self, is_last: bool) -> io::Result<Hash> {
-        let req = Request::GetId { is_last };
-        let data = serialize(&req).expect("serialize GetId");
-        self.socket.send_to(&data, &self.addr)?;
-        let mut buf = vec![0u8; 1024];
-        self.socket.recv_from(&mut buf)?;
-        let resp = deserialize(&buf).expect("deserialize Id");
-        if let Response::Id { id, .. } = resp {
-            return Ok(id);
-        }
-        Ok(Default::default())
-    }
-
-    pub fn get_last_id(&self) -> io::Result<Hash> {
-        self.get_id(true)
-    }
-
-    pub fn wait_on_signature(&mut self, wait_sig: &Signature) -> io::Result<()> {
-        let last_id = match self.last_id {
-            None => {
-                let first_id = self.get_id(false)?;
-                self.last_id = Some(first_id);
-                first_id
-            }
-            Some(last_id) => last_id,
-        };
-
-        let req = Request::GetEntries { last_id };
-        let data = serialize(&req).unwrap();
-        self.socket.send_to(&data, &self.addr).map(|_| ())?;
-
-        let mut buf = vec![0u8; 1024];
-        self.socket.recv_from(&mut buf)?;
-        let resp = deserialize(&buf).expect("deserialize signature");
-        if let Response::Entries { entries } = resp {
-            for Entry { id, event, .. } in entries {
-                self.last_id = Some(id);
-                if let Some(sig) = event.get_signature() {
-                    if sig == *wait_sig {
-                        return Ok(());
-                    }
-                }
-            }
-        }
-
-        // TODO: Loop until we found it.
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use accountant::Accountant;
-    use accountant_skel::AccountantSkel;
-    use std::thread::{sleep, spawn};
-    use std::time::Duration;
-    use mint::Mint;
-    use signature::{KeyPair, KeyPairUtil};
-
-    #[test]
-    fn test_accountant_stub() {
-        let addr = "127.0.0.1:9000";
-        let send_addr = "127.0.0.1:9001";
-        let alice = Mint::new(10_000);
-        let acc = Accountant::new(&alice, None);
-        let bob_pubkey = KeyPair::new().pubkey();
-        spawn(move || AccountantSkel::new(acc).serve(addr).unwrap());
-        sleep(Duration::from_millis(30));
-
-        let socket = UdpSocket::bind(send_addr).unwrap();
-        let mut acc = AccountantStub::new(addr, socket);
-        let last_id = acc.get_last_id().unwrap();
-        let sig = acc.transfer(500, &alice.keypair(), bob_pubkey, &last_id)
-            .unwrap();
-        acc.wait_on_signature(&sig).unwrap();
-        assert_eq!(acc.get_balance(&bob_pubkey).unwrap().unwrap(), 500);
-    }
-}

src/bank.rs

@@ -0,0 +1,885 @@
+//! The `bank` module tracks client balances and the progress of smart
+//! contracts. It offers a high-level API that signs transactions
+//! on behalf of the caller, and a low-level API for when they have
+//! already been signed and verified.
+
+extern crate libc;
+
+use chrono::prelude::*;
+use counter::Counter;
+use entry::Entry;
+use hash::Hash;
+use itertools::Itertools;
+use ledger::Block;
+use mint::Mint;
+use payment_plan::{Payment, PaymentPlan, Witness};
+use signature::{KeyPair, PublicKey, Signature};
+use std::collections::hash_map::Entry::Occupied;
+use std::collections::{HashMap, HashSet, VecDeque};
+use std::result;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::RwLock;
+use std::time::Instant;
+use streamer::WINDOW_SIZE;
+use timing::duration_as_us;
+use transaction::{Instruction, Plan, Transaction};
+
+/// The number of most recent `last_id` values that the bank will track the signatures
+/// of. Once the bank discards a `last_id`, it will reject any transactions that use
+/// that `last_id` in a transaction. Lowering this value reduces memory consumption,
+/// but requires clients to update its `last_id` more frequently. Raising the value
+/// lengthens the time a client must wait to be certain a missing transaction will
+/// not be processed by the network.
+pub const MAX_ENTRY_IDS: usize = 1024 * 16;
+
+pub const VERIFY_BLOCK_SIZE: usize = 16;
+
+/// Reasons a transaction might be rejected.
+#[derive(Debug, PartialEq, Eq)]
+pub enum BankError {
+    /// Attempt to debit from `PublicKey`, but no found no record of a prior credit.
+    AccountNotFound(PublicKey),
+
+    /// The requested debit from `PublicKey` has the potential to draw the balance
+    /// below zero. This can occur when a debit and credit are processed in parallel.
+    /// The bank may reject the debit or push it to a future entry.
+    InsufficientFunds(PublicKey),
+
+    /// The bank has seen `Signature` before. This can occur under normal operation
+    /// when a UDP packet is duplicated, as a user error from a client not updating
+    /// its `last_id`, or as a double-spend attack.
+    DuplicateSignature(Signature),
+
+    /// The bank has not seen the given `last_id` or the transaction is too old and
+    /// the `last_id` has been discarded.
+    LastIdNotFound(Hash),
+
+    /// The transaction is invalid and has requested a debit or credit of negative
+    /// tokens.
+    NegativeTokens,
+
+    /// Proof of History verification failed.
+    LedgerVerificationFailed,
+}
+
+pub type Result<T> = result::Result<T, BankError>;
+
+/// The state of all accounts and contracts after processing its entries.
+pub struct Bank {
+    /// A map of account public keys to the balance in that account.
+    balances: RwLock<HashMap<PublicKey, i64>>,
+
+    /// A map of smart contract transaction signatures to what remains of its payment
+    /// plan. Each transaction that targets the plan should cause it to be reduced.
+    /// Once it cannot be reduced, final payments are made and it is discarded.
+    pending: RwLock<HashMap<Signature, Plan>>,
+
+    /// A FIFO queue of `last_id` items, where each item is a set of signatures
+    /// that have been processed using that `last_id`. Rejected `last_id`
+    /// values are so old that the `last_id` has been pulled out of the queue.
+    last_ids: RwLock<VecDeque<Hash>>,
+
+    // Mapping of hashes to signature sets. The bank uses this data to
+    /// reject transactions with signatures its seen before
+    last_ids_sigs: RwLock<HashMap<Hash, HashSet<Signature>>>,
+
+    /// The number of transactions the bank has processed without error since the
+    /// start of the ledger.
+    transaction_count: AtomicUsize,
+}
+
+impl Default for Bank {
+    fn default() -> Self {
+        Bank {
+            balances: RwLock::new(HashMap::new()),
+            pending: RwLock::new(HashMap::new()),
+            last_ids: RwLock::new(VecDeque::new()),
+            last_ids_sigs: RwLock::new(HashMap::new()),
+            transaction_count: AtomicUsize::new(0),
+        }
+    }
+}
+
+impl Bank {
+    /// Create an Bank using a deposit.
+    pub fn new_from_deposit(deposit: &Payment) -> Self {
+        let bank = Self::default();
+        bank.apply_payment(deposit, &mut bank.balances.write().unwrap());
+        bank
+    }
+
+    /// Create an Bank with only a Mint. Typically used by unit tests.
+    pub fn new(mint: &Mint) -> Self {
+        let deposit = Payment {
+            to: mint.pubkey(),
+            tokens: mint.tokens,
+        };
+        let bank = Self::new_from_deposit(&deposit);
+        bank.register_entry_id(&mint.last_id());
+        bank
+    }
+
+    /// Commit funds to the `payment.to` party.
+    fn apply_payment(&self, payment: &Payment, balances: &mut HashMap<PublicKey, i64>) {
+        *balances.entry(payment.to).or_insert(0) += payment.tokens;
+    }
+
+    /// Return the last entry ID registered.
+    pub fn last_id(&self) -> Hash {
+        let last_ids = self.last_ids.read().expect("'last_ids' read lock");
+        let last_item = last_ids
+            .iter()
+            .last()
+            .expect("get last item from 'last_ids' list");
+        *last_item
+    }
+
+    /// Store the given signature. The bank will reject any transaction with the same signature.
+    fn reserve_signature(signatures: &mut HashSet<Signature>, sig: &Signature) -> Result<()> {
+        if let Some(sig) = signatures.get(sig) {
+            return Err(BankError::DuplicateSignature(*sig));
+        }
+        signatures.insert(*sig);
+        Ok(())
+    }
+
+    /// Forget the given `signature` because its transaction was rejected.
+    fn forget_signature(signatures: &mut HashSet<Signature>, signature: &Signature) {
+        signatures.remove(signature);
+    }
+
+    /// Forget the given `signature` with `last_id` because the transaction was rejected.
+    fn forget_signature_with_last_id(&self, signature: &Signature, last_id: &Hash) {
+        if let Some(entry) = self.last_ids_sigs
+            .write()
+            .expect("'last_ids' read lock in forget_signature_with_last_id")
+            .get_mut(last_id)
+        {
+            Self::forget_signature(entry, signature);
+        }
+    }
+
+    /// Forget all signatures. Useful for benchmarking.
+    pub fn clear_signatures(&self) {
+        for (_, sigs) in self.last_ids_sigs.write().unwrap().iter_mut() {
+            sigs.clear();
+        }
+    }
+
+    fn reserve_signature_with_last_id(&self, signature: &Signature, last_id: &Hash) -> Result<()> {
+        if let Some(entry) = self.last_ids_sigs
+            .write()
+            .expect("'last_ids' read lock in reserve_signature_with_last_id")
+            .get_mut(last_id)
+        {
+            return Self::reserve_signature(entry, signature);
+        }
+        Err(BankError::LastIdNotFound(*last_id))
+    }
+
+    /// Tell the bank which Entry IDs exist on the ledger. This function
+    /// assumes subsequent calls correspond to later entries, and will boot
+    /// the oldest ones once its internal cache is full. Once boot, the
+    /// bank will reject transactions using that `last_id`.
+    pub fn register_entry_id(&self, last_id: &Hash) {
+        let mut last_ids = self.last_ids
+            .write()
+            .expect("'last_ids' write lock in register_entry_id");
+        let mut last_ids_sigs = self.last_ids_sigs
+            .write()
+            .expect("last_ids_sigs write lock");
+        if last_ids.len() >= MAX_ENTRY_IDS {
+            let id = last_ids.pop_front().unwrap();
+            last_ids_sigs.remove(&id);
+        }
+        last_ids_sigs.insert(*last_id, HashSet::new());
+        last_ids.push_back(*last_id);
+    }
+
+    /// Deduct tokens from the 'from' address the account has sufficient
+    /// funds and isn't a duplicate.
+    fn apply_debits(&self, tx: &Transaction, bals: &mut HashMap<PublicKey, i64>) -> Result<()> {
+        let mut purge = false;
+        {
+            let option = bals.get_mut(&tx.from);
+            if option.is_none() {
+                if let Instruction::NewVote(_) = &tx.instruction {
+                    inc_new_counter!("bank-appy_debits-vote_account_not_found", 1);
+                } else {
+                    inc_new_counter!("bank-appy_debits-generic_account_not_found", 1);
+                }
+                return Err(BankError::AccountNotFound(tx.from));
+            }
+            let bal = option.unwrap();
+
+            self.reserve_signature_with_last_id(&tx.sig, &tx.last_id)?;
+
+            if let Instruction::NewContract(contract) = &tx.instruction {
+                if contract.tokens < 0 {
+                    return Err(BankError::NegativeTokens);
+                }
+
+                if *bal < contract.tokens {
+                    self.forget_signature_with_last_id(&tx.sig, &tx.last_id);
+                    return Err(BankError::InsufficientFunds(tx.from));
+                } else if *bal == contract.tokens {
+                    purge = true;
+                } else {
+                    *bal -= contract.tokens;
+                }
+            };
+        }
+
+        if purge {
+            bals.remove(&tx.from);
+        }
+
+        Ok(())
+    }
+
+    /// Apply only a transaction's credits. Credits from multiple transactions
+    /// may safely be applied in parallel.
+    fn apply_credits(&self, tx: &Transaction, balances: &mut HashMap<PublicKey, i64>) {
+        match &tx.instruction {
+            Instruction::NewContract(contract) => {
+                let plan = contract.plan.clone();
+                if let Some(payment) = plan.final_payment() {
+                    self.apply_payment(&payment, balances);
+                } else {
+                    let mut pending = self.pending
+                        .write()
+                        .expect("'pending' write lock in apply_credits");
+                    pending.insert(tx.sig, plan);
+                }
+            }
+            Instruction::ApplyTimestamp(dt) => {
+                let _ = self.apply_timestamp(tx.from, *dt);
+            }
+            Instruction::ApplySignature(tx_sig) => {
+                let _ = self.apply_signature(tx.from, *tx_sig);
+            }
+            Instruction::NewVote(_vote) => {
+                info!("GOT VOTE!");
+                // TODO: record the vote in the stake table...
+            }
+        }
+    }
+
+    /// Process a Transaction. If it contains a payment plan that requires a witness
+    /// to progress, the payment plan will be stored in the bank.
+    pub fn process_transaction(&self, tx: &Transaction) -> Result<()> {
+        let bals = &mut self.balances.write().unwrap();
+        self.apply_debits(tx, bals)?;
+        self.apply_credits(tx, bals);
+        self.transaction_count.fetch_add(1, Ordering::Relaxed);
+        Ok(())
+    }
+
+    /// Process a batch of transactions.
+    #[must_use]
+    pub fn process_transactions(&self, txs: Vec<Transaction>) -> Vec<Result<Transaction>> {
+        let bals = &mut self.balances.write().unwrap();
+        debug!("processing Transactions {}", txs.len());
+        let txs_len = txs.len();
+        let now = Instant::now();
+        let results: Vec<_> = txs.into_iter()
+            .map(|tx| self.apply_debits(&tx, bals).map(|_| tx))
+            .collect(); // Calling collect() here forces all debits to complete before moving on.
+
+        let debits = now.elapsed();
+        let now = Instant::now();
+
+        let res: Vec<_> = results
+            .into_iter()
+            .map(|result| {
+                result.map(|tx| {
+                    self.apply_credits(&tx, bals);
+                    tx
+                })
+            })
+            .collect();
+
+        debug!(
+            "debits: {} us credits: {:?} us tx: {}",
+            duration_as_us(&debits),
+            duration_as_us(&now.elapsed()),
+            txs_len
+        );
+
+        let mut tx_count = 0;
+        for r in &res {
+            if r.is_ok() {
+                tx_count += 1;
+            } else {
+                info!("tx error: {:?}", r);
+            }
+        }
+        self.transaction_count
+            .fetch_add(tx_count, Ordering::Relaxed);
+        res
+    }
+
+    fn process_entry(&self, entry: Entry) -> Result<()> {
+        if !entry.transactions.is_empty() {
+            for result in self.process_transactions(entry.transactions) {
+                result?;
+            }
+        }
+        if !entry.has_more {
+            self.register_entry_id(&entry.id);
+        }
+        Ok(())
+    }
+
+    /// Process an ordered list of entries, populating a circular buffer "tail"
+    ///   as we go.
+    fn process_entries_tail(
+        &self,
+        entries: Vec<Entry>,
+        tail: &mut Vec<Entry>,
+        tail_idx: &mut usize,
+    ) -> Result<u64> {
+        let mut entry_count = 0;
+
+        for entry in entries {
+            if tail.len() > *tail_idx {
+                tail[*tail_idx] = entry.clone();
+            } else {
+                tail.push(entry.clone());
+            }
+            *tail_idx = (*tail_idx + 1) % WINDOW_SIZE as usize;
+
+            entry_count += 1;
+            self.process_entry(entry)?;
+        }
+
+        Ok(entry_count)
+    }
+
+    /// Process an ordered list of entries.
+    pub fn process_entries(&self, entries: Vec<Entry>) -> Result<u64> {
+        let mut entry_count = 0;
+        for entry in entries {
+            entry_count += 1;
+            self.process_entry(entry)?;
+        }
+        Ok(entry_count)
+    }
+
+    /// Append entry blocks to the ledger, verifying them along the way.
+    fn process_blocks<I>(
+        &self,
+        entries: I,
+        tail: &mut Vec<Entry>,
+        tail_idx: &mut usize,
+    ) -> Result<u64>
+    where
+        I: IntoIterator<Item = Entry>,
+    {
+        // Ledger verification needs to be parallelized, but we can't pull the whole
+        // thing into memory. We therefore chunk it.
+        let mut entry_count = 0;
+        for block in &entries.into_iter().chunks(VERIFY_BLOCK_SIZE) {
+            let block: Vec<_> = block.collect();
+            if !block.verify(&self.last_id()) {
+                return Err(BankError::LedgerVerificationFailed);
+            }
+            entry_count += self.process_entries_tail(block, tail, tail_idx)?;
+        }
+        Ok(entry_count)
+    }
+
+    /// Process a full ledger.
+    pub fn process_ledger<I>(&self, entries: I) -> Result<(u64, Vec<Entry>)>
+    where
+        I: IntoIterator<Item = Entry>,
+    {
+        let mut entries = entries.into_iter();
+
+        // The first item in the ledger is required to be an entry with zero num_hashes,
+        // which implies its id can be used as the ledger's seed.
+        let entry0 = entries.next().expect("invalid ledger: empty");
+
+        // The second item in the ledger is a special transaction where the to and from
+        // fields are the same. That entry should be treated as a deposit, not a
+        // transfer to oneself.
+        let entry1 = entries
+            .next()
+            .expect("invalid ledger: need at least 2 entries");
+        {
+            let tx = &entry1.transactions[0];
+            let deposit = if let Instruction::NewContract(contract) = &tx.instruction {
+                contract.plan.final_payment()
+            } else {
+                None
+            }.expect("invalid ledger, needs to start with a contract");
+
+            self.apply_payment(&deposit, &mut self.balances.write().unwrap());
+        }
+        self.register_entry_id(&entry0.id);
+        self.register_entry_id(&entry1.id);
+
+        let mut tail = Vec::with_capacity(WINDOW_SIZE as usize);
+        tail.push(entry0);
+        tail.push(entry1);
+        let mut tail_idx = 2;
+        let entry_count = 2 + self.process_blocks(entries, &mut tail, &mut tail_idx)?;
+
+        // check f we need to rotate tail
+        let tail = if tail.len() == WINDOW_SIZE as usize {
+            rotate_vector(tail, tail_idx)
+        } else {
+            tail
+        };
+
+        Ok((entry_count, tail))
+    }
+
+    /// Process a Witness Signature. Any payment plans waiting on this signature
+    /// will progress one step.
+    fn apply_signature(&self, from: PublicKey, tx_sig: Signature) -> Result<()> {
+        if let Occupied(mut e) = self.pending
+            .write()
+            .expect("write() in apply_signature")
+            .entry(tx_sig)
+        {
+            e.get_mut().apply_witness(&Witness::Signature, &from);
+            if let Some(payment) = e.get().final_payment() {
+                self.apply_payment(&payment, &mut self.balances.write().unwrap());
+                e.remove_entry();
+            }
+        };
+
+        Ok(())
+    }
+
+    /// Process a Witness Timestamp. Any payment plans waiting on this timestamp
+    /// will progress one step.
+    fn apply_timestamp(&self, from: PublicKey, dt: DateTime<Utc>) -> Result<()> {
+        // Check to see if any timelocked transactions can be completed.
+        let mut completed = vec![];
+
+        // Hold 'pending' write lock until the end of this function. Otherwise another thread can
+        // double-spend if it enters before the modified plan is removed from 'pending'.
+        let mut pending = self.pending
+            .write()
+            .expect("'pending' write lock in apply_timestamp");
+        for (key, plan) in pending.iter_mut() {
+            plan.apply_witness(&Witness::Timestamp(dt), &from);
+            if let Some(payment) = plan.final_payment() {
+                self.apply_payment(&payment, &mut self.balances.write().unwrap());
+                completed.push(key.clone());
+            }
+        }
+
+        for key in completed {
+            pending.remove(&key);
+        }
+
+        Ok(())
+    }
+
+    /// Create, sign, and process a Transaction from `keypair` to `to` of
+    /// `n` tokens where `last_id` is the last Entry ID observed by the client.
+    pub fn transfer(
+        &self,
+        n: i64,
+        keypair: &KeyPair,
+        to: PublicKey,
+        last_id: Hash,
+    ) -> Result<Signature> {
+        let tx = Transaction::new(keypair, to, n, last_id);
+        let sig = tx.sig;
+        self.process_transaction(&tx).map(|_| sig)
+    }
+
+    /// Create, sign, and process a postdated Transaction from `keypair`
+    /// to `to` of `n` tokens on `dt` where `last_id` is the last Entry ID
+    /// observed by the client.
+    pub fn transfer_on_date(
+        &self,
+        n: i64,
+        keypair: &KeyPair,
+        to: PublicKey,
+        dt: DateTime<Utc>,
+        last_id: Hash,
+    ) -> Result<Signature> {
+        let tx = Transaction::new_on_date(keypair, to, dt, n, last_id);
+        let sig = tx.sig;
+        self.process_transaction(&tx).map(|_| sig)
+    }
+
+    pub fn get_balance(&self, pubkey: &PublicKey) -> i64 {
+        let bals = self.balances
+            .read()
+            .expect("'balances' read lock in get_balance");
+        bals.get(pubkey).cloned().unwrap_or(0)
+    }
+
+    pub fn transaction_count(&self) -> usize {
+        self.transaction_count.load(Ordering::Relaxed)
+    }
+
+    pub fn has_signature(&self, signature: &Signature) -> bool {
+        let last_ids_sigs = self.last_ids_sigs
+            .read()
+            .expect("'last_ids_sigs' read lock");
+        for (_hash, signatures) in last_ids_sigs.iter() {
+            if signatures.contains(signature) {
+                return true;
+            }
+        }
+        false
+    }
+}
+
+fn rotate_vector<T: Clone>(v: Vec<T>, at: usize) -> Vec<T> {
+    if at != 0 {
+        let mut ret = Vec::with_capacity(v.len());
+        ret.extend_from_slice(&v[at..]);
+        ret.extend_from_slice(&v[0..at]);
+        ret
+    } else {
+        v
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bincode::serialize;
+    use entry::next_entry;
+    use entry::Entry;
+    use entry_writer::{self, EntryWriter};
+    use hash::hash;
+    use signature::KeyPairUtil;
+    use std::io::{BufReader, Cursor, Seek, SeekFrom};
+
+    #[test]
+    fn test_two_payments_to_one_party() {
+        let mint = Mint::new(10_000);
+        let pubkey = KeyPair::new().pubkey();
+        let bank = Bank::new(&mint);
+        assert_eq!(bank.last_id(), mint.last_id());
+
+        bank.transfer(1_000, &mint.keypair(), pubkey, mint.last_id())
+            .unwrap();
+        assert_eq!(bank.get_balance(&pubkey), 1_000);
+
+        bank.transfer(500, &mint.keypair(), pubkey, mint.last_id())
+            .unwrap();
+        assert_eq!(bank.get_balance(&pubkey), 1_500);
+        assert_eq!(bank.transaction_count(), 2);
+    }
+
+    #[test]
+    fn test_negative_tokens() {
+        let mint = Mint::new(1);
+        let pubkey = KeyPair::new().pubkey();
+        let bank = Bank::new(&mint);
+        assert_eq!(
+            bank.transfer(-1, &mint.keypair(), pubkey, mint.last_id()),
+            Err(BankError::NegativeTokens)
+        );
+        assert_eq!(bank.transaction_count(), 0);
+    }
+
+    #[test]
+    fn test_account_not_found() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let keypair = KeyPair::new();
+        assert_eq!(
+            bank.transfer(1, &keypair, mint.pubkey(), mint.last_id()),
+            Err(BankError::AccountNotFound(keypair.pubkey()))
+        );
+        assert_eq!(bank.transaction_count(), 0);
+    }
+
+    #[test]
+    fn test_insufficient_funds() {
+        let mint = Mint::new(11_000);
+        let bank = Bank::new(&mint);
+        let pubkey = KeyPair::new().pubkey();
+        bank.transfer(1_000, &mint.keypair(), pubkey, mint.last_id())
+            .unwrap();
+        assert_eq!(bank.transaction_count(), 1);
+        assert_eq!(
+            bank.transfer(10_001, &mint.keypair(), pubkey, mint.last_id()),
+            Err(BankError::InsufficientFunds(mint.pubkey()))
+        );
+        assert_eq!(bank.transaction_count(), 1);
+
+        let mint_pubkey = mint.keypair().pubkey();
+        assert_eq!(bank.get_balance(&mint_pubkey), 10_000);
+        assert_eq!(bank.get_balance(&pubkey), 1_000);
+    }
+
+    #[test]
+    fn test_transfer_to_newb() {
+        let mint = Mint::new(10_000);
+        let bank = Bank::new(&mint);
+        let pubkey = KeyPair::new().pubkey();
+        bank.transfer(500, &mint.keypair(), pubkey, mint.last_id())
+            .unwrap();
+        assert_eq!(bank.get_balance(&pubkey), 500);
+    }
+
+    #[test]
+    fn test_transfer_on_date() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let pubkey = KeyPair::new().pubkey();
+        let dt = Utc::now();
+        bank.transfer_on_date(1, &mint.keypair(), pubkey, dt, mint.last_id())
+            .unwrap();
+
+        // Mint's balance will be zero because all funds are locked up.
+        assert_eq!(bank.get_balance(&mint.pubkey()), 0);
+
+        // tx count is 1, because debits were applied.
+        assert_eq!(bank.transaction_count(), 1);
+
+        // pubkey's balance will be None because the funds have not been
+        // sent.
+        assert_eq!(bank.get_balance(&pubkey), 0);
+
+        // Now, acknowledge the time in the condition occurred and
+        // that pubkey's funds are now available.
+        bank.apply_timestamp(mint.pubkey(), dt).unwrap();
+        assert_eq!(bank.get_balance(&pubkey), 1);
+
+        // tx count is still 1, because we chose not to count timestamp transactions
+        // tx count.
+        assert_eq!(bank.transaction_count(), 1);
+
+        bank.apply_timestamp(mint.pubkey(), dt).unwrap(); // <-- Attack! Attempt to process completed transaction.
+        assert_ne!(bank.get_balance(&pubkey), 2);
+    }
+
+    #[test]
+    fn test_cancel_transfer() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let pubkey = KeyPair::new().pubkey();
+        let dt = Utc::now();
+        let sig = bank.transfer_on_date(1, &mint.keypair(), pubkey, dt, mint.last_id())
+            .unwrap();
+
+        // Assert the debit counts as a transaction.
+        assert_eq!(bank.transaction_count(), 1);
+
+        // Mint's balance will be zero because all funds are locked up.
+        assert_eq!(bank.get_balance(&mint.pubkey()), 0);
+
+        // pubkey's balance will be None because the funds have not been
+        // sent.
+        assert_eq!(bank.get_balance(&pubkey), 0);
+
+        // Now, cancel the trancaction. Mint gets her funds back, pubkey never sees them.
+        bank.apply_signature(mint.pubkey(), sig).unwrap();
+        assert_eq!(bank.get_balance(&mint.pubkey()), 1);
+        assert_eq!(bank.get_balance(&pubkey), 0);
+
+        // Assert cancel doesn't cause count to go backward.
+        assert_eq!(bank.transaction_count(), 1);
+
+        bank.apply_signature(mint.pubkey(), sig).unwrap(); // <-- Attack! Attempt to cancel completed transaction.
+        assert_ne!(bank.get_balance(&mint.pubkey()), 2);
+    }
+
+    #[test]
+    fn test_duplicate_transaction_signature() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let sig = Signature::default();
+        assert!(
+            bank.reserve_signature_with_last_id(&sig, &mint.last_id())
+                .is_ok()
+        );
+        assert_eq!(
+            bank.reserve_signature_with_last_id(&sig, &mint.last_id()),
+            Err(BankError::DuplicateSignature(sig))
+        );
+    }
+
+    #[test]
+    fn test_forget_signature() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let sig = Signature::default();
+        bank.reserve_signature_with_last_id(&sig, &mint.last_id())
+            .unwrap();
+        bank.forget_signature_with_last_id(&sig, &mint.last_id());
+        assert!(
+            bank.reserve_signature_with_last_id(&sig, &mint.last_id())
+                .is_ok()
+        );
+    }
+
+    #[test]
+    fn test_has_signature() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let sig = Signature::default();
+        bank.reserve_signature_with_last_id(&sig, &mint.last_id())
+            .expect("reserve signature");
+        assert!(bank.has_signature(&sig));
+    }
+
+    #[test]
+    fn test_reject_old_last_id() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let sig = Signature::default();
+        for i in 0..MAX_ENTRY_IDS {
+            let last_id = hash(&serialize(&i).unwrap()); // Unique hash
+            bank.register_entry_id(&last_id);
+        }
+        // Assert we're no longer able to use the oldest entry ID.
+        assert_eq!(
+            bank.reserve_signature_with_last_id(&sig, &mint.last_id()),
+            Err(BankError::LastIdNotFound(mint.last_id()))
+        );
+    }
+
+    #[test]
+    fn test_debits_before_credits() {
+        let mint = Mint::new(2);
+        let bank = Bank::new(&mint);
+        let keypair = KeyPair::new();
+        let tx0 = Transaction::new(&mint.keypair(), keypair.pubkey(), 2, mint.last_id());
+        let tx1 = Transaction::new(&keypair, mint.pubkey(), 1, mint.last_id());
+        let txs = vec![tx0, tx1];
+        let results = bank.process_transactions(txs);
+        assert!(results[1].is_err());
+
+        // Assert bad transactions aren't counted.
+        assert_eq!(bank.transaction_count(), 1);
+    }
+
+    #[test]
+    fn test_process_empty_entry_is_registered() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+        let keypair = KeyPair::new();
+        let entry = next_entry(&mint.last_id(), 1, vec![]);
+        let tx = Transaction::new(&mint.keypair(), keypair.pubkey(), 1, entry.id);
+
+        // First, ensure the TX is rejected because of the unregistered last ID
+        assert_eq!(
+            bank.process_transaction(&tx),
+            Err(BankError::LastIdNotFound(entry.id))
+        );
+
+        // Now ensure the TX is accepted despite pointing to the ID of an empty entry.
+        bank.process_entries(vec![entry]).unwrap();
+        assert!(bank.process_transaction(&tx).is_ok());
+    }
+
+    #[test]
+    fn test_process_genesis() {
+        let mint = Mint::new(1);
+        let genesis = mint.create_entries();
+        let bank = Bank::default();
+        bank.process_ledger(genesis).unwrap();
+        assert_eq!(bank.get_balance(&mint.pubkey()), 1);
+    }
+
+    fn create_sample_block(mint: &Mint, length: usize) -> impl Iterator<Item = Entry> {
+        let mut entries = Vec::with_capacity(length);
+        let mut hash = mint.last_id();
+        let mut cur_hashes = 0;
+        for _ in 0..length {
+            let keypair = KeyPair::new();
+            let tx = Transaction::new(&mint.keypair(), keypair.pubkey(), 1, hash);
+            let entry = Entry::new_mut(&mut hash, &mut cur_hashes, vec![tx], false);
+            entries.push(entry);
+        }
+        entries.into_iter()
+    }
+    fn create_sample_ledger(length: usize) -> (impl Iterator<Item = Entry>, PublicKey) {
+        let mint = Mint::new(1 + length as i64);
+        let genesis = mint.create_entries();
+        let block = create_sample_block(&mint, length);
+        (genesis.into_iter().chain(block), mint.pubkey())
+    }
+
+    #[test]
+    fn test_process_ledger() {
+        let (ledger, pubkey) = create_sample_ledger(1);
+        let (ledger, dup) = ledger.tee();
+        let bank = Bank::default();
+        let (ledger_height, tail) = bank.process_ledger(ledger).unwrap();
+        assert_eq!(bank.get_balance(&pubkey), 1);
+        assert_eq!(ledger_height, 3);
+        assert_eq!(tail.len(), 3);
+        assert_eq!(tail, dup.collect_vec());
+        let last_entry = &tail[tail.len() - 1];
+        assert_eq!(bank.last_id(), last_entry.id);
+    }
+
+    #[test]
+    fn test_process_ledger_around_window_size() {
+        // TODO: put me back in when Criterion is up
+        //        for _ in 0..10 {
+        //            let (ledger, _) = create_sample_ledger(WINDOW_SIZE as usize);
+        //            let bank = Bank::default();
+        //            let (_, _) = bank.process_ledger(ledger).unwrap();
+        //        }
+
+        let window_size = WINDOW_SIZE as usize;
+        for entry_count in window_size - 3..window_size + 2 {
+            let (ledger, pubkey) = create_sample_ledger(entry_count);
+            let bank = Bank::default();
+            let (ledger_height, tail) = bank.process_ledger(ledger).unwrap();
+            assert_eq!(bank.get_balance(&pubkey), 1);
+            assert_eq!(ledger_height, entry_count as u64 + 2);
+            assert!(tail.len() <= window_size);
+            let last_entry = &tail[tail.len() - 1];
+            assert_eq!(bank.last_id(), last_entry.id);
+        }
+    }
+
+    // Write the given entries to a file and then return a file iterator to them.
+    fn to_file_iter(entries: impl Iterator<Item = Entry>) -> impl Iterator<Item = Entry> {
+        let mut file = Cursor::new(vec![]);
+        EntryWriter::write_entries(&mut file, entries).unwrap();
+        file.seek(SeekFrom::Start(0)).unwrap();
+
+        let reader = BufReader::new(file);
+        entry_writer::read_entries(reader).map(|x| x.unwrap())
+    }
+
+    #[test]
+    fn test_process_ledger_from_file() {
+        let (ledger, pubkey) = create_sample_ledger(1);
+        let ledger = to_file_iter(ledger);
+
+        let bank = Bank::default();
+        bank.process_ledger(ledger).unwrap();
+        assert_eq!(bank.get_balance(&pubkey), 1);
+    }
+
+    #[test]
+    fn test_process_ledger_from_files() {
+        let mint = Mint::new(2);
+        let genesis = to_file_iter(mint.create_entries().into_iter());
+        let block = to_file_iter(create_sample_block(&mint, 1));
+
+        let bank = Bank::default();
+        bank.process_ledger(genesis.chain(block)).unwrap();
+        assert_eq!(bank.get_balance(&mint.pubkey()), 1);
+    }
+
+    #[test]
+    fn test_rotate_vector() {
+        let expect = vec![1, 2, 3, 4];
+
+        assert_eq!(rotate_vector(vec![4, 1, 2, 3], 1), expect);
+        assert_eq!(rotate_vector(vec![1, 2, 3, 4], 0), expect);
+        assert_eq!(rotate_vector(vec![2, 3, 4, 1], 3), expect);
+        assert_eq!(rotate_vector(vec![3, 4, 1, 2], 2), expect);
+    }
+
+}

src/banking_stage.rs

@@ -0,0 +1,201 @@
+//! The `banking_stage` processes Transaction messages. It is intended to be used
+//! to contruct a software pipeline. The stage uses all available CPU cores and
+//! can do its processing in parallel with signature verification on the GPU.
+
+use bank::Bank;
+use bincode::deserialize;
+use counter::Counter;
+use packet::{PacketRecycler, Packets, SharedPackets};
+use rayon::prelude::*;
+use record_stage::Signal;
+use result::{Error, Result};
+use service::Service;
+use std::net::SocketAddr;
+use std::sync::atomic::AtomicUsize;
+use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender};
+use std::sync::Arc;
+use std::thread::{self, Builder, JoinHandle};
+use std::time::Duration;
+use std::time::Instant;
+use timing;
+use transaction::Transaction;
+
+/// Stores the stage's thread handle and output receiver.
+pub struct BankingStage {
+    /// Handle to the stage's thread.
+    thread_hdl: JoinHandle<()>,
+}
+
+impl BankingStage {
+    /// Create the stage using `bank`. Exit when `verified_receiver` is dropped.
+    /// Discard input packets using `packet_recycler` to minimize memory
+    /// allocations in a previous stage such as the `fetch_stage`.
+    pub fn new(
+        bank: Arc<Bank>,
+        verified_receiver: Receiver<Vec<(SharedPackets, Vec<u8>)>>,
+        packet_recycler: PacketRecycler,
+    ) -> (Self, Receiver<Signal>) {
+        let (signal_sender, signal_receiver) = channel();
+        let thread_hdl = Builder::new()
+            .name("solana-banking-stage".to_string())
+            .spawn(move || loop {
+                if let Err(e) = Self::process_packets(
+                    &bank,
+                    &verified_receiver,
+                    &signal_sender,
+                    &packet_recycler,
+                ) {
+                    match e {
+                        Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => break,
+                        Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
+                        _ => error!("{:?}", e),
+                    }
+                }
+            })
+            .unwrap();
+        (BankingStage { thread_hdl }, signal_receiver)
+    }
+
+    /// Convert the transactions from a blob of binary data to a vector of transactions and
+    /// an unused `SocketAddr` that could be used to send a response.
+    fn deserialize_transactions(p: &Packets) -> Vec<Option<(Transaction, SocketAddr)>> {
+        p.packets
+            .par_iter()
+            .map(|x| {
+                deserialize(&x.data[0..x.meta.size])
+                    .map(|req| (req, x.meta.addr()))
+                    .ok()
+            })
+            .collect()
+    }
+
+    /// Process the incoming packets and send output `Signal` messages to `signal_sender`.
+    /// Discard packets via `packet_recycler`.
+    pub fn process_packets(
+        bank: &Arc<Bank>,
+        verified_receiver: &Receiver<Vec<(SharedPackets, Vec<u8>)>>,
+        signal_sender: &Sender<Signal>,
+        packet_recycler: &PacketRecycler,
+    ) -> Result<()> {
+        let timer = Duration::new(1, 0);
+        let recv_start = Instant::now();
+        let mms = verified_receiver.recv_timeout(timer)?;
+        let mut reqs_len = 0;
+        let mms_len = mms.len();
+        info!(
+            "@{:?} process start stalled for: {:?}ms batches: {}",
+            timing::timestamp(),
+            timing::duration_as_ms(&recv_start.elapsed()),
+            mms.len(),
+        );
+        let count = mms.iter().map(|x| x.1.len()).sum();
+        let proc_start = Instant::now();
+        for (msgs, vers) in mms {
+            let transactions = Self::deserialize_transactions(&msgs.read().unwrap());
+            reqs_len += transactions.len();
+            let transactions = transactions
+                .into_iter()
+                .zip(vers)
+                .filter_map(|(tx, ver)| match tx {
+                    None => None,
+                    Some((tx, _addr)) => if tx.verify_plan() && ver != 0 {
+                        Some(tx)
+                    } else {
+                        None
+                    },
+                })
+                .collect();
+
+            debug!("process_transactions");
+            let results = bank.process_transactions(transactions);
+            let transactions = results.into_iter().filter_map(|x| x.ok()).collect();
+            signal_sender.send(Signal::Transactions(transactions))?;
+            debug!("done process_transactions");
+
+            packet_recycler.recycle(msgs);
+        }
+        let total_time_s = timing::duration_as_s(&proc_start.elapsed());
+        let total_time_ms = timing::duration_as_ms(&proc_start.elapsed());
+        info!(
+            "@{:?} done processing transaction batches: {} time: {:?}ms reqs: {} reqs/s: {}",
+            timing::timestamp(),
+            mms_len,
+            total_time_ms,
+            reqs_len,
+            (reqs_len as f32) / (total_time_s)
+        );
+        inc_new_counter!("banking_stage-process_packets", count);
+        Ok(())
+    }
+}
+
+impl Service for BankingStage {
+    fn thread_hdls(self) -> Vec<JoinHandle<()>> {
+        vec![self.thread_hdl]
+    }
+
+    fn join(self) -> thread::Result<()> {
+        self.thread_hdl.join()
+    }
+}
+
+// TODO: When banking is pulled out of RequestStage, add this test back in.
+
+//use bank::Bank;
+//use entry::Entry;
+//use hash::Hash;
+//use record_stage::RecordStage;
+//use record_stage::Signal;
+//use result::Result;
+//use std::sync::mpsc::{channel, Sender};
+//use std::sync::{Arc, Mutex};
+//use std::time::Duration;
+//use transaction::Transaction;
+//
+//#[cfg(test)]
+//mod tests {
+//    use bank::Bank;
+//    use mint::Mint;
+//    use signature::{KeyPair, KeyPairUtil};
+//    use transaction::Transaction;
+//
+//    #[test]
+//    // TODO: Move this test banking_stage. Calling process_transactions() directly
+//    // defeats the purpose of this test.
+//    fn test_banking_sequential_consistency() {
+//        // In this attack we'll demonstrate that a verifier can interpret the ledger
+//        // differently if either the server doesn't signal the ledger to add an
+//        // Entry OR if the verifier tries to parallelize across multiple Entries.
+//        let mint = Mint::new(2);
+//        let bank = Bank::new(&mint);
+//        let banking_stage = EventProcessor::new(bank, &mint.last_id(), None);
+//
+//        // Process a batch that includes a transaction that receives two tokens.
+//        let alice = KeyPair::new();
+//        let tx = Transaction::new(&mint.keypair(), alice.pubkey(), 2, mint.last_id());
+//        let transactions = vec![tx];
+//        let entry0 = banking_stage.process_transactions(transactions).unwrap();
+//
+//        // Process a second batch that spends one of those tokens.
+//        let tx = Transaction::new(&alice, mint.pubkey(), 1, mint.last_id());
+//        let transactions = vec![tx];
+//        let entry1 = banking_stage.process_transactions(transactions).unwrap();
+//
+//        // Collect the ledger and feed it to a new bank.
+//        let entries = vec![entry0, entry1];
+//
+//        // Assert the user holds one token, not two. If the server only output one
+//        // entry, then the second transaction will be rejected, because it drives
+//        // the account balance below zero before the credit is added.
+//        let bank = Bank::new(&mint);
+//        for entry in entries {
+//            assert!(
+//                bank
+//                    .process_transactions(entry.transactions)
+//                    .into_iter()
+//                    .all(|x| x.is_ok())
+//            );
+//        }
+//        assert_eq!(bank.get_balance(&alice.pubkey()), Some(1));
+//    }
+//}

src/bin/client-demo.rs

@@ -1,77 +1,453 @@
+extern crate bincode;
+extern crate clap;
+extern crate env_logger;
+extern crate rayon;
 extern crate serde_json;
-extern crate silk;
+extern crate solana;
 
-use silk::accountant_stub::AccountantStub;
-use silk::signature::{KeyPair, KeyPairUtil};
-use silk::transaction::Transaction;
-use silk::mint::Mint;
+use bincode::serialize;
+use clap::{App, Arg};
+use rayon::prelude::*;
+use solana::crdt::{Crdt, NodeInfo};
+use solana::drone::{DroneRequest, DRONE_PORT};
+use solana::fullnode::Config;
+use solana::hash::Hash;
+use solana::nat::{udp_public_bind, udp_random_bind};
+use solana::ncp::Ncp;
+use solana::service::Service;
+use solana::signature::{read_keypair, GenKeys, KeyPair, KeyPairUtil};
+use solana::streamer::default_window;
+use solana::thin_client::ThinClient;
+use solana::timing::{duration_as_ms, duration_as_s};
+use solana::transaction::Transaction;
+use std::error;
+use std::fs::File;
+use std::io::Write;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpStream, UdpSocket};
+use std::process::exit;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, RwLock};
+use std::thread::sleep;
+use std::thread::Builder;
+use std::thread::JoinHandle;
+use std::time::Duration;
 use std::time::Instant;
-use std::net::UdpSocket;
-use std::io::stdin;
 
-fn main() {
-    let addr = "127.0.0.1:8000";
-    let send_addr = "127.0.0.1:8001";
+fn sample_tx_count(
+    exit: &Arc<AtomicBool>,
+    maxes: &Arc<RwLock<Vec<(f64, u64)>>>,
+    first_count: u64,
+    v: &NodeInfo,
+    sample_period: u64,
+) {
+    let mut client = mk_client(&v);
+    let mut now = Instant::now();
+    let mut initial_tx_count = client.transaction_count();
+    let mut max_tps = 0.0;
+    let mut total;
+    loop {
+        let tx_count = client.transaction_count();
+        let duration = now.elapsed();
+        now = Instant::now();
+        let sample = tx_count - initial_tx_count;
+        initial_tx_count = tx_count;
+        println!("{}: Transactions processed {}", v.contact_info.tpu, sample);
+        let ns = duration.as_secs() * 1_000_000_000 + u64::from(duration.subsec_nanos());
+        let tps = (sample * 1_000_000_000) as f64 / ns as f64;
+        if tps > max_tps {
+            max_tps = tps;
+        }
+        println!("{}: {:.2} tps", v.contact_info.tpu, tps);
+        total = tx_count - first_count;
+        println!(
+            "{}: Total Transactions processed {}",
+            v.contact_info.tpu, total
+        );
+        sleep(Duration::new(sample_period, 0));
 
-    let mint: Mint = serde_json::from_reader(stdin()).unwrap();
-    let mint_keypair = mint.keypair();
-    let mint_pubkey = mint.pubkey();
+        if exit.load(Ordering::Relaxed) {
+            println!("exiting validator thread");
+            maxes.write().unwrap().push((max_tps, total));
+            break;
+        }
+    }
+}
 
-    let socket = UdpSocket::bind(send_addr).unwrap();
-    let mut acc = AccountantStub::new(addr, socket);
-    let last_id = acc.get_last_id().unwrap();
+fn generate_and_send_txs(
+    client: &mut ThinClient,
+    tx_clients: &[ThinClient],
+    id: &KeyPair,
+    keypairs: &[KeyPair],
+    leader: &NodeInfo,
+    txs: i64,
+    last_id: &mut Hash,
+    threads: usize,
+    reclaim: bool,
+) {
+    println!("Signing transactions... {}", txs / 2,);
+    let signing_start = Instant::now();
 
-    let txs = acc.get_balance(&mint_pubkey).unwrap().unwrap();
-    println!("Mint's Initial Balance {}", txs);
+    let transactions: Vec<_> = if !reclaim {
+        keypairs
+            .par_iter()
+            .map(|keypair| Transaction::new(&id, keypair.pubkey(), 1, *last_id))
+            .collect()
+    } else {
+        keypairs
+            .par_iter()
+            .map(|keypair| Transaction::new(keypair, id.pubkey(), 1, *last_id))
+            .collect()
+    };
 
-    println!("Signing transactions...");
-    let now = Instant::now();
-    let transactions: Vec<_> = (0..txs)
-        .map(|_| {
-            let rando_pubkey = KeyPair::new().pubkey();
-            Transaction::new(&mint_keypair, rando_pubkey, 1, last_id)
-        })
-        .collect();
-    let duration = now.elapsed();
-    let ns = duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64;
+    let duration = signing_start.elapsed();
+    let ns = duration.as_secs() * 1_000_000_000 + u64::from(duration.subsec_nanos());
     let bsps = txs as f64 / ns as f64;
     let nsps = ns as f64 / txs as f64;
     println!(
-        "Done. {} thousand signatures per second, {}us per signature",
+        "Done. {:.2} thousand signatures per second, {:.2} us per signature, {} ms total time",
         bsps * 1_000_000_f64,
-        nsps / 1_000_f64
+        nsps / 1_000_f64,
+        duration_as_ms(&duration),
     );
 
-    println!("Verify signatures...");
-    let now = Instant::now();
-    for tr in &transactions {
-        assert!(tr.verify());
-    }
-    let duration = now.elapsed();
-    let ns = duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64;
-    let bsvps = txs as f64 / ns as f64;
-    let nspsv = ns as f64 / txs as f64;
     println!(
-        "Done. {} thousand signature verifications per second, {}us per signature verification",
-        bsvps * 1_000_000_f64,
-        nspsv / 1_000_f64
+        "Transfering {} transactions in {} batches",
+        txs / 2,
+        threads
+    );
+    let transfer_start = Instant::now();
+    let sz = transactions.len() / threads;
+    let chunks: Vec<_> = transactions.chunks(sz).collect();
+    chunks
+        .into_par_iter()
+        .zip(tx_clients)
+        .for_each(|(txs, client)| {
+            println!(
+                "Transferring 1 unit {} times... to {:?}",
+                txs.len(),
+                leader.contact_info.tpu
+            );
+            for tx in txs {
+                client.transfer_signed(tx).unwrap();
+            }
+        });
+    println!(
+        "Transfer done. {:?} ms {} tps",
+        duration_as_ms(&transfer_start.elapsed()),
+        txs as f32 / (duration_as_s(&transfer_start.elapsed()))
     );
 
-    println!("Transferring 1 unit {} times...", txs);
-    let now = Instant::now();
-    let mut sig = Default::default();
-    for tr in transactions {
-        sig = tr.sig;
-        acc.transfer_signed(tr).unwrap();
+    loop {
+        let new_id = client.get_last_id();
+        if *last_id != new_id {
+            *last_id = new_id;
+            break;
+        }
+        sleep(Duration::from_millis(100));
     }
-    println!("Waiting for last transaction to be confirmed...",);
-    acc.wait_on_signature(&sig).unwrap();
-
-    let duration = now.elapsed();
-    let ns = duration.as_secs() * 1_000_000_000 + duration.subsec_nanos() as u64;
-    let tps = (txs * 1_000_000_000) as f64 / ns as f64;
-    println!("Done. {} tps!", tps);
-    let val = acc.get_balance(&mint_pubkey).unwrap().unwrap();
-    println!("Mint's Final Balance {}", val);
-    assert_eq!(val, 0);
+}
+
+fn main() {
+    env_logger::init();
+    let mut threads = 4usize;
+    let mut num_nodes = 1usize;
+    let mut time_sec = 90;
+
+    let matches = App::new("solana-client-demo")
+        .arg(
+            Arg::with_name("leader")
+                .short("l")
+                .long("leader")
+                .value_name("PATH")
+                .takes_value(true)
+                .help("/path/to/leader.json"),
+        )
+        .arg(
+            Arg::with_name("keypair")
+                .short("k")
+                .long("keypair")
+                .value_name("PATH")
+                .takes_value(true)
+                .default_value("~/.config/solana/id.json")
+                .help("/path/to/id.json"),
+        )
+        .arg(
+            Arg::with_name("num_nodes")
+                .short("n")
+                .long("nodes")
+                .value_name("NUMBER")
+                .takes_value(true)
+                .help("number of nodes to converge to"),
+        )
+        .arg(
+            Arg::with_name("threads")
+                .short("t")
+                .long("threads")
+                .value_name("NUMBER")
+                .takes_value(true)
+                .help("number of threads"),
+        )
+        .arg(
+            Arg::with_name("seconds")
+                .short("s")
+                .long("sec")
+                .value_name("NUMBER")
+                .takes_value(true)
+                .help("send transactions for this many seconds"),
+        )
+        .get_matches();
+
+    let leader: NodeInfo;
+    if let Some(l) = matches.value_of("leader") {
+        leader = read_leader(l).node_info;
+    } else {
+        let server_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8000);
+        leader = NodeInfo::new_leader(&server_addr);
+    };
+
+    let id = read_keypair(matches.value_of("keypair").unwrap()).expect("client keypair");
+
+    if let Some(t) = matches.value_of("threads") {
+        threads = t.to_string().parse().expect("integer");
+    }
+
+    if let Some(n) = matches.value_of("num_nodes") {
+        num_nodes = n.to_string().parse().expect("integer");
+    }
+
+    if let Some(s) = matches.value_of("seconds") {
+        time_sec = s.to_string().parse().expect("integer");
+    }
+
+    let mut drone_addr = leader.contact_info.tpu;
+    drone_addr.set_port(DRONE_PORT);
+
+    let signal = Arc::new(AtomicBool::new(false));
+    let mut c_threads = vec![];
+    let validators = converge(&leader, &signal, num_nodes, &mut c_threads);
+    println!("Network has {} node(s)", validators.len());
+    assert!(validators.len() >= num_nodes);
+
+    let mut client = mk_client(&leader);
+
+    let starting_balance = client.poll_get_balance(&id.pubkey()).unwrap();
+    let txs: i64 = 500_000;
+
+    if starting_balance < txs {
+        let airdrop_amount = txs - starting_balance;
+        println!("Airdropping {:?} tokens", airdrop_amount);
+        request_airdrop(&drone_addr, &id, airdrop_amount as u64).unwrap();
+        // TODO: return airdrop Result from Drone
+        sleep(Duration::from_millis(100));
+
+        let balance = client.poll_get_balance(&id.pubkey()).unwrap();
+        println!("Your balance is: {:?}", balance);
+
+        if balance < txs || (starting_balance == balance) {
+            println!("TPS airdrop limit reached; wait 60sec to retry");
+            exit(1);
+        }
+    }
+
+    println!("Get last ID...");
+    let mut last_id = client.get_last_id();
+    println!("Got last ID {:?}", last_id);
+
+    let mut seed = [0u8; 32];
+    seed.copy_from_slice(&id.public_key_bytes()[..32]);
+    let rnd = GenKeys::new(seed);
+
+    println!("Creating keypairs...");
+    let keypairs = rnd.gen_n_keypairs(txs / 2);
+
+    let first_count = client.transaction_count();
+    println!("initial count {}", first_count);
+
+    println!("Sampling tps every second...",);
+
+    // Setup a thread per validator to sample every period
+    // collect the max transaction rate and total tx count seen
+    let maxes = Arc::new(RwLock::new(Vec::new()));
+    let sample_period = 1; // in seconds
+    let v_threads: Vec<_> = validators
+        .into_iter()
+        .map(|v| {
+            let exit = signal.clone();
+            let maxes = maxes.clone();
+            Builder::new()
+                .name("solana-client-sample".to_string())
+                .spawn(move || {
+                    sample_tx_count(&exit, &maxes, first_count, &v, sample_period);
+                })
+                .unwrap()
+        })
+        .collect();
+
+    let clients: Vec<_> = (0..threads).map(|_| mk_client(&leader)).collect();
+
+    // generate and send transactions for the specified duration
+    let time = Duration::new(time_sec / 2, 0);
+    let mut now = Instant::now();
+    while now.elapsed() < time {
+        generate_and_send_txs(
+            &mut client,
+            &clients,
+            &id,
+            &keypairs,
+            &leader,
+            txs,
+            &mut last_id,
+            threads,
+            false,
+        );
+    }
+    last_id = client.get_last_id();
+    now = Instant::now();
+    while now.elapsed() < time {
+        generate_and_send_txs(
+            &mut client,
+            &clients,
+            &id,
+            &keypairs,
+            &leader,
+            txs,
+            &mut last_id,
+            threads,
+            true,
+        );
+    }
+
+    // Stop the sampling threads so it will collect the stats
+    signal.store(true, Ordering::Relaxed);
+    for t in v_threads {
+        t.join().unwrap();
+    }
+
+    // Compute/report stats
+    let mut max_of_maxes = 0.0;
+    let mut total_txs = 0;
+    for (max, txs) in maxes.read().unwrap().iter() {
+        if *max > max_of_maxes {
+            max_of_maxes = *max;
+        }
+        total_txs += *txs;
+    }
+    println!(
+        "\nHighest TPS: {:.2} sampling period {}s total transactions: {} clients: {}",
+        max_of_maxes,
+        sample_period,
+        total_txs,
+        maxes.read().unwrap().len()
+    );
+
+    // join the crdt client threads
+    for t in c_threads {
+        t.join().unwrap();
+    }
+}
+
+fn mk_client(r: &NodeInfo) -> ThinClient {
+    let requests_socket = udp_random_bind(8000, 10000, 5).unwrap();
+    let transactions_socket = udp_random_bind(8000, 10000, 5).unwrap();
+
+    requests_socket
+        .set_read_timeout(Some(Duration::new(1, 0)))
+        .unwrap();
+
+    ThinClient::new(
+        r.contact_info.rpu,
+        requests_socket,
+        r.contact_info.tpu,
+        transactions_socket,
+    )
+}
+
+fn spy_node() -> (NodeInfo, UdpSocket) {
+    let gossip_socket_pair = udp_public_bind("gossip", 8000, 10000);
+    let pubkey = KeyPair::new().pubkey();
+    let daddr = "0.0.0.0:0".parse().unwrap();
+    assert!(!gossip_socket_pair.addr.ip().is_unspecified());
+    assert!(!gossip_socket_pair.addr.ip().is_multicast());
+    let node = NodeInfo::new(
+        pubkey,
+        //gossip.local_addr().unwrap(),
+        gossip_socket_pair.addr,
+        daddr,
+        daddr,
+        daddr,
+        daddr,
+    );
+    (node, gossip_socket_pair.receiver)
+}
+
+fn converge(
+    leader: &NodeInfo,
+    exit: &Arc<AtomicBool>,
+    num_nodes: usize,
+    threads: &mut Vec<JoinHandle<()>>,
+) -> Vec<NodeInfo> {
+    //lets spy on the network
+    let daddr = "0.0.0.0:0".parse().unwrap();
+    let (spy, spy_gossip) = spy_node();
+    let mut spy_crdt = Crdt::new(spy).expect("Crdt::new");
+    spy_crdt.insert(&leader);
+    spy_crdt.set_leader(leader.id);
+    let spy_ref = Arc::new(RwLock::new(spy_crdt));
+    let window = default_window();
+    let gossip_send_socket = udp_random_bind(8000, 10000, 5).unwrap();
+    let ncp = Ncp::new(
+        &spy_ref,
+        window.clone(),
+        spy_gossip,
+        gossip_send_socket,
+        exit.clone(),
+    ).expect("DataReplicator::new");
+    let mut rv = vec![];
+    //wait for the network to converge, 30 seconds should be plenty
+    for _ in 0..30 {
+        let v: Vec<NodeInfo> = spy_ref
+            .read()
+            .unwrap()
+            .table
+            .values()
+            .into_iter()
+            .filter(|x| x.contact_info.rpu != daddr)
+            .cloned()
+            .collect();
+        if v.len() >= num_nodes {
+            println!("CONVERGED!");
+            rv.extend(v.into_iter());
+            break;
+        } else {
+            println!(
+                "{} node(s) discovered (looking for {} or more)",
+                v.len(),
+                num_nodes
+            );
+        }
+        sleep(Duration::new(1, 0));
+    }
+    threads.extend(ncp.thread_hdls().into_iter());
+    rv
+}
+
+fn read_leader(path: &str) -> Config {
+    let file = File::open(path).unwrap_or_else(|_| panic!("file not found: {}", path));
+    serde_json::from_reader(file).unwrap_or_else(|_| panic!("failed to parse {}", path))
+}
+
+fn request_airdrop(
+    drone_addr: &SocketAddr,
+    id: &KeyPair,
+    tokens: u64,
+) -> Result<(), Box<error::Error>> {
+    let mut stream = TcpStream::connect(drone_addr)?;
+    let req = DroneRequest::GetAirdrop {
+        airdrop_request_amount: tokens,
+        client_public_key: id.pubkey(),
+    };
+    let tx = serialize(&req).expect("serialize drone request");
+    stream.write_all(&tx).unwrap();
+    // TODO: add timeout to this function, in case of unresponsive drone
+    Ok(())
 }

src/bin/drone.rs

@@ -0,0 +1,149 @@
+extern crate bincode;
+extern crate clap;
+extern crate env_logger;
+extern crate serde_json;
+extern crate solana;
+extern crate tokio;
+extern crate tokio_codec;
+extern crate tokio_io;
+
+use bincode::deserialize;
+use clap::{App, Arg};
+use solana::crdt::NodeInfo;
+use solana::drone::{Drone, DroneRequest, DRONE_PORT};
+use solana::fullnode::Config;
+use solana::metrics::set_panic_hook;
+use solana::signature::read_keypair;
+use std::fs::File;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::sync::{Arc, Mutex};
+use std::thread;
+use tokio::net::TcpListener;
+use tokio::prelude::*;
+use tokio_codec::{BytesCodec, Decoder};
+
+fn main() {
+    env_logger::init();
+    set_panic_hook("drone");
+    let matches = App::new("drone")
+        .arg(
+            Arg::with_name("leader")
+                .short("l")
+                .long("leader")
+                .value_name("PATH")
+                .takes_value(true)
+                .help("/path/to/leader.json"),
+        )
+        .arg(
+            Arg::with_name("keypair")
+                .short("k")
+                .long("keypair")
+                .value_name("PATH")
+                .takes_value(true)
+                .required(true)
+                .help("/path/to/mint.json"),
+        )
+        .arg(
+            Arg::with_name("time")
+                .short("t")
+                .long("time")
+                .value_name("SECONDS")
+                .takes_value(true)
+                .help("time slice over which to limit requests to drone"),
+        )
+        .arg(
+            Arg::with_name("cap")
+                .short("c")
+                .long("cap")
+                .value_name("NUMBER")
+                .takes_value(true)
+                .help("request limit for time slice"),
+        )
+        .get_matches();
+
+    let leader: NodeInfo;
+    if let Some(l) = matches.value_of("leader") {
+        leader = read_leader(l).node_info;
+    } else {
+        let server_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8000);
+        leader = NodeInfo::new_leader(&server_addr);
+    };
+
+    let mint_keypair =
+        read_keypair(matches.value_of("keypair").expect("keypair")).expect("client keypair");
+
+    let time_slice: Option<u64>;
+    if let Some(t) = matches.value_of("time") {
+        time_slice = Some(t.to_string().parse().expect("integer"));
+    } else {
+        time_slice = None;
+    }
+    let request_cap: Option<u64>;
+    if let Some(c) = matches.value_of("cap") {
+        request_cap = Some(c.to_string().parse().expect("integer"));
+    } else {
+        request_cap = None;
+    }
+
+    let drone_addr: SocketAddr = format!("0.0.0.0:{}", DRONE_PORT).parse().unwrap();
+
+    let drone = Arc::new(Mutex::new(Drone::new(
+        mint_keypair,
+        drone_addr,
+        leader.contact_info.tpu,
+        leader.contact_info.rpu,
+        time_slice,
+        request_cap,
+    )));
+
+    let drone1 = drone.clone();
+    thread::spawn(move || loop {
+        let time = drone1.lock().unwrap().time_slice;
+        thread::sleep(time);
+        drone1.lock().unwrap().clear_request_count();
+    });
+
+    let socket = TcpListener::bind(&drone_addr).unwrap();
+    println!("Drone started. Listening on: {}", drone_addr);
+    let done = socket
+        .incoming()
+        .map_err(|e| println!("failed to accept socket; error = {:?}", e))
+        .for_each(move |socket| {
+            let drone2 = drone.clone();
+            // let client_ip = socket.peer_addr().expect("drone peer_addr").ip();
+            let framed = BytesCodec::new().framed(socket);
+            let (_writer, reader) = framed.split();
+
+            let processor = reader
+                .for_each(move |bytes| {
+                    let req: DroneRequest =
+                        deserialize(&bytes).expect("deserialize packet in drone");
+                    println!("Airdrop requested...");
+                    // let res = drone2.lock().unwrap().check_rate_limit(client_ip);
+                    let res1 = drone2.lock().unwrap().send_airdrop(req);
+                    match res1 {
+                        Ok(_) => println!("Airdrop sent!"),
+                        Err(_) => println!("Request limit reached for this time slice"),
+                    }
+                    Ok(())
+                })
+                .and_then(|()| {
+                    println!("Socket received FIN packet and closed connection");
+                    Ok(())
+                })
+                .or_else(|err| {
+                    println!("Socket closed with error: {:?}", err);
+                    Err(err)
+                })
+                .then(|result| {
+                    println!("Socket closed with result: {:?}", result);
+                    Ok(())
+                });
+            tokio::spawn(processor)
+        });
+    tokio::run(done);
+}
+fn read_leader(path: &str) -> Config {
+    let file = File::open(path).unwrap_or_else(|_| panic!("file not found: {}", path));
+    serde_json::from_reader(file).unwrap_or_else(|_| panic!("failed to parse {}", path))
+}

src/bin/fullnode-config.rs

@@ -0,0 +1,81 @@
+extern crate clap;
+extern crate dirs;
+extern crate serde_json;
+extern crate solana;
+
+use clap::{App, Arg};
+use solana::crdt::{get_ip_addr, parse_port_or_addr};
+use solana::fullnode::Config;
+use solana::nat::get_public_ip_addr;
+use solana::signature::read_pkcs8;
+use std::io;
+use std::net::SocketAddr;
+
+fn main() {
+    let matches = App::new("fullnode-config")
+        .arg(
+            Arg::with_name("local")
+                .short("l")
+                .long("local")
+                .takes_value(false)
+                .help("detect network address from local machine configuration"),
+        )
+        .arg(
+            Arg::with_name("keypair")
+                .short("k")
+                .long("keypair")
+                .value_name("PATH")
+                .takes_value(true)
+                .help("/path/to/id.json"),
+        )
+        .arg(
+            Arg::with_name("public")
+                .short("p")
+                .long("public")
+                .takes_value(false)
+                .help("detect public network address using public servers"),
+        )
+        .arg(
+            Arg::with_name("bind")
+                .short("b")
+                .long("bind")
+                .value_name("PORT")
+                .takes_value(true)
+                .help("bind to port or address"),
+        )
+        .get_matches();
+
+    let bind_addr: SocketAddr = {
+        let mut bind_addr = parse_port_or_addr({
+            if let Some(b) = matches.value_of("bind") {
+                Some(b.to_string())
+            } else {
+                None
+            }
+        });
+        if matches.is_present("local") {
+            let ip = get_ip_addr().unwrap();
+            bind_addr.set_ip(ip);
+        }
+        if matches.is_present("public") {
+            let ip = get_public_ip_addr().unwrap();
+            bind_addr.set_ip(ip);
+        }
+        bind_addr
+    };
+
+    let mut path = dirs::home_dir().expect("home directory");
+    let id_path = if matches.is_present("keypair") {
+        matches.value_of("keypair").unwrap()
+    } else {
+        path.extend(&[".config", "solana", "id.json"]);
+        path.to_str().unwrap()
+    };
+    let pkcs8 = read_pkcs8(id_path).expect("client keypair");
+
+    // we need all the receiving sockets to be bound within the expected
+    // port range that we open on aws
+    let config = Config::new(&bind_addr, pkcs8);
+    let stdout = io::stdout();
+    serde_json::to_writer(stdout, &config).expect("serialize");
+}

src/bin/fullnode.rs

@@ -0,0 +1,86 @@
+extern crate clap;
+extern crate env_logger;
+extern crate getopts;
+extern crate log;
+extern crate serde_json;
+extern crate solana;
+
+use clap::{App, Arg};
+use solana::crdt::{NodeInfo, TestNode};
+use solana::fullnode::{Config, FullNode, LedgerFile};
+use solana::metrics::set_panic_hook;
+use solana::service::Service;
+use solana::signature::{KeyPair, KeyPairUtil};
+use std::fs::File;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::process::exit;
+//use std::time::Duration;
+
+fn main() -> () {
+    env_logger::init();
+    set_panic_hook("fullnode");
+    let matches = App::new("fullnode")
+        .arg(
+            Arg::with_name("identity")
+                .short("i")
+                .long("identity")
+                .value_name("FILE")
+                .takes_value(true)
+                .help("run with the identity found in FILE"),
+        )
+        .arg(
+            Arg::with_name("testnet")
+                .short("t")
+                .long("testnet")
+                .value_name("HOST:PORT")
+                .takes_value(true)
+                .help("connect to the network at this gossip entry point"),
+        )
+        .arg(
+            Arg::with_name("ledger")
+                .short("L")
+                .long("ledger")
+                .value_name("FILE")
+                .takes_value(true)
+                .help("use FILE as persistent ledger (defaults to stdin/stdout)"),
+        )
+        .get_matches();
+
+    let bind_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8000);
+    let mut keypair = KeyPair::new();
+    let mut repl_data = NodeInfo::new_leader_with_pubkey(keypair.pubkey(), &bind_addr);
+    if let Some(i) = matches.value_of("identity") {
+        let path = i.to_string();
+        if let Ok(file) = File::open(path.clone()) {
+            let parse: serde_json::Result<Config> = serde_json::from_reader(file);
+            if let Ok(data) = parse {
+                keypair = data.keypair();
+                repl_data = data.node_info;
+            } else {
+                eprintln!("failed to parse {}", path);
+                exit(1);
+            }
+        } else {
+            eprintln!("failed to read {}", path);
+            exit(1);
+        }
+    }
+    let ledger = if let Some(l) = matches.value_of("ledger") {
+        LedgerFile::Path(l.to_string())
+    } else {
+        LedgerFile::StdInOut
+    };
+
+    let mut node = TestNode::new_with_bind_addr(repl_data, bind_addr);
+    let fullnode = if let Some(t) = matches.value_of("testnet") {
+        let testnet_address_string = t.to_string();
+        let testnet_addr = testnet_address_string.parse().unwrap();
+
+        FullNode::new(node, false, ledger, Some(keypair), Some(testnet_addr))
+    } else {
+        node.data.leader_id = node.data.id;
+
+        FullNode::new(node, true, ledger, None, None)
+    };
+    fullnode.join().expect("join");
+}

src/bin/genesis-demo.rs

@@ -1,30 +0,0 @@
-extern crate serde_json;
-extern crate silk;
-
-use silk::mint::Mint;
-use silk::event::Event;
-use silk::transaction::Transaction;
-use silk::log::create_entries;
-use silk::signature::{KeyPair, KeyPairUtil, PublicKey};
-use silk::hash::Hash;
-use std::io::stdin;
-
-fn transfer(from: &KeyPair, (to, tokens): (PublicKey, i64), last_id: Hash) -> Event {
-    Event::Transaction(Transaction::new(&from, to, tokens, last_id))
-}
-
-fn main() {
-    let alice = (KeyPair::new().pubkey(), 200);
-    let bob = (KeyPair::new().pubkey(), 100);
-
-    let mint: Mint = serde_json::from_reader(stdin()).unwrap();
-    let from = mint.keypair();
-    let seed = mint.seed();
-    let mut events = mint.create_events();
-    events.push(transfer(&from, alice, seed));
-    events.push(transfer(&from, bob, seed));
-
-    for entry in create_entries(&seed, events) {
-        println!("{}", serde_json::to_string(&entry).unwrap());
-    }
-}

src/bin/genesis.rs

@@ -1,14 +1,50 @@
 //! A command-line executable for generating the chain's genesis block.
 
+extern crate atty;
+#[macro_use]
+extern crate clap;
 extern crate serde_json;
-extern crate silk;
+extern crate solana;
 
-use silk::mint::Mint;
-use std::io::stdin;
+use atty::{is, Stream};
+use clap::{App, Arg};
+use solana::entry_writer::EntryWriter;
+use solana::mint::Mint;
+use std::error;
+use std::io::{stdin, stdout, Read};
+use std::process::exit;
 
-fn main() {
-    let mint: Mint = serde_json::from_reader(stdin()).unwrap();
-    for x in mint.create_entries() {
-        println!("{}", serde_json::to_string(&x).unwrap());
+fn main() -> Result<(), Box<error::Error>> {
+    let matches = App::new("solana-genesis")
+        .arg(
+            Arg::with_name("tokens")
+                .short("t")
+                .long("tokens")
+                .value_name("NUMBER")
+                .takes_value(true)
+                .required(true)
+                .help("Number of tokens with which to initialize mint"),
+        )
+        .get_matches();
+
+    let tokens = value_t_or_exit!(matches, "tokens", i64);
+
+    if is(Stream::Stdin) {
+        eprintln!("nothing found on stdin, expected a json file");
+        exit(1);
     }
+
+    let mut buffer = String::new();
+    let num_bytes = stdin().read_to_string(&mut buffer)?;
+    if num_bytes == 0 {
+        eprintln!("empty file on stdin, expected a json file");
+        exit(1);
+    }
+
+    let pkcs8: Vec<u8> = serde_json::from_str(&buffer)?;
+    let mint = Mint::new_with_pkcs8(tokens, pkcs8);
+
+    let mut writer = stdout();
+    EntryWriter::write_entries(&mut writer, mint.create_entries())?;
+    Ok(())
 }

src/bin/historian-demo.rs

@@ -1,36 +0,0 @@
-extern crate silk;
-
-use silk::historian::Historian;
-use silk::hash::Hash;
-use silk::entry::Entry;
-use silk::log::verify_slice;
-use silk::signature::{KeyPair, KeyPairUtil};
-use silk::transaction::Transaction;
-use silk::event::Event;
-use std::thread::sleep;
-use std::time::Duration;
-use std::sync::mpsc::SendError;
-
-fn create_log(hist: &Historian, seed: &Hash) -> Result<(), SendError<Event>> {
-    sleep(Duration::from_millis(15));
-    let keypair = KeyPair::new();
-    let tr = Transaction::new(&keypair, keypair.pubkey(), 42, *seed);
-    let event0 = Event::Transaction(tr);
-    hist.sender.send(event0)?;
-    sleep(Duration::from_millis(10));
-    Ok(())
-}
-
-fn main() {
-    let seed = Hash::default();
-    let hist = Historian::new(&seed, Some(10));
-    create_log(&hist, &seed).expect("send error");
-    drop(hist.sender);
-    let entries: Vec<Entry> = hist.receiver.iter().collect();
-    for entry in &entries {
-        println!("{:?}", entry);
-    }
-    // Proof-of-History: Verify the historian learned about the events
-    // in the same order they appear in the vector.
-    assert!(verify_slice(&entries, &seed));
-}

src/bin/keygen.rs

@@ -0,0 +1,49 @@
+extern crate clap;
+extern crate dirs;
+extern crate ring;
+extern crate serde_json;
+
+use clap::{App, Arg};
+use ring::rand::SystemRandom;
+use ring::signature::Ed25519KeyPair;
+use std::error;
+use std::fs::{self, File};
+use std::io::Write;
+use std::path::Path;
+
+fn main() -> Result<(), Box<error::Error>> {
+    let matches = App::new("solana-keygen")
+        .arg(
+            Arg::with_name("outfile")
+                .short("o")
+                .long("outfile")
+                .value_name("PATH")
+                .takes_value(true)
+                .help("path to generated file"),
+        )
+        .get_matches();
+
+    let rnd = SystemRandom::new();
+    let pkcs8_bytes = Ed25519KeyPair::generate_pkcs8(&rnd)?;
+    let serialized = serde_json::to_string(&pkcs8_bytes.to_vec())?;
+
+    let mut path = dirs::home_dir().expect("home directory");
+    let outfile = if matches.is_present("outfile") {
+        matches.value_of("outfile").unwrap()
+    } else {
+        path.extend(&[".config", "solana", "id.json"]);
+        path.to_str().unwrap()
+    };
+
+    if outfile == "-" {
+        println!("{}", serialized);
+    } else {
+        if let Some(outdir) = Path::new(outfile).parent() {
+            fs::create_dir_all(outdir)?;
+        }
+        let mut f = File::create(outfile)?;
+        f.write_all(&serialized.into_bytes())?;
+    }
+
+    Ok(())
+}

src/bin/mint.rs

@@ -1,15 +0,0 @@
-extern crate serde_json;
-extern crate silk;
-
-use silk::mint::Mint;
-use std::io;
-
-fn main() {
-    let mut input_text = String::new();
-    io::stdin().read_line(&mut input_text).unwrap();
-    let trimmed = input_text.trim();
-    let tokens = trimmed.parse::<i64>().unwrap();
-
-    let mint = Mint::new(tokens);
-    println!("{}", serde_json::to_string(&mint).unwrap());
-}

src/bin/testnode.rs

@@ -1,19 +0,0 @@
-extern crate serde_json;
-extern crate silk;
-
-use silk::accountant_skel::AccountantSkel;
-use silk::accountant::Accountant;
-use std::io::{self, BufRead};
-
-fn main() {
-    let addr = "127.0.0.1:8000";
-    let stdin = io::stdin();
-    let entries = stdin
-        .lock()
-        .lines()
-        .map(|line| serde_json::from_str(&line.unwrap()).unwrap());
-    let acc = Accountant::new_from_entries(entries, Some(1000));
-    let mut skel = AccountantSkel::new(acc);
-    eprintln!("Listening on {}", addr);
-    skel.serve(addr).unwrap();
-}

src/bin/wallet.rs

@@ -0,0 +1,356 @@
+extern crate atty;
+extern crate bincode;
+extern crate bs58;
+extern crate clap;
+extern crate dirs;
+extern crate env_logger;
+extern crate serde_json;
+extern crate solana;
+
+use bincode::serialize;
+use clap::{App, Arg, SubCommand};
+use solana::crdt::NodeInfo;
+use solana::drone::{DroneRequest, DRONE_PORT};
+use solana::fullnode::Config;
+use solana::signature::{read_keypair, KeyPair, KeyPairUtil, PublicKey, Signature};
+use solana::thin_client::ThinClient;
+use std::error;
+use std::fmt;
+use std::fs::File;
+use std::io;
+use std::io::prelude::*;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpStream, UdpSocket};
+use std::thread::sleep;
+use std::time::Duration;
+
+enum WalletCommand {
+    Address,
+    Balance,
+    AirDrop(i64),
+    Pay(i64, PublicKey),
+    Confirm(Signature),
+}
+
+#[derive(Debug, Clone)]
+enum WalletError {
+    CommandNotRecognized(String),
+    BadParameter(String),
+}
+
+impl fmt::Display for WalletError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "invalid")
+    }
+}
+
+impl error::Error for WalletError {
+    fn description(&self) -> &str {
+        "invalid"
+    }
+
+    fn cause(&self) -> Option<&error::Error> {
+        // Generic error, underlying cause isn't tracked.
+        None
+    }
+}
+
+struct WalletConfig {
+    leader: NodeInfo,
+    id: KeyPair,
+    drone_addr: SocketAddr,
+    command: WalletCommand,
+}
+
+impl Default for WalletConfig {
+    fn default() -> WalletConfig {
+        let default_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8000);
+        WalletConfig {
+            leader: NodeInfo::new_leader(&default_addr),
+            id: KeyPair::new(),
+            drone_addr: default_addr,
+            command: WalletCommand::Balance,
+        }
+    }
+}
+
+fn parse_args() -> Result<WalletConfig, Box<error::Error>> {
+    let matches = App::new("solana-wallet")
+        .arg(
+            Arg::with_name("leader")
+                .short("l")
+                .long("leader")
+                .value_name("PATH")
+                .takes_value(true)
+                .help("/path/to/leader.json"),
+        )
+        .arg(
+            Arg::with_name("keypair")
+                .short("k")
+                .long("keypair")
+                .value_name("PATH")
+                .takes_value(true)
+                .help("/path/to/id.json"),
+        )
+        .subcommand(
+            SubCommand::with_name("airdrop")
+                .about("Request a batch of tokens")
+                .arg(
+                    Arg::with_name("tokens")
+                        // .index(1)
+                        .long("tokens")
+                        .value_name("NUMBER")
+                        .takes_value(true)
+                        .required(true)
+                        .help("The number of tokens to request"),
+                ),
+        )
+        .subcommand(
+            SubCommand::with_name("pay")
+                .about("Send a payment")
+                .arg(
+                    Arg::with_name("tokens")
+                        // .index(2)
+                        .long("tokens")
+                        .value_name("NUMBER")
+                        .takes_value(true)
+                        .required(true)
+                        .help("the number of tokens to send"),
+                )
+                .arg(
+                    Arg::with_name("to")
+                        // .index(1)
+                        .long("to")
+                        .value_name("PUBKEY")
+                        .takes_value(true)
+                        .help("The pubkey of recipient"),
+                ),
+        )
+        .subcommand(
+            SubCommand::with_name("confirm")
+                .about("Confirm your payment by signature")
+                .arg(
+                    Arg::with_name("signature")
+                        .index(1)
+                        .value_name("SIGNATURE")
+                        .required(true)
+                        .help("The transaction signature to confirm"),
+                ),
+        )
+        .subcommand(SubCommand::with_name("balance").about("Get your balance"))
+        .subcommand(SubCommand::with_name("address").about("Get your public key"))
+        .get_matches();
+
+    let leader: NodeInfo;
+    if let Some(l) = matches.value_of("leader") {
+        leader = read_leader(l)?.node_info;
+    } else {
+        let server_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8000);
+        leader = NodeInfo::new_leader(&server_addr);
+    };
+
+    let mut path = dirs::home_dir().expect("home directory");
+    let id_path = if matches.is_present("keypair") {
+        matches.value_of("keypair").unwrap()
+    } else {
+        path.extend(&[".config", "solana", "id.json"]);
+        path.to_str().unwrap()
+    };
+    let id = read_keypair(id_path).or_else(|err| {
+        display_actions();
+        Err(WalletError::BadParameter(format!(
+            "{}: Unable to open keypair file: {}",
+            err, id_path
+        )))
+    })?;
+
+    let mut drone_addr = leader.contact_info.tpu;
+    drone_addr.set_port(DRONE_PORT);
+
+    let command = match matches.subcommand() {
+        ("airdrop", Some(airdrop_matches)) => {
+            let tokens = airdrop_matches.value_of("tokens").unwrap().parse()?;
+            Ok(WalletCommand::AirDrop(tokens))
+        }
+        ("pay", Some(pay_matches)) => {
+            let to = if pay_matches.is_present("to") {
+                let pubkey_vec = bs58::decode(pay_matches.value_of("to").unwrap())
+                    .into_vec()
+                    .expect("base58-encoded public key");
+
+                if pubkey_vec.len() != std::mem::size_of::<PublicKey>() {
+                    display_actions();
+                    Err(WalletError::BadParameter("Invalid public key".to_string()))?;
+                }
+                PublicKey::clone_from_slice(&pubkey_vec)
+            } else {
+                id.pubkey()
+            };
+
+            let tokens = pay_matches.value_of("tokens").unwrap().parse()?;
+
+            Ok(WalletCommand::Pay(tokens, to))
+        }
+        ("confirm", Some(confirm_matches)) => {
+            let sig_vec = bs58::decode(confirm_matches.value_of("signature").unwrap())
+                .into_vec()
+                .expect("base58-encoded signature");
+
+            if sig_vec.len() == std::mem::size_of::<Signature>() {
+                let sig = Signature::clone_from_slice(&sig_vec);
+                Ok(WalletCommand::Confirm(sig))
+            } else {
+                display_actions();
+                Err(WalletError::BadParameter("Invalid signature".to_string()))
+            }
+        }
+        ("balance", Some(_balance_matches)) => Ok(WalletCommand::Balance),
+        ("address", Some(_address_matches)) => Ok(WalletCommand::Address),
+        ("", None) => {
+            display_actions();
+            Err(WalletError::CommandNotRecognized(
+                "no subcommand given".to_string(),
+            ))
+        }
+        _ => unreachable!(),
+    }?;
+
+    Ok(WalletConfig {
+        leader,
+        id,
+        drone_addr, // TODO: Add an option for this.
+        command,
+    })
+}
+
+fn process_command(
+    config: &WalletConfig,
+    client: &mut ThinClient,
+) -> Result<(), Box<error::Error>> {
+    match config.command {
+        // Check client balance
+        WalletCommand::Address => {
+            println!("{}", bs58::encode(config.id.pubkey()).into_string());
+        }
+        WalletCommand::Balance => {
+            println!("Balance requested...");
+            let balance = client.poll_get_balance(&config.id.pubkey());
+            match balance {
+                Ok(balance) => {
+                    println!("Your balance is: {:?}", balance);
+                }
+                Err(ref e) if e.kind() == std::io::ErrorKind::Other => {
+                    println!("No account found! Request an airdrop to get started.");
+                }
+                Err(error) => {
+                    println!("An error occurred: {:?}", error);
+                }
+            }
+        }
+        // Request an airdrop from Solana Drone;
+        // Request amount is set in request_airdrop function
+        WalletCommand::AirDrop(tokens) => {
+            println!(
+                "Requesting airdrop of {:?} tokens from {}",
+                tokens, config.drone_addr
+            );
+            let previous_balance = client.poll_get_balance(&config.id.pubkey())?;
+            request_airdrop(&config.drone_addr, &config.id, tokens as u64)?;
+
+            // TODO: return airdrop Result from Drone instead of polling the
+            //       network
+            let mut current_balance = previous_balance;
+            for _ in 0..20 {
+                sleep(Duration::from_millis(500));
+                current_balance = client.poll_get_balance(&config.id.pubkey())?;
+                if previous_balance != current_balance {
+                    break;
+                }
+                println!(".");
+            }
+            println!("Your balance is: {:?}", current_balance);
+            if current_balance - previous_balance != tokens {
+                Err("Airdrop failed!")?;
+            }
+        }
+        // If client has positive balance, spend tokens in {balance} number of transactions
+        WalletCommand::Pay(tokens, to) => {
+            let last_id = client.get_last_id();
+            let sig = client.transfer(tokens, &config.id, to, &last_id)?;
+            println!("{}", bs58::encode(sig).into_string());
+        }
+        // Confirm the last client transaction by signature
+        WalletCommand::Confirm(sig) => {
+            if client.check_signature(&sig) {
+                println!("Confirmed");
+            } else {
+                println!("Not found");
+            }
+        }
+    }
+    Ok(())
+}
+
+fn display_actions() {
+    println!();
+    println!("Commands:");
+    println!("  address   Get your public key");
+    println!("  balance   Get your account balance");
+    println!("  airdrop   Request a batch of tokens");
+    println!("  pay       Send tokens to a public key");
+    println!("  confirm   Confirm your last payment by signature");
+    println!();
+}
+
+fn read_leader(path: &str) -> Result<Config, WalletError> {
+    let file = File::open(path.to_string()).or_else(|err| {
+        Err(WalletError::BadParameter(format!(
+            "{}: Unable to open leader file: {}",
+            err, path
+        )))
+    })?;
+
+    serde_json::from_reader(file).or_else(|err| {
+        Err(WalletError::BadParameter(format!(
+            "{}: Failed to parse leader file: {}",
+            err, path
+        )))
+    })
+}
+
+fn mk_client(r: &NodeInfo) -> io::Result<ThinClient> {
+    let requests_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+    let transactions_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+    requests_socket
+        .set_read_timeout(Some(Duration::new(1, 0)))
+        .unwrap();
+
+    Ok(ThinClient::new(
+        r.contact_info.rpu,
+        requests_socket,
+        r.contact_info.tpu,
+        transactions_socket,
+    ))
+}
+
+fn request_airdrop(
+    drone_addr: &SocketAddr,
+    id: &KeyPair,
+    tokens: u64,
+) -> Result<(), Box<error::Error>> {
+    let mut stream = TcpStream::connect(drone_addr)?;
+    let req = DroneRequest::GetAirdrop {
+        airdrop_request_amount: tokens,
+        client_public_key: id.pubkey(),
+    };
+    let tx = serialize(&req).expect("serialize drone request");
+    stream.write_all(&tx).unwrap();
+    // TODO: add timeout to this function, in case of unresponsive drone
+    Ok(())
+}
+
+fn main() -> Result<(), Box<error::Error>> {
+    env_logger::init();
+    let config = parse_args()?;
+    let mut client = mk_client(&config.leader)?;
+    process_command(&config, &mut client)
+}

src/blob_fetch_stage.rs

@@ -0,0 +1,62 @@
+//! The `blob_fetch_stage` pulls blobs from UDP sockets and sends it to a channel.
+
+use packet::BlobRecycler;
+use service::Service;
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::channel;
+use std::sync::Arc;
+use std::thread::{self, JoinHandle};
+use streamer::{self, BlobReceiver};
+
+pub struct BlobFetchStage {
+    exit: Arc<AtomicBool>,
+    thread_hdls: Vec<JoinHandle<()>>,
+}
+
+impl BlobFetchStage {
+    pub fn new(
+        socket: UdpSocket,
+        exit: Arc<AtomicBool>,
+        blob_recycler: &BlobRecycler,
+    ) -> (Self, BlobReceiver) {
+        Self::new_multi_socket(vec![socket], exit, blob_recycler)
+    }
+    pub fn new_multi_socket(
+        sockets: Vec<UdpSocket>,
+        exit: Arc<AtomicBool>,
+        blob_recycler: &BlobRecycler,
+    ) -> (Self, BlobReceiver) {
+        let (blob_sender, blob_receiver) = channel();
+        let thread_hdls: Vec<_> = sockets
+            .into_iter()
+            .map(|socket| {
+                streamer::blob_receiver(
+                    exit.clone(),
+                    blob_recycler.clone(),
+                    socket,
+                    blob_sender.clone(),
+                ).expect("blob receiver init")
+            })
+            .collect();
+
+        (BlobFetchStage { exit, thread_hdls }, blob_receiver)
+    }
+
+    pub fn close(&self) {
+        self.exit.store(true, Ordering::Relaxed);
+    }
+}
+
+impl Service for BlobFetchStage {
+    fn thread_hdls(self) -> Vec<JoinHandle<()>> {
+        self.thread_hdls
+    }
+
+    fn join(self) -> thread::Result<()> {
+        for thread_hdl in self.thread_hdls() {
+            thread_hdl.join()?;
+        }
+        Ok(())
+    }
+}

src/budget.rs

@@ -0,0 +1,199 @@
+//! The `budget` module provides a domain-specific language for payment plans. Users create Budget objects that
+//! are given to an interpreter. The interpreter listens for `Witness` transactions,
+//! which it uses to reduce the payment plan. When the budget is reduced to a
+//! `Payment`, the payment is executed.
+
+use chrono::prelude::*;
+use payment_plan::{Payment, PaymentPlan, Witness};
+use signature::PublicKey;
+use std::mem;
+
+/// A data type representing a `Witness` that the payment plan is waiting on.
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub enum Condition {
+    /// Wait for a `Timestamp` `Witness` at or after the given `DateTime`.
+    Timestamp(DateTime<Utc>, PublicKey),
+
+    /// Wait for a `Signature` `Witness` from `PublicKey`.
+    Signature(PublicKey),
+}
+
+impl Condition {
+    /// Return true if the given Witness satisfies this Condition.
+    pub fn is_satisfied(&self, witness: &Witness, from: &PublicKey) -> bool {
+        match (self, witness) {
+            (Condition::Signature(pubkey), Witness::Signature) => pubkey == from,
+            (Condition::Timestamp(dt, pubkey), Witness::Timestamp(last_time)) => {
+                pubkey == from && dt <= last_time
+            }
+            _ => false,
+        }
+    }
+}
+
+/// A data type reprsenting a payment plan.
+#[repr(C)]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+pub enum Budget {
+    /// Make a payment.
+    Pay(Payment),
+
+    /// Make a payment after some condition.
+    After(Condition, Payment),
+
+    /// Either make a payment after one condition or a different payment after another
+    /// condition, which ever condition is satisfied first.
+    Or((Condition, Payment), (Condition, Payment)),
+}
+
+impl Budget {
+    /// Create the simplest budget - one that pays `tokens` to PublicKey.
+    pub fn new_payment(tokens: i64, to: PublicKey) -> Self {
+        Budget::Pay(Payment { tokens, to })
+    }
+
+    /// Create a budget that pays `tokens` to `to` after being witnessed by `from`.
+    pub fn new_authorized_payment(from: PublicKey, tokens: i64, to: PublicKey) -> Self {
+        Budget::After(Condition::Signature(from), Payment { tokens, to })
+    }
+
+    /// Create a budget that pays `tokens` to `to` after the given DateTime.
+    pub fn new_future_payment(
+        dt: DateTime<Utc>,
+        from: PublicKey,
+        tokens: i64,
+        to: PublicKey,
+    ) -> Self {
+        Budget::After(Condition::Timestamp(dt, from), Payment { tokens, to })
+    }
+
+    /// Create a budget that pays `tokens` to `to` after the given DateTime
+    /// unless cancelled by `from`.
+    pub fn new_cancelable_future_payment(
+        dt: DateTime<Utc>,
+        from: PublicKey,
+        tokens: i64,
+        to: PublicKey,
+    ) -> Self {
+        Budget::Or(
+            (Condition::Timestamp(dt, from), Payment { tokens, to }),
+            (Condition::Signature(from), Payment { tokens, to: from }),
+        )
+    }
+}
+
+impl PaymentPlan for Budget {
+    /// Return Payment if the budget requires no additional Witnesses.
+    fn final_payment(&self) -> Option<Payment> {
+        match self {
+            Budget::Pay(payment) => Some(payment.clone()),
+            _ => None,
+        }
+    }
+
+    /// Return true if the budget spends exactly `spendable_tokens`.
+    fn verify(&self, spendable_tokens: i64) -> bool {
+        match self {
+            Budget::Pay(payment) | Budget::After(_, payment) => payment.tokens == spendable_tokens,
+            Budget::Or(a, b) => a.1.tokens == spendable_tokens && b.1.tokens == spendable_tokens,
+        }
+    }
+
+    /// Apply a witness to the budget to see if the budget can be reduced.
+    /// If so, modify the budget in-place.
+    fn apply_witness(&mut self, witness: &Witness, from: &PublicKey) {
+        let new_payment = match self {
+            Budget::After(cond, payment) if cond.is_satisfied(witness, from) => Some(payment),
+            Budget::Or((cond, payment), _) if cond.is_satisfied(witness, from) => Some(payment),
+            Budget::Or(_, (cond, payment)) if cond.is_satisfied(witness, from) => Some(payment),
+            _ => None,
+        }.cloned();
+
+        if let Some(payment) = new_payment {
+            mem::replace(self, Budget::Pay(payment));
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use signature::{KeyPair, KeyPairUtil};
+
+    #[test]
+    fn test_signature_satisfied() {
+        let from = PublicKey::default();
+        assert!(Condition::Signature(from).is_satisfied(&Witness::Signature, &from));
+    }
+
+    #[test]
+    fn test_timestamp_satisfied() {
+        let dt1 = Utc.ymd(2014, 11, 14).and_hms(8, 9, 10);
+        let dt2 = Utc.ymd(2014, 11, 14).and_hms(10, 9, 8);
+        let from = PublicKey::default();
+        assert!(Condition::Timestamp(dt1, from).is_satisfied(&Witness::Timestamp(dt1), &from));
+        assert!(Condition::Timestamp(dt1, from).is_satisfied(&Witness::Timestamp(dt2), &from));
+        assert!(!Condition::Timestamp(dt2, from).is_satisfied(&Witness::Timestamp(dt1), &from));
+    }
+
+    #[test]
+    fn test_verify() {
+        let dt = Utc.ymd(2014, 11, 14).and_hms(8, 9, 10);
+        let from = PublicKey::default();
+        let to = PublicKey::default();
+        assert!(Budget::new_payment(42, to).verify(42));
+        assert!(Budget::new_authorized_payment(from, 42, to).verify(42));
+        assert!(Budget::new_future_payment(dt, from, 42, to).verify(42));
+        assert!(Budget::new_cancelable_future_payment(dt, from, 42, to).verify(42));
+    }
+
+    #[test]
+    fn test_authorized_payment() {
+        let from = PublicKey::default();
+        let to = PublicKey::default();
+
+        let mut budget = Budget::new_authorized_payment(from, 42, to);
+        budget.apply_witness(&Witness::Signature, &from);
+        assert_eq!(budget, Budget::new_payment(42, to));
+    }
+
+    #[test]
+    fn test_future_payment() {
+        let dt = Utc.ymd(2014, 11, 14).and_hms(8, 9, 10);
+        let from = KeyPair::new().pubkey();
+        let to = KeyPair::new().pubkey();
+
+        let mut budget = Budget::new_future_payment(dt, from, 42, to);
+        budget.apply_witness(&Witness::Timestamp(dt), &from);
+        assert_eq!(budget, Budget::new_payment(42, to));
+    }
+
+    #[test]
+    fn test_unauthorized_future_payment() {
+        // Ensure timestamp will only be acknowledged if it came from the
+        // whitelisted public key.
+        let dt = Utc.ymd(2014, 11, 14).and_hms(8, 9, 10);
+        let from = KeyPair::new().pubkey();
+        let to = KeyPair::new().pubkey();
+
+        let mut budget = Budget::new_future_payment(dt, from, 42, to);
+        let orig_budget = budget.clone();
+        budget.apply_witness(&Witness::Timestamp(dt), &to); // <-- Attack!
+        assert_eq!(budget, orig_budget);
+    }
+
+    #[test]
+    fn test_cancelable_future_payment() {
+        let dt = Utc.ymd(2014, 11, 14).and_hms(8, 9, 10);
+        let from = PublicKey::default();
+        let to = PublicKey::default();
+
+        let mut budget = Budget::new_cancelable_future_payment(dt, from, 42, to);
+        budget.apply_witness(&Witness::Timestamp(dt), &from);
+        assert_eq!(budget, Budget::new_payment(42, to));
+
+        let mut budget = Budget::new_cancelable_future_payment(dt, from, 42, to);
+        budget.apply_witness(&Witness::Signature, &from);
+        assert_eq!(budget, Budget::new_payment(42, from));
+    }
+}

src/choose_gossip_peer_strategy.rs

@@ -0,0 +1,333 @@
+use crdt::{CrdtError, NodeInfo};
+use rand::distributions::{Distribution, Weighted, WeightedChoice};
+use rand::thread_rng;
+use result::Result;
+use signature::PublicKey;
+use std;
+use std::collections::HashMap;
+
+pub const DEFAULT_WEIGHT: u32 = 1;
+
+pub trait ChooseGossipPeerStrategy {
+    fn choose_peer<'a>(&self, options: Vec<&'a NodeInfo>) -> Result<&'a NodeInfo>;
+}
+
+pub struct ChooseRandomPeerStrategy<'a> {
+    random: &'a Fn() -> u64,
+}
+
+// Given a source of randomness "random", this strategy will randomly pick a validator
+// from the input options. This strategy works in isolation, but doesn't leverage any
+// rumors from the rest of the gossip network to make more informed decisions about
+// which validators have more/less updates
+impl<'a, 'b> ChooseRandomPeerStrategy<'a> {
+    pub fn new(random: &'a Fn() -> u64) -> Self {
+        ChooseRandomPeerStrategy { random }
+    }
+}
+
+impl<'a> ChooseGossipPeerStrategy for ChooseRandomPeerStrategy<'a> {
+    fn choose_peer<'b>(&self, options: Vec<&'b NodeInfo>) -> Result<&'b NodeInfo> {
+        if options.is_empty() {
+            Err(CrdtError::NoPeers)?;
+        }
+
+        let n = ((self.random)() as usize) % options.len();
+        Ok(options[n])
+    }
+}
+
+// This strategy uses rumors accumulated from the rest of the network to weight
+// the importance of communicating with a particular validator based on cumulative network
+// perceiption of the number of updates the validator has to offer. A validator is randomly
+// picked based on a weighted sample from the pool of viable choices. The "weight", w, of a
+// particular validator "v" is calculated as follows:
+//
+//  w = [Sum for all i in I_v: (rumor_v(i) - observed(v)) * stake(i)] /
+//      [Sum for all i in I_v: Sum(stake(i))]
+//
+// where I_v is the set of all validators that returned a rumor about the update_index of
+// validator "v", stake(i) is the size of the stake of validator "i", observed(v) is the
+// observed update_index from the last direct communication validator "v", and
+// rumor_v(i) is the rumored update_index of validator "v" propagated by fellow validator "i".
+
+// This could be a problem if there are validators with large stakes lying about their
+// observed updates. There could also be a problem in network partitions, or even just
+// when certain validators are disproportionately active, where we hear more rumors about
+// certain clusters of nodes that then propagate more rumros about each other. Hopefully
+// this can be resolved with a good baseline DEFAULT_WEIGHT, or by implementing lockout
+// periods for very active validators in the future.
+
+pub struct ChooseWeightedPeerStrategy<'a> {
+    // The map of last directly observed update_index for each active validator.
+    // This is how we get observed(v) from the formula above.
+    remote: &'a HashMap<PublicKey, u64>,
+    // The map of rumored update_index for each active validator. Using the formula above,
+    // to find rumor_v(i), we would first look up "v" in the outer map, then look up
+    // "i" in the inner map, i.e. look up external_liveness[v][i]
+    external_liveness: &'a HashMap<PublicKey, HashMap<PublicKey, u64>>,
+    // A function returning the size of the stake for a particular validator, corresponds
+    // to stake(i) in the formula above.
+    get_stake: &'a Fn(PublicKey) -> f64,
+}
+
+impl<'a> ChooseWeightedPeerStrategy<'a> {
+    pub fn new(
+        remote: &'a HashMap<PublicKey, u64>,
+        external_liveness: &'a HashMap<PublicKey, HashMap<PublicKey, u64>>,
+        get_stake: &'a Fn(PublicKey) -> f64,
+    ) -> Self {
+        ChooseWeightedPeerStrategy {
+            remote,
+            external_liveness,
+            get_stake,
+        }
+    }
+
+    fn calculate_weighted_remote_index(&self, peer_id: PublicKey) -> u32 {
+        let mut last_seen_index = 0;
+        // If the peer is not in our remote table, then we leave last_seen_index as zero.
+        // Only happens when a peer appears in our crdt.table but not in our crdt.remote,
+        // which means a validator was directly injected into our crdt.table
+        if let Some(index) = self.remote.get(&peer_id) {
+            last_seen_index = *index;
+        }
+
+        let liveness_entry = self.external_liveness.get(&peer_id);
+        if liveness_entry.is_none() {
+            return DEFAULT_WEIGHT;
+        }
+
+        let votes = liveness_entry.unwrap();
+
+        if votes.is_empty() {
+            return DEFAULT_WEIGHT;
+        }
+
+        // Calculate the weighted average of the rumors
+        let mut relevant_votes = vec![];
+
+        let total_stake = votes.iter().fold(0.0, |total_stake, (&id, &vote)| {
+            let stake = (self.get_stake)(id);
+            // If the total stake is going to overflow u64, pick
+            // the larger of either the current total_stake, or the
+            // new stake, this way we are guaranteed to get at least u64/2
+            // sample of stake in our weighted calculation
+            if std::f64::MAX - total_stake < stake {
+                if stake > total_stake {
+                    relevant_votes = vec![(stake, vote)];
+                    stake
+                } else {
+                    total_stake
+                }
+            } else {
+                relevant_votes.push((stake, vote));
+                total_stake + stake
+            }
+        });
+
+        let weighted_vote = relevant_votes.iter().fold(0.0, |sum, &(stake, vote)| {
+            if vote < last_seen_index {
+                // This should never happen because we maintain the invariant that the indexes
+                // in the external_liveness table are always greater than the corresponding
+                // indexes in the remote table, if the index exists in the remote table at all.
+
+                // Case 1: Attempt to insert bigger index into the "external_liveness" table
+                // happens after an insertion into the "remote" table. In this case,
+                // (see apply_updates()) function, we prevent the insertion if the entry
+                // in the remote table >= the atempted insertion into the "external" liveness
+                // table.
+
+                // Case 2: Bigger index in the "external_liveness" table inserted before
+                // a smaller insertion into the "remote" table. We clear the corresponding
+                // "external_liveness" table entry on all insertions into the  "remote" table
+                // See apply_updates() function.
+
+                warn!("weighted peer index was smaller than local entry in remote table");
+                return sum;
+            }
+
+            let vote_difference = (vote - last_seen_index) as f64;
+            let new_weight = vote_difference * (stake / total_stake);
+
+            if std::f64::MAX - sum < new_weight {
+                return f64::max(new_weight, sum);
+            }
+
+            sum + new_weight
+        });
+
+        // Return u32 b/c the weighted sampling API from rand::distributions
+        // only takes u32 for weights
+        if weighted_vote >= f64::from(std::u32::MAX) {
+            return std::u32::MAX;
+        }
+
+        // If the weighted rumors we've heard about aren't any greater than
+        // what we've directly learned from the last time we communicated with the
+        // peer (i.e. weighted_vote == 0), then return a weight of 1.
+        // Otherwise, return the calculated weight.
+        weighted_vote as u32 + DEFAULT_WEIGHT
+    }
+}
+
+impl<'a> ChooseGossipPeerStrategy for ChooseWeightedPeerStrategy<'a> {
+    fn choose_peer<'b>(&self, options: Vec<&'b NodeInfo>) -> Result<&'b NodeInfo> {
+        if options.is_empty() {
+            Err(CrdtError::NoPeers)?;
+        }
+
+        let mut weighted_peers = vec![];
+        for peer in options {
+            let weight = self.calculate_weighted_remote_index(peer.id);
+            weighted_peers.push(Weighted { weight, item: peer });
+        }
+
+        let mut rng = thread_rng();
+        Ok(WeightedChoice::new(&mut weighted_peers).sample(&mut rng))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use choose_gossip_peer_strategy::{ChooseWeightedPeerStrategy, DEFAULT_WEIGHT};
+    use logger;
+    use signature::{KeyPair, KeyPairUtil, PublicKey};
+    use std;
+    use std::collections::HashMap;
+
+    fn get_stake(_id: PublicKey) -> f64 {
+        1.0
+    }
+
+    #[test]
+    fn test_default() {
+        logger::setup();
+
+        // Initialize the filler keys
+        let key1 = KeyPair::new().pubkey();
+
+        let remote: HashMap<PublicKey, u64> = HashMap::new();
+        let external_liveness: HashMap<PublicKey, HashMap<PublicKey, u64>> = HashMap::new();
+
+        let weighted_strategy =
+            ChooseWeightedPeerStrategy::new(&remote, &external_liveness, &get_stake);
+
+        // If external_liveness table doesn't contain this entry,
+        // return the default weight
+        let result = weighted_strategy.calculate_weighted_remote_index(key1);
+        assert_eq!(result, DEFAULT_WEIGHT);
+    }
+
+    #[test]
+    fn test_only_external_liveness() {
+        logger::setup();
+
+        // Initialize the filler keys
+        let key1 = KeyPair::new().pubkey();
+        let key2 = KeyPair::new().pubkey();
+
+        let remote: HashMap<PublicKey, u64> = HashMap::new();
+        let mut external_liveness: HashMap<PublicKey, HashMap<PublicKey, u64>> = HashMap::new();
+
+        // If only the liveness table contains the entry, should return the
+        // weighted liveness entries
+        let test_value: u32 = 5;
+        let mut rumors: HashMap<PublicKey, u64> = HashMap::new();
+        rumors.insert(key2, test_value as u64);
+        external_liveness.insert(key1, rumors);
+
+        let weighted_strategy =
+            ChooseWeightedPeerStrategy::new(&remote, &external_liveness, &get_stake);
+
+        let result = weighted_strategy.calculate_weighted_remote_index(key1);
+        assert_eq!(result, test_value + DEFAULT_WEIGHT);
+    }
+
+    #[test]
+    fn test_overflow_votes() {
+        logger::setup();
+
+        // Initialize the filler keys
+        let key1 = KeyPair::new().pubkey();
+        let key2 = KeyPair::new().pubkey();
+
+        let remote: HashMap<PublicKey, u64> = HashMap::new();
+        let mut external_liveness: HashMap<PublicKey, HashMap<PublicKey, u64>> = HashMap::new();
+
+        // If the vote index is greater than u32::MAX, default to u32::MAX
+        let test_value = (std::u32::MAX as u64) + 10;
+        let mut rumors: HashMap<PublicKey, u64> = HashMap::new();
+        rumors.insert(key2, test_value);
+        external_liveness.insert(key1, rumors);
+
+        let weighted_strategy =
+            ChooseWeightedPeerStrategy::new(&remote, &external_liveness, &get_stake);
+
+        let result = weighted_strategy.calculate_weighted_remote_index(key1);
+        assert_eq!(result, std::u32::MAX);
+    }
+
+    #[test]
+    fn test_many_validators() {
+        logger::setup();
+
+        // Initialize the filler keys
+        let key1 = KeyPair::new().pubkey();
+
+        let mut remote: HashMap<PublicKey, u64> = HashMap::new();
+        let mut external_liveness: HashMap<PublicKey, HashMap<PublicKey, u64>> = HashMap::new();
+
+        // Test many validators' rumors in external_liveness
+        let num_peers = 10;
+        let mut rumors: HashMap<PublicKey, u64> = HashMap::new();
+
+        remote.insert(key1, 0);
+
+        for i in 0..num_peers {
+            let pk = KeyPair::new().pubkey();
+            rumors.insert(pk, i);
+        }
+
+        external_liveness.insert(key1, rumors);
+
+        let weighted_strategy =
+            ChooseWeightedPeerStrategy::new(&remote, &external_liveness, &get_stake);
+
+        let result = weighted_strategy.calculate_weighted_remote_index(key1);
+        assert_eq!(result, (num_peers / 2) as u32);
+    }
+
+    #[test]
+    fn test_many_validators2() {
+        logger::setup();
+
+        // Initialize the filler keys
+        let key1 = KeyPair::new().pubkey();
+
+        let mut remote: HashMap<PublicKey, u64> = HashMap::new();
+        let mut external_liveness: HashMap<PublicKey, HashMap<PublicKey, u64>> = HashMap::new();
+
+        // Test many validators' rumors in external_liveness
+        let num_peers = 10;
+        let old_index = 20;
+        let mut rumors: HashMap<PublicKey, u64> = HashMap::new();
+
+        remote.insert(key1, old_index);
+
+        for _i in 0..num_peers {
+            let pk = KeyPair::new().pubkey();
+            rumors.insert(pk, old_index);
+        }
+
+        external_liveness.insert(key1, rumors);
+
+        let weighted_strategy =
+            ChooseWeightedPeerStrategy::new(&remote, &external_liveness, &get_stake);
+
+        let result = weighted_strategy.calculate_weighted_remote_index(key1);
+
+        // If nobody has seen a newer update then revert to default
+        assert_eq!(result, DEFAULT_WEIGHT);
+    }
+}

src/counter.rs

@@ -0,0 +1,155 @@
+use influx_db_client as influxdb;
+use metrics;
+use std::env;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use timing;
+
+const DEFAULT_METRICS_RATE: usize = 100;
+
+pub struct Counter {
+    pub name: &'static str,
+    /// total accumulated value
+    pub counts: AtomicUsize,
+    pub times: AtomicUsize,
+    /// last accumulated value logged
+    pub lastlog: AtomicUsize,
+    pub lograte: AtomicUsize,
+}
+
+macro_rules! create_counter {
+    ($name:expr, $lograte:expr) => {
+        Counter {
+            name: $name,
+            counts: AtomicUsize::new(0),
+            times: AtomicUsize::new(0),
+            lastlog: AtomicUsize::new(0),
+            lograte: AtomicUsize::new($lograte),
+        }
+    };
+}
+
+macro_rules! inc_counter {
+    ($name:expr, $count:expr) => {
+        unsafe { $name.inc($count) };
+    };
+}
+
+macro_rules! inc_new_counter {
+    ($name:expr, $count:expr) => {{
+        static mut INC_NEW_COUNTER: Counter = create_counter!($name, 0);
+        inc_counter!(INC_NEW_COUNTER, $count);
+    }};
+    ($name:expr, $count:expr, $lograte:expr) => {{
+        static mut INC_NEW_COUNTER: Counter = create_counter!($name, $lograte);
+        inc_counter!(INC_NEW_COUNTER, $count);
+    }};
+}
+
+impl Counter {
+    fn default_log_rate() -> usize {
+        let v = env::var("SOLANA_DEFAULT_METRICS_RATE")
+            .map(|x| x.parse().unwrap_or(DEFAULT_METRICS_RATE))
+            .unwrap_or(DEFAULT_METRICS_RATE);
+        if v == 0 {
+            DEFAULT_METRICS_RATE
+        } else {
+            v
+        }
+    }
+    pub fn inc(&mut self, events: usize) {
+        let counts = self.counts.fetch_add(events, Ordering::Relaxed);
+        let times = self.times.fetch_add(1, Ordering::Relaxed);
+        let mut lograte = self.lograte.load(Ordering::Relaxed);
+        if lograte == 0 {
+            lograte = Counter::default_log_rate();
+            self.lograte.store(lograte, Ordering::Relaxed);
+        }
+        if times % lograte == 0 && times > 0 {
+            let lastlog = self.lastlog.load(Ordering::Relaxed);
+            info!(
+                "COUNTER:{{\"name\": \"{}\", \"counts\": {}, \"samples\": {},  \"now\": {}}}",
+                self.name,
+                counts,
+                times,
+                timing::timestamp(),
+            );
+            metrics::submit(
+                influxdb::Point::new(&format!("counter-{}", self.name))
+                    .add_field(
+                        "count",
+                        influxdb::Value::Integer(counts as i64 - lastlog as i64),
+                    )
+                    .to_owned(),
+            );
+            self.lastlog
+                .compare_and_swap(lastlog, counts, Ordering::Relaxed);
+        }
+    }
+}
+#[cfg(test)]
+mod tests {
+    use counter::{Counter, DEFAULT_METRICS_RATE};
+    use std::env;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    #[test]
+    fn test_counter() {
+        static mut COUNTER: Counter = create_counter!("test", 100);
+        let count = 1;
+        inc_counter!(COUNTER, count);
+        unsafe {
+            assert_eq!(COUNTER.counts.load(Ordering::Relaxed), 1);
+            assert_eq!(COUNTER.times.load(Ordering::Relaxed), 1);
+            assert_eq!(COUNTER.lograte.load(Ordering::Relaxed), 100);
+            assert_eq!(COUNTER.lastlog.load(Ordering::Relaxed), 0);
+            assert_eq!(COUNTER.name, "test");
+        }
+        for _ in 0..199 {
+            inc_counter!(COUNTER, 2);
+        }
+        unsafe {
+            assert_eq!(COUNTER.lastlog.load(Ordering::Relaxed), 199);
+        }
+        inc_counter!(COUNTER, 2);
+        unsafe {
+            assert_eq!(COUNTER.lastlog.load(Ordering::Relaxed), 399);
+        }
+    }
+    #[test]
+    fn test_inc_new_counter() {
+        //make sure that macros are syntactically correct
+        //the variable is internal to the macro scope so there is no way to introspect it
+        inc_new_counter!("counter-1", 1);
+        inc_new_counter!("counter-2", 1, 2);
+    }
+    #[test]
+    fn test_lograte() {
+        static mut COUNTER: Counter = create_counter!("test_lograte", 0);
+        inc_counter!(COUNTER, 2);
+        unsafe {
+            assert_eq!(
+                COUNTER.lograte.load(Ordering::Relaxed),
+                DEFAULT_METRICS_RATE
+            );
+        }
+    }
+    #[test]
+    fn test_lograte_env() {
+        assert_ne!(DEFAULT_METRICS_RATE, 0);
+        static mut COUNTER: Counter = create_counter!("test_lograte_env", 0);
+        env::set_var("SOLANA_DEFAULT_METRICS_RATE", "50");
+        inc_counter!(COUNTER, 2);
+        unsafe {
+            assert_eq!(COUNTER.lograte.load(Ordering::Relaxed), 50);
+        }
+
+        static mut COUNTER2: Counter = create_counter!("test_lograte_env", 0);
+        env::set_var("SOLANA_DEFAULT_METRICS_RATE", "0");
+        inc_counter!(COUNTER2, 2);
+        unsafe {
+            assert_eq!(
+                COUNTER2.lograte.load(Ordering::Relaxed),
+                DEFAULT_METRICS_RATE
+            );
+        }
+    }
+}

src/drone.rs

@@ -0,0 +1,337 @@
+//! The `drone` module provides an object for launching a Solana Drone,
+//! which is the custodian of any remaining tokens in a mint.
+//! The Solana Drone builds and send airdrop transactions,
+//! checking requests against a request cap for a given time time_slice
+//! and (to come) an IP rate limit.
+
+use influx_db_client as influxdb;
+use metrics;
+use signature::{KeyPair, PublicKey};
+use std::io;
+use std::io::{Error, ErrorKind};
+use std::net::{IpAddr, SocketAddr, UdpSocket};
+use std::time::Duration;
+use thin_client::ThinClient;
+use transaction::Transaction;
+
+pub const TIME_SLICE: u64 = 60;
+pub const REQUEST_CAP: u64 = 1_000_000;
+pub const DRONE_PORT: u16 = 9900;
+
+#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
+pub enum DroneRequest {
+    GetAirdrop {
+        airdrop_request_amount: u64,
+        client_public_key: PublicKey,
+    },
+}
+
+pub struct Drone {
+    mint_keypair: KeyPair,
+    ip_cache: Vec<IpAddr>,
+    _airdrop_addr: SocketAddr,
+    transactions_addr: SocketAddr,
+    requests_addr: SocketAddr,
+    pub time_slice: Duration,
+    request_cap: u64,
+    pub request_current: u64,
+}
+
+impl Drone {
+    pub fn new(
+        mint_keypair: KeyPair,
+        _airdrop_addr: SocketAddr,
+        transactions_addr: SocketAddr,
+        requests_addr: SocketAddr,
+        time_input: Option<u64>,
+        request_cap_input: Option<u64>,
+    ) -> Drone {
+        let time_slice = match time_input {
+            Some(time) => Duration::new(time, 0),
+            None => Duration::new(TIME_SLICE, 0),
+        };
+        let request_cap = match request_cap_input {
+            Some(cap) => cap,
+            None => REQUEST_CAP,
+        };
+        Drone {
+            mint_keypair,
+            ip_cache: Vec::new(),
+            _airdrop_addr,
+            transactions_addr,
+            requests_addr,
+            time_slice,
+            request_cap,
+            request_current: 0,
+        }
+    }
+
+    pub fn check_request_limit(&mut self, request_amount: u64) -> bool {
+        (self.request_current + request_amount) <= self.request_cap
+    }
+
+    pub fn clear_request_count(&mut self) {
+        self.request_current = 0;
+    }
+
+    pub fn add_ip_to_cache(&mut self, ip: IpAddr) {
+        self.ip_cache.push(ip);
+    }
+
+    pub fn clear_ip_cache(&mut self) {
+        self.ip_cache.clear();
+    }
+
+    pub fn check_rate_limit(&mut self, ip: IpAddr) -> Result<IpAddr, IpAddr> {
+        // [WIP] This is placeholder code for a proper rate limiter.
+        // Right now it will only allow one total drone request per IP
+        if self.ip_cache.contains(&ip) {
+            // Add proper error handling here
+            Err(ip)
+        } else {
+            self.add_ip_to_cache(ip);
+            Ok(ip)
+        }
+    }
+
+    pub fn send_airdrop(&mut self, req: DroneRequest) -> Result<usize, io::Error> {
+        let tx: Transaction;
+        let request_amount: u64;
+        let requests_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+        let transactions_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
+
+        let mut client = ThinClient::new(
+            self.requests_addr,
+            requests_socket,
+            self.transactions_addr,
+            transactions_socket,
+        );
+        let last_id = client.get_last_id();
+
+        match req {
+            DroneRequest::GetAirdrop {
+                airdrop_request_amount,
+                client_public_key,
+            } => {
+                info!(
+                    "Requesting airdrop of {} to {:?}",
+                    airdrop_request_amount, client_public_key
+                );
+                request_amount = airdrop_request_amount;
+                tx = Transaction::new(
+                    &self.mint_keypair,
+                    client_public_key,
+                    airdrop_request_amount as i64,
+                    last_id,
+                );
+            }
+        }
+        if self.check_request_limit(request_amount) {
+            self.request_current += request_amount;
+            metrics::submit(
+                influxdb::Point::new("drone")
+                    .add_tag("op", influxdb::Value::String("airdrop".to_string()))
+                    .add_field(
+                        "request_amount",
+                        influxdb::Value::Integer(request_amount as i64),
+                    )
+                    .add_field(
+                        "request_current",
+                        influxdb::Value::Integer(self.request_current as i64),
+                    )
+                    .to_owned(),
+            );
+            client.transfer_signed(&tx)
+        } else {
+            Err(Error::new(ErrorKind::Other, "token limit reached"))
+        }
+    }
+}
+
+impl Drop for Drone {
+    fn drop(&mut self) {
+        metrics::flush();
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use bank::Bank;
+    use crdt::{get_ip_addr, TestNode};
+    use drone::{Drone, DroneRequest, REQUEST_CAP, TIME_SLICE};
+    use fullnode::FullNode;
+    use logger;
+    use mint::Mint;
+    use service::Service;
+    use signature::{KeyPair, KeyPairUtil};
+    use std::io::sink;
+    use std::net::{SocketAddr, UdpSocket};
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::Arc;
+    use std::thread::sleep;
+    use std::time::Duration;
+    use thin_client::ThinClient;
+
+    #[test]
+    fn test_check_request_limit() {
+        let keypair = KeyPair::new();
+        let mut addr: SocketAddr = "0.0.0.0:9900".parse().unwrap();
+        addr.set_ip(get_ip_addr().unwrap());
+        let transactions_addr = "0.0.0.0:0".parse().unwrap();
+        let requests_addr = "0.0.0.0:0".parse().unwrap();
+        let mut drone = Drone::new(
+            keypair,
+            addr,
+            transactions_addr,
+            requests_addr,
+            None,
+            Some(3),
+        );
+        assert!(drone.check_request_limit(1));
+        drone.request_current = 3;
+        assert!(!drone.check_request_limit(1));
+    }
+
+    #[test]
+    fn test_clear_request_count() {
+        let keypair = KeyPair::new();
+        let mut addr: SocketAddr = "0.0.0.0:9900".parse().unwrap();
+        addr.set_ip(get_ip_addr().unwrap());
+        let transactions_addr = "0.0.0.0:0".parse().unwrap();
+        let requests_addr = "0.0.0.0:0".parse().unwrap();
+        let mut drone = Drone::new(keypair, addr, transactions_addr, requests_addr, None, None);
+        drone.request_current = drone.request_current + 256;
+        assert_eq!(drone.request_current, 256);
+        drone.clear_request_count();
+        assert_eq!(drone.request_current, 0);
+    }
+
+    #[test]
+    fn test_add_ip_to_cache() {
+        let keypair = KeyPair::new();
+        let mut addr: SocketAddr = "0.0.0.0:9900".parse().unwrap();
+        addr.set_ip(get_ip_addr().unwrap());
+        let transactions_addr = "0.0.0.0:0".parse().unwrap();
+        let requests_addr = "0.0.0.0:0".parse().unwrap();
+        let mut drone = Drone::new(keypair, addr, transactions_addr, requests_addr, None, None);
+        let ip = "127.0.0.1".parse().expect("create IpAddr from string");
+        assert_eq!(drone.ip_cache.len(), 0);
+        drone.add_ip_to_cache(ip);
+        assert_eq!(drone.ip_cache.len(), 1);
+        assert!(drone.ip_cache.contains(&ip));
+    }
+
+    #[test]
+    fn test_clear_ip_cache() {
+        let keypair = KeyPair::new();
+        let mut addr: SocketAddr = "0.0.0.0:9900".parse().unwrap();
+        addr.set_ip(get_ip_addr().unwrap());
+        let transactions_addr = "0.0.0.0:0".parse().unwrap();
+        let requests_addr = "0.0.0.0:0".parse().unwrap();
+        let mut drone = Drone::new(keypair, addr, transactions_addr, requests_addr, None, None);
+        let ip = "127.0.0.1".parse().expect("create IpAddr from string");
+        assert_eq!(drone.ip_cache.len(), 0);
+        drone.add_ip_to_cache(ip);
+        assert_eq!(drone.ip_cache.len(), 1);
+        drone.clear_ip_cache();
+        assert_eq!(drone.ip_cache.len(), 0);
+        assert!(drone.ip_cache.is_empty());
+    }
+
+    #[test]
+    fn test_drone_default_init() {
+        let keypair = KeyPair::new();
+        let mut addr: SocketAddr = "0.0.0.0:9900".parse().unwrap();
+        addr.set_ip(get_ip_addr().unwrap());
+        let transactions_addr = "0.0.0.0:0".parse().unwrap();
+        let requests_addr = "0.0.0.0:0".parse().unwrap();
+        let time_slice: Option<u64> = None;
+        let request_cap: Option<u64> = None;
+        let drone = Drone::new(
+            keypair,
+            addr,
+            transactions_addr,
+            requests_addr,
+            time_slice,
+            request_cap,
+        );
+        assert_eq!(drone.time_slice, Duration::new(TIME_SLICE, 0));
+        assert_eq!(drone.request_cap, REQUEST_CAP);
+    }
+
+    #[test]
+    #[ignore]
+    fn test_send_airdrop() {
+        const SMALL_BATCH: i64 = 50;
+        const TPS_BATCH: i64 = 5_000_000;
+
+        logger::setup();
+        let leader = TestNode::new_localhost();
+
+        let alice = Mint::new(10_000_000);
+        let bank = Bank::new(&alice);
+        let bob_pubkey = KeyPair::new().pubkey();
+        let carlos_pubkey = KeyPair::new().pubkey();
+        let exit = Arc::new(AtomicBool::new(false));
+        let leader_data = leader.data.clone();
+
+        let server = FullNode::new_leader(
+            bank,
+            0,
+            None,
+            Some(Duration::from_millis(30)),
+            leader,
+            exit.clone(),
+            sink(),
+        );
+        //TODO: this seems unstable
+        sleep(Duration::from_millis(900));
+
+        let mut addr: SocketAddr = "0.0.0.0:9900".parse().expect("bind to drone socket");
+        addr.set_ip(get_ip_addr().expect("drone get_ip_addr"));
+        let mut drone = Drone::new(
+            alice.keypair(),
+            addr,
+            leader_data.contact_info.tpu,
+            leader_data.contact_info.rpu,
+            None,
+            Some(150_000),
+        );
+
+        let bob_req = DroneRequest::GetAirdrop {
+            airdrop_request_amount: 50,
+            client_public_key: bob_pubkey,
+        };
+        let bob_result = drone.send_airdrop(bob_req).expect("send airdrop test");
+        assert!(bob_result > 0);
+
+        let carlos_req = DroneRequest::GetAirdrop {
+            airdrop_request_amount: 5_000_000,
+            client_public_key: carlos_pubkey,
+        };
+        let carlos_result = drone.send_airdrop(carlos_req).expect("send airdrop test");
+        assert!(carlos_result > 0);
+
+        let requests_socket = UdpSocket::bind("0.0.0.0:0").expect("drone bind to requests socket");
+        let transactions_socket =
+            UdpSocket::bind("0.0.0.0:0").expect("drone bind to transactions socket");
+
+        let mut client = ThinClient::new(
+            leader_data.contact_info.rpu,
+            requests_socket,
+            leader_data.contact_info.tpu,
+            transactions_socket,
+        );
+
+        let bob_balance = client.poll_get_balance(&bob_pubkey);
+        info!("Small request balance: {:?}", bob_balance);
+        assert_eq!(bob_balance.unwrap(), SMALL_BATCH);
+
+        let carlos_balance = client.poll_get_balance(&carlos_pubkey);
+        info!("TPS request balance: {:?}", carlos_balance);
+        assert_eq!(carlos_balance.unwrap(), TPS_BATCH);
+
+        exit.store(true, Ordering::Relaxed);
+        server.join().unwrap();
+    }
+}

src/entry.rs

@@ -1,97 +1,236 @@
+//! The `entry` module is a fundamental building block of Proof of History. It contains a
+//! unique ID that is the hash of the Entry before it, plus the hash of the
+//! transactions within it. Entries cannot be reordered, and its field `num_hashes`
+//! represents an approximate amount of time since the last Entry was created.
+use bincode::serialized_size;
 use hash::{extend_and_hash, hash, Hash};
-use event::Event;
+use packet::BLOB_DATA_SIZE;
+use rayon::prelude::*;
+use transaction::Transaction;
+
+/// Each Entry contains three pieces of data. The `num_hashes` field is the number
+/// of hashes performed since the previous entry.  The `id` field is the result
+/// of hashing `id` from the previous entry `num_hashes` times.  The `transactions`
+/// field points to Transactions that took place shortly before `id` was generated.
+///
+/// If you divide `num_hashes` by the amount of time it takes to generate a new hash, you
+/// get a duration estimate since the last Entry. Since processing power increases
+/// over time, one should expect the duration `num_hashes` represents to decrease proportionally.
+/// An upper bound on Duration can be estimated by assuming each hash was generated by the
+/// world's fastest processor at the time the entry was recorded. Or said another way, it
+/// is physically not possible for a shorter duration to have occurred if one assumes the
+/// hash was computed by the world's fastest processor at that time. The hash chain is both
+/// a Verifiable Delay Function (VDF) and a Proof of Work (not to be confused with Proof or
+/// Work consensus!)
 
 #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
 pub struct Entry {
+    /// The number of hashes since the previous Entry ID.
     pub num_hashes: u64,
+
+    /// The SHA-256 hash `num_hashes` after the previous Entry ID.
     pub id: Hash,
-    pub event: Event,
+
+    /// An unordered list of transactions that were observed before the Entry ID was
+    /// generated. The may have been observed before a previous Entry ID but were
+    /// pushed back into this list to ensure deterministic interpretation of the ledger.
+    pub transactions: Vec<Transaction>,
+
+    /// Indication that:
+    ///  1. the next Entry in the ledger has transactions that can potentially
+    ///       be verified in parallel with these transactions
+    ///  2. this Entry can be left out of the bank's entry_id cache for
+    ///       purposes of duplicate rejection
+    pub has_more: bool,
+
+    /// Erasure requires that Entry be a multiple of 4 bytes in size
+    pad: [u8; 3],
 }
 
 impl Entry {
-    /// Creates a Entry from the number of hashes 'num_hashes' since the previous event
-    /// and that resulting 'id'.
+    /// Creates the next Entry `num_hashes` after `start_hash`.
+    pub fn new(
+        start_hash: &Hash,
+        cur_hashes: u64,
+        transactions: Vec<Transaction>,
+        has_more: bool,
+    ) -> Self {
+        let num_hashes = cur_hashes + if transactions.is_empty() { 0 } else { 1 };
+        let id = next_hash(start_hash, 0, &transactions);
+        let entry = Entry {
+            num_hashes,
+            id,
+            transactions,
+            has_more,
+            pad: [0, 0, 0],
+        };
+        assert!(serialized_size(&entry).unwrap() <= BLOB_DATA_SIZE as u64);
+        entry
+    }
+
+    pub fn will_fit(transactions: Vec<Transaction>) -> bool {
+        serialized_size(&Entry {
+            num_hashes: 0,
+            id: Hash::default(),
+            transactions,
+            has_more: false,
+            pad: [0, 0, 0],
+        }).unwrap() <= BLOB_DATA_SIZE as u64
+    }
+
+    /// Creates the next Tick Entry `num_hashes` after `start_hash`.
+    pub fn new_mut(
+        start_hash: &mut Hash,
+        cur_hashes: &mut u64,
+        transactions: Vec<Transaction>,
+        has_more: bool,
+    ) -> Self {
+        let entry = Self::new(start_hash, *cur_hashes, transactions, has_more);
+        *start_hash = entry.id;
+        *cur_hashes = 0;
+        assert!(serialized_size(&entry).unwrap() <= BLOB_DATA_SIZE as u64);
+        entry
+    }
+
+    /// Creates a Entry from the number of hashes `num_hashes` since the previous transaction
+    /// and that resulting `id`.
     pub fn new_tick(num_hashes: u64, id: &Hash) -> Self {
         Entry {
             num_hashes,
             id: *id,
-            event: Event::Tick,
+            transactions: vec![],
+            has_more: false,
+            pad: [0, 0, 0],
         }
     }
 
-    /// Verifies self.id is the result of hashing a 'start_hash' 'self.num_hashes' times.
-    /// If the event is not a Tick, then hash that as well.
+    /// Verifies self.id is the result of hashing a `start_hash` `self.num_hashes` times.
+    /// If the transaction is not a Tick, then hash that as well.
     pub fn verify(&self, start_hash: &Hash) -> bool {
-        if !self.event.verify() {
-            return false;
-        }
-        self.id == next_hash(start_hash, self.num_hashes, &self.event)
+        self.transactions.par_iter().all(|tx| tx.verify_plan())
+            && self.id == next_hash(start_hash, self.num_hashes, &self.transactions)
     }
 }
 
-/// Creates the hash 'num_hashes' after start_hash. If the event contains
-/// signature, the final hash will be a hash of both the previous ID and
-/// the signature.
-pub fn next_hash(start_hash: &Hash, num_hashes: u64, event: &Event) -> Hash {
+fn add_transaction_data(hash_data: &mut Vec<u8>, tx: &Transaction) {
+    hash_data.push(0u8);
+    hash_data.extend_from_slice(&tx.sig);
+}
+
+/// Creates the hash `num_hashes` after `start_hash`. If the transaction contains
+/// a signature, the final hash will be a hash of both the previous ID and
+/// the signature.  If num_hashes is zero and there's no transaction data,
+///  start_hash is returned.
+fn next_hash(start_hash: &Hash, num_hashes: u64, transactions: &[Transaction]) -> Hash {
     let mut id = *start_hash;
-    let sig = event.get_signature();
-    let start_index = if sig.is_some() { 1 } else { 0 };
-    for _ in start_index..num_hashes {
+    for _ in 1..num_hashes {
         id = hash(&id);
     }
-    if let Some(sig) = sig {
-        id = extend_and_hash(&id, &sig);
-    }
-    id
-}
 
-/// Creates the next Entry 'num_hashes' after 'start_hash'.
-pub fn create_entry(start_hash: &Hash, cur_hashes: u64, event: Event) -> Entry {
-    let sig = event.get_signature();
-    let num_hashes = cur_hashes + if sig.is_some() { 1 } else { 0 };
-    let id = next_hash(start_hash, 0, &event);
-    Entry {
-        num_hashes,
-        id,
-        event,
+    // Hash all the transaction data
+    let mut hash_data = vec![];
+    for tx in transactions {
+        add_transaction_data(&mut hash_data, tx);
+    }
+
+    if !hash_data.is_empty() {
+        extend_and_hash(&id, &hash_data)
+    } else if num_hashes != 0 {
+        hash(&id)
+    } else {
+        id
     }
 }
 
-/// Creates the next Tick Entry 'num_hashes' after 'start_hash'.
-pub fn create_entry_mut(start_hash: &mut Hash, cur_hashes: &mut u64, event: Event) -> Entry {
-    let entry = create_entry(start_hash, *cur_hashes, event);
-    *start_hash = entry.id;
-    *cur_hashes = 0;
-    entry
-}
-
-/// Creates the next Tick Entry 'num_hashes' after 'start_hash'.
-pub fn next_tick(start_hash: &Hash, num_hashes: u64) -> Entry {
-    let event = Event::Tick;
+/// Creates the next Tick or Transaction Entry `num_hashes` after `start_hash`.
+pub fn next_entry(start_hash: &Hash, num_hashes: u64, transactions: Vec<Transaction>) -> Entry {
+    assert!(num_hashes > 0 || transactions.is_empty());
     Entry {
         num_hashes,
-        id: next_hash(start_hash, num_hashes, &event),
-        event,
+        id: next_hash(start_hash, num_hashes, &transactions),
+        transactions,
+        has_more: false,
+        pad: [0, 0, 0],
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use chrono::prelude::*;
+    use entry::Entry;
+    use hash::hash;
+    use signature::{KeyPair, KeyPairUtil};
+    use transaction::Transaction;
 
     #[test]
-    fn test_event_verify() {
+    fn test_entry_verify() {
         let zero = Hash::default();
         let one = hash(&zero);
         assert!(Entry::new_tick(0, &zero).verify(&zero)); // base case
         assert!(!Entry::new_tick(0, &zero).verify(&one)); // base case, bad
-        assert!(next_tick(&zero, 1).verify(&zero)); // inductive step
-        assert!(!next_tick(&zero, 1).verify(&one)); // inductive step, bad
+        assert!(next_entry(&zero, 1, vec![]).verify(&zero)); // inductive step
+        assert!(!next_entry(&zero, 1, vec![]).verify(&one)); // inductive step, bad
     }
 
     #[test]
-    fn test_next_tick() {
+    fn test_transaction_reorder_attack() {
         let zero = Hash::default();
-        assert_eq!(next_tick(&zero, 1).num_hashes, 1)
+
+        // First, verify entries
+        let keypair = KeyPair::new();
+        let tx0 = Transaction::new(&keypair, keypair.pubkey(), 0, zero);
+        let tx1 = Transaction::new(&keypair, keypair.pubkey(), 1, zero);
+        let mut e0 = Entry::new(&zero, 0, vec![tx0.clone(), tx1.clone()], false);
+        assert!(e0.verify(&zero));
+
+        // Next, swap two transactions and ensure verification fails.
+        e0.transactions[0] = tx1; // <-- attack
+        e0.transactions[1] = tx0;
+        assert!(!e0.verify(&zero));
+    }
+
+    #[test]
+    fn test_witness_reorder_attack() {
+        let zero = Hash::default();
+
+        // First, verify entries
+        let keypair = KeyPair::new();
+        let tx0 = Transaction::new_timestamp(&keypair, Utc::now(), zero);
+        let tx1 = Transaction::new_signature(&keypair, Default::default(), zero);
+        let mut e0 = Entry::new(&zero, 0, vec![tx0.clone(), tx1.clone()], false);
+        assert!(e0.verify(&zero));
+
+        // Next, swap two witness transactions and ensure verification fails.
+        e0.transactions[0] = tx1; // <-- attack
+        e0.transactions[1] = tx0;
+        assert!(!e0.verify(&zero));
+    }
+
+    #[test]
+    fn test_next_entry() {
+        let zero = Hash::default();
+        let tick = next_entry(&zero, 1, vec![]);
+        assert_eq!(tick.num_hashes, 1);
+        assert_ne!(tick.id, zero);
+
+        let tick = next_entry(&zero, 0, vec![]);
+        assert_eq!(tick.num_hashes, 0);
+        assert_eq!(tick.id, zero);
+
+        let keypair = KeyPair::new();
+        let tx0 = Transaction::new_timestamp(&keypair, Utc::now(), zero);
+        let entry0 = next_entry(&zero, 1, vec![tx0.clone()]);
+        assert_eq!(entry0.num_hashes, 1);
+        assert_eq!(entry0.id, next_hash(&zero, 1, &vec![tx0]));
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_next_entry_panic() {
+        let zero = Hash::default();
+        let keypair = KeyPair::new();
+        let tx = Transaction::new(&keypair, keypair.pubkey(), 0, zero);
+        next_entry(&zero, 0, vec![tx]);
     }
 }

src/entry_writer.rs

@@ -0,0 +1,125 @@
+//! The `entry_writer` module helps implement the TPU's write stage. It
+//! writes entries to the given writer, which is typically a file or
+//! stdout, and then sends the Entry to its output channel.
+
+use bank::Bank;
+use entry::Entry;
+use serde_json;
+use std::io::{self, BufRead, Cursor, Error, ErrorKind, Write};
+
+pub struct EntryWriter<'a, W> {
+    bank: &'a Bank,
+    writer: W,
+}
+
+impl<'a, W: Write> EntryWriter<'a, W> {
+    /// Create a new Tpu that wraps the given Bank.
+    pub fn new(bank: &'a Bank, writer: W) -> Self {
+        EntryWriter { bank, writer }
+    }
+
+    fn write_entry(writer: &mut W, entry: &Entry) -> io::Result<()> {
+        let serialized = serde_json::to_string(entry).unwrap();
+        writeln!(writer, "{}", serialized)?;
+        writer.flush()
+    }
+
+    pub fn write_entries<I>(writer: &mut W, entries: I) -> io::Result<()>
+    where
+        I: IntoIterator<Item = Entry>,
+    {
+        for entry in entries {
+            Self::write_entry(writer, &entry)?;
+        }
+        Ok(())
+    }
+
+    fn write_and_register_entry(&mut self, entry: &Entry) -> io::Result<()> {
+        trace!("write_and_register_entry entry");
+        if !entry.has_more {
+            self.bank.register_entry_id(&entry.id);
+        }
+        Self::write_entry(&mut self.writer, entry)
+    }
+
+    pub fn write_and_register_entries(&mut self, entries: &[Entry]) -> io::Result<()> {
+        for entry in entries {
+            self.write_and_register_entry(&entry)?;
+        }
+        Ok(())
+    }
+}
+
+/// Parse a string containing an Entry.
+pub fn read_entry(s: &str) -> io::Result<Entry> {
+    serde_json::from_str(s).map_err(|e| Error::new(ErrorKind::Other, e.to_string()))
+}
+
+/// Return an iterator for all the entries in the given file.
+pub fn read_entries<R: BufRead>(reader: R) -> impl Iterator<Item = io::Result<Entry>> {
+    reader.lines().map(|s| read_entry(&s?))
+}
+
+/// Same as read_entries() but returning a vector. Handy for debugging short logs.
+pub fn read_entries_to_vec<R: BufRead>(reader: R) -> io::Result<Vec<Entry>> {
+    let mut result = vec![];
+    for x in read_entries(reader) {
+        result.push(x?);
+    }
+    Ok(result)
+}
+
+/// Same as read_entries() but parsing a string and returning a vector.
+pub fn read_entries_from_str(s: &str) -> io::Result<Vec<Entry>> {
+    read_entries_to_vec(Cursor::new(s))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use ledger;
+    use mint::Mint;
+    use packet::BLOB_DATA_SIZE;
+    use signature::{KeyPair, KeyPairUtil};
+    use std::str;
+    use transaction::Transaction;
+
+    #[test]
+    fn test_dont_register_partial_entries() {
+        let mint = Mint::new(1);
+        let bank = Bank::new(&mint);
+
+        let writer = io::sink();
+        let mut entry_writer = EntryWriter::new(&bank, writer);
+        let keypair = KeyPair::new();
+        let tx = Transaction::new(&mint.keypair(), keypair.pubkey(), 1, mint.last_id());
+
+        // NOTE: if Entry grows to larger than a transaction, the code below falls over
+        let threshold = (BLOB_DATA_SIZE / 256) - 1; // 256 is transaction size
+
+        // Verify large entries are split up and the first sets has_more.
+        let txs = vec![tx.clone(); threshold * 2];
+        let entries = ledger::next_entries(&mint.last_id(), 0, txs);
+        assert_eq!(entries.len(), 2);
+        assert!(entries[0].has_more);
+        assert!(!entries[1].has_more);
+
+        // Verify that write_and_register_entry doesn't register the first entries after a split.
+        assert_eq!(bank.last_id(), mint.last_id());
+        entry_writer.write_and_register_entry(&entries[0]).unwrap();
+        assert_eq!(bank.last_id(), mint.last_id());
+
+        // Verify that write_and_register_entry registers the final entry after a split.
+        entry_writer.write_and_register_entry(&entries[1]).unwrap();
+        assert_eq!(bank.last_id(), entries[1].id);
+    }
+
+    #[test]
+    fn test_read_entries_from_str() {
+        let mint = Mint::new(1);
+        let mut buf = vec![];
+        EntryWriter::write_entries(&mut buf, mint.create_entries()).unwrap();
+        let entries = read_entries_from_str(str::from_utf8(&buf).unwrap()).unwrap();
+        assert_eq!(entries, mint.create_entries());
+    }
+}

src/erasure.rs

@@ -0,0 +1,622 @@
+// Support erasure coding
+
+use packet::{BlobRecycler, SharedBlob, BLOB_HEADER_SIZE};
+use std::result;
+
+//TODO(sakridge) pick these values
+pub const NUM_CODED: usize = 20;
+pub const MAX_MISSING: usize = 4;
+const NUM_DATA: usize = NUM_CODED - MAX_MISSING;
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum ErasureError {
+    NotEnoughBlocksToDecode,
+    DecodeError,
+    EncodeError,
+    InvalidBlockSize,
+}
+
+pub type Result<T> = result::Result<T, ErasureError>;
+
+// k = number of data devices
+// m = number of coding devices
+// w = word size
+
+extern "C" {
+    fn jerasure_matrix_encode(
+        k: i32,
+        m: i32,
+        w: i32,
+        matrix: *const i32,
+        data_ptrs: *const *const u8,
+        coding_ptrs: *const *mut u8,
+        size: i32,
+    );
+    fn jerasure_matrix_decode(
+        k: i32,
+        m: i32,
+        w: i32,
+        matrix: *const i32,
+        row_k_ones: i32,
+        erasures: *const i32,
+        data_ptrs: *const *mut u8,
+        coding_ptrs: *const *const u8,
+        size: i32,
+    ) -> i32;
+    fn galois_single_divide(a: i32, b: i32, w: i32) -> i32;
+}
+
+fn get_matrix(m: i32, k: i32, w: i32) -> Vec<i32> {
+    let mut matrix = vec![0; (m * k) as usize];
+    for i in 0..m {
+        for j in 0..k {
+            unsafe {
+                matrix[(i * k + j) as usize] = galois_single_divide(1, i ^ (m + j), w);
+            }
+        }
+    }
+    matrix
+}
+
+pub const ERASURE_W: i32 = 32;
+
+// Generate coding blocks into coding
+//   There are some alignment restrictions, blocks should be aligned by 16 bytes
+//   which means their size should be >= 16 bytes
+pub fn generate_coding_blocks(coding: &mut [&mut [u8]], data: &[&[u8]]) -> Result<()> {
+    if data.len() == 0 {
+        return Ok(());
+    }
+    let m = coding.len() as i32;
+    let block_len = data[0].len();
+    let matrix: Vec<i32> = get_matrix(m, data.len() as i32, ERASURE_W);
+    let mut coding_arg = Vec::new();
+    let mut data_arg = Vec::new();
+    for block in data {
+        if block_len != block.len() {
+            trace!(
+                "data block size incorrect {} expected {}",
+                block.len(),
+                block_len
+            );
+            return Err(ErasureError::InvalidBlockSize);
+        }
+        data_arg.push(block.as_ptr());
+    }
+    for mut block in coding {
+        if block_len != block.len() {
+            trace!(
+                "coding block size incorrect {} expected {}",
+                block.len(),
+                block_len
+            );
+            return Err(ErasureError::InvalidBlockSize);
+        }
+        coding_arg.push(block.as_mut_ptr());
+    }
+
+    unsafe {
+        jerasure_matrix_encode(
+            data.len() as i32,
+            m,
+            ERASURE_W,
+            matrix.as_ptr(),
+            data_arg.as_ptr(),
+            coding_arg.as_ptr(),
+            data[0].len() as i32,
+        );
+    }
+    Ok(())
+}
+
+// Recover data + coding blocks into data blocks
+//   data: array of blocks to recover into
+//   coding: arry of coding blocks
+//   erasures: list of indices in data where blocks should be recovered
+pub fn decode_blocks(data: &mut [&mut [u8]], coding: &[&[u8]], erasures: &[i32]) -> Result<()> {
+    if data.len() == 0 {
+        return Ok(());
+    }
+    let block_len = data[0].len();
+    let matrix: Vec<i32> = get_matrix(coding.len() as i32, data.len() as i32, ERASURE_W);
+
+    // generate coding pointers, blocks should be the same size
+    let mut coding_arg: Vec<*const u8> = Vec::new();
+    for x in coding.iter() {
+        if x.len() != block_len {
+            return Err(ErasureError::InvalidBlockSize);
+        }
+        coding_arg.push(x.as_ptr());
+    }
+
+    // generate data pointers, blocks should be the same size
+    let mut data_arg: Vec<*mut u8> = Vec::new();
+    for x in data.iter_mut() {
+        if x.len() != block_len {
+            return Err(ErasureError::InvalidBlockSize);
+        }
+        data_arg.push(x.as_mut_ptr());
+    }
+    unsafe {
+        let ret = jerasure_matrix_decode(
+            data.len() as i32,
+            coding.len() as i32,
+            ERASURE_W,
+            matrix.as_ptr(),
+            0,
+            erasures.as_ptr(),
+            data_arg.as_ptr(),
+            coding_arg.as_ptr(),
+            data[0].len() as i32,
+        );
+        trace!("jerasure_matrix_decode ret: {}", ret);
+        for x in data[erasures[0] as usize][0..8].iter() {
+            trace!("{} ", x)
+        }
+        trace!("");
+        if ret < 0 {
+            return Err(ErasureError::DecodeError);
+        }
+    }
+    Ok(())
+}
+
+// Allocate some coding blobs and insert into the blobs array
+pub fn add_coding_blobs(recycler: &BlobRecycler, blobs: &mut Vec<SharedBlob>, consumed: u64) {
+    let mut added = 0;
+    let blobs_len = blobs.len() as u64;
+    for i in consumed..consumed + blobs_len {
+        let is = i as usize;
+        if is != 0 && ((is + MAX_MISSING) % NUM_CODED) == 0 {
+            for _ in 0..MAX_MISSING {
+                trace!("putting coding at {}", (i - consumed));
+                let new_blob = recycler.allocate();
+                let new_blob_clone = new_blob.clone();
+                let mut new_blob_l = new_blob_clone.write().unwrap();
+                new_blob_l.set_size(0);
+                new_blob_l.set_coding().unwrap();
+                drop(new_blob_l);
+                blobs.insert((i - consumed) as usize, new_blob);
+                added += 1;
+            }
+        }
+    }
+    info!(
+        "add_coding consumed: {} blobs.len(): {} added: {}",
+        consumed,
+        blobs.len(),
+        added
+    );
+}
+
+// Generate coding blocks in window starting from consumed
+pub fn generate_coding(
+    window: &mut Vec<Option<SharedBlob>>,
+    consumed: usize,
+    num_blobs: usize,
+) -> Result<()> {
+    let mut block_start = consumed - (consumed % NUM_CODED);
+
+    for i in consumed..consumed + num_blobs {
+        if (i % NUM_CODED) == (NUM_CODED - 1) {
+            let mut data_blobs = Vec::new();
+            let mut coding_blobs = Vec::new();
+            let mut data_locks = Vec::new();
+            let mut data_ptrs: Vec<&[u8]> = Vec::new();
+            let mut coding_locks = Vec::new();
+            let mut coding_ptrs: Vec<&mut [u8]> = Vec::new();
+
+            info!(
+                "generate_coding start: {} end: {} consumed: {} num_blobs: {}",
+                block_start,
+                block_start + NUM_DATA,
+                consumed,
+                num_blobs
+            );
+            for i in block_start..block_start + NUM_DATA {
+                let n = i % window.len();
+                trace!("window[{}] = {:?}", n, window[n]);
+                if window[n].is_none() {
+                    trace!("data block is null @ {}", n);
+                    return Ok(());
+                }
+                data_blobs.push(
+                    window[n]
+                        .clone()
+                        .expect("'data_blobs' arr in pub fn generate_coding"),
+                );
+            }
+            let mut max_data_size = 0;
+            for b in &data_blobs {
+                let lck = b.write().expect("'b' write lock in pub fn generate_coding");
+                if lck.meta.size > max_data_size {
+                    max_data_size = lck.meta.size;
+                }
+                data_locks.push(lck);
+            }
+            trace!("max_data_size: {}", max_data_size);
+            for (i, l) in data_locks.iter_mut().enumerate() {
+                trace!("i: {} data: {}", i, l.data[0]);
+                data_ptrs.push(&l.data[..max_data_size]);
+            }
+
+            // generate coding ptr array
+            let coding_start = block_start + NUM_DATA;
+            let coding_end = block_start + NUM_CODED;
+            for i in coding_start..coding_end {
+                let n = i % window.len();
+                if window[n].is_none() {
+                    trace!("coding block is null @ {}", n);
+                    return Ok(());
+                }
+                let w_l = window[n].clone().unwrap();
+                w_l.write().unwrap().set_size(max_data_size);
+                if w_l.write().unwrap().set_coding().is_err() {
+                    return Err(ErasureError::EncodeError);
+                }
+                coding_blobs.push(
+                    window[n]
+                        .clone()
+                        .expect("'coding_blobs' arr in pub fn generate_coding"),
+                );
+            }
+            for b in &coding_blobs {
+                coding_locks.push(
+                    b.write()
+                        .expect("'coding_locks' arr in pub fn generate_coding"),
+                );
+            }
+            for (i, l) in coding_locks.iter_mut().enumerate() {
+                trace!("i: {} coding: {} size: {}", i, l.data[0], max_data_size);
+                coding_ptrs.push(&mut l.data_mut()[..max_data_size]);
+            }
+
+            generate_coding_blocks(coding_ptrs.as_mut_slice(), &data_ptrs)?;
+            debug!(
+                "consumed: {} data: {}:{} coding: {}:{}",
+                consumed,
+                block_start,
+                block_start + NUM_DATA,
+                coding_start,
+                coding_end
+            );
+            block_start += NUM_CODED;
+        }
+    }
+    Ok(())
+}
+
+// Recover missing blocks into window
+//   missing blocks should be None, will use re
+//   to allocate new ones. Returns err if not enough
+//   coding blocks are present to restore
+pub fn recover(
+    re: &BlobRecycler,
+    window: &mut Vec<Option<SharedBlob>>,
+    consumed: usize,
+    received: usize,
+) -> Result<()> {
+    //recover with erasure coding
+    if received <= consumed {
+        return Ok(());
+    }
+    let num_blocks = (received - consumed) / NUM_CODED;
+    let mut block_start = consumed - (consumed % NUM_CODED);
+
+    if num_blocks > 0 {
+        debug!(
+            "num_blocks: {} received: {} consumed: {}",
+            num_blocks, received, consumed
+        );
+    }
+
+    for i in 0..num_blocks {
+        if i > 100 {
+            break;
+        }
+        let mut data_missing = 0;
+        let mut coded_missing = 0;
+        let coding_start = block_start + NUM_DATA;
+        let coding_end = block_start + NUM_CODED;
+        trace!(
+            "recover: block_start: {} coding_start: {} coding_end: {}",
+            block_start,
+            coding_start,
+            coding_end
+        );
+        for i in block_start..coding_end {
+            let n = i % window.len();
+            if window[n].is_none() {
+                if i >= coding_start {
+                    coded_missing += 1;
+                } else {
+                    data_missing += 1;
+                }
+            }
+        }
+        if (data_missing + coded_missing) != NUM_CODED && (data_missing + coded_missing) != 0 {
+            debug!(
+                "1: start: {} recovering: data: {} coding: {}",
+                block_start, data_missing, coded_missing
+            );
+        }
+        if data_missing > 0 {
+            if (data_missing + coded_missing) <= MAX_MISSING {
+                debug!(
+                    "2: recovering: data: {} coding: {}",
+                    data_missing, coded_missing
+                );
+                let mut blobs: Vec<SharedBlob> = Vec::new();
+                let mut locks = Vec::new();
+                let mut erasures: Vec<i32> = Vec::new();
+                let mut meta = None;
+                let mut size = None;
+                for i in block_start..coding_end {
+                    let j = i % window.len();
+                    let mut b = &mut window[j];
+                    if b.is_some() {
+                        if i >= NUM_DATA && size.is_none() {
+                            let bl = b.clone().unwrap();
+                            size = Some(bl.read().unwrap().meta.size - BLOB_HEADER_SIZE);
+                        }
+                        if meta.is_none() {
+                            let bl = b.clone().unwrap();
+                            meta = Some(bl.read().unwrap().meta.clone());
+                        }
+                        blobs.push(b.clone().expect("'blobs' arr in pb fn recover"));
+                        continue;
+                    }
+                    let n = re.allocate();
+                    *b = Some(n.clone());
+                    //mark the missing memory
+                    blobs.push(n);
+                    erasures.push((i - block_start) as i32);
+                }
+                erasures.push(-1);
+                trace!(
+                    "erasures: {:?} data_size: {} header_size: {}",
+                    erasures,
+                    size.unwrap(),
+                    BLOB_HEADER_SIZE
+                );
+                //lock everything
+                for b in &blobs {
+                    locks.push(b.write().expect("'locks' arr in pb fn recover"));
+                }
+                {
+                    let mut coding_ptrs: Vec<&[u8]> = Vec::new();
+                    let mut data_ptrs: Vec<&mut [u8]> = Vec::new();
+                    for (i, l) in locks.iter_mut().enumerate() {
+                        if i >= NUM_DATA {
+                            trace!("pushing coding: {}", i);
+                            coding_ptrs.push(&l.data()[..size.unwrap()]);
+                        } else {
+                            trace!("pushing data: {}", i);
+                            data_ptrs.push(&mut l.data[..size.unwrap()]);
+                        }
+                    }
+                    trace!(
+                        "coding_ptrs.len: {} data_ptrs.len {}",
+                        coding_ptrs.len(),
+                        data_ptrs.len()
+                    );
+                    decode_blocks(data_ptrs.as_mut_slice(), &coding_ptrs, &erasures)?;
+                }
+                for i in &erasures[..erasures.len() - 1] {
+                    let idx = *i as usize;
+                    let data_size = locks[idx].get_data_size().unwrap() - BLOB_HEADER_SIZE as u64;
+                    locks[idx].meta = meta.clone().unwrap();
+                    locks[idx].set_size(data_size as usize);
+                    trace!(
+                        "erasures[{}] size: {} data[0]: {}",
+                        *i,
+                        data_size,
+                        locks[idx].data()[0]
+                    );
+                }
+            }
+        }
+        block_start += NUM_CODED;
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod test {
+    use crdt;
+    use erasure;
+    use logger;
+    use packet::{BlobRecycler, SharedBlob, BLOB_HEADER_SIZE};
+    use signature::KeyPair;
+    use signature::KeyPairUtil;
+    use std::sync::{Arc, RwLock};
+
+    #[test]
+    pub fn test_coding() {
+        let zero_vec = vec![0; 16];
+        let mut vs: Vec<Vec<u8>> = (0..4).map(|i| (i..(16 + i)).collect()).collect();
+        let v_orig: Vec<u8> = vs[0].clone();
+
+        let m = 2;
+        let mut coding_blocks: Vec<_> = (0..m).map(|_| vec![0u8; 16]).collect();
+
+        {
+            let mut coding_blocks_slices: Vec<_> =
+                coding_blocks.iter_mut().map(|x| x.as_mut_slice()).collect();
+            let v_slices: Vec<_> = vs.iter().map(|x| x.as_slice()).collect();
+
+            assert!(
+                erasure::generate_coding_blocks(
+                    coding_blocks_slices.as_mut_slice(),
+                    v_slices.as_slice()
+                ).is_ok()
+            );
+        }
+        trace!("coding blocks:");
+        for b in &coding_blocks {
+            trace!("{:?}", b);
+        }
+        let erasure: i32 = 1;
+        let erasures = vec![erasure, -1];
+        // clear an entry
+        vs[erasure as usize].copy_from_slice(zero_vec.as_slice());
+
+        {
+            let coding_blocks_slices: Vec<_> = coding_blocks.iter().map(|x| x.as_slice()).collect();
+            let mut v_slices: Vec<_> = vs.iter_mut().map(|x| x.as_mut_slice()).collect();
+
+            assert!(
+                erasure::decode_blocks(
+                    v_slices.as_mut_slice(),
+                    coding_blocks_slices.as_slice(),
+                    erasures.as_slice(),
+                ).is_ok()
+            );
+        }
+
+        trace!("vs:");
+        for v in &vs {
+            trace!("{:?}", v);
+        }
+        assert_eq!(v_orig, vs[0]);
+    }
+
+    fn print_window(window: &Vec<Option<SharedBlob>>) {
+        for (i, w) in window.iter().enumerate() {
+            print!("window({}): ", i);
+            if w.is_some() {
+                let window_l1 = w.clone().unwrap();
+                let window_l2 = window_l1.read().unwrap();
+                print!(
+                    "index: {:?} meta.size: {} data: ",
+                    window_l2.get_index(),
+                    window_l2.meta.size
+                );
+                for i in 0..8 {
+                    print!("{} ", window_l2.data()[i]);
+                }
+            } else {
+                print!("null");
+            }
+            println!("");
+        }
+    }
+
+    fn generate_window(
+        data_len: usize,
+        blob_recycler: &BlobRecycler,
+        offset: usize,
+        num_blobs: usize,
+    ) -> (Vec<Option<SharedBlob>>, usize) {
+        let mut window = vec![None; 32];
+        let mut blobs = Vec::new();
+        for i in 0..num_blobs {
+            let b = blob_recycler.allocate();
+            let b_ = b.clone();
+            let mut w = b.write().unwrap();
+            w.set_size(data_len);
+            for k in 0..data_len {
+                w.data_mut()[k] = (k + i) as u8;
+            }
+            blobs.push(b_);
+        }
+        erasure::add_coding_blobs(blob_recycler, &mut blobs, offset as u64);
+        let blobs_len = blobs.len();
+
+        let d = crdt::NodeInfo::new(
+            KeyPair::new().pubkey(),
+            "127.0.0.1:1234".parse().unwrap(),
+            "127.0.0.1:1235".parse().unwrap(),
+            "127.0.0.1:1236".parse().unwrap(),
+            "127.0.0.1:1237".parse().unwrap(),
+            "127.0.0.1:1238".parse().unwrap(),
+        );
+        assert!(crdt::Crdt::index_blobs(&d, &blobs, &mut (offset as u64)).is_ok());
+        for b in blobs {
+            let idx = b.read().unwrap().get_index().unwrap() as usize;
+            window[idx] = Some(b);
+        }
+        (window, blobs_len)
+    }
+
+    #[test]
+    pub fn test_window_recover_basic() {
+        logger::setup();
+        let data_len = 16;
+        let blob_recycler = BlobRecycler::default();
+
+        // Generate a window
+        let offset = 1;
+        let num_blobs = erasure::NUM_DATA + 2;
+        let (mut window, blobs_len) = generate_window(data_len, &blob_recycler, 0, num_blobs);
+        println!("** after-gen-window:");
+        print_window(&window);
+
+        // Generate the coding blocks
+        assert!(erasure::generate_coding(&mut window, offset, blobs_len).is_ok());
+        println!("** after-gen-coding:");
+        print_window(&window);
+
+        let erase_offset = offset;
+        // Create a hole in the window
+        let refwindow = window[erase_offset].clone();
+        window[erase_offset] = None;
+
+        // Recover it from coding
+        assert!(erasure::recover(&blob_recycler, &mut window, offset, offset + blobs_len).is_ok());
+        println!("** after-recover:");
+        print_window(&window);
+
+        // Check the result
+        let window_l = window[erase_offset].clone().unwrap();
+        let window_l2 = window_l.read().unwrap();
+        let ref_l = refwindow.clone().unwrap();
+        let ref_l2 = ref_l.read().unwrap();
+        assert_eq!(
+            window_l2.data[..(data_len + BLOB_HEADER_SIZE)],
+            ref_l2.data[..(data_len + BLOB_HEADER_SIZE)]
+        );
+        assert_eq!(window_l2.meta.size, ref_l2.meta.size);
+        assert_eq!(window_l2.meta.addr, ref_l2.meta.addr);
+        assert_eq!(window_l2.meta.port, ref_l2.meta.port);
+        assert_eq!(window_l2.meta.v6, ref_l2.meta.v6);
+        assert_eq!(window_l2.get_index().unwrap(), erase_offset as u64);
+    }
+
+    //TODO This needs to be reworked
+    #[test]
+    #[ignore]
+    pub fn test_window_recover() {
+        logger::setup();
+        let blob_recycler = BlobRecycler::default();
+        let offset = 4;
+        let data_len = 16;
+        let num_blobs = erasure::NUM_DATA + 2;
+        let (mut window, blobs_len) = generate_window(data_len, &blob_recycler, offset, num_blobs);
+        println!("** after-gen:");
+        print_window(&window);
+        assert!(erasure::generate_coding(&mut window, offset, blobs_len).is_ok());
+        println!("** after-coding:");
+        print_window(&window);
+        let refwindow = window[offset + 1].clone();
+        window[offset + 1] = None;
+        window[offset + 2] = None;
+        window[offset + erasure::NUM_CODED + 3] = None;
+        window[offset + (2 * erasure::NUM_CODED) + 0] = None;
+        window[offset + (2 * erasure::NUM_CODED) + 1] = None;
+        window[offset + (2 * erasure::NUM_CODED) + 2] = None;
+        let window_l0 = &(window[offset + (3 * erasure::NUM_CODED)]).clone().unwrap();
+        window_l0.write().unwrap().data[0] = 55;
+        println!("** after-nulling:");
+        print_window(&window);
+        assert!(erasure::recover(&blob_recycler, &mut window, offset, offset + blobs_len).is_ok());
+        println!("** after-restore:");
+        print_window(&window);
+        let window_l = window[offset + 1].clone().unwrap();
+        let ref_l = refwindow.clone().unwrap();
+        assert_eq!(
+            window_l.read().unwrap().data()[..data_len],
+            ref_l.read().unwrap().data()[..data_len]
+        );
+    }
+}

src/event.rs

@@ -1,58 +0,0 @@
-//! The `event` crate provides the data structures for log events.
-
-use signature::{KeyPair, KeyPairUtil, PublicKey, Signature, SignatureUtil};
-use transaction::Transaction;
-use chrono::prelude::*;
-use bincode::serialize;
-
-/// When 'event' is Tick, the event represents a simple clock tick, and exists for the
-/// sole purpose of improving the performance of event log verification. A tick can
-/// be generated in 'num_hashes' hashes and verified in 'num_hashes' hashes.  By logging
-/// a hash alongside the tick, each tick and be verified in parallel using the 'id'
-/// of the preceding tick to seed its hashing.
-#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
-pub enum Event {
-    Tick,
-    Transaction(Transaction<i64>),
-    Signature {
-        from: PublicKey,
-        tx_sig: Signature,
-        sig: Signature,
-    },
-    Timestamp {
-        from: PublicKey,
-        dt: DateTime<Utc>,
-        sig: Signature,
-    },
-}
-
-impl Event {
-    pub fn new_timestamp(from: &KeyPair, dt: DateTime<Utc>) -> Self {
-        let sign_data = serialize(&dt).unwrap();
-        let sig = Signature::clone_from_slice(from.sign(&sign_data).as_ref());
-        Event::Timestamp {
-            from: from.pubkey(),
-            dt,
-            sig,
-        }
-    }
-
-    // TODO: Rename this to transaction_signature().
-    pub fn get_signature(&self) -> Option<Signature> {
-        match *self {
-            Event::Tick => None,
-            Event::Transaction(ref tr) => Some(tr.sig),
-            Event::Signature { .. } => None,
-            Event::Timestamp { .. } => None,
-        }
-    }
-
-    pub fn verify(&self) -> bool {
-        match *self {
-            Event::Tick => true,
-            Event::Transaction(ref tr) => tr.verify(),
-            Event::Signature { from, tx_sig, sig } => sig.verify(&from, &tx_sig),
-            Event::Timestamp { from, dt, sig } => sig.verify(&from, &serialize(&dt).unwrap()),
-        }
-    }
-}

src/fetch_stage.rs

@@ -0,0 +1,62 @@
+//! The `fetch_stage` batches input from a UDP socket and sends it to a channel.
+
+use packet::PacketRecycler;
+use service::Service;
+use std::net::UdpSocket;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::mpsc::channel;
+use std::sync::Arc;
+use std::thread::{self, JoinHandle};
+use streamer::{self, PacketReceiver};
+
+pub struct FetchStage {
+    exit: Arc<AtomicBool>,
+    thread_hdls: Vec<JoinHandle<()>>,
+}
+
+impl FetchStage {
+    pub fn new(
+        socket: UdpSocket,
+        exit: Arc<AtomicBool>,
+        packet_recycler: &PacketRecycler,
+    ) -> (Self, PacketReceiver) {
+        Self::new_multi_socket(vec![socket], exit, packet_recycler)
+    }
+    pub fn new_multi_socket(
+        sockets: Vec<UdpSocket>,
+        exit: Arc<AtomicBool>,
+        packet_recycler: &PacketRecycler,
+    ) -> (Self, PacketReceiver) {
+        let (packet_sender, packet_receiver) = channel();
+        let thread_hdls: Vec<_> = sockets
+            .into_iter()
+            .map(|socket| {
+                streamer::receiver(
+                    socket,
+                    exit.clone(),
+                    packet_recycler.clone(),
+                    packet_sender.clone(),
+                )
+            })
+            .collect();
+
+        (FetchStage { exit, thread_hdls }, packet_receiver)
+    }
+
+    pub fn close(&self) {
+        self.exit.store(true, Ordering::Relaxed);
+    }
+}
+
+impl Service for FetchStage {
+    fn thread_hdls(self) -> Vec<JoinHandle<()>> {
+        self.thread_hdls
+    }
+
+    fn join(self) -> thread::Result<()> {
+        for thread_hdl in self.thread_hdls() {
+            thread_hdl.join()?;
+        }
+        Ok(())
+    }
+}

src/fullnode.rs

@@ -0,0 +1,385 @@
+//! The `fullnode` module hosts all the fullnode microservices.
+
+use bank::Bank;
+use crdt::{Crdt, NodeInfo, TestNode};
+use entry::Entry;
+use entry_writer;
+use ledger::Block;
+use ncp::Ncp;
+use packet::BlobRecycler;
+use rpu::Rpu;
+use service::Service;
+use signature::{KeyPair, KeyPairUtil};
+use std::collections::VecDeque;
+use std::fs::{File, OpenOptions};
+use std::io::{stdin, stdout, BufReader};
+use std::io::{Read, Write};
+use std::net::SocketAddr;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, RwLock};
+use std::thread::{JoinHandle, Result};
+use std::time::Duration;
+use streamer;
+use tpu::Tpu;
+use tvu::Tvu;
+use untrusted::Input;
+
+//use std::time::Duration;
+pub struct FullNode {
+    exit: Arc<AtomicBool>,
+    thread_hdls: Vec<JoinHandle<()>>,
+}
+
+pub enum LedgerFile {
+    StdInOut,
+    Path(String),
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+/// Fullnode configuration to be stored in file
+pub struct Config {
+    pub node_info: NodeInfo,
+    pkcs8: Vec<u8>,
+}
+
+/// Structure to be replicated by the network
+impl Config {
+    pub fn new(bind_addr: &SocketAddr, pkcs8: Vec<u8>) -> Self {
+        let keypair =
+            KeyPair::from_pkcs8(Input::from(&pkcs8)).expect("from_pkcs8 in fullnode::Config new");
+        let pubkey = keypair.pubkey();
+        let node_info = NodeInfo::new_leader_with_pubkey(pubkey, bind_addr);
+        Config { node_info, pkcs8 }
+    }
+    pub fn keypair(&self) -> KeyPair {
+        KeyPair::from_pkcs8(Input::from(&self.pkcs8))
+            .expect("from_pkcs8 in fullnode::Config keypair")
+    }
+}
+
+impl FullNode {
+    pub fn new(
+        mut node: TestNode,
+        leader: bool,
+        ledger: LedgerFile,
+        keypair_for_validator: Option<KeyPair>,
+        network_entry_for_validator: Option<SocketAddr>,
+    ) -> FullNode {
+        info!("creating bank...");
+        let bank = Bank::default();
+        let (infile, outfile): (Box<Read>, Box<Write + Send>) = match ledger {
+            LedgerFile::Path(path) => (
+                Box::new(File::open(path.clone()).expect("opening ledger file")),
+                Box::new(
+                    OpenOptions::new()
+                        .create(true)
+                        .append(true)
+                        .open(path)
+                        .expect("opening ledger file"),
+                ),
+            ),
+            LedgerFile::StdInOut => (Box::new(stdin()), Box::new(stdout())),
+        };
+        let reader = BufReader::new(infile);
+        let entries = entry_writer::read_entries(reader).map(|e| e.expect("failed to parse entry"));
+
+        info!("processing ledger...");
+        let (entry_height, ledger_tail) = bank.process_ledger(entries).expect("process_ledger");
+        // entry_height is the network-wide agreed height of the ledger.
+        //  initialize it from the input ledger
+        info!("processed {} ledger...", entry_height);
+
+        info!("creating networking stack...");
+
+        let local_gossip_addr = node.sockets.gossip.local_addr().unwrap();
+        let local_requests_addr = node.sockets.requests.local_addr().unwrap();
+        info!(
+            "starting... local gossip address: {} (advertising {})",
+            local_gossip_addr, node.data.contact_info.ncp
+        );
+        let requests_addr = node.data.contact_info.rpu;
+        let exit = Arc::new(AtomicBool::new(false));
+        if !leader {
+            let testnet_addr = network_entry_for_validator.expect("validator requires entry");
+
+            let network_entry_point = NodeInfo::new_entry_point(testnet_addr);
+            let keypair = keypair_for_validator.expect("validator requires keypair");
+            let server = FullNode::new_validator(
+                keypair,
+                bank,
+                entry_height,
+                Some(ledger_tail),
+                node,
+                &network_entry_point,
+                exit.clone(),
+            );
+            info!(
+                "validator ready... local request address: {} (advertising {}) connected to: {}",
+                local_requests_addr, requests_addr, testnet_addr
+            );
+            server
+        } else {
+            node.data.leader_id = node.data.id;
+
+            let server = FullNode::new_leader(
+                bank,
+                entry_height,
+                Some(ledger_tail),
+                //Some(Duration::from_millis(1000)),
+                None,
+                node,
+                exit.clone(),
+                outfile,
+            );
+            info!(
+                "leader ready... local request address: {} (advertising {})",
+                local_requests_addr, requests_addr
+            );
+            server
+        }
+    }
+
+    fn new_window(
+        ledger_tail: Option<Vec<Entry>>,
+        entry_height: u64,
+        crdt: &Arc<RwLock<Crdt>>,
+        blob_recycler: &BlobRecycler,
+    ) -> streamer::Window {
+        match ledger_tail {
+            Some(ledger_tail) => {
+                // convert to blobs
+                let mut blobs = VecDeque::new();
+                ledger_tail.to_blobs(&blob_recycler, &mut blobs);
+
+                // flatten deque to vec
+                let blobs: Vec<_> = blobs.into_iter().collect();
+                streamer::initialized_window(&crdt, blobs, entry_height)
+            }
+            None => streamer::default_window(),
+        }
+    }
+
+    /// Create a server instance acting as a leader.
+    ///
+    /// ```text
+    ///              .---------------------.
+    ///              |  Leader             |
+    ///              |                     |
+    ///  .--------.  |  .-----.            |
+    ///  |        |---->|     |            |
+    ///  | Client |  |  | RPU |            |
+    ///  |        |<----|     |            |
+    ///  `----+---`  |  `-----`            |
+    ///       |      |     ^               |
+    ///       |      |     |               |
+    ///       |      |  .--+---.           |
+    ///       |      |  | Bank |           |
+    ///       |      |  `------`           |
+    ///       |      |     ^               |
+    ///       |      |     |               |    .------------.
+    ///       |      |  .--+--.   .-----.  |    |            |
+    ///       `-------->| TPU +-->| NCP +------>| Validators |
+    ///              |  `-----`   `-----`  |    |            |
+    ///              |                     |    `------------`
+    ///              `---------------------`
+    /// ```
+    pub fn new_leader<W: Write + Send + 'static>(
+        bank: Bank,
+        entry_height: u64,
+        ledger_tail: Option<Vec<Entry>>,
+        tick_duration: Option<Duration>,
+        node: TestNode,
+        exit: Arc<AtomicBool>,
+        writer: W,
+    ) -> Self {
+        let bank = Arc::new(bank);
+        let mut thread_hdls = vec![];
+        let rpu = Rpu::new(
+            &bank,
+            node.sockets.requests,
+            node.sockets.respond,
+            exit.clone(),
+        );
+        thread_hdls.extend(rpu.thread_hdls());
+
+        let blob_recycler = BlobRecycler::default();
+        let crdt = Arc::new(RwLock::new(Crdt::new(node.data).expect("Crdt::new")));
+        let (tpu, blob_receiver) = Tpu::new(
+            &bank,
+            &crdt,
+            tick_duration,
+            node.sockets.transaction,
+            &blob_recycler,
+            exit.clone(),
+            writer,
+        );
+        thread_hdls.extend(tpu.thread_hdls());
+        let window = FullNode::new_window(ledger_tail, entry_height, &crdt, &blob_recycler);
+        let ncp = Ncp::new(
+            &crdt,
+            window.clone(),
+            node.sockets.gossip,
+            node.sockets.gossip_send,
+            exit.clone(),
+        ).expect("Ncp::new");
+        thread_hdls.extend(ncp.thread_hdls());
+
+        let t_broadcast = streamer::broadcaster(
+            node.sockets.broadcast,
+            crdt,
+            window,
+            entry_height,
+            blob_recycler.clone(),
+            blob_receiver,
+        );
+        thread_hdls.extend(vec![t_broadcast]);
+
+        FullNode { exit, thread_hdls }
+    }
+
+    /// Create a server instance acting as a validator.
+    ///
+    /// ```text
+    ///               .-------------------------------.
+    ///               | Validator                     |
+    ///               |                               |
+    ///   .--------.  |            .-----.            |
+    ///   |        |-------------->|     |            |
+    ///   | Client |  |            | RPU |            |
+    ///   |        |<--------------|     |            |
+    ///   `--------`  |            `-----`            |
+    ///               |               ^               |
+    ///               |               |               |
+    ///               |            .--+---.           |
+    ///               |            | Bank |           |
+    ///               |            `------`           |
+    ///               |               ^               |
+    ///   .--------.  |               |               |    .------------.
+    ///   |        |  |            .--+--.            |    |            |
+    ///   | Leader |<------------->| TVU +<--------------->|            |
+    ///   |        |  |            `-----`            |    | Validators |
+    ///   |        |  |               ^               |    |            |
+    ///   |        |  |               |               |    |            |
+    ///   |        |  |            .--+--.            |    |            |
+    ///   |        |<------------->| NCP +<--------------->|            |
+    ///   |        |  |            `-----`            |    |            |
+    ///   `--------`  |                               |    `------------`
+    ///               `-------------------------------`
+    /// ```
+    pub fn new_validator(
+        keypair: KeyPair,
+        bank: Bank,
+        entry_height: u64,
+        ledger_tail: Option<Vec<Entry>>,
+        node: TestNode,
+        entry_point: &NodeInfo,
+        exit: Arc<AtomicBool>,
+    ) -> Self {
+        let bank = Arc::new(bank);
+        let mut thread_hdls = vec![];
+        let rpu = Rpu::new(
+            &bank,
+            node.sockets.requests,
+            node.sockets.respond,
+            exit.clone(),
+        );
+        thread_hdls.extend(rpu.thread_hdls());
+
+        let crdt = Arc::new(RwLock::new(Crdt::new(node.data).expect("Crdt::new")));
+        crdt.write()
+            .expect("'crdt' write lock before insert() in pub fn replicate")
+            .insert(&entry_point);
+
+        let blob_recycler = BlobRecycler::default();
+
+        let window = FullNode::new_window(ledger_tail, entry_height, &crdt, &blob_recycler);
+
+        let ncp = Ncp::new(
+            &crdt,
+            window.clone(),
+            node.sockets.gossip,
+            node.sockets.gossip_send,
+            exit.clone(),
+        ).expect("Ncp::new");
+
+        let tvu = Tvu::new(
+            keypair,
+            &bank,
+            entry_height,
+            crdt.clone(),
+            window.clone(),
+            node.sockets.replicate,
+            node.sockets.repair,
+            node.sockets.retransmit,
+            exit.clone(),
+        );
+        thread_hdls.extend(tvu.thread_hdls());
+        thread_hdls.extend(ncp.thread_hdls());
+        FullNode { exit, thread_hdls }
+    }
+
+    //used for notifying many nodes in parallel to exit
+    pub fn exit(&self) {
+        self.exit.store(true, Ordering::Relaxed);
+    }
+    pub fn close(self) -> Result<()> {
+        self.exit();
+        self.join()
+    }
+}
+
+impl Service for FullNode {
+    fn thread_hdls(self) -> Vec<JoinHandle<()>> {
+        self.thread_hdls
+    }
+
+    fn join(self) -> Result<()> {
+        for thread_hdl in self.thread_hdls() {
+            thread_hdl.join()?;
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use bank::Bank;
+    use crdt::TestNode;
+    use fullnode::FullNode;
+    use mint::Mint;
+    use service::Service;
+    use signature::{KeyPair, KeyPairUtil};
+    use std::sync::atomic::AtomicBool;
+    use std::sync::Arc;
+    #[test]
+    fn validator_exit() {
+        let kp = KeyPair::new();
+        let tn = TestNode::new_localhost_with_pubkey(kp.pubkey());
+        let alice = Mint::new(10_000);
+        let bank = Bank::new(&alice);
+        let exit = Arc::new(AtomicBool::new(false));
+        let entry = tn.data.clone();
+        let v = FullNode::new_validator(kp, bank, 0, None, tn, &entry, exit);
+        v.exit();
+        v.join().unwrap();
+    }
+    #[test]
+    fn validator_parallel_exit() {
+        let vals: Vec<FullNode> = (0..2)
+            .map(|_| {
+                let kp = KeyPair::new();
+                let tn = TestNode::new_localhost_with_pubkey(kp.pubkey());
+                let alice = Mint::new(10_000);
+                let bank = Bank::new(&alice);
+                let exit = Arc::new(AtomicBool::new(false));
+                let entry = tn.data.clone();
+                FullNode::new_validator(kp, bank, 0, None, tn, &entry, exit)
+            })
+            .collect();
+        //each validator can exit in parallel to speed many sequential calls to `join`
+        vals.iter().for_each(|v| v.exit());
+        //while join is called sequentially, the above exit call notified all the
+        //validators to exit from all their threads
+        vals.into_iter().for_each(|v| v.join().unwrap());
+    }
+}

src/hash.rs

@@ -1,5 +1,7 @@
-use generic_array::GenericArray;
+//! The `hash` module provides functions for creating SHA-256 hashes.
+
 use generic_array::typenum::U32;
+use generic_array::GenericArray;
 use sha2::{Digest, Sha256};
 
 pub type Hash = GenericArray<u8, U32>;
@@ -8,7 +10,10 @@ pub type Hash = GenericArray<u8, U32>;
 pub fn hash(val: &[u8]) -> Hash {
     let mut hasher = Sha256::default();
     hasher.input(val);
-    hasher.result()
+
+    // At the time of this writing, the sha2 library is stuck on an old version
+    // of generic_array (0.9.0). Decouple ourselves with a clone to our version.
+    GenericArray::clone_from_slice(hasher.result().as_slice())
 }
 
 /// Return the hash of the given hash extended with the given value.

src/historian.rs

@@ -1,140 +0,0 @@
-//! The `historian` crate provides a microservice for generating a Proof-of-History.
-//! It manages a thread containing a Proof-of-History Logger.
-
-use std::thread::{spawn, JoinHandle};
-use std::collections::HashSet;
-use std::sync::mpsc::{sync_channel, Receiver, SyncSender};
-use std::time::Instant;
-use hash::{hash, Hash};
-use entry::Entry;
-use logger::{ExitReason, Logger};
-use signature::Signature;
-use event::Event;
-
-pub struct Historian {
-    pub sender: SyncSender<Event>,
-    pub receiver: Receiver<Entry>,
-    pub thread_hdl: JoinHandle<ExitReason>,
-    pub signatures: HashSet<Signature>,
-}
-
-impl Historian {
-    pub fn new(start_hash: &Hash, ms_per_tick: Option<u64>) -> Self {
-        let (sender, event_receiver) = sync_channel(1000);
-        let (entry_sender, receiver) = sync_channel(1000);
-        let thread_hdl =
-            Historian::create_logger(*start_hash, ms_per_tick, event_receiver, entry_sender);
-        let signatures = HashSet::new();
-        Historian {
-            sender,
-            receiver,
-            thread_hdl,
-            signatures,
-        }
-    }
-
-    /// A background thread that will continue tagging received Event messages and
-    /// sending back Entry messages until either the receiver or sender channel is closed.
-    fn create_logger(
-        start_hash: Hash,
-        ms_per_tick: Option<u64>,
-        receiver: Receiver<Event>,
-        sender: SyncSender<Entry>,
-    ) -> JoinHandle<ExitReason> {
-        spawn(move || {
-            let mut logger = Logger::new(receiver, sender, start_hash);
-            let now = Instant::now();
-            loop {
-                if let Err(err) = logger.process_events(now, ms_per_tick) {
-                    return err;
-                }
-                if ms_per_tick.is_some() {
-                    logger.last_id = hash(&logger.last_id);
-                    logger.num_hashes += 1;
-                }
-            }
-        })
-    }
-}
-
-pub fn reserve_signature(sigs: &mut HashSet<Signature>, sig: &Signature) -> bool {
-    if sigs.contains(sig) {
-        return false;
-    }
-    sigs.insert(*sig);
-    true
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use log::*;
-    use event::*;
-    use std::thread::sleep;
-    use std::time::Duration;
-
-    #[test]
-    fn test_historian() {
-        let zero = Hash::default();
-        let hist = Historian::new(&zero, None);
-
-        hist.sender.send(Event::Tick).unwrap();
-        sleep(Duration::new(0, 1_000_000));
-        hist.sender.send(Event::Tick).unwrap();
-        sleep(Duration::new(0, 1_000_000));
-        hist.sender.send(Event::Tick).unwrap();
-
-        let entry0 = hist.receiver.recv().unwrap();
-        let entry1 = hist.receiver.recv().unwrap();
-        let entry2 = hist.receiver.recv().unwrap();
-
-        assert_eq!(entry0.num_hashes, 0);
-        assert_eq!(entry1.num_hashes, 0);
-        assert_eq!(entry2.num_hashes, 0);
-
-        drop(hist.sender);
-        assert_eq!(
-            hist.thread_hdl.join().unwrap(),
-            ExitReason::RecvDisconnected
-        );
-
-        assert!(verify_slice(&[entry0, entry1, entry2], &zero));
-    }
-
-    #[test]
-    fn test_historian_closed_sender() {
-        let zero = Hash::default();
-        let hist = Historian::new(&zero, None);
-        drop(hist.receiver);
-        hist.sender.send(Event::Tick).unwrap();
-        assert_eq!(
-            hist.thread_hdl.join().unwrap(),
-            ExitReason::SendDisconnected
-        );
-    }
-
-    #[test]
-    fn test_duplicate_event_signature() {
-        let mut sigs = HashSet::new();
-        let sig = Signature::default();
-        assert!(reserve_signature(&mut sigs, &sig));
-        assert!(!reserve_signature(&mut sigs, &sig));
-    }
-
-    #[test]
-    fn test_ticking_historian() {
-        let zero = Hash::default();
-        let hist = Historian::new(&zero, Some(20));
-        sleep(Duration::from_millis(30));
-        hist.sender.send(Event::Tick).unwrap();
-        drop(hist.sender);
-        let entries: Vec<Entry> = hist.receiver.iter().collect();
-
-        // Ensure one entry is sent back for each tick sent in.
-        assert_eq!(entries.len(), 1);
-
-        // Ensure the ID is not the seed, which indicates another Tick
-        // was logged before the one we sent.
-        assert_ne!(entries[0].id, zero);
-    }
-}

src/ledger.rs

@@ -0,0 +1,198 @@
+//! The `ledger` module provides functions for parallel verification of the
+//! Proof of History ledger.
+
+use bincode::{self, deserialize, serialize_into};
+use entry::Entry;
+use hash::Hash;
+use packet::{self, SharedBlob, BLOB_SIZE};
+use rayon::prelude::*;
+use std::collections::VecDeque;
+use std::io::Cursor;
+use transaction::Transaction;
+
+// a Block is a slice of Entries
+
+pub trait Block {
+    /// Verifies the hashes and counts of a slice of transactions are all consistent.
+    fn verify(&self, start_hash: &Hash) -> bool;
+    fn to_blobs(&self, blob_recycler: &packet::BlobRecycler, q: &mut VecDeque<SharedBlob>);
+}
+
+impl Block for [Entry] {
+    fn verify(&self, start_hash: &Hash) -> bool {
+        let genesis = [Entry::new_tick(0, start_hash)];
+        let entry_pairs = genesis.par_iter().chain(self).zip(self);
+        entry_pairs.all(|(x0, x1)| x1.verify(&x0.id))
+    }
+
+    fn to_blobs(&self, blob_recycler: &packet::BlobRecycler, q: &mut VecDeque<SharedBlob>) {
+        for entry in self {
+            let blob = blob_recycler.allocate();
+            let pos = {
+                let mut bd = blob.write().unwrap();
+                let mut out = Cursor::new(bd.data_mut());
+                serialize_into(&mut out, &entry).expect("failed to serialize output");
+                out.position() as usize
+            };
+            assert!(pos < BLOB_SIZE);
+            blob.write().unwrap().set_size(pos);
+            q.push_back(blob);
+        }
+    }
+}
+
+pub fn reconstruct_entries_from_blobs(blobs: VecDeque<SharedBlob>) -> bincode::Result<Vec<Entry>> {
+    let mut entries: Vec<Entry> = Vec::with_capacity(blobs.len());
+
+    for blob in blobs {
+        let entry = {
+            let msg = blob.read().unwrap();
+            deserialize(&msg.data()[..msg.meta.size])
+        };
+
+        match entry {
+            Ok(entry) => entries.push(entry),
+            Err(err) => {
+                trace!("reconstruct_entry_from_blobs: {}", err);
+                return Err(err);
+            }
+        }
+    }
+    Ok(entries)
+}
+
+/// Creates the next entries for given transactions, outputs
+/// updates start_hash to id of last Entry, sets cur_hashes to 0
+pub fn next_entries_mut(
+    start_hash: &mut Hash,
+    cur_hashes: &mut u64,
+    transactions: Vec<Transaction>,
+) -> Vec<Entry> {
+    if transactions.is_empty() {
+        vec![Entry::new_mut(start_hash, cur_hashes, transactions, false)]
+    } else {
+        let mut chunk_len = transactions.len();
+
+        // check for fit, make sure they can be serialized
+        while !Entry::will_fit(transactions[0..chunk_len].to_vec()) {
+            chunk_len /= 2;
+        }
+
+        let mut num_chunks = if transactions.len() % chunk_len == 0 {
+            transactions.len() / chunk_len
+        } else {
+            transactions.len() / chunk_len + 1
+        };
+
+        let mut entries = Vec::with_capacity(num_chunks);
+
+        for chunk in transactions.chunks(chunk_len) {
+            num_chunks -= 1;
+            entries.push(Entry::new_mut(
+                start_hash,
+                cur_hashes,
+                chunk.to_vec(),
+                num_chunks > 0,
+            ));
+        }
+        entries
+    }
+}
+
+/// Creates the next Entries for given transactions
+pub fn next_entries(
+    start_hash: &Hash,
+    cur_hashes: u64,
+    transactions: Vec<Transaction>,
+) -> Vec<Entry> {
+    let mut id = *start_hash;
+    let mut num_hashes = cur_hashes;
+    next_entries_mut(&mut id, &mut num_hashes, transactions)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use entry::{next_entry, Entry};
+    use hash::hash;
+    use packet::{BlobRecycler, BLOB_DATA_SIZE};
+    use signature::{KeyPair, KeyPairUtil};
+    use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+    use transaction::Transaction;
+
+    #[test]
+    fn test_verify_slice() {
+        let zero = Hash::default();
+        let one = hash(&zero);
+        assert!(vec![][..].verify(&zero)); // base case
+        assert!(vec![Entry::new_tick(0, &zero)][..].verify(&zero)); // singleton case 1
+        assert!(!vec![Entry::new_tick(0, &zero)][..].verify(&one)); // singleton case 2, bad
+        assert!(vec![next_entry(&zero, 0, vec![]); 2][..].verify(&zero)); // inductive step
+
+        let mut bad_ticks = vec![next_entry(&zero, 0, vec![]); 2];
+        bad_ticks[1].id = one;
+        assert!(!bad_ticks.verify(&zero)); // inductive step, bad
+    }
+
+    #[test]
+    fn test_entries_to_blobs() {
+        let zero = Hash::default();
+        let one = hash(&zero);
+        let keypair = KeyPair::new();
+        let tx0 = Transaction::new(&keypair, keypair.pubkey(), 1, one);
+        let transactions = vec![tx0; 10_000];
+        let entries = next_entries(&zero, 0, transactions);
+
+        let blob_recycler = BlobRecycler::default();
+        let mut blob_q = VecDeque::new();
+        entries.to_blobs(&blob_recycler, &mut blob_q);
+
+        assert_eq!(reconstruct_entries_from_blobs(blob_q).unwrap(), entries);
+    }
+
+    #[test]
+    fn test_bad_blobs_attack() {
+        let blob_recycler = BlobRecycler::default();
+        let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 8000);
+        let blobs_q = packet::to_blobs(vec![(0, addr)], &blob_recycler).unwrap(); // <-- attack!
+        assert!(reconstruct_entries_from_blobs(blobs_q).is_err());
+    }
+
+    #[test]
+    fn test_next_entries() {
+        let id = Hash::default();
+        let next_id = hash(&id);
+        let keypair = KeyPair::new();
+        let tx0 = Transaction::new(&keypair, keypair.pubkey(), 1, next_id);
+
+        // NOTE: if Entry grows to larger than a transaction, the code below falls over
+        let threshold = (BLOB_DATA_SIZE / 256) - 1; // 256 is transaction size
+
+        // verify no split
+        let transactions = vec![tx0.clone(); threshold];
+        let entries0 = next_entries(&id, 0, transactions.clone());
+        assert_eq!(entries0.len(), 1);
+        assert!(entries0.verify(&id));
+
+        // verify the split
+        let transactions = vec![tx0.clone(); threshold * 2];
+        let entries0 = next_entries(&id, 0, transactions.clone());
+        assert_eq!(entries0.len(), 2);
+        assert!(entries0[0].has_more);
+        assert!(!entries0[entries0.len() - 1].has_more);
+
+        assert!(entries0.verify(&id));
+        // test hand-construction... brittle, changes if split method changes... ?
+        //        let mut entries1 = vec![];
+        //        entries1.push(Entry::new(&id, 1, transactions[..threshold].to_vec(), true));
+        //        id = entries1[0].id;
+        //        entries1.push(Entry::new(
+        //            &id,
+        //            1,
+        //            transactions[threshold..].to_vec(),
+        //            false,
+        //        ));
+        //
+        //        assert_eq!(entries0, entries1);
+    }
+}

src/lib.rs

@@ -1,24 +1,78 @@
+//! The `solana` library implements the Solana high-performance blockchain architecture.
+//! It includes a full Rust implementation of the architecture (see
+//! [Server](server/struct.Server.html)) as well as hooks to GPU implementations of its most
+//! paralellizable components (i.e. [SigVerify](sigverify/index.html)).  It also includes
+//! command-line tools to spin up fullnodes and a Rust library
+//! (see [ThinClient](thin_client/struct.ThinClient.html)) to interact with them.
+//!
+
 #![cfg_attr(feature = "unstable", feature(test))]
-pub mod signature;
-pub mod hash;
-pub mod transaction;
-pub mod event;
+#[macro_use]
+pub mod counter;
+pub mod bank;
+pub mod banking_stage;
+pub mod blob_fetch_stage;
+pub mod budget;
+pub mod choose_gossip_peer_strategy;
+pub mod crdt;
+pub mod drone;
 pub mod entry;
-pub mod log;
-pub mod mint;
+pub mod entry_writer;
+#[cfg(feature = "erasure")]
+pub mod erasure;
+pub mod fetch_stage;
+pub mod fullnode;
+pub mod hash;
+pub mod ledger;
 pub mod logger;
-pub mod historian;
-pub mod accountant;
-pub mod accountant_skel;
-pub mod accountant_stub;
+pub mod metrics;
+pub mod mint;
+pub mod nat;
+pub mod ncp;
+pub mod packet;
+pub mod payment_plan;
+pub mod record_stage;
+pub mod recorder;
+pub mod replicate_stage;
+pub mod request;
+pub mod request_processor;
+pub mod request_stage;
+pub mod result;
+pub mod rpu;
+pub mod service;
+pub mod signature;
+pub mod sigverify;
+pub mod sigverify_stage;
+pub mod streamer;
+pub mod thin_client;
+pub mod timing;
+pub mod tpu;
+pub mod transaction;
+pub mod tvu;
+pub mod voting;
+pub mod window_stage;
+pub mod write_stage;
 extern crate bincode;
+extern crate byteorder;
 extern crate chrono;
 extern crate generic_array;
+extern crate itertools;
+extern crate libc;
+#[macro_use]
+extern crate log;
 extern crate rayon;
 extern crate ring;
 extern crate serde;
 #[macro_use]
 extern crate serde_derive;
+extern crate pnet_datalink;
 extern crate serde_json;
 extern crate sha2;
 extern crate untrusted;
+
+#[cfg(test)]
+#[macro_use]
+extern crate matches;
+
+extern crate influx_db_client;
+extern crate rand;