params: release go-ethereum v1.8.11

This PR fixes a retriever logic bug. When a peer had a soft timeout
and then a response arrived, it always assumed it was the same peer
even though it could have been a later requested one that did not time
out at all yet. In this case the logic went to an illegal state and
deadlocked, causing a goroutine leak.

Fixes #16243 and replaces #16359.
Thanks to @riceke for finding the bug in the logic.

.dockerignore

@@ -1,4 +1,7 @@
 **/.git
+.git
+!.git/HEAD
+!.git/refs/heads
 **/*_test.go
 
 build/_workspace

.gitattributes

@@ -1,2 +1,3 @@
 # Auto detect text files and perform LF normalization
 * text=auto
+*.sol linguist-language=Solidity

.github/CODEOWNERS

@@ -0,0 +1,12 @@
+# Lines starting with '#' are comments.
+# Each line is a file pattern followed by one or more owners.
+
+accounts/usbwallet @karalabe
+consensus          @karalabe
+core/              @karalabe @holiman
+eth/               @karalabe
+les/               @zsfelfoldi
+light/             @zsfelfoldi
+mobile/            @karalabe
+p2p/               @fjl @zsfelfoldi
+whisper/           @gballet @gluk256

.github/CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 Before you do a feature request please check and make sure that it isn't possible
 through some other means. The JavaScript enabled console is a powerful feature
-in the right hands. Please check our [Bitchin' tricks](https://github.com/ethereum/go-ethereum/wiki/bitchin-tricks) wiki page for more info
+in the right hands. Please check our [Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
 and help.
 
 ## Contributing

.github/ISSUE_TEMPLATE.md

@@ -1,3 +1,9 @@
+Hi there,
+
+please note that this is an issue tracker reserved for bug reports and feature requests.
+
+For general questions please use the gitter channel or the Ethereum stack exchange at https://ethereum.stackexchange.com.
+
 #### System information
 
 Geth version: `geth version`

.github/no-response.yml

@@ -0,0 +1,11 @@
+# Number of days of inactivity before an Issue is closed for lack of response
+daysUntilClose: 30
+# Label requiring a response
+responseRequiredLabel: more-information-needed
+# Comment to post when closing an Issue for lack of response. Set to `false` to disable
+closeComment: >
+  This issue has been automatically closed because there has been no response
+  to our request for more information from the original author. With only the
+  information that is currently in the issue, we don't have enough information
+  to take action. Please reach out if you have or find the answers we need so
+  that we can investigate further.

.github/stale.yml

@@ -0,0 +1,17 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 366
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 42
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: stale
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

.gitignore

@@ -30,3 +30,18 @@ build/_vendor/pkg
 # travis
 profile.tmp
 profile.cov
+
+# IdeaIDE
+.idea
+
+# VS Code
+.vscode
+
+# dashboard
+/dashboard/assets/flow-typed
+/dashboard/assets/node_modules
+/dashboard/assets/stats.json
+/dashboard/assets/bundle.js
+/dashboard/assets/package-lock.json
+
+**/yarn-error.log

.mailmap

@@ -65,7 +65,8 @@ Enrique Fynn <enriquefynn@gmail.com>
 
 Vincent G <caktux@gmail.com>
 
-RJ Catalano <rj@erisindustries.com>
+RJ Catalano <catalanor0220@gmail.com>
+RJ Catalano <catalanor0220@gmail.com> <rj@erisindustries.com>
 
 Nchinda Nchinda <nchinda2@gmail.com>
 
@@ -109,3 +110,14 @@ Frank Wang <eternnoir@gmail.com>
 Gary Rong <garyrong0905@gmail.com>
 
 Guillaume Nicolas <guin56@gmail.com>
+
+Sorin Neacsu <sorin.neacsu@gmail.com>
+Sorin Neacsu <sorin.neacsu@gmail.com> <sorin@users.noreply.github.com>
+
+Valentin Wüstholz <wuestholz@gmail.com>
+Valentin Wüstholz <wuestholz@gmail.com> <wuestholz@users.noreply.github.com>
+
+Armin Braun <me@obrown.io>
+
+Ernesto del Toro <ernesto.deltoro@gmail.com>
+Ernesto del Toro <ernesto.deltoro@gmail.com> <ernestodeltoro@users.noreply.github.com>

.travis.yml

@@ -6,57 +6,79 @@ matrix:
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.7.6
+      go: 1.9.x
       script:
-        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
         - go run build/ci.go install
-        - go run build/ci.go test -coverage
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # These are the latest Go versions.
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.8.3
+      go: 1.10.x
       script:
-        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
         - go run build/ci.go install
-        - go run build/ci.go test -coverage -misspell
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     - os: osx
-      go: 1.8.3
-      sudo: required
+      go: 1.10.x
       script:
+        - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
         - brew update
-        - brew install caskroom/cask/brew-cask
         - brew cask install osxfuse
         - go run build/ci.go install
-        - go run build/ci.go test -coverage -misspell
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
-    # This builder does the Ubuntu PPA and Linux Azure uploads
+    # This builder only tests code linters on latest version of Go
     - os: linux
       dist: trusty
-      sudo: required
-      go: 1.8.3
+      go: 1.10.x
+      env:
+        - lint
+      git:
+        submodules: false # avoid cloning ethereum/tests
+      script:
+        - go run build/ci.go lint
+
+    # This builder does the Ubuntu PPA upload
+    - os: linux
+      dist: trusty
+      go: 1.10.x
       env:
         - ubuntu-ppa
-        - azure-linux
+      git:
+        submodules: false # avoid cloning ethereum/tests
       addons:
         apt:
           packages:
             - devscripts
             - debhelper
             - dput
-            - gcc-multilib
             - fakeroot
       script:
-        # Build for the primary platforms that Trusty can manage
         - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
+
+    # This builder does the Linux Azure uploads
+    - os: linux
+      dist: trusty
+      sudo: required
+      go: 1.10.x
+      env:
+        - azure-linux
+      git:
+        submodules: false # avoid cloning ethereum/tests
+      addons:
+        apt:
+          packages:
+            - gcc-multilib
+      script:
+        # Build for the primary platforms that Trusty can manage
         - go run build/ci.go install
         - go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
         - go run build/ci.go install -arch 386
@@ -66,24 +88,25 @@ matrix:
         - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
         - sudo ln -s /usr/include/asm-generic /usr/include/asm
 
-        - GOARM=5 CC=arm-linux-gnueabi-gcc go run build/ci.go install -arch arm
+        - GOARM=5 go run build/ci.go install -arch arm -cc arm-linux-gnueabi-gcc
         - GOARM=5 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
-        - GOARM=6 CC=arm-linux-gnueabi-gcc go run build/ci.go install -arch arm
+        - GOARM=6 go run build/ci.go install -arch arm -cc arm-linux-gnueabi-gcc
         - GOARM=6 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
-        - GOARM=7 CC=arm-linux-gnueabihf-gcc go run build/ci.go install -arch arm
+        - GOARM=7 go run build/ci.go install -arch arm -cc arm-linux-gnueabihf-gcc
         - GOARM=7 go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
-        - CC=aarch64-linux-gnu-gcc go run build/ci.go install -arch arm64
+        - go run build/ci.go install -arch arm64 -cc aarch64-linux-gnu-gcc
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Linux Azure MIPS xgo uploads
     - os: linux
       dist: trusty
-      sudo: required
       services:
         - docker
-      go: 1.8.3
+      go: 1.10.x
       env:
         - azure-linux-mips
+      git:
+        submodules: false # avoid cloning ethereum/tests
       script:
         - go run build/ci.go xgo --alltools -- --targets=linux/mips --ldflags '-extldflags "-static"' -v
         - for bin in build/bin/*-linux-mips; do mv -f "${bin}" "${bin/-linux-mips/}"; done
@@ -103,7 +126,7 @@ matrix:
 
     # This builder does the Android Maven and Azure uploads
     - os: linux
-      dist: precise # Needed for the android tools
+      dist: trusty
       addons:
         apt:
           packages:
@@ -120,17 +143,19 @@ matrix:
       env:
         - azure-android
         - maven-android
+      git:
+        submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.10.2.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
       script:
         # Build the Android archive and upload it to Maven Central and Azure
-        - curl https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip -o android-ndk-r14b.zip
-        - unzip -q android-ndk-r14b.zip && rm android-ndk-r14b.zip
-        - mv android-ndk-r14b $HOME
-        - export ANDROID_NDK=$HOME/android-ndk-r14b
+        - curl https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip -o android-ndk-r16b.zip
+        - unzip -q android-ndk-r16b.zip && rm android-ndk-r16b.zip
+        - mv android-ndk-r16b $HOME
+        - export ANDROID_NDK=$HOME/android-ndk-r16b
 
         - mkdir -p $GOPATH/src/github.com/ethereum
         - ln -s `pwd` $GOPATH/src/github.com/ethereum
@@ -138,11 +163,13 @@ matrix:
 
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
     - os: osx
-      go: 1.8.3
+      go: 1.10.x
       env:
         - azure-osx
         - azure-ios
         - cocoapods-ios
+      git:
+        submodules: false # avoid cloning ethereum/tests
       script:
         - go run build/ci.go install
         - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -upload gethstore/builds
@@ -158,24 +185,21 @@ matrix:
         - xctool -version
         - xcrun simctl list
 
+        # Workaround for https://github.com/golang/go/issues/23749
+        - export CGO_CFLAGS_ALLOW='-fmodules|-fblocks|-fobjc-arc'
         - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
 
     # This builder does the Azure archive purges to avoid accumulating junk
     - os: linux
       dist: trusty
-      sudo: required
-      go: 1.8.3
+      go: 1.10.x
       env:
         - azure-purge
+      git:
+        submodules: false # avoid cloning ethereum/tests
       script:
         - go run build/ci.go purge -store gethstore/builds -days 14
 
-install:
-  - go get golang.org/x/tools/cmd/cover
-script:
-  - go run build/ci.go install
-  - go run build/ci.go test -coverage
-
 notifications:
   webhooks:
     urls:

AUTHORS

@@ -1,85 +1,173 @@
 # This is the official list of go-ethereum authors for copyright purposes.
 
+Afri Schoedon <5chdn@users.noreply.github.com>
+Agustin Armellini Fischer <armellini13@gmail.com>
+Airead <fgh1987168@gmail.com>
+Alan Chen <alanchchen@users.noreply.github.com>
+Alejandro Isaza <alejandro.isaza@gmail.com>
 Ales Katona <ales@coinbase.com>
 Alex Leverington <alex@ethdev.com>
+Alex Wu <wuyiding@gmail.com>
 Alexandre Van de Sande <alex.vandesande@ethdev.com>
+Ali Hajimirza <Ali92hm@users.noreply.github.com>
+Anton Evangelatov <anton.evangelatov@gmail.com>
+Arba Sasmoyo <arba.sasmoyo@gmail.com>
+Armani Ferrante <armaniferrante@berkeley.edu>
+Armin Braun <me@obrown.io>
 Aron Fischer <github@aron.guru>
 Bas van Kervel <bas@ethdev.com>
 Benjamin Brent <benjamin@benjaminbrent.com>
+Benoit Verkindt <benoit.verkindt@gmail.com>
+Bo <bohende@gmail.com>
+Bo Ye <boy.e.computer.1982@outlook.com>
+Bob Glickstein <bobg@users.noreply.github.com>
 Brian Schroeder <bts@gmail.com>
 Casey Detrio <cdetrio@gmail.com>
+Chase Wright <mysticryuujin@gmail.com>
 Christoph Jentzsch <jentzsch.software@gmail.com>
 Daniel A. Nagy <nagy.da@gmail.com>
+Daniel Sloof <goapsychadelic@gmail.com>
+Darrel Herbst <dherbst@gmail.com>
+Dave Appleton <calistralabs@gmail.com>
 Diego Siqueira <DiSiqueira@users.noreply.github.com>
+Dmitry Shulyak <yashulyak@gmail.com>
+Egon Elbre <egonelbre@gmail.com>
+Elias Naur <elias.naur@gmail.com>
 Elliot Shepherd <elliot@identitii.com>
 Enrique Fynn <enriquefynn@gmail.com>
+Ernesto del Toro <ernesto.deltoro@gmail.com>
 Ethan Buchman <ethan@coinculture.info>
+Eugene Valeyev <evgen.povt@gmail.com>
+Evangelos Pappas <epappas@evalonlabs.com>
+Evgeny Danilenko <6655321@bk.ru>
 Fabian Vogelsteller <fabian@frozeman.de>
+Fabio Barone <fabio.barone.co@gmail.com>
 Fabio Berger <fabioberger1991@gmail.com>
+FaceHo <facehoshi@gmail.com>
 Felix Lange <fjl@twurst.com>
+Fiisio <liangcszzu@163.com>
 Frank Wang <eternnoir@gmail.com>
+Furkan KAMACI <furkankamaci@gmail.com>
 Gary Rong <garyrong0905@gmail.com>
+George Ornbo <george@shapeshed.com>
 Gregg Dourgarian <greggd@tempworks.com>
+Guillaume Ballet <gballet@gmail.com>
 Guillaume Nicolas <guin56@gmail.com>
 Gustav Simonsson <gustav.simonsson@gmail.com>
 Hao Bryan Cheng <haobcheng@gmail.com>
 Henning Diedrich <hd@eonblast.com>
 Isidoro Ghezzi <isidoro.ghezzi@icloud.com>
+Ivan Daniluk <ivan.daniluk@gmail.com>
 Jae Kwon <jkwon.work@gmail.com>
 Jamie Pitts <james.pitts@gmail.com>
+Janoš Guljaš <janos@users.noreply.github.com>
 Jason Carver <jacarver@linkedin.com>
+Jay Guo <guojiannan1101@gmail.com>
 Jeff R. Allen <jra@nella.org>
 Jeffrey Wilcke <jeffrey@ethereum.org>
 Jens Agerberg <github@agerberg.me>
+Jia Chenhui <jiachenhui1989@gmail.com>
+Jim McDonald <Jim@mcdee.net>
+Joel Burget <joelburget@gmail.com>
 Jonathan Brown <jbrown@bluedroplet.com>
 Joseph Chow <ethereum@outlook.com>
 Justin Clark-Casey <justincc@justincc.org>
 Justin Drake <drakefjustin@gmail.com>
 Kenji Siu <kenji@isuntv.com>
 Kobi Gurkan <kobigurk@gmail.com>
+Konrad Feldmeier <konrad@brainbot.com>
+Kurkó Mihály <kurkomisi@users.noreply.github.com>
+Kyuntae Ethan Kim <ethan.kyuntae.kim@gmail.com>
 Lefteris Karapetsas <lefteris@refu.co>
 Leif Jurvetson <leijurv@gmail.com>
+Leo Shklovskii <leo@thermopylae.net>
 Lewis Marshall <lewis@lmars.net>
+Lio李欧 <lionello@users.noreply.github.com>
 Louis Holbrook <dev@holbrook.no>
 Luca Zeug <luclu@users.noreply.github.com>
+Magicking <s@6120.eu>
 Maran Hidskes <maran.hidskes@gmail.com>
 Marek Kotewicz <marek.kotewicz@gmail.com>
+Mark <markya0616@gmail.com>
 Martin Holst Swende <martin@swende.se>
 Matthew Di Ferrante <mattdf@users.noreply.github.com>
 Matthew Wampler-Doty <matthew.wampler.doty@gmail.com>
+Maximilian Meister <mmeister@suse.de>
 Micah Zoltu <micah@zoltu.net>
+Michael Ruminer <michael.ruminer+github@gmail.com>
+Miguel Mota <miguelmota2@gmail.com>
+Miya Chen <miyatlchen@gmail.com>
 Nchinda Nchinda <nchinda2@gmail.com>
 Nick Dodson <silentcicero@outlook.com>
 Nick Johnson <arachnid@notdot.net>
+Nicolas Guillaume <gunicolas@sqli.com>
+Noman <noman@noman.land>
+Oli Bye <olibye@users.noreply.github.com>
+Paul Litvak <litvakpol@012.net.il>
 Paulo L F Casaretto <pcasaretto@gmail.com>
+Paweł Bylica <chfast@gmail.com>
 Peter Pratscher <pratscher@gmail.com>
+Petr Mikusek <petr@mikusek.info>
 Péter Szilágyi <peterke@gmail.com>
-RJ Catalano <rj@erisindustries.com>
+RJ Catalano <catalanor0220@gmail.com>
 Ramesh Nair <ram@hiddentao.com>
 Ricardo Catalinas Jiménez <r@untroubled.be>
+Ricardo Domingos <ricardohsd@gmail.com>
+Richard Hart <richardhart92@gmail.com>
+Rob <robert@rojotek.com>
+Robert Zaremba <robert.zaremba@scale-it.pl>
+Russ Cox <rsc@golang.org>
 Rémy Roy <remyroy@remyroy.com>
+S. Matthew English <s-matthew-english@users.noreply.github.com>
 Shintaro Kaneko <kaneshin0120@gmail.com>
+Sorin Neacsu <sorin.neacsu@gmail.com>
 Stein Dekker <dekker.stein@gmail.com>
+Steve Waldman <swaldman@mchange.com>
 Steven Roose <stevenroose@gmail.com>
 Taylor Gerring <taylor.gerring@gmail.com>
 Thomas Bocek <tom@tomp2p.net>
+Ti Zhou <tizhou1986@gmail.com>
 Tosh Camille <tochecamille@gmail.com>
-Valentin Wüstholz <wuestholz@users.noreply.github.com>
+Valentin Wüstholz <wuestholz@gmail.com>
 Victor Farazdagi <simple.square@gmail.com>
 Victor Tran <vu.tran54@gmail.com>
 Viktor Trón <viktor.tron@gmail.com>
 Ville Sundell <github@solarius.fi>
 Vincent G <caktux@gmail.com>
 Vitalik Buterin <v@buterin.com>
+Vitaly V <vvelikodny@gmail.com>
 Vivek Anand <vivekanand1101@users.noreply.github.com>
 Vlad Gluhovsky <gluk256@users.noreply.github.com>
 Yohann Léon <sybiload@gmail.com>
 Yoichi Hirai <i@yoichihirai.com>
+Yondon Fu <yondon.fu@gmail.com>
+Zach <zach.ramsay@gmail.com>
 Zahoor Mohamed <zahoor@zahoor.in>
+Zoe Nolan <github@zoenolan.org>
 Zsolt Felföldi <zsfelfoldi@gmail.com>
+am2rican5 <am2rican5@gmail.com>
+ayeowch <ayeowch@gmail.com>
+b00ris <b00ris@mail.ru>
+bailantaotao <Edwin@maicoin.com>
+baizhenxuan <nkbai@163.com>
+bloonfield <bloonfield@163.com>
+changhong <changhong.yu@shanbay.com>
+evgk <evgeniy.kamyshev@gmail.com>
+ferhat elmas <elmas.ferhat@gmail.com>
 holisticode <holistic.computing@gmail.com>
+jtakalai <juuso.takalainen@streamr.com>
 ken10100147 <sunhongping@kanjian.com>
 ligi <ligi@ligi.de>
+mark.lin <mark@maicoin.com>
+necaremus <necaremus@gmail.com>
+njupt-moon <1015041018@njupt.edu.cn>
+nkbai <nkbai@163.com>
+rhaps107 <dod-source@yandex.ru>
+slumber1122 <slumber1122@gmail.com>
+sunxiaojun2014 <sunxiaojun-xy@360.cn>
+terasum <terasum@163.com>
+tsarpaul <Litvakpol@012.net.il>
 xiekeyang <xiekeyang@users.noreply.github.com>
+yoza <yoza.is12s@gmail.com>
 ΞTHΞЯSPHΞЯΞ <{viktor.tron,nagydani,zsfelfoldi}@gmail.com>
 Максим Чусовлянов <mchusovlianov@gmail.com>

Dockerfile

@@ -1,15 +1,16 @@
-FROM alpine:3.5
+# Build Geth in a stock Go builder container
+FROM golang:1.10-alpine as builder
+
+RUN apk add --no-cache make gcc musl-dev linux-headers
 
 ADD . /go-ethereum
-RUN \
-  apk add --update git go make gcc musl-dev linux-headers && \
-  (cd go-ethereum && make geth)                           && \
-  cp go-ethereum/build/bin/geth /usr/local/bin/           && \
-  apk del git go make gcc musl-dev linux-headers          && \
-  rm -rf /go-ethereum && rm -rf /var/cache/apk/*
+RUN cd /go-ethereum && make geth
 
-EXPOSE 8545
-EXPOSE 30303
-EXPOSE 30303/udp
+# Pull Geth into a second stage deploy alpine container
+FROM alpine:latest
 
+RUN apk add --no-cache ca-certificates
+COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
+
+EXPOSE 8545 8546 30303 30303/udp
 ENTRYPOINT ["geth"]

Dockerfile.alltools

@@ -0,0 +1,15 @@
+# Build Geth in a stock Go builder container
+FROM golang:1.10-alpine as builder
+
+RUN apk add --no-cache make gcc musl-dev linux-headers
+
+ADD . /go-ethereum
+RUN cd /go-ethereum && make all
+
+# Pull all binaries into a second stage deploy alpine container
+FROM alpine:latest
+
+RUN apk add --no-cache ca-certificates
+COPY --from=builder /go-ethereum/build/bin/* /usr/local/bin/
+
+EXPOSE 8545 8546 30303 30303/udp

Makefile

@@ -8,7 +8,7 @@
 .PHONY: geth-darwin geth-darwin-386 geth-darwin-amd64
 .PHONY: geth-windows geth-windows-386 geth-windows-amd64
 
-GOBIN = build/bin
+GOBIN = $(shell pwd)/build/bin
 GO ?= latest
 
 geth:
@@ -21,11 +21,6 @@ swarm:
 	@echo "Done building."
 	@echo "Run \"$(GOBIN)/swarm\" to launch swarm."
 
-evm:
-	build/env.sh go run build/ci.go install ./cmd/evm
-	@echo "Done building."
-	@echo "Run \"$(GOBIN)/evm\" to start the evm."
-
 all:
 	build/env.sh go run build/ci.go install
 
@@ -42,6 +37,9 @@ ios:
 test: all
 	build/env.sh go run build/ci.go test
 
+lint: ## Run linters.
+	build/env.sh go run build/ci.go lint
+
 clean:
 	rm -fr build/_workspace/pkg/ $(GOBIN)/*
 
@@ -50,9 +48,13 @@ clean:
 
 devtools:
 	env GOBIN= go get -u golang.org/x/tools/cmd/stringer
-	env GOBIN= go get -u github.com/jteeuwen/go-bindata/go-bindata
+	env GOBIN= go get -u github.com/kevinburke/go-bindata/go-bindata
 	env GOBIN= go get -u github.com/fjl/gencodec
+	env GOBIN= go get -u github.com/golang/protobuf/protoc-gen-go
 	env GOBIN= go install ./cmd/abigen
+	@type "npm" 2> /dev/null || echo 'Please install node.js and npm'
+	@type "solc" 2> /dev/null || echo 'Please install solc'
+	@type "protoc" 2> /dev/null || echo 'Please install protoc'
 
 # Cross Compilation Targets (xgo)
 

README.md

@@ -5,6 +5,8 @@ Official golang implementation of the Ethereum protocol.
 [![API Reference](
 https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
 )](https://godoc.org/github.com/ethereum/go-ethereum)
+[![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum)
+[![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethereum/go-ethereum?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 
 Automated builds are available for stable releases and the unstable master branch.
@@ -35,11 +37,11 @@ The go-ethereum project comes with several wrappers/executables found in the `cm
 | **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default) archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. |
 | `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. |
 | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. |
-| `disasm` | Bytecode disassembler to convert EVM (Ethereum Virtual Machine) bytecode into more user friendly assembly-like opcodes (e.g. `echo "6001" | disasm`). For details on the individual opcodes, please see pages 22-30 of the [Ethereum Yellow Paper](http://gavwood.com/paper.pdf). |
-| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow insolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
+| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
 | `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. |
 | `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). |
 | `swarm`    | swarm daemon and tools. This is the entrypoint for the swarm network. `swarm --help` for command line options and subcommands. See https://swarm-guide.readthedocs.io for swarm documentation. |
+| `puppeth`    | a CLI wizard that aids in creating a new Ethereum network. |
 
 ## Running geth
 
@@ -56,16 +58,14 @@ the user doesn't care about years-old historical data, so we can fast-sync quick
 state of the network. To do so:
 
 ```
-$ geth --fast --cache=512 console
+$ geth console
 ```
 
 This command will:
 
- * Start geth in fast sync mode (`--fast`), causing it to download more data in exchange for avoiding
-   processing the entire history of the Ethereum network, which is very CPU intensive.
- * Bump the memory allowance of the database to 512MB (`--cache=512`), which can help significantly in
-   sync times especially for HDD users. This flag is optional and you can set it as high or as low as
-   you'd like, though we'd recommend the 512MB - 2GB range.
+ * Start geth in fast sync mode (default, can be changed with the `--syncmode` flag), causing it to
+   download more data in exchange for avoiding processing the entire history of the Ethereum network,
+   which is very CPU intensive.
  * Start up Geth's built-in interactive [JavaScript console](https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console),
    (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API)
    as well as Geth's own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs).
@@ -80,12 +80,11 @@ entire system. In other words, instead of attaching to the main network, you wan
 network with your node, which is fully equivalent to the main network, but with play-Ether only.
 
 ```
-$ geth --testnet --fast --cache=512 console
+$ geth --testnet console
 ```
 
-The `--fast`, `--cache` flags and `console` subcommand have the exact same meaning as above and they
-are equally useful on the testnet too. Please see above for their explanations if you've skipped to
-here.
+The `console` subcommand have the exact same meaning as above and they are equally useful on the
+testnet too. Please see above for their explanations if you've skipped to here.
 
 Specifying the `--testnet` flag however will reconfigure your Geth instance a bit:
 
@@ -102,6 +101,14 @@ over between the main network and test network, you should make sure to always u
 for play-money and real-money. Unless you manually move accounts, Geth will by default correctly
 separate the two networks and will not make any accounts available between them.*
 
+### Full node on the Rinkeby test network
+
+The above test network is a cross client one based on the ethash proof-of-work consensus algorithm. As such, it has certain extra overhead and is more susceptible to reorganization attacks due to the network's low difficulty / security. Go Ethereum also supports connecting to a proof-of-authority based test network called [*Rinkeby*](https://www.rinkeby.io) (operated by members of the community). This network is lighter, more secure, but is only supported by go-ethereum.
+
+```
+$ geth --rinkeby console
+```
+
 ### Configuration
 
 As an alternative to passing the numerous flags to the `geth` binary, you can also pass a configuration file via:
@@ -116,7 +123,7 @@ To get an idea how the file should look like you can use the `dumpconfig` subcom
 $ geth --your-favourite-flags dumpconfig
 ```
 
-*Note: This works only with geth v1.6.0 and above*
+*Note: This works only with geth v1.6.0 and above.*
 
 #### Docker quick start
 
@@ -125,15 +132,17 @@ One of the quickest ways to get Ethereum up and running on your machine is by us
 ```
 docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \
            -p 8545:8545 -p 30303:30303 \
-           ethereum/client-go --fast --cache=512
+           ethereum/client-go
 ```
 
-This will start geth in fast sync mode with a DB memory allowance of 512MB just as the above command does.  It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image.
+This will start geth in fast-sync mode with a DB memory allowance of 1GB just as the above command does.  It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image.
+
+Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containers and/or hosts. By default, `geth` binds to the local interface and RPC endpoints is not accessible from the outside.
 
 ### Programatically interfacing Geth nodes
 
 As a developer, sooner rather than later you'll want to start interacting with Geth and the Ethereum
-network via your own programs and not manually through the console. To aid this, Geth has built in
+network via your own programs and not manually through the console. To aid this, Geth has built-in
 support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC) and
 [Geth specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)). These can be
 exposed via HTTP, WebSockets and IPC (unix sockets on unix based platforms, and named pipes on Windows).
@@ -264,7 +273,7 @@ instance for mining, run it with all your usual flags, extended by:
 $ geth <usual-flags> --mine --minerthreads=1 --etherbase=0x0000000000000000000000000000000000000000
 ```
 
-Which will start mining bocks and transactions on a single CPU thread, crediting all proceedings to
+Which will start mining blocks and transactions on a single CPU thread, crediting all proceedings to
 the account specified by `--etherbase`. You can further tune the mining by changing the default gas
 limit blocks converge to (`--targetgaslimit`) and the price transactions are accepted at (`--gasprice`).
 

VERSION

@@ -1 +1 @@
-1.6.7
+1.8.11

accounts/abi/abi.go

@@ -17,13 +17,10 @@
 package abi
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
-	"reflect"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common"
 )
 
 // The ABI holds information about a contract's context and available
@@ -54,136 +51,52 @@ func JSON(reader io.Reader) (ABI, error) {
 // methods string signature. (signature = baz(uint32,string32))
 func (abi ABI) Pack(name string, args ...interface{}) ([]byte, error) {
 	// Fetch the ABI of the requested method
-	var method Method
-
 	if name == "" {
-		method = abi.Constructor
-	} else {
-		m, exist := abi.Methods[name]
-		if !exist {
-			return nil, fmt.Errorf("method '%s' not found", name)
+		// constructor
+		arguments, err := abi.Constructor.Inputs.Pack(args...)
+		if err != nil {
+			return nil, err
 		}
-		method = m
+		return arguments, nil
+
 	}
-	arguments, err := method.pack(args...)
+	method, exist := abi.Methods[name]
+	if !exist {
+		return nil, fmt.Errorf("method '%s' not found", name)
+	}
+
+	arguments, err := method.Inputs.Pack(args...)
 	if err != nil {
 		return nil, err
 	}
 	// Pack up the method ID too if not a constructor and return
-	if name == "" {
-		return arguments, nil
-	}
 	return append(method.Id(), arguments...), nil
 }
 
-// these variable are used to determine certain types during type assertion for
-// assignment.
-var (
-	r_interSlice = reflect.TypeOf([]interface{}{})
-	r_hash       = reflect.TypeOf(common.Hash{})
-	r_bytes      = reflect.TypeOf([]byte{})
-	r_byte       = reflect.TypeOf(byte(0))
-)
-
 // Unpack output in v according to the abi specification
-func (abi ABI) Unpack(v interface{}, name string, output []byte) error {
-	var method = abi.Methods[name]
-
+func (abi ABI) Unpack(v interface{}, name string, output []byte) (err error) {
 	if len(output) == 0 {
 		return fmt.Errorf("abi: unmarshalling empty output")
 	}
-
-	// make sure the passed value is a pointer
-	valueOf := reflect.ValueOf(v)
-	if reflect.Ptr != valueOf.Kind() {
-		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
+	// since there can't be naming collisions with contracts and events,
+	// we need to decide whether we're calling a method or an event
+	if method, ok := abi.Methods[name]; ok {
+		if len(output)%32 != 0 {
+			return fmt.Errorf("abi: improperly formatted output")
+		}
+		return method.Outputs.Unpack(v, output)
+	} else if event, ok := abi.Events[name]; ok {
+		return event.Inputs.Unpack(v, output)
 	}
-
-	var (
-		value = valueOf.Elem()
-		typ   = value.Type()
-	)
-
-	if len(method.Outputs) > 1 {
-		switch value.Kind() {
-		// struct will match named return values to the struct's field
-		// names
-		case reflect.Struct:
-			for i := 0; i < len(method.Outputs); i++ {
-				marshalledValue, err := toGoType(i, method.Outputs[i], output)
-				if err != nil {
-					return err
-				}
-				reflectValue := reflect.ValueOf(marshalledValue)
-
-				for j := 0; j < typ.NumField(); j++ {
-					field := typ.Field(j)
-					// TODO read tags: `abi:"fieldName"`
-					if field.Name == strings.ToUpper(method.Outputs[i].Name[:1])+method.Outputs[i].Name[1:] {
-						if err := set(value.Field(j), reflectValue, method.Outputs[i]); err != nil {
-							return err
-						}
-					}
-				}
-			}
-		case reflect.Slice:
-			if !value.Type().AssignableTo(r_interSlice) {
-				return fmt.Errorf("abi: cannot marshal tuple in to slice %T (only []interface{} is supported)", v)
-			}
-
-			// if the slice already contains values, set those instead of the interface slice itself.
-			if value.Len() > 0 {
-				if len(method.Outputs) > value.Len() {
-					return fmt.Errorf("abi: cannot marshal in to slices of unequal size (require: %v, got: %v)", len(method.Outputs), value.Len())
-				}
-
-				for i := 0; i < len(method.Outputs); i++ {
-					marshalledValue, err := toGoType(i, method.Outputs[i], output)
-					if err != nil {
-						return err
-					}
-					reflectValue := reflect.ValueOf(marshalledValue)
-					if err := set(value.Index(i).Elem(), reflectValue, method.Outputs[i]); err != nil {
-						return err
-					}
-				}
-				return nil
-			}
-
-			// create a new slice and start appending the unmarshalled
-			// values to the new interface slice.
-			z := reflect.MakeSlice(typ, 0, len(method.Outputs))
-			for i := 0; i < len(method.Outputs); i++ {
-				marshalledValue, err := toGoType(i, method.Outputs[i], output)
-				if err != nil {
-					return err
-				}
-				z = reflect.Append(z, reflect.ValueOf(marshalledValue))
-			}
-			value.Set(z)
-		default:
-			return fmt.Errorf("abi: cannot unmarshal tuple in to %v", typ)
-		}
-
-	} else {
-		marshalledValue, err := toGoType(0, method.Outputs[0], output)
-		if err != nil {
-			return err
-		}
-		if err := set(value, reflect.ValueOf(marshalledValue), method.Outputs[0]); err != nil {
-			return err
-		}
-	}
-
-	return nil
+	return fmt.Errorf("abi: could not locate named method or event")
 }
 
+// UnmarshalJSON implements json.Unmarshaler interface
 func (abi *ABI) UnmarshalJSON(data []byte) error {
 	var fields []struct {
 		Type      string
 		Name      string
 		Constant  bool
-		Indexed   bool
 		Anonymous bool
 		Inputs    []Argument
 		Outputs   []Argument
@@ -220,3 +133,14 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
 
 	return nil
 }
+
+// MethodById looks up a method by the 4-byte id
+// returns nil if none found
+func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
+	for _, method := range abi.Methods {
+		if bytes.Equal(method.Id(), sigdata[:4]) {
+			return &method, nil
+		}
+	}
+	return nil, fmt.Errorf("no method with id: %#x", sigdata[:4])
+}

accounts/abi/abi_test.go

@@ -18,36 +18,19 @@ package abi
 
 import (
 	"bytes"
+	"encoding/hex"
 	"fmt"
 	"log"
 	"math/big"
-	"reflect"
 	"strings"
 	"testing"
 
+	"reflect"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
-// formatSilceOutput add padding to the value and adds a size
-func formatSliceOutput(v ...[]byte) []byte {
-	off := common.LeftPadBytes(big.NewInt(int64(len(v))).Bytes(), 32)
-	output := append(off, make([]byte, 0, len(v)*32)...)
-
-	for _, value := range v {
-		output = append(output, common.LeftPadBytes(value, 32)...)
-	}
-	return output
-}
-
-// quick helper padding
-func pad(input []byte, size int, left bool) []byte {
-	if left {
-		return common.LeftPadBytes(input, size)
-	}
-	return common.RightPadBytes(input, size)
-}
-
 const jsondata = `
 [
 	{ "type" : "function", "name" : "balance", "constant" : true },
@@ -93,9 +76,24 @@ func TestReader(t *testing.T) {
 	}
 
 	// deep equal fails for some reason
-	t.Skip()
-	if !reflect.DeepEqual(abi, exp) {
-		t.Errorf("\nabi: %v\ndoes not match exp: %v", abi, exp)
+	for name, expM := range exp.Methods {
+		gotM, exist := abi.Methods[name]
+		if !exist {
+			t.Errorf("Missing expected method %v", name)
+		}
+		if !reflect.DeepEqual(gotM, expM) {
+			t.Errorf("\nGot abi method: \n%v\ndoes not match expected method\n%v", gotM, expM)
+		}
+	}
+
+	for name, gotM := range abi.Methods {
+		expM, exist := exp.Methods[name]
+		if !exist {
+			t.Errorf("Found extra method %v", name)
+		}
+		if !reflect.DeepEqual(gotM, expM) {
+			t.Errorf("\nGot abi method: \n%v\ndoes not match expected method\n%v", gotM, expM)
+		}
 	}
 }
 
@@ -191,7 +189,7 @@ func TestMethodSignature(t *testing.T) {
 		t.Errorf("expected ids to match %x != %x", m.Id(), idexp)
 	}
 
-	uintt, _ := NewType("uint")
+	uintt, _ := NewType("uint256")
 	m = Method{"foo", false, []Argument{{"bar", uintt, false}}, nil}
 	exp = "foo(uint256)"
 	if m.Sig() != exp {
@@ -367,6 +365,188 @@ func TestInputVariableInputLength(t *testing.T) {
 	}
 }
 
+func TestInputFixedArrayAndVariableInputLength(t *testing.T) {
+	const definition = `[
+	{ "type" : "function", "name" : "fixedArrStr", "constant" : true, "inputs" : [ { "name" : "str", "type" : "string" }, { "name" : "fixedArr", "type" : "uint256[2]" } ] },
+	{ "type" : "function", "name" : "fixedArrBytes", "constant" : true, "inputs" : [ { "name" : "str", "type" : "bytes" }, { "name" : "fixedArr", "type" : "uint256[2]" } ] },
+    { "type" : "function", "name" : "mixedArrStr", "constant" : true, "inputs" : [ { "name" : "str", "type" : "string" }, { "name" : "fixedArr", "type": "uint256[2]" }, { "name" : "dynArr", "type": "uint256[]" } ] },
+    { "type" : "function", "name" : "doubleFixedArrStr", "constant" : true, "inputs" : [ { "name" : "str", "type" : "string" }, { "name" : "fixedArr1", "type": "uint256[2]" }, { "name" : "fixedArr2", "type": "uint256[3]" } ] },
+    { "type" : "function", "name" : "multipleMixedArrStr", "constant" : true, "inputs" : [ { "name" : "str", "type" : "string" }, { "name" : "fixedArr1", "type": "uint256[2]" }, { "name" : "dynArr", "type" : "uint256[]" }, { "name" : "fixedArr2", "type" : "uint256[3]" } ] }
+	]`
+
+	abi, err := JSON(strings.NewReader(definition))
+	if err != nil {
+		t.Error(err)
+	}
+
+	// test string, fixed array uint256[2]
+	strin := "hello world"
+	arrin := [2]*big.Int{big.NewInt(1), big.NewInt(2)}
+	fixedArrStrPack, err := abi.Pack("fixedArrStr", strin, arrin)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// generate expected output
+	offset := make([]byte, 32)
+	offset[31] = 96
+	length := make([]byte, 32)
+	length[31] = byte(len(strin))
+	strvalue := common.RightPadBytes([]byte(strin), 32)
+	arrinvalue1 := common.LeftPadBytes(arrin[0].Bytes(), 32)
+	arrinvalue2 := common.LeftPadBytes(arrin[1].Bytes(), 32)
+	exp := append(offset, arrinvalue1...)
+	exp = append(exp, arrinvalue2...)
+	exp = append(exp, append(length, strvalue...)...)
+
+	// ignore first 4 bytes of the output. This is the function identifier
+	fixedArrStrPack = fixedArrStrPack[4:]
+	if !bytes.Equal(fixedArrStrPack, exp) {
+		t.Errorf("expected %x, got %x\n", exp, fixedArrStrPack)
+	}
+
+	// test byte array, fixed array uint256[2]
+	bytesin := []byte(strin)
+	arrin = [2]*big.Int{big.NewInt(1), big.NewInt(2)}
+	fixedArrBytesPack, err := abi.Pack("fixedArrBytes", bytesin, arrin)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// generate expected output
+	offset = make([]byte, 32)
+	offset[31] = 96
+	length = make([]byte, 32)
+	length[31] = byte(len(strin))
+	strvalue = common.RightPadBytes([]byte(strin), 32)
+	arrinvalue1 = common.LeftPadBytes(arrin[0].Bytes(), 32)
+	arrinvalue2 = common.LeftPadBytes(arrin[1].Bytes(), 32)
+	exp = append(offset, arrinvalue1...)
+	exp = append(exp, arrinvalue2...)
+	exp = append(exp, append(length, strvalue...)...)
+
+	// ignore first 4 bytes of the output. This is the function identifier
+	fixedArrBytesPack = fixedArrBytesPack[4:]
+	if !bytes.Equal(fixedArrBytesPack, exp) {
+		t.Errorf("expected %x, got %x\n", exp, fixedArrBytesPack)
+	}
+
+	// test string, fixed array uint256[2], dynamic array uint256[]
+	strin = "hello world"
+	fixedarrin := [2]*big.Int{big.NewInt(1), big.NewInt(2)}
+	dynarrin := []*big.Int{big.NewInt(1), big.NewInt(2), big.NewInt(3)}
+	mixedArrStrPack, err := abi.Pack("mixedArrStr", strin, fixedarrin, dynarrin)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// generate expected output
+	stroffset := make([]byte, 32)
+	stroffset[31] = 128
+	strlength := make([]byte, 32)
+	strlength[31] = byte(len(strin))
+	strvalue = common.RightPadBytes([]byte(strin), 32)
+	fixedarrinvalue1 := common.LeftPadBytes(fixedarrin[0].Bytes(), 32)
+	fixedarrinvalue2 := common.LeftPadBytes(fixedarrin[1].Bytes(), 32)
+	dynarroffset := make([]byte, 32)
+	dynarroffset[31] = byte(160 + ((len(strin)/32)+1)*32)
+	dynarrlength := make([]byte, 32)
+	dynarrlength[31] = byte(len(dynarrin))
+	dynarrinvalue1 := common.LeftPadBytes(dynarrin[0].Bytes(), 32)
+	dynarrinvalue2 := common.LeftPadBytes(dynarrin[1].Bytes(), 32)
+	dynarrinvalue3 := common.LeftPadBytes(dynarrin[2].Bytes(), 32)
+	exp = append(stroffset, fixedarrinvalue1...)
+	exp = append(exp, fixedarrinvalue2...)
+	exp = append(exp, dynarroffset...)
+	exp = append(exp, append(strlength, strvalue...)...)
+	dynarrarg := append(dynarrlength, dynarrinvalue1...)
+	dynarrarg = append(dynarrarg, dynarrinvalue2...)
+	dynarrarg = append(dynarrarg, dynarrinvalue3...)
+	exp = append(exp, dynarrarg...)
+
+	// ignore first 4 bytes of the output. This is the function identifier
+	mixedArrStrPack = mixedArrStrPack[4:]
+	if !bytes.Equal(mixedArrStrPack, exp) {
+		t.Errorf("expected %x, got %x\n", exp, mixedArrStrPack)
+	}
+
+	// test string, fixed array uint256[2], fixed array uint256[3]
+	strin = "hello world"
+	fixedarrin1 := [2]*big.Int{big.NewInt(1), big.NewInt(2)}
+	fixedarrin2 := [3]*big.Int{big.NewInt(1), big.NewInt(2), big.NewInt(3)}
+	doubleFixedArrStrPack, err := abi.Pack("doubleFixedArrStr", strin, fixedarrin1, fixedarrin2)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// generate expected output
+	stroffset = make([]byte, 32)
+	stroffset[31] = 192
+	strlength = make([]byte, 32)
+	strlength[31] = byte(len(strin))
+	strvalue = common.RightPadBytes([]byte(strin), 32)
+	fixedarrin1value1 := common.LeftPadBytes(fixedarrin1[0].Bytes(), 32)
+	fixedarrin1value2 := common.LeftPadBytes(fixedarrin1[1].Bytes(), 32)
+	fixedarrin2value1 := common.LeftPadBytes(fixedarrin2[0].Bytes(), 32)
+	fixedarrin2value2 := common.LeftPadBytes(fixedarrin2[1].Bytes(), 32)
+	fixedarrin2value3 := common.LeftPadBytes(fixedarrin2[2].Bytes(), 32)
+	exp = append(stroffset, fixedarrin1value1...)
+	exp = append(exp, fixedarrin1value2...)
+	exp = append(exp, fixedarrin2value1...)
+	exp = append(exp, fixedarrin2value2...)
+	exp = append(exp, fixedarrin2value3...)
+	exp = append(exp, append(strlength, strvalue...)...)
+
+	// ignore first 4 bytes of the output. This is the function identifier
+	doubleFixedArrStrPack = doubleFixedArrStrPack[4:]
+	if !bytes.Equal(doubleFixedArrStrPack, exp) {
+		t.Errorf("expected %x, got %x\n", exp, doubleFixedArrStrPack)
+	}
+
+	// test string, fixed array uint256[2], dynamic array uint256[], fixed array uint256[3]
+	strin = "hello world"
+	fixedarrin1 = [2]*big.Int{big.NewInt(1), big.NewInt(2)}
+	dynarrin = []*big.Int{big.NewInt(1), big.NewInt(2)}
+	fixedarrin2 = [3]*big.Int{big.NewInt(1), big.NewInt(2), big.NewInt(3)}
+	multipleMixedArrStrPack, err := abi.Pack("multipleMixedArrStr", strin, fixedarrin1, dynarrin, fixedarrin2)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// generate expected output
+	stroffset = make([]byte, 32)
+	stroffset[31] = 224
+	strlength = make([]byte, 32)
+	strlength[31] = byte(len(strin))
+	strvalue = common.RightPadBytes([]byte(strin), 32)
+	fixedarrin1value1 = common.LeftPadBytes(fixedarrin1[0].Bytes(), 32)
+	fixedarrin1value2 = common.LeftPadBytes(fixedarrin1[1].Bytes(), 32)
+	dynarroffset = U256(big.NewInt(int64(256 + ((len(strin)/32)+1)*32)))
+	dynarrlength = make([]byte, 32)
+	dynarrlength[31] = byte(len(dynarrin))
+	dynarrinvalue1 = common.LeftPadBytes(dynarrin[0].Bytes(), 32)
+	dynarrinvalue2 = common.LeftPadBytes(dynarrin[1].Bytes(), 32)
+	fixedarrin2value1 = common.LeftPadBytes(fixedarrin2[0].Bytes(), 32)
+	fixedarrin2value2 = common.LeftPadBytes(fixedarrin2[1].Bytes(), 32)
+	fixedarrin2value3 = common.LeftPadBytes(fixedarrin2[2].Bytes(), 32)
+	exp = append(stroffset, fixedarrin1value1...)
+	exp = append(exp, fixedarrin1value2...)
+	exp = append(exp, dynarroffset...)
+	exp = append(exp, fixedarrin2value1...)
+	exp = append(exp, fixedarrin2value2...)
+	exp = append(exp, fixedarrin2value3...)
+	exp = append(exp, append(strlength, strvalue...)...)
+	dynarrarg = append(dynarrlength, dynarrinvalue1...)
+	dynarrarg = append(dynarrarg, dynarrinvalue2...)
+	exp = append(exp, dynarrarg...)
+
+	// ignore first 4 bytes of the output. This is the function identifier
+	multipleMixedArrStrPack = multipleMixedArrStrPack[4:]
+	if !bytes.Equal(multipleMixedArrStrPack, exp) {
+		t.Errorf("expected %x, got %x\n", exp, multipleMixedArrStrPack)
+	}
+}
+
 func TestDefaultFunctionParsing(t *testing.T) {
 	const definition = `[{ "name" : "balance" }]`
 
@@ -437,3 +617,99 @@ func TestBareEvents(t *testing.T) {
 		}
 	}
 }
+
+// TestUnpackEvent is based on this contract:
+//    contract T {
+//      event received(address sender, uint amount, bytes memo);
+//      event receivedAddr(address sender);
+//      function receive(bytes memo) external payable {
+//        received(msg.sender, msg.value, memo);
+//        receivedAddr(msg.sender);
+//      }
+//    }
+// When receive("X") is called with sender 0x00... and value 1, it produces this tx receipt:
+//   receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]}
+func TestUnpackEvent(t *testing.T) {
+	const abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"receive","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"memo","type":"bytes"}],"name":"received","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"}],"name":"receivedAddr","type":"event"}]`
+	abi, err := JSON(strings.NewReader(abiJSON))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	const hexdata = `000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158`
+	data, err := hex.DecodeString(hexdata)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(data)%32 == 0 {
+		t.Errorf("len(data) is %d, want a non-multiple of 32", len(data))
+	}
+
+	type ReceivedEvent struct {
+		Address common.Address
+		Amount  *big.Int
+		Memo    []byte
+	}
+	var ev ReceivedEvent
+
+	err = abi.Unpack(&ev, "received", data)
+	if err != nil {
+		t.Error(err)
+	} else {
+		t.Logf("len(data): %d; received event: %+v", len(data), ev)
+	}
+
+	type ReceivedAddrEvent struct {
+		Address common.Address
+	}
+	var receivedAddrEv ReceivedAddrEvent
+	err = abi.Unpack(&receivedAddrEv, "receivedAddr", data)
+	if err != nil {
+		t.Error(err)
+	} else {
+		t.Logf("len(data): %d; received event: %+v", len(data), receivedAddrEv)
+	}
+}
+
+func TestABI_MethodById(t *testing.T) {
+	const abiJSON = `[
+		{"type":"function","name":"receive","constant":false,"inputs":[{"name":"memo","type":"bytes"}],"outputs":[],"payable":true,"stateMutability":"payable"},
+		{"type":"event","name":"received","anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"memo","type":"bytes"}]},
+		{"type":"function","name":"fixedArrStr","constant":true,"inputs":[{"name":"str","type":"string"},{"name":"fixedArr","type":"uint256[2]"}]},
+		{"type":"function","name":"fixedArrBytes","constant":true,"inputs":[{"name":"str","type":"bytes"},{"name":"fixedArr","type":"uint256[2]"}]},
+		{"type":"function","name":"mixedArrStr","constant":true,"inputs":[{"name":"str","type":"string"},{"name":"fixedArr","type":"uint256[2]"},{"name":"dynArr","type":"uint256[]"}]},
+		{"type":"function","name":"doubleFixedArrStr","constant":true,"inputs":[{"name":"str","type":"string"},{"name":"fixedArr1","type":"uint256[2]"},{"name":"fixedArr2","type":"uint256[3]"}]},
+		{"type":"function","name":"multipleMixedArrStr","constant":true,"inputs":[{"name":"str","type":"string"},{"name":"fixedArr1","type":"uint256[2]"},{"name":"dynArr","type":"uint256[]"},{"name":"fixedArr2","type":"uint256[3]"}]},
+		{"type":"function","name":"balance","constant":true},
+		{"type":"function","name":"send","constant":false,"inputs":[{"name":"amount","type":"uint256"}]},
+		{"type":"function","name":"test","constant":false,"inputs":[{"name":"number","type":"uint32"}]},
+		{"type":"function","name":"string","constant":false,"inputs":[{"name":"inputs","type":"string"}]},
+		{"type":"function","name":"bool","constant":false,"inputs":[{"name":"inputs","type":"bool"}]},
+		{"type":"function","name":"address","constant":false,"inputs":[{"name":"inputs","type":"address"}]},
+		{"type":"function","name":"uint64[2]","constant":false,"inputs":[{"name":"inputs","type":"uint64[2]"}]},
+		{"type":"function","name":"uint64[]","constant":false,"inputs":[{"name":"inputs","type":"uint64[]"}]},
+		{"type":"function","name":"foo","constant":false,"inputs":[{"name":"inputs","type":"uint32"}]},
+		{"type":"function","name":"bar","constant":false,"inputs":[{"name":"inputs","type":"uint32"},{"name":"string","type":"uint16"}]},
+		{"type":"function","name":"_slice","constant":false,"inputs":[{"name":"inputs","type":"uint32[2]"}]},
+		{"type":"function","name":"__slice256","constant":false,"inputs":[{"name":"inputs","type":"uint256[2]"}]},
+		{"type":"function","name":"sliceAddress","constant":false,"inputs":[{"name":"inputs","type":"address[]"}]},
+		{"type":"function","name":"sliceMultiAddress","constant":false,"inputs":[{"name":"a","type":"address[]"},{"name":"b","type":"address[]"}]}
+	]
+`
+	abi, err := JSON(strings.NewReader(abiJSON))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for name, m := range abi.Methods {
+		a := fmt.Sprintf("%v", m)
+		m2, err := abi.MethodById(m.Id())
+		if err != nil {
+			t.Fatalf("Failed to look up ABI method: %v", err)
+		}
+		b := fmt.Sprintf("%v", m2)
+		if a != b {
+			t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.Id()))
+		}
+	}
+
+}

accounts/abi/argument.go

@@ -19,6 +19,8 @@ package abi
 import (
 	"encoding/json"
 	"fmt"
+	"reflect"
+	"strings"
 )
 
 // Argument holds the name of the argument and the corresponding type.
@@ -29,7 +31,10 @@ type Argument struct {
 	Indexed bool // indexed is only used by events
 }
 
-func (a *Argument) UnmarshalJSON(data []byte) error {
+type Arguments []Argument
+
+// UnmarshalJSON implements json.Unmarshaler interface
+func (argument *Argument) UnmarshalJSON(data []byte) error {
 	var extarg struct {
 		Name    string
 		Type    string
@@ -40,12 +45,246 @@ func (a *Argument) UnmarshalJSON(data []byte) error {
 		return fmt.Errorf("argument json err: %v", err)
 	}
 
-	a.Type, err = NewType(extarg.Type)
+	argument.Type, err = NewType(extarg.Type)
 	if err != nil {
 		return err
 	}
-	a.Name = extarg.Name
-	a.Indexed = extarg.Indexed
+	argument.Name = extarg.Name
+	argument.Indexed = extarg.Indexed
 
 	return nil
 }
+
+// LengthNonIndexed returns the number of arguments when not counting 'indexed' ones. Only events
+// can ever have 'indexed' arguments, it should always be false on arguments for method input/output
+func (arguments Arguments) LengthNonIndexed() int {
+	out := 0
+	for _, arg := range arguments {
+		if !arg.Indexed {
+			out++
+		}
+	}
+	return out
+}
+
+// NonIndexed returns the arguments with indexed arguments filtered out
+func (arguments Arguments) NonIndexed() Arguments {
+	var ret []Argument
+	for _, arg := range arguments {
+		if !arg.Indexed {
+			ret = append(ret, arg)
+		}
+	}
+	return ret
+}
+
+// isTuple returns true for non-atomic constructs, like (uint,uint) or uint[]
+func (arguments Arguments) isTuple() bool {
+	return len(arguments) > 1
+}
+
+// Unpack performs the operation hexdata -> Go format
+func (arguments Arguments) Unpack(v interface{}, data []byte) error {
+
+	// make sure the passed value is arguments pointer
+	if reflect.Ptr != reflect.ValueOf(v).Kind() {
+		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
+	}
+	marshalledValues, err := arguments.UnpackValues(data)
+	if err != nil {
+		return err
+	}
+	if arguments.isTuple() {
+		return arguments.unpackTuple(v, marshalledValues)
+	}
+	return arguments.unpackAtomic(v, marshalledValues)
+}
+
+func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interface{}) error {
+
+	var (
+		value = reflect.ValueOf(v).Elem()
+		typ   = value.Type()
+		kind  = value.Kind()
+	)
+
+	if err := requireUnpackKind(value, typ, kind, arguments); err != nil {
+		return err
+	}
+
+	// If the interface is a struct, get of abi->struct_field mapping
+
+	var abi2struct map[string]string
+	if kind == reflect.Struct {
+		var err error
+		abi2struct, err = mapAbiToStructFields(arguments, value)
+		if err != nil {
+			return err
+		}
+	}
+	for i, arg := range arguments.NonIndexed() {
+
+		reflectValue := reflect.ValueOf(marshalledValues[i])
+
+		switch kind {
+		case reflect.Struct:
+			if structField, ok := abi2struct[arg.Name]; ok {
+				if err := set(value.FieldByName(structField), reflectValue, arg); err != nil {
+					return err
+				}
+			}
+		case reflect.Slice, reflect.Array:
+			if value.Len() < i {
+				return fmt.Errorf("abi: insufficient number of arguments for unpack, want %d, got %d", len(arguments), value.Len())
+			}
+			v := value.Index(i)
+			if err := requireAssignable(v, reflectValue); err != nil {
+				return err
+			}
+
+			if err := set(v.Elem(), reflectValue, arg); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("abi:[2] cannot unmarshal tuple in to %v", typ)
+		}
+	}
+	return nil
+}
+
+// unpackAtomic unpacks ( hexdata -> go ) a single value
+func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues []interface{}) error {
+	if len(marshalledValues) != 1 {
+		return fmt.Errorf("abi: wrong length, expected single value, got %d", len(marshalledValues))
+	}
+
+	elem := reflect.ValueOf(v).Elem()
+	kind := elem.Kind()
+	reflectValue := reflect.ValueOf(marshalledValues[0])
+
+	var abi2struct map[string]string
+	if kind == reflect.Struct {
+		var err error
+		if abi2struct, err = mapAbiToStructFields(arguments, elem); err != nil {
+			return err
+		}
+		arg := arguments.NonIndexed()[0]
+		if structField, ok := abi2struct[arg.Name]; ok {
+			return set(elem.FieldByName(structField), reflectValue, arg)
+		}
+		return nil
+	}
+
+	return set(elem, reflectValue, arguments.NonIndexed()[0])
+
+}
+
+// Computes the full size of an array;
+// i.e. counting nested arrays, which count towards size for unpacking.
+func getArraySize(arr *Type) int {
+	size := arr.Size
+	// Arrays can be nested, with each element being the same size
+	arr = arr.Elem
+	for arr.T == ArrayTy {
+		// Keep multiplying by elem.Size while the elem is an array.
+		size *= arr.Size
+		arr = arr.Elem
+	}
+	// Now we have the full array size, including its children.
+	return size
+}
+
+// UnpackValues can be used to unpack ABI-encoded hexdata according to the ABI-specification,
+// without supplying a struct to unpack into. Instead, this method returns a list containing the
+// values. An atomic argument will be a list with one element.
+func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) {
+	retval := make([]interface{}, 0, arguments.LengthNonIndexed())
+	virtualArgs := 0
+	for index, arg := range arguments.NonIndexed() {
+		marshalledValue, err := toGoType((index+virtualArgs)*32, arg.Type, data)
+		if arg.Type.T == ArrayTy {
+			// If we have a static array, like [3]uint256, these are coded as
+			// just like uint256,uint256,uint256.
+			// This means that we need to add two 'virtual' arguments when
+			// we count the index from now on.
+			//
+			// Array values nested multiple levels deep are also encoded inline:
+			// [2][3]uint256: uint256,uint256,uint256,uint256,uint256,uint256
+			//
+			// Calculate the full array size to get the correct offset for the next argument.
+			// Decrement it by 1, as the normal index increment is still applied.
+			virtualArgs += getArraySize(&arg.Type) - 1
+		}
+		if err != nil {
+			return nil, err
+		}
+		retval = append(retval, marshalledValue)
+	}
+	return retval, nil
+}
+
+// PackValues performs the operation Go format -> Hexdata
+// It is the semantic opposite of UnpackValues
+func (arguments Arguments) PackValues(args []interface{}) ([]byte, error) {
+	return arguments.Pack(args...)
+}
+
+// Pack performs the operation Go format -> Hexdata
+func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) {
+	// Make sure arguments match up and pack them
+	abiArgs := arguments
+	if len(args) != len(abiArgs) {
+		return nil, fmt.Errorf("argument count mismatch: %d for %d", len(args), len(abiArgs))
+	}
+	// variable input is the output appended at the end of packed
+	// output. This is used for strings and bytes types input.
+	var variableInput []byte
+
+	// input offset is the bytes offset for packed output
+	inputOffset := 0
+	for _, abiArg := range abiArgs {
+		if abiArg.Type.T == ArrayTy {
+			inputOffset += 32 * abiArg.Type.Size
+		} else {
+			inputOffset += 32
+		}
+	}
+	var ret []byte
+	for i, a := range args {
+		input := abiArgs[i]
+		// pack the input
+		packed, err := input.Type.pack(reflect.ValueOf(a))
+		if err != nil {
+			return nil, err
+		}
+		// check for a slice type (string, bytes, slice)
+		if input.Type.requiresLengthPrefix() {
+			// calculate the offset
+			offset := inputOffset + len(variableInput)
+			// set the offset
+			ret = append(ret, packNum(reflect.ValueOf(offset))...)
+			// Append the packed output to the variable input. The variable input
+			// will be appended at the end of the input.
+			variableInput = append(variableInput, packed...)
+		} else {
+			// append the packed value to the input
+			ret = append(ret, packed...)
+		}
+	}
+	// append the variable input at the end of the packed input
+	ret = append(ret, variableInput...)
+
+	return ret, nil
+}
+
+// capitalise makes the first character of a string upper case, also removing any
+// prefixing underscores from the variable names.
+func capitalise(input string) string {
+	for len(input) > 0 && input[0] == '_' {
+		input = input[1:]
+	}
+	if len(input) == 0 {
+		return ""
+	}
+	return strings.ToUpper(input[:1]) + input[1:]
+}

accounts/abi/bind/backend.go

@@ -52,12 +52,6 @@ type ContractCaller interface {
 	CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error)
 }
 
-// DeployBackend wraps the operations needed by WaitMined and WaitDeployed.
-type DeployBackend interface {
-	TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error)
-	CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error)
-}
-
 // PendingContractCaller defines methods to perform contract calls on the pending state.
 // Call will try to discover this interface when access to the pending state is requested.
 // If the backend does not support the pending state, Call returns ErrNoPendingState.
@@ -85,13 +79,34 @@ type ContractTransactor interface {
 	// There is no guarantee that this is the true gas limit requirement as other
 	// transactions may be added or removed by miners, but it should provide a basis
 	// for setting a reasonable default.
-	EstimateGas(ctx context.Context, call ethereum.CallMsg) (usedGas *big.Int, err error)
+	EstimateGas(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error)
 	// SendTransaction injects the transaction into the pending pool for execution.
 	SendTransaction(ctx context.Context, tx *types.Transaction) error
 }
 
+// ContractFilterer defines the methods needed to access log events using one-off
+// queries or continuous event subscriptions.
+type ContractFilterer interface {
+	// FilterLogs executes a log filter operation, blocking during execution and
+	// returning all the results in one batch.
+	//
+	// TODO(karalabe): Deprecate when the subscription one can return past data too.
+	FilterLogs(ctx context.Context, query ethereum.FilterQuery) ([]types.Log, error)
+
+	// SubscribeFilterLogs creates a background log filtering operation, returning
+	// a subscription immediately, which can be used to stream the found events.
+	SubscribeFilterLogs(ctx context.Context, query ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error)
+}
+
+// DeployBackend wraps the operations needed by WaitMined and WaitDeployed.
+type DeployBackend interface {
+	TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error)
+	CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error)
+}
+
 // ContractBackend defines the methods needed to work with contracts on a read-write basis.
 type ContractBackend interface {
 	ContractCaller
 	ContractTransactor
+	ContractFilterer
 }

accounts/abi/bind/backends/simulated.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"math/big"
 	"sync"
+	"time"
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
@@ -29,18 +30,23 @@ import (
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/bloombits"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/eth/filters"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
 )
 
 // This nil assignment ensures compile time that SimulatedBackend implements bind.ContractBackend.
 var _ bind.ContractBackend = (*SimulatedBackend)(nil)
 
 var errBlockNumberUnsupported = errors.New("SimulatedBackend cannot access blocks other than the latest block")
+var errGasEstimationFailed = errors.New("gas required exceeds allowance or always failing transaction")
 
 // SimulatedBackend implements bind.ContractBackend, simulating a blockchain in
 // the background. Its main purpose is to allow easily testing contract bindings.
@@ -52,17 +58,25 @@ type SimulatedBackend struct {
 	pendingBlock *types.Block   // Currently pending block that will be imported on request
 	pendingState *state.StateDB // Currently pending state that will be the active on on request
 
+	events *filters.EventSystem // Event system for filtering log events live
+
 	config *params.ChainConfig
 }
 
 // NewSimulatedBackend creates a new binding backend using a simulated blockchain
 // for testing purposes.
 func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend {
-	database, _ := ethdb.NewMemDatabase()
-	genesis := core.Genesis{Config: params.AllProtocolChanges, Alloc: alloc}
+	database := ethdb.NewMemDatabase()
+	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc}
 	genesis.MustCommit(database)
-	blockchain, _ := core.NewBlockChain(database, genesis.Config, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
-	backend := &SimulatedBackend{database: database, blockchain: blockchain, config: genesis.Config}
+	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{})
+
+	backend := &SimulatedBackend{
+		database:   database,
+		blockchain: blockchain,
+		config:     genesis.Config,
+		events:     filters.NewEventSystem(new(event.TypeMux), &filterBackend{database, blockchain}, false),
+	}
 	backend.rollback()
 	return backend
 }
@@ -88,9 +102,11 @@ func (b *SimulatedBackend) Rollback() {
 }
 
 func (b *SimulatedBackend) rollback() {
-	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), b.database, 1, func(int, *core.BlockGen) {})
+	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(int, *core.BlockGen) {})
+	statedb, _ := b.blockchain.State()
+
 	b.pendingBlock = blocks[0]
-	b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database))
+	b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
 }
 
 // CodeAt returns the code associated with a certain account in the blockchain.
@@ -144,7 +160,8 @@ func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Addres
 
 // TransactionReceipt returns the receipt of a transaction.
 func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) {
-	return core.GetReceipt(b.database, txHash), nil
+	receipt, _, _, _ := rawdb.ReadReceipt(b.database, txHash)
+	return receipt, nil
 }
 
 // PendingCodeAt returns the code associated with an account in the pending state.
@@ -167,7 +184,7 @@ func (b *SimulatedBackend) CallContract(ctx context.Context, call ethereum.CallM
 	if err != nil {
 		return nil, err
 	}
-	rval, _, err := b.callContract(ctx, call, b.blockchain.CurrentBlock(), state)
+	rval, _, _, err := b.callContract(ctx, call, b.blockchain.CurrentBlock(), state)
 	return rval, err
 }
 
@@ -177,7 +194,7 @@ func (b *SimulatedBackend) PendingCallContract(ctx context.Context, call ethereu
 	defer b.mu.Unlock()
 	defer b.pendingState.RevertToSnapshot(b.pendingState.Snapshot())
 
-	rval, _, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
+	rval, _, _, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
 	return rval, err
 }
 
@@ -198,46 +215,63 @@ func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error
 
 // EstimateGas executes the requested code against the currently pending block/state and
 // returns the used amount of gas.
-func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMsg) (*big.Int, error) {
+func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMsg) (uint64, error) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	// Binary search the gas requirement, as it may be higher than the amount used
-	var lo, hi uint64
-	if call.Gas != nil {
-		hi = call.Gas.Uint64()
+	// Determine the lowest and highest possible gas limits to binary search in between
+	var (
+		lo  uint64 = params.TxGas - 1
+		hi  uint64
+		cap uint64
+	)
+	if call.Gas >= params.TxGas {
+		hi = call.Gas
 	} else {
-		hi = b.pendingBlock.GasLimit().Uint64()
+		hi = b.pendingBlock.GasLimit()
 	}
-	for lo+1 < hi {
-		// Take a guess at the gas, and check transaction validity
-		mid := (hi + lo) / 2
-		call.Gas = new(big.Int).SetUint64(mid)
+	cap = hi
+
+	// Create a helper to check if a gas allowance results in an executable transaction
+	executable := func(gas uint64) bool {
+		call.Gas = gas
 
 		snapshot := b.pendingState.Snapshot()
-		_, gas, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
+		_, _, failed, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
 		b.pendingState.RevertToSnapshot(snapshot)
 
-		// If the transaction became invalid or used all the gas (failed), raise the gas limit
-		if err != nil || gas.Cmp(call.Gas) == 0 {
-			lo = mid
-			continue
+		if err != nil || failed {
+			return false
 		}
-		// Otherwise assume the transaction succeeded, lower the gas limit
-		hi = mid
+		return true
 	}
-	return new(big.Int).SetUint64(hi), nil
+	// Execute the binary search and hone in on an executable gas limit
+	for lo+1 < hi {
+		mid := (hi + lo) / 2
+		if !executable(mid) {
+			lo = mid
+		} else {
+			hi = mid
+		}
+	}
+	// Reject the transaction as invalid if it still fails at the highest allowance
+	if hi == cap {
+		if !executable(hi) {
+			return 0, errGasEstimationFailed
+		}
+	}
+	return hi, nil
 }
 
-// callContract implemens common code between normal and pending contract calls.
+// callContract implements common code between normal and pending contract calls.
 // state is modified during execution, make sure to copy it if necessary.
-func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallMsg, block *types.Block, statedb *state.StateDB) ([]byte, *big.Int, error) {
+func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallMsg, block *types.Block, statedb *state.StateDB) ([]byte, uint64, bool, error) {
 	// Ensure message is initialized properly.
 	if call.GasPrice == nil {
 		call.GasPrice = big.NewInt(1)
 	}
-	if call.Gas == nil || call.Gas.Sign() == 0 {
-		call.Gas = big.NewInt(50000000)
+	if call.Gas == 0 {
+		call.Gas = 50000000
 	}
 	if call.Value == nil {
 		call.Value = new(big.Int)
@@ -252,9 +286,9 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 	// Create a new environment which holds all relevant information
 	// about the transaction and calling mechanisms.
 	vmenv := vm.NewEVM(evmContext, statedb, b.config, vm.Config{})
-	gaspool := new(core.GasPool).AddGas(math.MaxBig256)
-	ret, gasUsed, _, err := core.NewStateTransition(vmenv, msg, gaspool).TransitionDb()
-	return ret, gasUsed, err
+	gaspool := new(core.GasPool).AddGas(math.MaxUint64)
+
+	return core.NewStateTransition(vmenv, msg, gaspool).TransitionDb()
 }
 
 // SendTransaction updates the pending block to include the given transaction.
@@ -272,14 +306,96 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 		panic(fmt.Errorf("invalid transaction nonce: got %d, want %d", tx.Nonce(), nonce))
 	}
 
-	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), b.database, 1, func(number int, block *core.BlockGen) {
+	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) {
+		for _, tx := range b.pendingBlock.Transactions() {
+			block.AddTxWithChain(b.blockchain, tx)
+		}
+		block.AddTxWithChain(b.blockchain, tx)
+	})
+	statedb, _ := b.blockchain.State()
+
+	b.pendingBlock = blocks[0]
+	b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
+	return nil
+}
+
+// FilterLogs executes a log filter operation, blocking during execution and
+// returning all the results in one batch.
+//
+// TODO(karalabe): Deprecate when the subscription one can return past data too.
+func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.FilterQuery) ([]types.Log, error) {
+	// Initialize unset filter boundaried to run from genesis to chain head
+	from := int64(0)
+	if query.FromBlock != nil {
+		from = query.FromBlock.Int64()
+	}
+	to := int64(-1)
+	if query.ToBlock != nil {
+		to = query.ToBlock.Int64()
+	}
+	// Construct and execute the filter
+	filter := filters.New(&filterBackend{b.database, b.blockchain}, from, to, query.Addresses, query.Topics)
+
+	logs, err := filter.Logs(ctx)
+	if err != nil {
+		return nil, err
+	}
+	res := make([]types.Log, len(logs))
+	for i, log := range logs {
+		res[i] = *log
+	}
+	return res, nil
+}
+
+// SubscribeFilterLogs creates a background log filtering operation, returning a
+// subscription immediately, which can be used to stream the found events.
+func (b *SimulatedBackend) SubscribeFilterLogs(ctx context.Context, query ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) {
+	// Subscribe to contract events
+	sink := make(chan []*types.Log)
+
+	sub, err := b.events.SubscribeLogs(query, sink)
+	if err != nil {
+		return nil, err
+	}
+	// Since we're getting logs in batches, we need to flatten them into a plain stream
+	return event.NewSubscription(func(quit <-chan struct{}) error {
+		defer sub.Unsubscribe()
+		for {
+			select {
+			case logs := <-sink:
+				for _, log := range logs {
+					select {
+					case ch <- *log:
+					case err := <-sub.Err():
+						return err
+					case <-quit:
+						return nil
+					}
+				}
+			case err := <-sub.Err():
+				return err
+			case <-quit:
+				return nil
+			}
+		}
+	}), nil
+}
+
+// AdjustTime adds a time shift to the simulated clock.
+func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) {
 		for _, tx := range b.pendingBlock.Transactions() {
 			block.AddTx(tx)
 		}
-		block.AddTx(tx)
+		block.OffsetTime(int64(adjustment.Seconds()))
 	})
+	statedb, _ := b.blockchain.State()
+
 	b.pendingBlock = blocks[0]
-	b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database))
+	b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database())
+
 	return nil
 }
 
@@ -293,6 +409,68 @@ func (m callmsg) Nonce() uint64        { return 0 }
 func (m callmsg) CheckNonce() bool     { return false }
 func (m callmsg) To() *common.Address  { return m.CallMsg.To }
 func (m callmsg) GasPrice() *big.Int   { return m.CallMsg.GasPrice }
-func (m callmsg) Gas() *big.Int        { return m.CallMsg.Gas }
+func (m callmsg) Gas() uint64          { return m.CallMsg.Gas }
 func (m callmsg) Value() *big.Int      { return m.CallMsg.Value }
 func (m callmsg) Data() []byte         { return m.CallMsg.Data }
+
+// filterBackend implements filters.Backend to support filtering for logs without
+// taking bloom-bits acceleration structures into account.
+type filterBackend struct {
+	db ethdb.Database
+	bc *core.BlockChain
+}
+
+func (fb *filterBackend) ChainDb() ethdb.Database  { return fb.db }
+func (fb *filterBackend) EventMux() *event.TypeMux { panic("not supported") }
+
+func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumber) (*types.Header, error) {
+	if block == rpc.LatestBlockNumber {
+		return fb.bc.CurrentHeader(), nil
+	}
+	return fb.bc.GetHeaderByNumber(uint64(block.Int64())), nil
+}
+
+func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) {
+	number := rawdb.ReadHeaderNumber(fb.db, hash)
+	if number == nil {
+		return nil, nil
+	}
+	return rawdb.ReadReceipts(fb.db, hash, *number), nil
+}
+
+func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) {
+	number := rawdb.ReadHeaderNumber(fb.db, hash)
+	if number == nil {
+		return nil, nil
+	}
+	receipts := rawdb.ReadReceipts(fb.db, hash, *number)
+	if receipts == nil {
+		return nil, nil
+	}
+	logs := make([][]*types.Log, len(receipts))
+	for i, receipt := range receipts {
+		logs[i] = receipt.Logs
+	}
+	return logs, nil
+}
+
+func (fb *filterBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription {
+	return event.NewSubscription(func(quit <-chan struct{}) error {
+		<-quit
+		return nil
+	})
+}
+func (fb *filterBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
+	return fb.bc.SubscribeChainEvent(ch)
+}
+func (fb *filterBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription {
+	return fb.bc.SubscribeRemovedLogsEvent(ch)
+}
+func (fb *filterBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription {
+	return fb.bc.SubscribeLogsEvent(ch)
+}
+
+func (fb *filterBackend) BloomStatus() (uint64, uint64) { return 4096, 0 }
+func (fb *filterBackend) ServiceFilter(ctx context.Context, ms *bloombits.MatcherSession) {
+	panic("not supported")
+}

accounts/abi/bind/base.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/event"
 )
 
 // SignerFn is a signer function callback when a contract requires a method to
@@ -50,11 +51,27 @@ type TransactOpts struct {
 
 	Value    *big.Int // Funds to transfer along along the transaction (nil = 0 = no funds)
 	GasPrice *big.Int // Gas price to use for the transaction execution (nil = gas price oracle)
-	GasLimit *big.Int // Gas limit to set for the transaction execution (nil = estimate + 10%)
+	GasLimit uint64   // Gas limit to set for the transaction execution (0 = estimate)
 
 	Context context.Context // Network context to support cancellation and timeouts (nil = no timeout)
 }
 
+// FilterOpts is the collection of options to fine tune filtering for events
+// within a bound contract.
+type FilterOpts struct {
+	Start uint64  // Start of the queried range
+	End   *uint64 // End of the range (nil = latest)
+
+	Context context.Context // Network context to support cancellation and timeouts (nil = no timeout)
+}
+
+// WatchOpts is the collection of options to fine tune subscribing for events
+// within a bound contract.
+type WatchOpts struct {
+	Start   *uint64         // Start of the queried range (nil = latest)
+	Context context.Context // Network context to support cancellation and timeouts (nil = no timeout)
+}
+
 // BoundContract is the base wrapper object that reflects a contract on the
 // Ethereum network. It contains a collection of methods that are used by the
 // higher level contract bindings to operate.
@@ -63,16 +80,18 @@ type BoundContract struct {
 	abi        abi.ABI            // Reflect based ABI to access the correct Ethereum methods
 	caller     ContractCaller     // Read interface to interact with the blockchain
 	transactor ContractTransactor // Write interface to interact with the blockchain
+	filterer   ContractFilterer   // Event filtering to interact with the blockchain
 }
 
 // NewBoundContract creates a low level contract interface through which calls
 // and transactions may be made through.
-func NewBoundContract(address common.Address, abi abi.ABI, caller ContractCaller, transactor ContractTransactor) *BoundContract {
+func NewBoundContract(address common.Address, abi abi.ABI, caller ContractCaller, transactor ContractTransactor, filterer ContractFilterer) *BoundContract {
 	return &BoundContract{
 		address:    address,
 		abi:        abi,
 		caller:     caller,
 		transactor: transactor,
+		filterer:   filterer,
 	}
 }
 
@@ -80,7 +99,7 @@ func NewBoundContract(address common.Address, abi abi.ABI, caller ContractCaller
 // deployment address with a Go wrapper.
 func DeployContract(opts *TransactOpts, abi abi.ABI, bytecode []byte, backend ContractBackend, params ...interface{}) (common.Address, *types.Transaction, *BoundContract, error) {
 	// Otherwise try to deploy the contract
-	c := NewBoundContract(common.Address{}, abi, backend, backend)
+	c := NewBoundContract(common.Address{}, abi, backend, backend, backend)
 
 	input, err := c.abi.Pack("", params...)
 	if err != nil {
@@ -189,7 +208,7 @@ func (c *BoundContract) transact(opts *TransactOpts, contract *common.Address, i
 		}
 	}
 	gasLimit := opts.GasLimit
-	if gasLimit == nil {
+	if gasLimit == 0 {
 		// Gas estimation cannot succeed without code for method invocations
 		if contract != nil {
 			if code, err := c.transactor.PendingCodeAt(ensureContext(opts.Context), c.address); err != nil {
@@ -225,6 +244,104 @@ func (c *BoundContract) transact(opts *TransactOpts, contract *common.Address, i
 	return signedTx, nil
 }
 
+// FilterLogs filters contract logs for past blocks, returning the necessary
+// channels to construct a strongly typed bound iterator on top of them.
+func (c *BoundContract) FilterLogs(opts *FilterOpts, name string, query ...[]interface{}) (chan types.Log, event.Subscription, error) {
+	// Don't crash on a lazy user
+	if opts == nil {
+		opts = new(FilterOpts)
+	}
+	// Append the event selector to the query parameters and construct the topic set
+	query = append([][]interface{}{{c.abi.Events[name].Id()}}, query...)
+
+	topics, err := makeTopics(query...)
+	if err != nil {
+		return nil, nil, err
+	}
+	// Start the background filtering
+	logs := make(chan types.Log, 128)
+
+	config := ethereum.FilterQuery{
+		Addresses: []common.Address{c.address},
+		Topics:    topics,
+		FromBlock: new(big.Int).SetUint64(opts.Start),
+	}
+	if opts.End != nil {
+		config.ToBlock = new(big.Int).SetUint64(*opts.End)
+	}
+	/* TODO(karalabe): Replace the rest of the method below with this when supported
+	sub, err := c.filterer.SubscribeFilterLogs(ensureContext(opts.Context), config, logs)
+	*/
+	buff, err := c.filterer.FilterLogs(ensureContext(opts.Context), config)
+	if err != nil {
+		return nil, nil, err
+	}
+	sub, err := event.NewSubscription(func(quit <-chan struct{}) error {
+		for _, log := range buff {
+			select {
+			case logs <- log:
+			case <-quit:
+				return nil
+			}
+		}
+		return nil
+	}), nil
+
+	if err != nil {
+		return nil, nil, err
+	}
+	return logs, sub, nil
+}
+
+// WatchLogs filters subscribes to contract logs for future blocks, returning a
+// subscription object that can be used to tear down the watcher.
+func (c *BoundContract) WatchLogs(opts *WatchOpts, name string, query ...[]interface{}) (chan types.Log, event.Subscription, error) {
+	// Don't crash on a lazy user
+	if opts == nil {
+		opts = new(WatchOpts)
+	}
+	// Append the event selector to the query parameters and construct the topic set
+	query = append([][]interface{}{{c.abi.Events[name].Id()}}, query...)
+
+	topics, err := makeTopics(query...)
+	if err != nil {
+		return nil, nil, err
+	}
+	// Start the background filtering
+	logs := make(chan types.Log, 128)
+
+	config := ethereum.FilterQuery{
+		Addresses: []common.Address{c.address},
+		Topics:    topics,
+	}
+	if opts.Start != nil {
+		config.FromBlock = new(big.Int).SetUint64(*opts.Start)
+	}
+	sub, err := c.filterer.SubscribeFilterLogs(ensureContext(opts.Context), config, logs)
+	if err != nil {
+		return nil, nil, err
+	}
+	return logs, sub, nil
+}
+
+// UnpackLog unpacks a retrieved log into the provided output structure.
+func (c *BoundContract) UnpackLog(out interface{}, event string, log types.Log) error {
+	if len(log.Data) > 0 {
+		if err := c.abi.Unpack(out, event, log.Data); err != nil {
+			return err
+		}
+	}
+	var indexed abi.Arguments
+	for _, arg := range c.abi.Events[event].Inputs {
+		if arg.Indexed {
+			indexed = append(indexed, arg)
+		}
+	}
+	return parseTopics(out, indexed, log.Topics[1:])
+}
+
+// ensureContext is a helper method to ensure a context is not nil, even if the
+// user specified it as such.
 func ensureContext(ctx context.Context) context.Context {
 	if ctx == nil {
 		return context.TODO()

accounts/abi/bind/bind.go

@@ -63,10 +63,11 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 			return r
 		}, abis[i])
 
-		// Extract the call and transact methods, and sort them alphabetically
+		// Extract the call and transact methods; events; and sort them alphabetically
 		var (
 			calls     = make(map[string]*tmplMethod)
 			transacts = make(map[string]*tmplMethod)
+			events    = make(map[string]*tmplEvent)
 		)
 		for _, original := range evmABI.Methods {
 			// Normalize the method for capital cases and non-anonymous inputs/outputs
@@ -89,11 +90,33 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 			}
 			// Append the methods to the call or transact lists
 			if original.Const {
-				calls[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original)}
+				calls[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original.Outputs)}
 			} else {
-				transacts[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original)}
+				transacts[original.Name] = &tmplMethod{Original: original, Normalized: normalized, Structured: structured(original.Outputs)}
 			}
 		}
+		for _, original := range evmABI.Events {
+			// Skip anonymous events as they don't support explicit filtering
+			if original.Anonymous {
+				continue
+			}
+			// Normalize the event for capital cases and non-anonymous outputs
+			normalized := original
+			normalized.Name = methodNormalizer[lang](original.Name)
+
+			normalized.Inputs = make([]abi.Argument, len(original.Inputs))
+			copy(normalized.Inputs, original.Inputs)
+			for j, input := range normalized.Inputs {
+				// Indexed fields are input, non-indexed ones are outputs
+				if input.Indexed {
+					if input.Name == "" {
+						normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j)
+					}
+				}
+			}
+			// Append the event to the accumulator list
+			events[original.Name] = &tmplEvent{Original: original, Normalized: normalized}
+		}
 		contracts[types[i]] = &tmplContract{
 			Type:        capitalise(types[i]),
 			InputABI:    strings.Replace(strippedABI, "\"", "\\\"", -1),
@@ -101,6 +124,7 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 			Constructor: evmABI.Constructor,
 			Calls:       calls,
 			Transacts:   transacts,
+			Events:      events,
 		}
 	}
 	// Generate the contract template data content and render it
@@ -111,10 +135,11 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 	buffer := new(bytes.Buffer)
 
 	funcs := map[string]interface{}{
-		"bindtype":     bindType[lang],
-		"namedtype":    namedType[lang],
-		"capitalise":   capitalise,
-		"decapitalise": decapitalise,
+		"bindtype":      bindType[lang],
+		"bindtopictype": bindTopicType[lang],
+		"namedtype":     namedType[lang],
+		"capitalise":    capitalise,
+		"decapitalise":  decapitalise,
 	}
 	tmpl := template.Must(template.New("").Funcs(funcs).Parse(tmplSource[lang]))
 	if err := tmpl.Execute(buffer, data); err != nil {
@@ -122,138 +147,194 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 	}
 	// For Go bindings pass the code through goimports to clean it up and double check
 	if lang == LangGo {
-		code, err := imports.Process("", buffer.Bytes(), nil)
+		code, err := imports.Process(".", buffer.Bytes(), nil)
 		if err != nil {
 			return "", fmt.Errorf("%v\n%s", err, buffer)
 		}
 		return string(code), nil
 	}
 	// For all others just return as is for now
-	return string(buffer.Bytes()), nil
+	return buffer.String(), nil
 }
 
 // bindType is a set of type binders that convert Solidity types to some supported
-// programming language.
+// programming language types.
 var bindType = map[Lang]func(kind abi.Type) string{
 	LangGo:   bindTypeGo,
 	LangJava: bindTypeJava,
 }
 
+// Helper function for the binding generators.
+// It reads the unmatched characters after the inner type-match,
+//  (since the inner type is a prefix of the total type declaration),
+//  looks for valid arrays (possibly a dynamic one) wrapping the inner type,
+//  and returns the sizes of these arrays.
+//
+// Returned array sizes are in the same order as solidity signatures; inner array size first.
+// Array sizes may also be "", indicating a dynamic array.
+func wrapArray(stringKind string, innerLen int, innerMapping string) (string, []string) {
+	remainder := stringKind[innerLen:]
+	//find all the sizes
+	matches := regexp.MustCompile(`\[(\d*)\]`).FindAllStringSubmatch(remainder, -1)
+	parts := make([]string, 0, len(matches))
+	for _, match := range matches {
+		//get group 1 from the regex match
+		parts = append(parts, match[1])
+	}
+	return innerMapping, parts
+}
+
+// Translates the array sizes to a Go-lang declaration of a (nested) array of the inner type.
+// Simply returns the inner type if arraySizes is empty.
+func arrayBindingGo(inner string, arraySizes []string) string {
+	out := ""
+	//prepend all array sizes, from outer (end arraySizes) to inner (start arraySizes)
+	for i := len(arraySizes) - 1; i >= 0; i-- {
+		out += "[" + arraySizes[i] + "]"
+	}
+	out += inner
+	return out
+}
+
 // bindTypeGo converts a Solidity type to a Go one. Since there is no clear mapping
 // from all Solidity types to Go ones (e.g. uint17), those that cannot be exactly
 // mapped will use an upscaled type (e.g. *big.Int).
 func bindTypeGo(kind abi.Type) string {
 	stringKind := kind.String()
+	innerLen, innerMapping := bindUnnestedTypeGo(stringKind)
+	return arrayBindingGo(wrapArray(stringKind, innerLen, innerMapping))
+}
+
+// The inner function of bindTypeGo, this finds the inner type of stringKind.
+// (Or just the type itself if it is not an array or slice)
+// The length of the matched part is returned, with the the translated type.
+func bindUnnestedTypeGo(stringKind string) (int, string) {
 
 	switch {
 	case strings.HasPrefix(stringKind, "address"):
-		parts := regexp.MustCompile(`address(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
-		if len(parts) != 2 {
-			return stringKind
-		}
-		return fmt.Sprintf("%scommon.Address", parts[1])
+		return len("address"), "common.Address"
 
 	case strings.HasPrefix(stringKind, "bytes"):
-		parts := regexp.MustCompile(`bytes([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
-		if len(parts) != 3 {
-			return stringKind
-		}
-		return fmt.Sprintf("%s[%s]byte", parts[2], parts[1])
+		parts := regexp.MustCompile(`bytes([0-9]*)`).FindStringSubmatch(stringKind)
+		return len(parts[0]), fmt.Sprintf("[%s]byte", parts[1])
 
 	case strings.HasPrefix(stringKind, "int") || strings.HasPrefix(stringKind, "uint"):
-		parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
-		if len(parts) != 4 {
-			return stringKind
-		}
+		parts := regexp.MustCompile(`(u)?int([0-9]*)`).FindStringSubmatch(stringKind)
 		switch parts[2] {
 		case "8", "16", "32", "64":
-			return fmt.Sprintf("%s%sint%s", parts[3], parts[1], parts[2])
+			return len(parts[0]), fmt.Sprintf("%sint%s", parts[1], parts[2])
 		}
-		return fmt.Sprintf("%s*big.Int", parts[3])
+		return len(parts[0]), "*big.Int"
 
-	case strings.HasPrefix(stringKind, "bool") || strings.HasPrefix(stringKind, "string"):
-		parts := regexp.MustCompile(`([a-z]+)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
-		if len(parts) != 3 {
-			return stringKind
-		}
-		return fmt.Sprintf("%s%s", parts[2], parts[1])
+	case strings.HasPrefix(stringKind, "bool"):
+		return len("bool"), "bool"
+
+	case strings.HasPrefix(stringKind, "string"):
+		return len("string"), "string"
 
 	default:
-		return stringKind
+		return len(stringKind), stringKind
 	}
 }
 
+// Translates the array sizes to a Java declaration of a (nested) array of the inner type.
+// Simply returns the inner type if arraySizes is empty.
+func arrayBindingJava(inner string, arraySizes []string) string {
+	// Java array type declarations do not include the length.
+	return inner + strings.Repeat("[]", len(arraySizes))
+}
+
 // bindTypeJava converts a Solidity type to a Java one. Since there is no clear mapping
 // from all Solidity types to Java ones (e.g. uint17), those that cannot be exactly
 // mapped will use an upscaled type (e.g. BigDecimal).
 func bindTypeJava(kind abi.Type) string {
 	stringKind := kind.String()
+	innerLen, innerMapping := bindUnnestedTypeJava(stringKind)
+	return arrayBindingJava(wrapArray(stringKind, innerLen, innerMapping))
+}
+
+// The inner function of bindTypeJava, this finds the inner type of stringKind.
+// (Or just the type itself if it is not an array or slice)
+// The length of the matched part is returned, with the the translated type.
+func bindUnnestedTypeJava(stringKind string) (int, string) {
 
 	switch {
 	case strings.HasPrefix(stringKind, "address"):
 		parts := regexp.MustCompile(`address(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
 		if len(parts) != 2 {
-			return stringKind
+			return len(stringKind), stringKind
 		}
 		if parts[1] == "" {
-			return fmt.Sprintf("Address")
+			return len("address"), "Address"
 		}
-		return fmt.Sprintf("Addresses")
+		return len(parts[0]), "Addresses"
 
 	case strings.HasPrefix(stringKind, "bytes"):
-		parts := regexp.MustCompile(`bytes([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
-		if len(parts) != 3 {
-			return stringKind
+		parts := regexp.MustCompile(`bytes([0-9]*)`).FindStringSubmatch(stringKind)
+		if len(parts) != 2 {
+			return len(stringKind), stringKind
 		}
-		if parts[2] != "" {
-			return "byte[][]"
-		}
-		return "byte[]"
+		return len(parts[0]), "byte[]"
 
 	case strings.HasPrefix(stringKind, "int") || strings.HasPrefix(stringKind, "uint"):
-		parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
-		if len(parts) != 4 {
-			return stringKind
+		//Note that uint and int (without digits) are also matched,
+		// these are size 256, and will translate to BigInt (the default).
+		parts := regexp.MustCompile(`(u)?int([0-9]*)`).FindStringSubmatch(stringKind)
+		if len(parts) != 3 {
+			return len(stringKind), stringKind
 		}
-		switch parts[2] {
-		case "8", "16", "32", "64":
-			if parts[1] == "" {
-				if parts[3] == "" {
-					return fmt.Sprintf("int%s", parts[2])
-				}
-				return fmt.Sprintf("int%s[]", parts[2])
-			}
+
+		namedSize := map[string]string{
+			"8":  "byte",
+			"16": "short",
+			"32": "int",
+			"64": "long",
+		}[parts[2]]
+
+		//default to BigInt
+		if namedSize == "" {
+			namedSize = "BigInt"
 		}
-		if parts[3] == "" {
-			return fmt.Sprintf("BigInt")
-		}
-		return fmt.Sprintf("BigInts")
+		return len(parts[0]), namedSize
 
 	case strings.HasPrefix(stringKind, "bool"):
-		parts := regexp.MustCompile(`bool(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
-		if len(parts) != 2 {
-			return stringKind
-		}
-		if parts[1] == "" {
-			return fmt.Sprintf("bool")
-		}
-		return fmt.Sprintf("bool[]")
+		return len("bool"), "boolean"
 
 	case strings.HasPrefix(stringKind, "string"):
-		parts := regexp.MustCompile(`string(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
-		if len(parts) != 2 {
-			return stringKind
-		}
-		if parts[1] == "" {
-			return fmt.Sprintf("String")
-		}
-		return fmt.Sprintf("String[]")
+		return len("string"), "String"
 
 	default:
-		return stringKind
+		return len(stringKind), stringKind
 	}
 }
 
+// bindTopicType is a set of type binders that convert Solidity types to some
+// supported programming language topic types.
+var bindTopicType = map[Lang]func(kind abi.Type) string{
+	LangGo:   bindTopicTypeGo,
+	LangJava: bindTopicTypeJava,
+}
+
+// bindTypeGo converts a Solidity topic type to a Go one. It is almost the same
+// funcionality as for simple types, but dynamic types get converted to hashes.
+func bindTopicTypeGo(kind abi.Type) string {
+	bound := bindTypeGo(kind)
+	if bound == "string" || bound == "[]byte" {
+		bound = "common.Hash"
+	}
+	return bound
+}
+
+// bindTypeGo converts a Solidity topic type to a Java one. It is almost the same
+// funcionality as for simple types, but dynamic types get converted to hashes.
+func bindTopicTypeJava(kind abi.Type) string {
+	bound := bindTypeJava(kind)
+	if bound == "String" || bound == "Bytes" {
+		bound = "Hash"
+	}
+	return bound
+}
+
 // namedType is a set of functions that transform language specific types to
 // named versions that my be used inside method names.
 var namedType = map[Lang]func(string, abi.Type) string{
@@ -273,11 +354,13 @@ func namedTypeJava(javaKind string, solKind abi.Type) string {
 		return "String"
 	case "string[]":
 		return "Strings"
-	case "bool":
+	case "boolean":
 		return "Bool"
-	case "bool[]":
+	case "boolean[]":
 		return "Bools"
-	case "BigInt":
+	case "BigInt[]":
+		return "BigInts"
+	default:
 		parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(solKind.String())
 		if len(parts) != 4 {
 			return javaKind
@@ -292,8 +375,6 @@ func namedTypeJava(javaKind string, solKind abi.Type) string {
 		default:
 			return javaKind
 		}
-	default:
-		return javaKind
 	}
 }
 
@@ -304,26 +385,71 @@ var methodNormalizer = map[Lang]func(string) string{
 	LangJava: decapitalise,
 }
 
-// capitalise makes the first character of a string upper case.
+// capitalise makes a camel-case string which starts with an upper case character.
 func capitalise(input string) string {
-	return strings.ToUpper(input[:1]) + input[1:]
+	for len(input) > 0 && input[0] == '_' {
+		input = input[1:]
+	}
+	if len(input) == 0 {
+		return ""
+	}
+	return toCamelCase(strings.ToUpper(input[:1]) + input[1:])
 }
 
-// decapitalise makes the first character of a string lower case.
+// decapitalise makes a camel-case string which starts with a lower case character.
 func decapitalise(input string) string {
-	return strings.ToLower(input[:1]) + input[1:]
+	for len(input) > 0 && input[0] == '_' {
+		input = input[1:]
+	}
+	if len(input) == 0 {
+		return ""
+	}
+	return toCamelCase(strings.ToLower(input[:1]) + input[1:])
 }
 
-// structured checks whether a method has enough information to return a proper
-// Go struct ot if flat returns are needed.
-func structured(method abi.Method) bool {
-	if len(method.Outputs) < 2 {
+// toCamelCase converts an under-score string to a camel-case string
+func toCamelCase(input string) string {
+	toupper := false
+
+	result := ""
+	for k, v := range input {
+		switch {
+		case k == 0:
+			result = strings.ToUpper(string(input[0]))
+
+		case toupper:
+			result += strings.ToUpper(string(v))
+			toupper = false
+
+		case v == '_':
+			toupper = true
+
+		default:
+			result += string(v)
+		}
+	}
+	return result
+}
+
+// structured checks whether a list of ABI data types has enough information to
+// operate through a proper Go struct or if flat returns are needed.
+func structured(args abi.Arguments) bool {
+	if len(args) < 2 {
 		return false
 	}
-	for _, out := range method.Outputs {
+	exists := make(map[string]bool)
+	for _, out := range args {
+		// If the name is anonymous, we can't organize into a struct
 		if out.Name == "" {
 			return false
 		}
+		// If the field name is empty when normalized or collides (var, Var, _var, _Var),
+		// we can't organize into a struct
+		field := capitalise(out.Name)
+		if field == "" || exists[field] {
+			return false
+		}
+		exists[field] = true
 	}
 	return true
 }

accounts/abi/bind/bind_test.go

@@ -126,6 +126,7 @@ var bindTests = []struct {
 				{"type":"function","name":"namedOutput","constant":true,"inputs":[],"outputs":[{"name":"str","type":"string"}]},
 				{"type":"function","name":"anonOutput","constant":true,"inputs":[],"outputs":[{"name":"","type":"string"}]},
 				{"type":"function","name":"namedOutputs","constant":true,"inputs":[],"outputs":[{"name":"str1","type":"string"},{"name":"str2","type":"string"}]},
+				{"type":"function","name":"collidingOutputs","constant":true,"inputs":[],"outputs":[{"name":"str","type":"string"},{"name":"Str","type":"string"}]},
 				{"type":"function","name":"anonOutputs","constant":true,"inputs":[],"outputs":[{"name":"","type":"string"},{"name":"","type":"string"}]},
 				{"type":"function","name":"mixedOutputs","constant":true,"inputs":[],"outputs":[{"name":"","type":"string"},{"name":"str","type":"string"}]}
 			]
@@ -140,12 +141,71 @@ var bindTests = []struct {
 			 str1, err        = b.NamedOutput(nil)
 			 str1, err        = b.AnonOutput(nil)
 			 res, _          := b.NamedOutputs(nil)
+			 str1, str2, err  = b.CollidingOutputs(nil)
 			 str1, str2, err  = b.AnonOutputs(nil)
 			 str1, str2, err  = b.MixedOutputs(nil)
 
 			 fmt.Println(str1, str2, res.Str1, res.Str2, err)
 		 }`,
 	},
+	// Tests that named, anonymous and indexed events are handled correctly
+	{
+		`EventChecker`, ``, ``,
+		`
+			[
+				{"type":"event","name":"empty","inputs":[]},
+				{"type":"event","name":"indexed","inputs":[{"name":"addr","type":"address","indexed":true},{"name":"num","type":"int256","indexed":true}]},
+				{"type":"event","name":"mixed","inputs":[{"name":"addr","type":"address","indexed":true},{"name":"num","type":"int256"}]},
+				{"type":"event","name":"anonymous","anonymous":true,"inputs":[]},
+				{"type":"event","name":"dynamic","inputs":[{"name":"idxStr","type":"string","indexed":true},{"name":"idxDat","type":"bytes","indexed":true},{"name":"str","type":"string"},{"name":"dat","type":"bytes"}]}
+			]
+		`,
+		`if e, err := NewEventChecker(common.Address{}, nil); e == nil || err != nil {
+			 t.Fatalf("binding (%v) nil or error (%v) not nil", e, nil)
+		 } else if false { // Don't run, just compile and test types
+			 var (
+				 err  error
+			   res  bool
+				 str  string
+				 dat  []byte
+				 hash common.Hash
+			 )
+			 _, err = e.FilterEmpty(nil)
+			 _, err = e.FilterIndexed(nil, []common.Address{}, []*big.Int{})
+
+			 mit, err := e.FilterMixed(nil, []common.Address{})
+
+			 res = mit.Next()  // Make sure the iterator has a Next method
+			 err = mit.Error() // Make sure the iterator has an Error method
+			 err = mit.Close() // Make sure the iterator has a Close method
+
+			 fmt.Println(mit.Event.Raw.BlockHash) // Make sure the raw log is contained within the results
+			 fmt.Println(mit.Event.Num)           // Make sure the unpacked non-indexed fields are present
+			 fmt.Println(mit.Event.Addr)          // Make sure the reconstructed indexed fields are present
+
+			 dit, err := e.FilterDynamic(nil, []string{}, [][]byte{})
+
+			 str  = dit.Event.Str    // Make sure non-indexed strings retain their type
+			 dat  = dit.Event.Dat    // Make sure non-indexed bytes retain their type
+			 hash = dit.Event.IdxStr // Make sure indexed strings turn into hashes
+			 hash = dit.Event.IdxDat // Make sure indexed bytes turn into hashes
+
+			 sink := make(chan *EventCheckerMixed)
+			 sub, err := e.WatchMixed(nil, sink, []common.Address{})
+			 defer sub.Unsubscribe()
+
+			 event := <-sink
+			 fmt.Println(event.Raw.BlockHash) // Make sure the raw log is contained within the results
+			 fmt.Println(event.Num)           // Make sure the unpacked non-indexed fields are present
+			 fmt.Println(event.Addr)          // Make sure the reconstructed indexed fields are present
+
+			 fmt.Println(res, str, dat, hash, err)
+		 }
+		 // Run a tiny reflection test to ensure disallowed methods don't appear
+		 if _, ok := reflect.TypeOf(&EventChecker{}).MethodByName("FilterAnonymous"); ok {
+		 	t.Errorf("binding has disallowed method (FilterAnonymous)")
+		 }`,
+	},
 	// Test that contract interactions (deploy, transact and call) generate working code
 	{
 		`Interactor`,
@@ -365,7 +425,7 @@ var bindTests = []struct {
 			}
 		`,
 	},
-	// Tests that gas estimation works for contracts with	weird gas mechanics too.
+	// Tests that gas estimation works for contracts with weird gas mechanics too.
 	{
 		`FunkyGasPattern`,
 		`
@@ -397,7 +457,6 @@ var bindTests = []struct {
 			sim.Commit()
 
 			// Set the field with automatic estimation and check that it succeeds
-			auth.GasLimit = nil
 			if _, err := limiter.SetField(auth, "automatic"); err != nil {
 				t.Fatalf("Failed to call automatically gased transaction: %v", err)
 			}
@@ -447,6 +506,309 @@ var bindTests = []struct {
 			}
 		`,
 	},
+	// Tests that methods and returns with underscores inside work correctly.
+	{
+		`Underscorer`,
+		`
+		contract Underscorer {
+			function UnderscoredOutput() constant returns (int _int, string _string) {
+				return (314, "pi");
+			}
+			function LowerLowerCollision() constant returns (int _res, int res) {
+				return (1, 2);
+			}
+			function LowerUpperCollision() constant returns (int _res, int Res) {
+				return (1, 2);
+			}
+			function UpperLowerCollision() constant returns (int _Res, int res) {
+				return (1, 2);
+			}
+			function UpperUpperCollision() constant returns (int _Res, int Res) {
+				return (1, 2);
+			}
+			function PurelyUnderscoredOutput() constant returns (int _, int res) {
+				return (1, 2);
+			}
+			function AllPurelyUnderscoredOutput() constant returns (int _, int __) {
+				return (1, 2);
+			}
+			function _under_scored_func() constant returns (int _int) {
+				return 0;
+			}
+		}
+		`, `6060604052341561000f57600080fd5b6103858061001e6000396000f30060606040526004361061008e576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806303a592131461009357806346546dbe146100c357806367e6633d146100ec5780639df4848514610181578063af7486ab146101b1578063b564b34d146101e1578063e02ab24d14610211578063e409ca4514610241575b600080fd5b341561009e57600080fd5b6100a6610271565b604051808381526020018281526020019250505060405180910390f35b34156100ce57600080fd5b6100d6610286565b6040518082815260200191505060405180910390f35b34156100f757600080fd5b6100ff61028e565b6040518083815260200180602001828103825283818151815260200191508051906020019080838360005b8381101561014557808201518184015260208101905061012a565b50505050905090810190601f1680156101725780820380516001836020036101000a031916815260200191505b50935050505060405180910390f35b341561018c57600080fd5b6101946102dc565b604051808381526020018281526020019250505060405180910390f35b34156101bc57600080fd5b6101c46102f1565b604051808381526020018281526020019250505060405180910390f35b34156101ec57600080fd5b6101f4610306565b604051808381526020018281526020019250505060405180910390f35b341561021c57600080fd5b61022461031b565b604051808381526020018281526020019250505060405180910390f35b341561024c57600080fd5b610254610330565b604051808381526020018281526020019250505060405180910390f35b60008060016002819150809050915091509091565b600080905090565b6000610298610345565b61013a8090506040805190810160405280600281526020017f7069000000000000000000000000000000000000000000000000000000000000815250915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b60008060016002819150809050915091509091565b6020604051908101604052806000815250905600a165627a7a72305820d1a53d9de9d1e3d55cb3dc591900b63c4f1ded79114f7b79b332684840e186a40029`,
+		`[{"constant":true,"inputs":[],"name":"LowerUpperCollision","outputs":[{"name":"_res","type":"int256"},{"name":"Res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"_under_scored_func","outputs":[{"name":"_int","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"UnderscoredOutput","outputs":[{"name":"_int","type":"int256"},{"name":"_string","type":"string"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"PurelyUnderscoredOutput","outputs":[{"name":"_","type":"int256"},{"name":"res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"UpperLowerCollision","outputs":[{"name":"_Res","type":"int256"},{"name":"res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"AllPurelyUnderscoredOutput","outputs":[{"name":"_","type":"int256"},{"name":"__","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"UpperUpperCollision","outputs":[{"name":"_Res","type":"int256"},{"name":"Res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"LowerLowerCollision","outputs":[{"name":"_res","type":"int256"},{"name":"res","type":"int256"}],"payable":false,"stateMutability":"view","type":"function"}]`,
+		`
+			// Generate a new random account and a funded simulator
+			key, _ := crypto.GenerateKey()
+			auth := bind.NewKeyedTransactor(key)
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+
+			// Deploy a underscorer tester contract and execute a structured call on it
+			_, _, underscorer, err := DeployUnderscorer(auth, sim)
+			if err != nil {
+				t.Fatalf("Failed to deploy underscorer contract: %v", err)
+			}
+			sim.Commit()
+
+			// Verify that underscored return values correctly parse into structs
+			if res, err := underscorer.UnderscoredOutput(nil); err != nil {
+				t.Errorf("Failed to call constant function: %v", err)
+			} else if res.Int.Cmp(big.NewInt(314)) != 0 || res.String != "pi" {
+				t.Errorf("Invalid result, want: {314, \"pi\"}, got: %+v", res)
+			}
+			// Verify that underscored and non-underscored name collisions force tuple outputs
+			var a, b *big.Int
+
+			a, b, _ = underscorer.LowerLowerCollision(nil)
+			a, b, _ = underscorer.LowerUpperCollision(nil)
+			a, b, _ = underscorer.UpperLowerCollision(nil)
+			a, b, _ = underscorer.UpperUpperCollision(nil)
+			a, b, _ = underscorer.PurelyUnderscoredOutput(nil)
+			a, b, _ = underscorer.AllPurelyUnderscoredOutput(nil)
+			a, _ = underscorer.UnderScoredFunc(nil)
+
+			fmt.Println(a, b, err)
+		`,
+	},
+	// Tests that logs can be successfully filtered and decoded.
+	{
+		`Eventer`,
+		`
+			contract Eventer {
+					event SimpleEvent (
+					address indexed Addr,
+					bytes32 indexed Id,
+					bool    indexed Flag,
+					uint    Value
+				);
+				function raiseSimpleEvent(address addr, bytes32 id, bool flag, uint value) {
+					SimpleEvent(addr, id, flag, value);
+				}
+
+				event NodataEvent (
+					uint   indexed Number,
+					int16  indexed Short,
+					uint32 indexed Long
+				);
+				function raiseNodataEvent(uint number, int16 short, uint32 long) {
+					NodataEvent(number, short, long);
+				}
+
+				event DynamicEvent (
+					string indexed IndexedString,
+					bytes  indexed IndexedBytes,
+					string NonIndexedString,
+					bytes  NonIndexedBytes
+				);
+				function raiseDynamicEvent(string str, bytes blob) {
+					DynamicEvent(str, blob, str, blob);
+				}
+			}
+		`,
+		`6060604052341561000f57600080fd5b61042c8061001e6000396000f300606060405260043610610057576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063528300ff1461005c578063630c31e2146100fc578063c7d116dd14610156575b600080fd5b341561006757600080fd5b6100fa600480803590602001908201803590602001908080601f0160208091040260200160405190810160405280939291908181526020018383808284378201915050505050509190803590602001908201803590602001908080601f01602080910402602001604051908101604052809392919081815260200183838082843782019150505050505091905050610194565b005b341561010757600080fd5b610154600480803573ffffffffffffffffffffffffffffffffffffffff16906020019091908035600019169060200190919080351515906020019091908035906020019091905050610367565b005b341561016157600080fd5b610192600480803590602001909190803560010b90602001909190803563ffffffff169060200190919050506103c3565b005b806040518082805190602001908083835b6020831015156101ca57805182526020820191506020810190506020830392506101a5565b6001836020036101000a0380198251168184511680821785525050505050509050019150506040518091039020826040518082805190602001908083835b60208310151561022d5780518252602082019150602081019050602083039250610208565b6001836020036101000a03801982511681845116808217855250505050505090500191505060405180910390207f3281fd4f5e152dd3385df49104a3f633706e21c9e80672e88d3bcddf33101f008484604051808060200180602001838103835285818151815260200191508051906020019080838360005b838110156102c15780820151818401526020810190506102a6565b50505050905090810190601f1680156102ee5780820380516001836020036101000a031916815260200191505b50838103825284818151815260200191508051906020019080838360005b8381101561032757808201518184015260208101905061030c565b50505050905090810190601f1680156103545780820380516001836020036101000a031916815260200191505b5094505050505060405180910390a35050565b81151583600019168573ffffffffffffffffffffffffffffffffffffffff167f1f097de4289df643bd9c11011cc61367aa12983405c021056e706eb5ba1250c8846040518082815260200191505060405180910390a450505050565b8063ffffffff168260010b847f3ca7f3a77e5e6e15e781850bc82e32adfa378a2a609370db24b4d0fae10da2c960405160405180910390a45050505600a165627a7a72305820d1f8a8bbddbc5bb29f285891d6ae1eef8420c52afdc05e1573f6114d8e1714710029`,
+		`[{"constant":false,"inputs":[{"name":"str","type":"string"},{"name":"blob","type":"bytes"}],"name":"raiseDynamicEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"addr","type":"address"},{"name":"id","type":"bytes32"},{"name":"flag","type":"bool"},{"name":"value","type":"uint256"}],"name":"raiseSimpleEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"number","type":"uint256"},{"name":"short","type":"int16"},{"name":"long","type":"uint32"}],"name":"raiseNodataEvent","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"anonymous":false,"inputs":[{"indexed":true,"name":"Addr","type":"address"},{"indexed":true,"name":"Id","type":"bytes32"},{"indexed":true,"name":"Flag","type":"bool"},{"indexed":false,"name":"Value","type":"uint256"}],"name":"SimpleEvent","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"Number","type":"uint256"},{"indexed":true,"name":"Short","type":"int16"},{"indexed":true,"name":"Long","type":"uint32"}],"name":"NodataEvent","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"name":"IndexedString","type":"string"},{"indexed":true,"name":"IndexedBytes","type":"bytes"},{"indexed":false,"name":"NonIndexedString","type":"string"},{"indexed":false,"name":"NonIndexedBytes","type":"bytes"}],"name":"DynamicEvent","type":"event"}]`,
+		`
+			// Generate a new random account and a funded simulator
+			key, _ := crypto.GenerateKey()
+			auth := bind.NewKeyedTransactor(key)
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+
+			// Deploy an eventer contract
+			_, _, eventer, err := DeployEventer(auth, sim)
+			if err != nil {
+				t.Fatalf("Failed to deploy eventer contract: %v", err)
+			}
+			sim.Commit()
+
+			// Inject a few events into the contract, gradually more in each block
+			for i := 1; i <= 3; i++ {
+				for j := 1; j <= i; j++ {
+					if _, err := eventer.RaiseSimpleEvent(auth, common.Address{byte(j)}, [32]byte{byte(j)}, true, big.NewInt(int64(10*i+j))); err != nil {
+						t.Fatalf("block %d, event %d: raise failed: %v", i, j, err)
+					}
+				}
+				sim.Commit()
+			}
+			// Test filtering for certain events and ensure they can be found
+			sit, err := eventer.FilterSimpleEvent(nil, []common.Address{common.Address{1}, common.Address{3}}, [][32]byte{{byte(1)}, {byte(2)}, {byte(3)}}, []bool{true})
+			if err != nil {
+				t.Fatalf("failed to filter for simple events: %v", err)
+			}
+			defer sit.Close()
+
+			sit.Next()
+			if sit.Event.Value.Uint64() != 11 || !sit.Event.Flag {
+				t.Errorf("simple log content mismatch: have %v, want {11, true}", sit.Event)
+			}
+			sit.Next()
+			if sit.Event.Value.Uint64() != 21 || !sit.Event.Flag {
+				t.Errorf("simple log content mismatch: have %v, want {21, true}", sit.Event)
+			}
+			sit.Next()
+			if sit.Event.Value.Uint64() != 31 || !sit.Event.Flag {
+				t.Errorf("simple log content mismatch: have %v, want {31, true}", sit.Event)
+			}
+			sit.Next()
+			if sit.Event.Value.Uint64() != 33 || !sit.Event.Flag {
+				t.Errorf("simple log content mismatch: have %v, want {33, true}", sit.Event)
+			}
+
+			if sit.Next() {
+				t.Errorf("unexpected simple event found: %+v", sit.Event)
+			}
+			if err = sit.Error(); err != nil {
+				t.Fatalf("simple event iteration failed: %v", err)
+			}
+			// Test raising and filtering for an event with no data component
+			if _, err := eventer.RaiseNodataEvent(auth, big.NewInt(314), 141, 271); err != nil {
+				t.Fatalf("failed to raise nodata event: %v", err)
+			}
+			sim.Commit()
+
+			nit, err := eventer.FilterNodataEvent(nil, []*big.Int{big.NewInt(314)}, []int16{140, 141, 142}, []uint32{271})
+			if err != nil {
+				t.Fatalf("failed to filter for nodata events: %v", err)
+			}
+			defer nit.Close()
+
+			if !nit.Next() {
+				t.Fatalf("nodata log not found: %v", nit.Error())
+			}
+			if nit.Event.Number.Uint64() != 314 {
+				t.Errorf("nodata log content mismatch: have %v, want 314", nit.Event.Number)
+			}
+			if nit.Next() {
+				t.Errorf("unexpected nodata event found: %+v", nit.Event)
+			}
+			if err = nit.Error(); err != nil {
+				t.Fatalf("nodata event iteration failed: %v", err)
+			}
+			// Test raising and filtering for events with dynamic indexed components
+			if _, err := eventer.RaiseDynamicEvent(auth, "Hello", []byte("World")); err != nil {
+				t.Fatalf("failed to raise dynamic event: %v", err)
+			}
+			sim.Commit()
+
+			dit, err := eventer.FilterDynamicEvent(nil, []string{"Hi", "Hello", "Bye"}, [][]byte{[]byte("World")})
+			if err != nil {
+				t.Fatalf("failed to filter for dynamic events: %v", err)
+			}
+			defer dit.Close()
+
+			if !dit.Next() {
+				t.Fatalf("dynamic log not found: %v", dit.Error())
+			}
+			if dit.Event.NonIndexedString != "Hello" || string(dit.Event.NonIndexedBytes) != "World" ||	dit.Event.IndexedString != common.HexToHash("0x06b3dfaec148fb1bb2b066f10ec285e7c9bf402ab32aa78a5d38e34566810cd2") || dit.Event.IndexedBytes != common.HexToHash("0xf2208c967df089f60420785795c0a9ba8896b0f6f1867fa7f1f12ad6f79c1a18") {
+				t.Errorf("dynamic log content mismatch: have %v, want {'0x06b3dfaec148fb1bb2b066f10ec285e7c9bf402ab32aa78a5d38e34566810cd2, '0xf2208c967df089f60420785795c0a9ba8896b0f6f1867fa7f1f12ad6f79c1a18', 'Hello', 'World'}", dit.Event)
+			}
+			if dit.Next() {
+				t.Errorf("unexpected dynamic event found: %+v", dit.Event)
+			}
+			if err = dit.Error(); err != nil {
+				t.Fatalf("dynamic event iteration failed: %v", err)
+			}
+			// Test subscribing to an event and raising it afterwards
+			ch := make(chan *EventerSimpleEvent, 16)
+			sub, err := eventer.WatchSimpleEvent(nil, ch, nil, nil, nil)
+			if err != nil {
+				t.Fatalf("failed to subscribe to simple events: %v", err)
+			}
+			if _, err := eventer.RaiseSimpleEvent(auth, common.Address{255}, [32]byte{255}, true, big.NewInt(255)); err != nil {
+				t.Fatalf("failed to raise subscribed simple event: %v", err)
+			}
+			sim.Commit()
+
+			select {
+			case event := <-ch:
+				if event.Value.Uint64() != 255 {
+					t.Errorf("simple log content mismatch: have %v, want 255", event)
+				}
+			case <-time.After(250 * time.Millisecond):
+				t.Fatalf("subscribed simple event didn't arrive")
+			}
+			// Unsubscribe from the event and make sure we're not delivered more
+			sub.Unsubscribe()
+
+			if _, err := eventer.RaiseSimpleEvent(auth, common.Address{254}, [32]byte{254}, true, big.NewInt(254)); err != nil {
+				t.Fatalf("failed to raise subscribed simple event: %v", err)
+			}
+			sim.Commit()
+
+			select {
+			case event := <-ch:
+				t.Fatalf("unsubscribed simple event arrived: %v", event)
+			case <-time.After(250 * time.Millisecond):
+			}
+		`,
+	},
+	{
+		`DeeplyNestedArray`,
+		`
+			contract DeeplyNestedArray {
+				uint64[3][4][5] public deepUint64Array;
+				function storeDeepUintArray(uint64[3][4][5] arr) public {
+					deepUint64Array = arr;
+				}
+				function retrieveDeepArray() public view returns (uint64[3][4][5]) {
+					return deepUint64Array;
+				}
+			}
+		`,
+		`6060604052341561000f57600080fd5b6106438061001e6000396000f300606060405260043610610057576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063344248551461005c5780638ed4573a1461011457806398ed1856146101ab575b600080fd5b341561006757600080fd5b610112600480806107800190600580602002604051908101604052809291906000905b828210156101055783826101800201600480602002604051908101604052809291906000905b828210156100f25783826060020160038060200260405190810160405280929190826003602002808284378201915050505050815260200190600101906100b0565b505050508152602001906001019061008a565b5050505091905050610208565b005b341561011f57600080fd5b61012761021d565b604051808260056000925b8184101561019b578284602002015160046000925b8184101561018d5782846020020151600360200280838360005b8381101561017c578082015181840152602081019050610161565b505050509050019260010192610147565b925050509260010192610132565b9250505091505060405180910390f35b34156101b657600080fd5b6101de6004808035906020019091908035906020019091908035906020019091905050610309565b604051808267ffffffffffffffff1667ffffffffffffffff16815260200191505060405180910390f35b80600090600561021992919061035f565b5050565b6102256103b0565b6000600580602002604051908101604052809291906000905b8282101561030057838260040201600480602002604051908101604052809291906000905b828210156102ed578382016003806020026040519081016040528092919082600380156102d9576020028201916000905b82829054906101000a900467ffffffffffffffff1667ffffffffffffffff16815260200190600801906020826007010492830192600103820291508084116102945790505b505050505081526020019060010190610263565b505050508152602001906001019061023e565b50505050905090565b60008360058110151561031857fe5b600402018260048110151561032957fe5b018160038110151561033757fe5b6004918282040191900660080292509250509054906101000a900467ffffffffffffffff1681565b826005600402810192821561039f579160200282015b8281111561039e5782518290600461038e9291906103df565b5091602001919060040190610375565b5b5090506103ac919061042d565b5090565b610780604051908101604052806005905b6103c9610459565b8152602001906001900390816103c15790505090565b826004810192821561041c579160200282015b8281111561041b5782518290600361040b929190610488565b50916020019190600101906103f2565b5b5090506104299190610536565b5090565b61045691905b8082111561045257600081816104499190610562565b50600401610433565b5090565b90565b610180604051908101604052806004905b6104726105a7565b81526020019060019003908161046a5790505090565b82600380016004900481019282156105255791602002820160005b838211156104ef57835183826101000a81548167ffffffffffffffff021916908367ffffffffffffffff16021790555092602001926008016020816007010492830192600103026104a3565b80156105235782816101000a81549067ffffffffffffffff02191690556008016020816007010492830192600103026104ef565b505b50905061053291906105d9565b5090565b61055f91905b8082111561055b57600081816105529190610610565b5060010161053c565b5090565b90565b50600081816105719190610610565b50600101600081816105839190610610565b50600101600081816105959190610610565b5060010160006105a59190610610565b565b6060604051908101604052806003905b600067ffffffffffffffff168152602001906001900390816105b75790505090565b61060d91905b8082111561060957600081816101000a81549067ffffffffffffffff0219169055506001016105df565b5090565b90565b50600090555600a165627a7a7230582087e5a43f6965ab6ef7a4ff056ab80ed78fd8c15cff57715a1bf34ec76a93661c0029`,
+		`[{"constant":false,"inputs":[{"name":"arr","type":"uint64[3][4][5]"}],"name":"storeDeepUintArray","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"retrieveDeepArray","outputs":[{"name":"","type":"uint64[3][4][5]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"}],"name":"deepUint64Array","outputs":[{"name":"","type":"uint64"}],"payable":false,"stateMutability":"view","type":"function"}]`,
+		`
+			// Generate a new random account and a funded simulator
+			key, _ := crypto.GenerateKey()
+			auth := bind.NewKeyedTransactor(key)
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+
+			//deploy the test contract
+			_, _, testContract, err := DeployDeeplyNestedArray(auth, sim)
+			if err != nil {
+				t.Fatalf("Failed to deploy test contract: %v", err)
+			}
+
+			// Finish deploy.
+			sim.Commit()
+
+			//Create coordinate-filled array, for testing purposes.
+			testArr := [5][4][3]uint64{}
+			for i := 0; i < 5; i++ {
+				testArr[i] = [4][3]uint64{}
+				for j := 0; j < 4; j++ {
+					testArr[i][j] = [3]uint64{}
+					for k := 0; k < 3; k++ {
+						//pack the coordinates, each array value will be unique, and can be validated easily.
+						testArr[i][j][k] = uint64(i) << 16 | uint64(j) << 8 | uint64(k)
+					}
+				}
+			}
+
+			if _, err := testContract.StoreDeepUintArray(&bind.TransactOpts{
+				From: auth.From,
+				Signer: auth.Signer,
+			}, testArr); err != nil {
+				t.Fatalf("Failed to store nested array in test contract: %v", err)
+			}
+
+			sim.Commit()
+
+			retrievedArr, err := testContract.RetrieveDeepArray(&bind.CallOpts{
+				From: auth.From,
+				Pending: false,
+			})
+			if err != nil {
+				t.Fatalf("Failed to retrieve nested array from test contract: %v", err)
+			}
+
+			//quick check to see if contents were copied
+			// (See accounts/abi/unpack_test.go for more extensive testing)
+			if retrievedArr[4][3][2] != testArr[4][3][2] {
+				t.Fatalf("Retrieved value does not match expected value! got: %d, expected: %d. %v", retrievedArr[4][3][2], testArr[4][3][2], err)
+			}
+		`,
+	},
 }
 
 // Tests that packages generated by the binder can be successfully compiled and
@@ -459,7 +821,7 @@ func TestBindings(t *testing.T) {
 	}
 	// Skip the test if the go-ethereum sources are symlinked (https://github.com/golang/go/issues/14845)
 	linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil))\n}")
-	linkTestDeps, err := imports.Process("", []byte(linkTestCode), nil)
+	linkTestDeps, err := imports.Process(os.TempDir(), []byte(linkTestCode), nil)
 	if err != nil {
 		t.Fatalf("failed check for goimports symlink bug: %v", err)
 	}
@@ -498,7 +860,7 @@ func TestBindings(t *testing.T) {
 		}
 	}
 	// Test the entire package and report any failures
-	cmd := exec.Command(gocmd, "test", "-v")
+	cmd := exec.Command(gocmd, "test", "-v", "-count", "1")
 	cmd.Dir = pkg
 	if out, err := cmd.CombinedOutput(); err != nil {
 		t.Fatalf("failed to run binding test: %v\n%s", err, out)

accounts/abi/bind/template.go

@@ -32,6 +32,7 @@ type tmplContract struct {
 	Constructor abi.Method             // Contract constructor for deploy parametrization
 	Calls       map[string]*tmplMethod // Contract calls that only read state data
 	Transacts   map[string]*tmplMethod // Contract calls that write state data
+	Events      map[string]*tmplEvent  // Contract events accessors
 }
 
 // tmplMethod is a wrapper around an abi.Method that contains a few preprocessed
@@ -39,7 +40,13 @@ type tmplContract struct {
 type tmplMethod struct {
 	Original   abi.Method // Original method as parsed by the abi package
 	Normalized abi.Method // Normalized version of the parsed method (capitalized names, non-anonymous args/returns)
-	Structured bool       // Whether the returns should be accumulated into a contract
+	Structured bool       // Whether the returns should be accumulated into a struct
+}
+
+// tmplEvent is a wrapper around an a
+type tmplEvent struct {
+	Original   abi.Event // Original event as parsed by the abi package
+	Normalized abi.Event // Normalized version of the parsed fields
 }
 
 // tmplSource is language to template mapping containing all the supported
@@ -52,8 +59,8 @@ var tmplSource = map[Lang]string{
 // tmplSourceGo is the Go source template use to generate the contract binding
 // based on.
 const tmplSourceGo = `
-// This file is an automatically generated Go binding. Do not modify as any
-// change will likely be lost upon the next re-generation!
+// Code generated - DO NOT EDIT.
+// This file is a generated binding and any manual changes will be lost.
 
 package {{.Package}}
 
@@ -75,7 +82,7 @@ package {{.Package}}
 		  if err != nil {
 		    return common.Address{}, nil, nil, err
 		  }
-		  return address, tx, &{{.Type}}{ {{.Type}}Caller: {{.Type}}Caller{contract: contract}, {{.Type}}Transactor: {{.Type}}Transactor{contract: contract} }, nil
+		  return address, tx, &{{.Type}}{ {{.Type}}Caller: {{.Type}}Caller{contract: contract}, {{.Type}}Transactor: {{.Type}}Transactor{contract: contract}, {{.Type}}Filterer: {{.Type}}Filterer{contract: contract} }, nil
 		}
 	{{end}}
 
@@ -83,6 +90,7 @@ package {{.Package}}
 	type {{.Type}} struct {
 	  {{.Type}}Caller     // Read-only binding to the contract
 	  {{.Type}}Transactor // Write-only binding to the contract
+		{{.Type}}Filterer   // Log filterer for contract events
 	}
 
 	// {{.Type}}Caller is an auto generated read-only Go binding around an Ethereum contract.
@@ -95,6 +103,11 @@ package {{.Package}}
 	  contract *bind.BoundContract // Generic contract wrapper for the low level calls
 	}
 
+	// {{.Type}}Filterer is an auto generated log filtering Go binding around an Ethereum contract events.
+	type {{.Type}}Filterer struct {
+	  contract *bind.BoundContract // Generic contract wrapper for the low level calls
+	}
+
 	// {{.Type}}Session is an auto generated Go binding around an Ethereum contract,
 	// with pre-set call and transact options.
 	type {{.Type}}Session struct {
@@ -134,16 +147,16 @@ package {{.Package}}
 
 	// New{{.Type}} creates a new instance of {{.Type}}, bound to a specific deployed contract.
 	func New{{.Type}}(address common.Address, backend bind.ContractBackend) (*{{.Type}}, error) {
-	  contract, err := bind{{.Type}}(address, backend, backend)
+	  contract, err := bind{{.Type}}(address, backend, backend, backend)
 	  if err != nil {
 	    return nil, err
 	  }
-	  return &{{.Type}}{ {{.Type}}Caller: {{.Type}}Caller{contract: contract}, {{.Type}}Transactor: {{.Type}}Transactor{contract: contract} }, nil
+	  return &{{.Type}}{ {{.Type}}Caller: {{.Type}}Caller{contract: contract}, {{.Type}}Transactor: {{.Type}}Transactor{contract: contract}, {{.Type}}Filterer: {{.Type}}Filterer{contract: contract} }, nil
 	}
 
 	// New{{.Type}}Caller creates a new read-only instance of {{.Type}}, bound to a specific deployed contract.
 	func New{{.Type}}Caller(address common.Address, caller bind.ContractCaller) (*{{.Type}}Caller, error) {
-	  contract, err := bind{{.Type}}(address, caller, nil)
+	  contract, err := bind{{.Type}}(address, caller, nil, nil)
 	  if err != nil {
 	    return nil, err
 	  }
@@ -152,20 +165,29 @@ package {{.Package}}
 
 	// New{{.Type}}Transactor creates a new write-only instance of {{.Type}}, bound to a specific deployed contract.
 	func New{{.Type}}Transactor(address common.Address, transactor bind.ContractTransactor) (*{{.Type}}Transactor, error) {
-	  contract, err := bind{{.Type}}(address, nil, transactor)
+	  contract, err := bind{{.Type}}(address, nil, transactor, nil)
 	  if err != nil {
 	    return nil, err
 	  }
 	  return &{{.Type}}Transactor{contract: contract}, nil
 	}
 
+	// New{{.Type}}Filterer creates a new log filterer instance of {{.Type}}, bound to a specific deployed contract.
+ 	func New{{.Type}}Filterer(address common.Address, filterer bind.ContractFilterer) (*{{.Type}}Filterer, error) {
+ 	  contract, err := bind{{.Type}}(address, nil, nil, filterer)
+ 	  if err != nil {
+ 	    return nil, err
+ 	  }
+ 	  return &{{.Type}}Filterer{contract: contract}, nil
+ 	}
+
 	// bind{{.Type}} binds a generic wrapper to an already deployed contract.
-	func bind{{.Type}}(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor) (*bind.BoundContract, error) {
+	func bind{{.Type}}(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor, filterer bind.ContractFilterer) (*bind.BoundContract, error) {
 	  parsed, err := abi.JSON(strings.NewReader({{.Type}}ABI))
 	  if err != nil {
 	    return nil, err
 	  }
-	  return bind.NewBoundContract(address, parsed, caller, transactor), nil
+	  return bind.NewBoundContract(address, parsed, caller, transactor, filterer), nil
 	}
 
 	// Call invokes the (constant) contract method with params as input values and
@@ -263,6 +285,137 @@ package {{.Package}}
 		  return _{{$contract.Type}}.Contract.{{.Normalized.Name}}(&_{{$contract.Type}}.TransactOpts {{range $i, $_ := .Normalized.Inputs}}, {{.Name}}{{end}})
 		}
 	{{end}}
+
+	{{range .Events}}
+		// {{$contract.Type}}{{.Normalized.Name}}Iterator is returned from Filter{{.Normalized.Name}} and is used to iterate over the raw logs and unpacked data for {{.Normalized.Name}} events raised by the {{$contract.Type}} contract.
+		type {{$contract.Type}}{{.Normalized.Name}}Iterator struct {
+			Event *{{$contract.Type}}{{.Normalized.Name}} // Event containing the contract specifics and raw log
+
+			contract *bind.BoundContract // Generic contract to use for unpacking event data
+			event    string              // Event name to use for unpacking event data
+
+			logs chan types.Log        // Log channel receiving the found contract events
+			sub  ethereum.Subscription // Subscription for errors, completion and termination
+			done bool                  // Whether the subscription completed delivering logs
+			fail error                 // Occurred error to stop iteration
+		}
+		// Next advances the iterator to the subsequent event, returning whether there
+		// are any more events found. In case of a retrieval or parsing error, false is
+		// returned and Error() can be queried for the exact failure.
+		func (it *{{$contract.Type}}{{.Normalized.Name}}Iterator) Next() bool {
+			// If the iterator failed, stop iterating
+			if (it.fail != nil) {
+				return false
+			}
+			// If the iterator completed, deliver directly whatever's available
+			if (it.done) {
+				select {
+				case log := <-it.logs:
+					it.Event = new({{$contract.Type}}{{.Normalized.Name}})
+					if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+						it.fail = err
+						return false
+					}
+					it.Event.Raw = log
+					return true
+
+				default:
+					return false
+				}
+			}
+			// Iterator still in progress, wait for either a data or an error event
+			select {
+			case log := <-it.logs:
+				it.Event = new({{$contract.Type}}{{.Normalized.Name}})
+				if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
+					it.fail = err
+					return false
+				}
+				it.Event.Raw = log
+				return true
+
+			case err := <-it.sub.Err():
+				it.done = true
+				it.fail = err
+				return it.Next()
+			}
+		}
+		// Error returns any retrieval or parsing error occurred during filtering.
+		func (it *{{$contract.Type}}{{.Normalized.Name}}Iterator) Error() error {
+			return it.fail
+		}
+		// Close terminates the iteration process, releasing any pending underlying
+		// resources.
+		func (it *{{$contract.Type}}{{.Normalized.Name}}Iterator) Close() error {
+			it.sub.Unsubscribe()
+			return nil
+		}
+
+		// {{$contract.Type}}{{.Normalized.Name}} represents a {{.Normalized.Name}} event raised by the {{$contract.Type}} contract.
+		type {{$contract.Type}}{{.Normalized.Name}} struct { {{range .Normalized.Inputs}}
+			{{capitalise .Name}} {{if .Indexed}}{{bindtopictype .Type}}{{else}}{{bindtype .Type}}{{end}}; {{end}}
+			Raw types.Log // Blockchain specific contextual infos
+		}
+
+		// Filter{{.Normalized.Name}} is a free log retrieval operation binding the contract event 0x{{printf "%x" .Original.Id}}.
+		//
+		// Solidity: {{.Original.String}}
+ 		func (_{{$contract.Type}} *{{$contract.Type}}Filterer) Filter{{.Normalized.Name}}(opts *bind.FilterOpts{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}} []{{bindtype .Type}}{{end}}{{end}}) (*{{$contract.Type}}{{.Normalized.Name}}Iterator, error) {
+			{{range .Normalized.Inputs}}
+			{{if .Indexed}}var {{.Name}}Rule []interface{}
+			for _, {{.Name}}Item := range {{.Name}} {
+				{{.Name}}Rule = append({{.Name}}Rule, {{.Name}}Item)
+			}{{end}}{{end}}
+
+			logs, sub, err := _{{$contract.Type}}.contract.FilterLogs(opts, "{{.Original.Name}}"{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}}Rule{{end}}{{end}})
+			if err != nil {
+				return nil, err
+			}
+			return &{{$contract.Type}}{{.Normalized.Name}}Iterator{contract: _{{$contract.Type}}.contract, event: "{{.Original.Name}}", logs: logs, sub: sub}, nil
+ 		}
+
+		// Watch{{.Normalized.Name}} is a free log subscription operation binding the contract event 0x{{printf "%x" .Original.Id}}.
+		//
+		// Solidity: {{.Original.String}}
+		func (_{{$contract.Type}} *{{$contract.Type}}Filterer) Watch{{.Normalized.Name}}(opts *bind.WatchOpts, sink chan<- *{{$contract.Type}}{{.Normalized.Name}}{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}} []{{bindtype .Type}}{{end}}{{end}}) (event.Subscription, error) {
+			{{range .Normalized.Inputs}}
+			{{if .Indexed}}var {{.Name}}Rule []interface{}
+			for _, {{.Name}}Item := range {{.Name}} {
+				{{.Name}}Rule = append({{.Name}}Rule, {{.Name}}Item)
+			}{{end}}{{end}}
+
+			logs, sub, err := _{{$contract.Type}}.contract.WatchLogs(opts, "{{.Original.Name}}"{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}}Rule{{end}}{{end}})
+			if err != nil {
+				return nil, err
+			}
+			return event.NewSubscription(func(quit <-chan struct{}) error {
+				defer sub.Unsubscribe()
+				for {
+					select {
+					case log := <-logs:
+						// New log arrived, parse the event and forward to the user
+						event := new({{$contract.Type}}{{.Normalized.Name}})
+						if err := _{{$contract.Type}}.contract.UnpackLog(event, "{{.Original.Name}}", log); err != nil {
+							return err
+						}
+						event.Raw = log
+
+						select {
+						case sink <- event:
+						case err := <-sub.Err():
+							return err
+						case <-quit:
+							return nil
+						}
+					case err := <-sub.Err():
+						return err
+					case <-quit:
+						return nil
+					}
+				}
+			}), nil
+		}
+ 	{{end}}
 {{end}}
 `
 

accounts/abi/bind/topics.go

@@ -0,0 +1,189 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package bind
+
+import (
+	"errors"
+	"fmt"
+	"math/big"
+	"reflect"
+
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+// makeTopics converts a filter query argument list into a filter topic set.
+func makeTopics(query ...[]interface{}) ([][]common.Hash, error) {
+	topics := make([][]common.Hash, len(query))
+	for i, filter := range query {
+		for _, rule := range filter {
+			var topic common.Hash
+
+			// Try to generate the topic based on simple types
+			switch rule := rule.(type) {
+			case common.Hash:
+				copy(topic[:], rule[:])
+			case common.Address:
+				copy(topic[common.HashLength-common.AddressLength:], rule[:])
+			case *big.Int:
+				blob := rule.Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case bool:
+				if rule {
+					topic[common.HashLength-1] = 1
+				}
+			case int8:
+				blob := big.NewInt(int64(rule)).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case int16:
+				blob := big.NewInt(int64(rule)).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case int32:
+				blob := big.NewInt(int64(rule)).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case int64:
+				blob := big.NewInt(rule).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case uint8:
+				blob := new(big.Int).SetUint64(uint64(rule)).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case uint16:
+				blob := new(big.Int).SetUint64(uint64(rule)).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case uint32:
+				blob := new(big.Int).SetUint64(uint64(rule)).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case uint64:
+				blob := new(big.Int).SetUint64(rule).Bytes()
+				copy(topic[common.HashLength-len(blob):], blob)
+			case string:
+				hash := crypto.Keccak256Hash([]byte(rule))
+				copy(topic[:], hash[:])
+			case []byte:
+				hash := crypto.Keccak256Hash(rule)
+				copy(topic[:], hash[:])
+
+			default:
+				// Attempt to generate the topic from funky types
+				val := reflect.ValueOf(rule)
+
+				switch {
+				case val.Kind() == reflect.Array && reflect.TypeOf(rule).Elem().Kind() == reflect.Uint8:
+					reflect.Copy(reflect.ValueOf(topic[common.HashLength-val.Len():]), val)
+
+				default:
+					return nil, fmt.Errorf("unsupported indexed type: %T", rule)
+				}
+			}
+			topics[i] = append(topics[i], topic)
+		}
+	}
+	return topics, nil
+}
+
+// Big batch of reflect types for topic reconstruction.
+var (
+	reflectHash    = reflect.TypeOf(common.Hash{})
+	reflectAddress = reflect.TypeOf(common.Address{})
+	reflectBigInt  = reflect.TypeOf(new(big.Int))
+)
+
+// parseTopics converts the indexed topic fields into actual log field values.
+//
+// Note, dynamic types cannot be reconstructed since they get mapped to Keccak256
+// hashes as the topic value!
+func parseTopics(out interface{}, fields abi.Arguments, topics []common.Hash) error {
+	// Sanity check that the fields and topics match up
+	if len(fields) != len(topics) {
+		return errors.New("topic/field count mismatch")
+	}
+	// Iterate over all the fields and reconstruct them from topics
+	for _, arg := range fields {
+		if !arg.Indexed {
+			return errors.New("non-indexed field in topic reconstruction")
+		}
+		field := reflect.ValueOf(out).Elem().FieldByName(capitalise(arg.Name))
+
+		// Try to parse the topic back into the fields based on primitive types
+		switch field.Kind() {
+		case reflect.Bool:
+			if topics[0][common.HashLength-1] == 1 {
+				field.Set(reflect.ValueOf(true))
+			}
+		case reflect.Int8:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(int8(num.Int64())))
+
+		case reflect.Int16:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(int16(num.Int64())))
+
+		case reflect.Int32:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(int32(num.Int64())))
+
+		case reflect.Int64:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(num.Int64()))
+
+		case reflect.Uint8:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(uint8(num.Uint64())))
+
+		case reflect.Uint16:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(uint16(num.Uint64())))
+
+		case reflect.Uint32:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(uint32(num.Uint64())))
+
+		case reflect.Uint64:
+			num := new(big.Int).SetBytes(topics[0][:])
+			field.Set(reflect.ValueOf(num.Uint64()))
+
+		default:
+			// Ran out of plain primitive types, try custom types
+			switch field.Type() {
+			case reflectHash: // Also covers all dynamic types
+				field.Set(reflect.ValueOf(topics[0]))
+
+			case reflectAddress:
+				var addr common.Address
+				copy(addr[:], topics[0][common.HashLength-common.AddressLength:])
+				field.Set(reflect.ValueOf(addr))
+
+			case reflectBigInt:
+				num := new(big.Int).SetBytes(topics[0][:])
+				field.Set(reflect.ValueOf(num))
+
+			default:
+				// Ran out of custom types, try the crazies
+				switch {
+				case arg.Type.T == abi.FixedBytesTy:
+					reflect.Copy(field, reflect.ValueOf(topics[0][common.HashLength-arg.Type.Size:]))
+
+				default:
+					return fmt.Errorf("unsupported indexed type: %v", arg.Type)
+				}
+			}
+		}
+		topics = topics[1:]
+	}
+	return nil
+}

accounts/abi/bind/util_test.go

@@ -34,18 +34,18 @@ var testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d
 
 var waitDeployedTests = map[string]struct {
 	code        string
-	gas         *big.Int
+	gas         uint64
 	wantAddress common.Address
 	wantErr     error
 }{
 	"successful deploy": {
 		code:        `6060604052600a8060106000396000f360606040526008565b00`,
-		gas:         big.NewInt(3000000),
+		gas:         3000000,
 		wantAddress: common.HexToAddress("0x3a220f351252089d385b29beca14e27f204c296a"),
 	},
 	"empty code": {
 		code:        ``,
-		gas:         big.NewInt(300000),
+		gas:         300000,
 		wantErr:     bind.ErrNoCodeAfterDeploy,
 		wantAddress: common.HexToAddress("0x3a220f351252089d385b29beca14e27f204c296a"),
 	},

accounts/abi/error.go

@@ -39,22 +39,23 @@ func formatSliceString(kind reflect.Kind, sliceSize int) string {
 // type in t.
 func sliceTypeCheck(t Type, val reflect.Value) error {
 	if val.Kind() != reflect.Slice && val.Kind() != reflect.Array {
-		return typeErr(formatSliceString(t.Kind, t.SliceSize), val.Type())
-	}
-	if t.IsArray && val.Len() != t.SliceSize {
-		return typeErr(formatSliceString(t.Elem.Kind, t.SliceSize), formatSliceString(val.Type().Elem().Kind(), val.Len()))
+		return typeErr(formatSliceString(t.Kind, t.Size), val.Type())
 	}
 
-	if t.Elem.IsSlice {
+	if t.T == ArrayTy && val.Len() != t.Size {
+		return typeErr(formatSliceString(t.Elem.Kind, t.Size), formatSliceString(val.Type().Elem().Kind(), val.Len()))
+	}
+
+	if t.Elem.T == SliceTy {
 		if val.Len() > 0 {
 			return sliceTypeCheck(*t.Elem, val.Index(0))
 		}
-	} else if t.Elem.IsArray {
+	} else if t.Elem.T == ArrayTy {
 		return sliceTypeCheck(*t.Elem, val.Index(0))
 	}
 
 	if elemKind := val.Type().Elem().Kind(); elemKind != t.Elem.Kind {
-		return typeErr(formatSliceString(t.Elem.Kind, t.SliceSize), val.Type())
+		return typeErr(formatSliceString(t.Elem.Kind, t.Size), val.Type())
 	}
 	return nil
 }
@@ -62,20 +63,19 @@ func sliceTypeCheck(t Type, val reflect.Value) error {
 // typeCheck checks that the given reflection value can be assigned to the reflection
 // type in t.
 func typeCheck(t Type, value reflect.Value) error {
-	if t.IsSlice || t.IsArray {
+	if t.T == SliceTy || t.T == ArrayTy {
 		return sliceTypeCheck(t, value)
 	}
 
 	// Check base type validity. Element types will be checked later on.
 	if t.Kind != value.Kind() {
 		return typeErr(t.Kind, value.Kind())
+	} else if t.T == FixedBytesTy && t.Size != value.Len() {
+		return typeErr(t.Type, value.Type())
+	} else {
+		return nil
 	}
-	return nil
-}
 
-// varErr returns a formatted error.
-func varErr(expected, got reflect.Kind) error {
-	return typeErr(expected, got)
 }
 
 // typeErr returns a formatted type casting error.

accounts/abi/event.go

@@ -30,7 +30,18 @@ import (
 type Event struct {
 	Name      string
 	Anonymous bool
-	Inputs    []Argument
+	Inputs    Arguments
+}
+
+func (e Event) String() string {
+	inputs := make([]string, len(e.Inputs))
+	for i, input := range e.Inputs {
+		inputs[i] = fmt.Sprintf("%v %v", input.Name, input.Type)
+		if input.Indexed {
+			inputs[i] = fmt.Sprintf("%v indexed %v", input.Name, input.Type)
+		}
+	}
+	return fmt.Sprintf("e %v(%v)", e.Name, strings.Join(inputs, ", "))
 }
 
 // Id returns the canonical representation of the event's signature used by the

accounts/abi/event_test.go

@@ -17,13 +17,69 @@
 package abi
 
 import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"math/big"
+	"reflect"
 	"strings"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
+var jsonEventTransfer = []byte(`{
+  "anonymous": false,
+  "inputs": [
+    {
+      "indexed": true, "name": "from", "type": "address"
+    }, {
+      "indexed": true, "name": "to", "type": "address"
+    }, {
+      "indexed": false, "name": "value", "type": "uint256"
+  }],
+  "name": "Transfer",
+  "type": "event"
+}`)
+
+var jsonEventPledge = []byte(`{
+  "anonymous": false,
+  "inputs": [{
+      "indexed": false, "name": "who", "type": "address"
+    }, {
+      "indexed": false, "name": "wad", "type": "uint128"
+    }, {
+      "indexed": false, "name": "currency", "type": "bytes3"
+  }],
+  "name": "Pledge",
+  "type": "event"
+}`)
+
+var jsonEventMixedCase = []byte(`{
+	"anonymous": false,
+	"inputs": [{
+		"indexed": false, "name": "value", "type": "uint256"
+	  }, {
+		"indexed": false, "name": "_value", "type": "uint256"
+	  }, {
+		"indexed": false, "name": "Value", "type": "uint256"
+	}],
+	"name": "MixedCase",
+	"type": "event"
+  }`)
+
+// 1000000
+var transferData1 = "00000000000000000000000000000000000000000000000000000000000f4240"
+
+// "0x00Ce0d46d924CC8437c806721496599FC3FFA268", 2218516807680, "usd"
+var pledgeData1 = "00000000000000000000000000ce0d46d924cc8437c806721496599fc3ffa2680000000000000000000000000000000000000000000000000000020489e800007573640000000000000000000000000000000000000000000000000000000000"
+
+// 1000000,2218516807680,1000001
+var mixedCaseData1 = "00000000000000000000000000000000000000000000000000000000000f42400000000000000000000000000000000000000000000000000000020489e8000000000000000000000000000000000000000000000000000000000000000f4241"
+
 func TestEventId(t *testing.T) {
 	var table = []struct {
 		definition   string
@@ -31,7 +87,7 @@ func TestEventId(t *testing.T) {
 	}{
 		{
 			definition: `[
-			{ "type" : "event", "name" : "balance", "inputs": [{ "name" : "in", "type": "uint" }] },
+			{ "type" : "event", "name" : "balance", "inputs": [{ "name" : "in", "type": "uint256" }] },
 			{ "type" : "event", "name" : "check", "inputs": [{ "name" : "t", "type": "address" }, { "name": "b", "type": "uint256" }] }
 			]`,
 			expectations: map[string]common.Hash{
@@ -54,3 +110,286 @@ func TestEventId(t *testing.T) {
 		}
 	}
 }
+
+// TestEventMultiValueWithArrayUnpack verifies that array fields will be counted after parsing array.
+func TestEventMultiValueWithArrayUnpack(t *testing.T) {
+	definition := `[{"name": "test", "type": "event", "inputs": [{"indexed": false, "name":"value1", "type":"uint8[2]"},{"indexed": false, "name":"value2", "type":"uint8"}]}]`
+	type testStruct struct {
+		Value1 [2]uint8
+		Value2 uint8
+	}
+	abi, err := JSON(strings.NewReader(definition))
+	require.NoError(t, err)
+	var b bytes.Buffer
+	var i uint8 = 1
+	for ; i <= 3; i++ {
+		b.Write(packNum(reflect.ValueOf(i)))
+	}
+	var rst testStruct
+	require.NoError(t, abi.Unpack(&rst, "test", b.Bytes()))
+	require.Equal(t, [2]uint8{1, 2}, rst.Value1)
+	require.Equal(t, uint8(3), rst.Value2)
+}
+
+func TestEventTupleUnpack(t *testing.T) {
+
+	type EventTransfer struct {
+		Value *big.Int
+	}
+
+	type EventTransferWithTag struct {
+		// this is valid because `value` is not exportable,
+		// so value is only unmarshalled into `Value1`.
+		value  *big.Int
+		Value1 *big.Int `abi:"value"`
+	}
+
+	type BadEventTransferWithSameFieldAndTag struct {
+		Value  *big.Int
+		Value1 *big.Int `abi:"value"`
+	}
+
+	type BadEventTransferWithDuplicatedTag struct {
+		Value1 *big.Int `abi:"value"`
+		Value2 *big.Int `abi:"value"`
+	}
+
+	type BadEventTransferWithEmptyTag struct {
+		Value *big.Int `abi:""`
+	}
+
+	type EventPledge struct {
+		Who      common.Address
+		Wad      *big.Int
+		Currency [3]byte
+	}
+
+	type BadEventPledge struct {
+		Who      string
+		Wad      int
+		Currency [3]byte
+	}
+
+	type EventMixedCase struct {
+		Value1 *big.Int `abi:"value"`
+		Value2 *big.Int `abi:"_value"`
+		Value3 *big.Int `abi:"Value"`
+	}
+
+	bigint := new(big.Int)
+	bigintExpected := big.NewInt(1000000)
+	bigintExpected2 := big.NewInt(2218516807680)
+	bigintExpected3 := big.NewInt(1000001)
+	addr := common.HexToAddress("0x00Ce0d46d924CC8437c806721496599FC3FFA268")
+	var testCases = []struct {
+		data     string
+		dest     interface{}
+		expected interface{}
+		jsonLog  []byte
+		error    string
+		name     string
+	}{{
+		transferData1,
+		&EventTransfer{},
+		&EventTransfer{Value: bigintExpected},
+		jsonEventTransfer,
+		"",
+		"Can unpack ERC20 Transfer event into structure",
+	}, {
+		transferData1,
+		&[]interface{}{&bigint},
+		&[]interface{}{&bigintExpected},
+		jsonEventTransfer,
+		"",
+		"Can unpack ERC20 Transfer event into slice",
+	}, {
+		transferData1,
+		&EventTransferWithTag{},
+		&EventTransferWithTag{Value1: bigintExpected},
+		jsonEventTransfer,
+		"",
+		"Can unpack ERC20 Transfer event into structure with abi: tag",
+	}, {
+		transferData1,
+		&BadEventTransferWithDuplicatedTag{},
+		&BadEventTransferWithDuplicatedTag{},
+		jsonEventTransfer,
+		"struct: abi tag in 'Value2' already mapped",
+		"Can not unpack ERC20 Transfer event with duplicated abi tag",
+	}, {
+		transferData1,
+		&BadEventTransferWithSameFieldAndTag{},
+		&BadEventTransferWithSameFieldAndTag{},
+		jsonEventTransfer,
+		"abi: multiple variables maps to the same abi field 'value'",
+		"Can not unpack ERC20 Transfer event with a field and a tag mapping to the same abi variable",
+	}, {
+		transferData1,
+		&BadEventTransferWithEmptyTag{},
+		&BadEventTransferWithEmptyTag{},
+		jsonEventTransfer,
+		"struct: abi tag in 'Value' is empty",
+		"Can not unpack ERC20 Transfer event with an empty tag",
+	}, {
+		pledgeData1,
+		&EventPledge{},
+		&EventPledge{
+			addr,
+			bigintExpected2,
+			[3]byte{'u', 's', 'd'}},
+		jsonEventPledge,
+		"",
+		"Can unpack Pledge event into structure",
+	}, {
+		pledgeData1,
+		&[]interface{}{&common.Address{}, &bigint, &[3]byte{}},
+		&[]interface{}{
+			&addr,
+			&bigintExpected2,
+			&[3]byte{'u', 's', 'd'}},
+		jsonEventPledge,
+		"",
+		"Can unpack Pledge event into slice",
+	}, {
+		pledgeData1,
+		&[3]interface{}{&common.Address{}, &bigint, &[3]byte{}},
+		&[3]interface{}{
+			&addr,
+			&bigintExpected2,
+			&[3]byte{'u', 's', 'd'}},
+		jsonEventPledge,
+		"",
+		"Can unpack Pledge event into an array",
+	}, {
+		pledgeData1,
+		&[]interface{}{new(int), 0, 0},
+		&[]interface{}{},
+		jsonEventPledge,
+		"abi: cannot unmarshal common.Address in to int",
+		"Can not unpack Pledge event into slice with wrong types",
+	}, {
+		pledgeData1,
+		&BadEventPledge{},
+		&BadEventPledge{},
+		jsonEventPledge,
+		"abi: cannot unmarshal common.Address in to string",
+		"Can not unpack Pledge event into struct with wrong filed types",
+	}, {
+		pledgeData1,
+		&[]interface{}{common.Address{}, new(big.Int)},
+		&[]interface{}{},
+		jsonEventPledge,
+		"abi: insufficient number of elements in the list/array for unpack, want 3, got 2",
+		"Can not unpack Pledge event into too short slice",
+	}, {
+		pledgeData1,
+		new(map[string]interface{}),
+		&[]interface{}{},
+		jsonEventPledge,
+		"abi: cannot unmarshal tuple into map[string]interface {}",
+		"Can not unpack Pledge event into map",
+	}, {
+		mixedCaseData1,
+		&EventMixedCase{},
+		&EventMixedCase{Value1: bigintExpected, Value2: bigintExpected2, Value3: bigintExpected3},
+		jsonEventMixedCase,
+		"",
+		"Can unpack abi variables with mixed case",
+	}}
+
+	for _, tc := range testCases {
+		assert := assert.New(t)
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			err := unpackTestEventData(tc.dest, tc.data, tc.jsonLog, assert)
+			if tc.error == "" {
+				assert.Nil(err, "Should be able to unpack event data.")
+				assert.Equal(tc.expected, tc.dest, tc.name)
+			} else {
+				assert.EqualError(err, tc.error, tc.name)
+			}
+		})
+	}
+}
+
+func unpackTestEventData(dest interface{}, hexData string, jsonEvent []byte, assert *assert.Assertions) error {
+	data, err := hex.DecodeString(hexData)
+	assert.NoError(err, "Hex data should be a correct hex-string")
+	var e Event
+	assert.NoError(json.Unmarshal(jsonEvent, &e), "Should be able to unmarshal event ABI")
+	a := ABI{Events: map[string]Event{"e": e}}
+	return a.Unpack(dest, "e", data)
+}
+
+/*
+Taken from
+https://github.com/ethereum/go-ethereum/pull/15568
+*/
+
+type testResult struct {
+	Values [2]*big.Int
+	Value1 *big.Int
+	Value2 *big.Int
+}
+
+type testCase struct {
+	definition string
+	want       testResult
+}
+
+func (tc testCase) encoded(intType, arrayType Type) []byte {
+	var b bytes.Buffer
+	if tc.want.Value1 != nil {
+		val, _ := intType.pack(reflect.ValueOf(tc.want.Value1))
+		b.Write(val)
+	}
+
+	if !reflect.DeepEqual(tc.want.Values, [2]*big.Int{nil, nil}) {
+		val, _ := arrayType.pack(reflect.ValueOf(tc.want.Values))
+		b.Write(val)
+	}
+	if tc.want.Value2 != nil {
+		val, _ := intType.pack(reflect.ValueOf(tc.want.Value2))
+		b.Write(val)
+	}
+	return b.Bytes()
+}
+
+// TestEventUnpackIndexed verifies that indexed field will be skipped by event decoder.
+func TestEventUnpackIndexed(t *testing.T) {
+	definition := `[{"name": "test", "type": "event", "inputs": [{"indexed": true, "name":"value1", "type":"uint8"},{"indexed": false, "name":"value2", "type":"uint8"}]}]`
+	type testStruct struct {
+		Value1 uint8
+		Value2 uint8
+	}
+	abi, err := JSON(strings.NewReader(definition))
+	require.NoError(t, err)
+	var b bytes.Buffer
+	b.Write(packNum(reflect.ValueOf(uint8(8))))
+	var rst testStruct
+	require.NoError(t, abi.Unpack(&rst, "test", b.Bytes()))
+	require.Equal(t, uint8(0), rst.Value1)
+	require.Equal(t, uint8(8), rst.Value2)
+}
+
+// TestEventIndexedWithArrayUnpack verifies that decoder will not overlow when static array is indexed input.
+func TestEventIndexedWithArrayUnpack(t *testing.T) {
+	definition := `[{"name": "test", "type": "event", "inputs": [{"indexed": true, "name":"value1", "type":"uint8[2]"},{"indexed": false, "name":"value2", "type":"string"}]}]`
+	type testStruct struct {
+		Value1 [2]uint8
+		Value2 string
+	}
+	abi, err := JSON(strings.NewReader(definition))
+	require.NoError(t, err)
+	var b bytes.Buffer
+	stringOut := "abc"
+	// number of fields that will be encoded * 32
+	b.Write(packNum(reflect.ValueOf(32)))
+	b.Write(packNum(reflect.ValueOf(len(stringOut))))
+	b.Write(common.RightPadBytes([]byte(stringOut), 32))
+
+	var rst testStruct
+	require.NoError(t, abi.Unpack(&rst, "test", b.Bytes()))
+	require.Equal(t, [2]uint8{0, 0}, rst.Value1)
+	require.Equal(t, stringOut, rst.Value2)
+}

accounts/abi/method.go

@@ -18,13 +18,12 @@ package abi
 
 import (
 	"fmt"
-	"reflect"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
-// Callable method given a `Name` and whether the method is a constant.
+// Method represents a callable given a `Name` and whether the method is a constant.
 // If the method is `Const` no transaction needs to be created for this
 // particular Method call. It can easily be simulated using a local VM.
 // For example a `Balance()` method only needs to retrieve something
@@ -35,46 +34,8 @@ import (
 type Method struct {
 	Name    string
 	Const   bool
-	Inputs  []Argument
-	Outputs []Argument
-}
-
-func (method Method) pack(args ...interface{}) ([]byte, error) {
-	// Make sure arguments match up and pack them
-	if len(args) != len(method.Inputs) {
-		return nil, fmt.Errorf("argument count mismatch: %d for %d", len(args), len(method.Inputs))
-	}
-	// variable input is the output appended at the end of packed
-	// output. This is used for strings and bytes types input.
-	var variableInput []byte
-
-	var ret []byte
-	for i, a := range args {
-		input := method.Inputs[i]
-		// pack the input
-		packed, err := input.Type.pack(reflect.ValueOf(a))
-		if err != nil {
-			return nil, fmt.Errorf("`%s` %v", method.Name, err)
-		}
-
-		// check for a slice type (string, bytes, slice)
-		if input.Type.requiresLengthPrefix() {
-			// calculate the offset
-			offset := len(method.Inputs)*32 + len(variableInput)
-			// set the offset
-			ret = append(ret, packNum(reflect.ValueOf(offset))...)
-			// Append the packed output to the variable input. The variable input
-			// will be appended at the end of the input.
-			variableInput = append(variableInput, packed...)
-		} else {
-			// append the packed value to the input
-			ret = append(ret, packed...)
-		}
-	}
-	// append the variable input at the end of the packed input
-	ret = append(ret, variableInput...)
-
-	return ret, nil
+	Inputs  Arguments
+	Outputs Arguments
 }
 
 // Sig returns the methods string signature according to the ABI spec.
@@ -84,35 +45,35 @@ func (method Method) pack(args ...interface{}) ([]byte, error) {
 //     function foo(uint32 a, int b)    =    "foo(uint32,int256)"
 //
 // Please note that "int" is substitute for its canonical representation "int256"
-func (m Method) Sig() string {
-	types := make([]string, len(m.Inputs))
+func (method Method) Sig() string {
+	types := make([]string, len(method.Inputs))
 	i := 0
-	for _, input := range m.Inputs {
+	for _, input := range method.Inputs {
 		types[i] = input.Type.String()
 		i++
 	}
-	return fmt.Sprintf("%v(%v)", m.Name, strings.Join(types, ","))
+	return fmt.Sprintf("%v(%v)", method.Name, strings.Join(types, ","))
 }
 
-func (m Method) String() string {
-	inputs := make([]string, len(m.Inputs))
-	for i, input := range m.Inputs {
+func (method Method) String() string {
+	inputs := make([]string, len(method.Inputs))
+	for i, input := range method.Inputs {
 		inputs[i] = fmt.Sprintf("%v %v", input.Name, input.Type)
 	}
-	outputs := make([]string, len(m.Outputs))
-	for i, output := range m.Outputs {
+	outputs := make([]string, len(method.Outputs))
+	for i, output := range method.Outputs {
 		if len(output.Name) > 0 {
 			outputs[i] = fmt.Sprintf("%v ", output.Name)
 		}
 		outputs[i] += output.Type.String()
 	}
 	constant := ""
-	if m.Const {
+	if method.Const {
 		constant = "constant "
 	}
-	return fmt.Sprintf("function %v(%v) %sreturns(%v)", m.Name, strings.Join(inputs, ", "), constant, strings.Join(outputs, ", "))
+	return fmt.Sprintf("function %v(%v) %sreturns(%v)", method.Name, strings.Join(inputs, ", "), constant, strings.Join(outputs, ", "))
 }
 
-func (m Method) Id() []byte {
-	return crypto.Keccak256([]byte(m.Sig()))[:4]
+func (method Method) Id() []byte {
+	return crypto.Keccak256([]byte(method.Sig()))[:4]
 }

accounts/abi/numbers.go

@@ -25,36 +25,23 @@ import (
 )
 
 var (
-	big_t     = reflect.TypeOf(big.Int{})
-	ubig_t    = reflect.TypeOf(big.Int{})
-	byte_t    = reflect.TypeOf(byte(0))
-	byte_ts   = reflect.TypeOf([]byte(nil))
-	uint_t    = reflect.TypeOf(uint(0))
-	uint8_t   = reflect.TypeOf(uint8(0))
-	uint16_t  = reflect.TypeOf(uint16(0))
-	uint32_t  = reflect.TypeOf(uint32(0))
-	uint64_t  = reflect.TypeOf(uint64(0))
-	int_t     = reflect.TypeOf(int(0))
-	int8_t    = reflect.TypeOf(int8(0))
-	int16_t   = reflect.TypeOf(int16(0))
-	int32_t   = reflect.TypeOf(int32(0))
-	int64_t   = reflect.TypeOf(int64(0))
-	hash_t    = reflect.TypeOf(common.Hash{})
-	address_t = reflect.TypeOf(common.Address{})
-
-	uint_ts   = reflect.TypeOf([]uint(nil))
-	uint8_ts  = reflect.TypeOf([]uint8(nil))
-	uint16_ts = reflect.TypeOf([]uint16(nil))
-	uint32_ts = reflect.TypeOf([]uint32(nil))
-	uint64_ts = reflect.TypeOf([]uint64(nil))
-	ubig_ts   = reflect.TypeOf([]*big.Int(nil))
-
-	int_ts   = reflect.TypeOf([]int(nil))
-	int8_ts  = reflect.TypeOf([]int8(nil))
-	int16_ts = reflect.TypeOf([]int16(nil))
-	int32_ts = reflect.TypeOf([]int32(nil))
-	int64_ts = reflect.TypeOf([]int64(nil))
-	big_ts   = reflect.TypeOf([]*big.Int(nil))
+	bigT      = reflect.TypeOf(&big.Int{})
+	derefbigT = reflect.TypeOf(big.Int{})
+	uint8T    = reflect.TypeOf(uint8(0))
+	uint16T   = reflect.TypeOf(uint16(0))
+	uint32T   = reflect.TypeOf(uint32(0))
+	uint64T   = reflect.TypeOf(uint64(0))
+	intT      = reflect.TypeOf(int(0))
+	int8T     = reflect.TypeOf(int8(0))
+	int16T    = reflect.TypeOf(int16(0))
+	int32T    = reflect.TypeOf(int32(0))
+	int64T    = reflect.TypeOf(int64(0))
+	addressT  = reflect.TypeOf(common.Address{})
+	intTS     = reflect.TypeOf([]int(nil))
+	int8TS    = reflect.TypeOf([]int8(nil))
+	int16TS   = reflect.TypeOf([]int16(nil))
+	int32TS   = reflect.TypeOf([]int32(nil))
+	int64TS   = reflect.TypeOf([]int64(nil))
 )
 
 // U256 converts a big Int into a 256bit EVM number.
@@ -65,7 +52,7 @@ func U256(n *big.Int) []byte {
 // checks whether the given reflect value is signed. This also works for slices with a number type
 func isSigned(v reflect.Value) bool {
 	switch v.Type() {
-	case int_ts, int8_ts, int16_ts, int32_ts, int64_ts, int_t, int8_t, int16_t, int32_t, int64_t:
+	case intTS, int8TS, int16TS, int32TS, int64TS, intT, int8T, int16T, int32T, int64T:
 		return true
 	}
 	return false

accounts/abi/pack.go

@@ -48,9 +48,8 @@ func packElement(t Type, reflectValue reflect.Value) []byte {
 	case BoolTy:
 		if reflectValue.Bool() {
 			return math.PaddedBigBytes(common.Big1, 32)
-		} else {
-			return math.PaddedBigBytes(common.Big0, 32)
 		}
+		return math.PaddedBigBytes(common.Big0, 32)
 	case BytesTy:
 		if reflectValue.Kind() == reflect.Array {
 			reflectValue = mustArrayToByteSlice(reflectValue)
@@ -61,8 +60,9 @@ func packElement(t Type, reflectValue reflect.Value) []byte {
 			reflectValue = mustArrayToByteSlice(reflectValue)
 		}
 		return common.RightPadBytes(reflectValue.Bytes(), 32)
+	default:
+		panic("abi: fatal error")
 	}
-	panic("abi: fatal error")
 }
 
 // packNum packs the given number (using the reflect value) and will cast it to appropriate number representation
@@ -74,6 +74,8 @@ func packNum(value reflect.Value) []byte {
 		return U256(big.NewInt(value.Int()))
 	case reflect.Ptr:
 		return U256(value.Interface().(*big.Int))
+	default:
+		panic("abi: fatal error")
 	}
-	return nil
+
 }

accounts/abi/pack_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -299,6 +299,11 @@ func TestPack(t *testing.T) {
 			[32]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
+		{
+			"uint32[2][3][4]",
+			[4][3][2]uint32{{{1, 2}, {3, 4}, {5, 6}}, {{7, 8}, {9, 10}, {11, 12}}, {{13, 14}, {15, 16}, {17, 18}}, {{19, 20}, {21, 22}, {23, 24}}},
+			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000b000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000d000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000f000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000110000000000000000000000000000000000000000000000000000000000000012000000000000000000000000000000000000000000000000000000000000001300000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000000000000000000000000000015000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000170000000000000000000000000000000000000000000000000000000000000018"),
+		},
 		{
 			"address[]",
 			[]common.Address{{1}, {2}},
@@ -322,12 +327,12 @@ func TestPack(t *testing.T) {
 	} {
 		typ, err := NewType(test.typ)
 		if err != nil {
-			t.Fatal("unexpected parse error:", err)
+			t.Fatalf("%v failed. Unexpected parse error: %v", i, err)
 		}
 
 		output, err := typ.pack(reflect.ValueOf(test.input))
 		if err != nil {
-			t.Fatal("unexpected pack error:", err)
+			t.Fatalf("%v failed. Unexpected pack error: %v", i, err)
 		}
 
 		if !bytes.Equal(output, test.output) {
@@ -435,7 +440,4 @@ func TestPackNumber(t *testing.T) {
 			t.Errorf("test %d: pack mismatch: have %x, want %x", i, packed, tt.packed)
 		}
 	}
-	if packed := packNum(reflect.ValueOf("string")); packed != nil {
-		t.Errorf("expected 'string' to pack to nil. got %x instead", packed)
-	}
 }

accounts/abi/reflect.go

@@ -19,12 +19,13 @@ package abi
 import (
 	"fmt"
 	"reflect"
+	"strings"
 )
 
 // indirect recursively dereferences the value until it either gets the value
 // or finds a big.Int
 func indirect(v reflect.Value) reflect.Value {
-	if v.Kind() == reflect.Ptr && v.Elem().Type() != big_t {
+	if v.Kind() == reflect.Ptr && v.Elem().Type() != derefbigT {
 		return indirect(v.Elem())
 	}
 	return v
@@ -36,26 +37,26 @@ func reflectIntKindAndType(unsigned bool, size int) (reflect.Kind, reflect.Type)
 	switch size {
 	case 8:
 		if unsigned {
-			return reflect.Uint8, uint8_t
+			return reflect.Uint8, uint8T
 		}
-		return reflect.Int8, int8_t
+		return reflect.Int8, int8T
 	case 16:
 		if unsigned {
-			return reflect.Uint16, uint16_t
+			return reflect.Uint16, uint16T
 		}
-		return reflect.Int16, int16_t
+		return reflect.Int16, int16T
 	case 32:
 		if unsigned {
-			return reflect.Uint32, uint32_t
+			return reflect.Uint32, uint32T
 		}
-		return reflect.Int32, int32_t
+		return reflect.Int32, int32T
 	case 64:
 		if unsigned {
-			return reflect.Uint64, uint64_t
+			return reflect.Uint64, uint64T
 		}
-		return reflect.Int64, int64_t
+		return reflect.Int64, int64T
 	}
-	return reflect.Ptr, big_t
+	return reflect.Ptr, bigT
 }
 
 // mustArrayToBytesSlice creates a new byte slice with the exact same size as value
@@ -73,15 +74,9 @@ func mustArrayToByteSlice(value reflect.Value) reflect.Value {
 func set(dst, src reflect.Value, output Argument) error {
 	dstType := dst.Type()
 	srcType := src.Type()
-
 	switch {
-	case dstType.AssignableTo(src.Type()):
+	case dstType.AssignableTo(srcType):
 		dst.Set(src)
-	case dstType.Kind() == reflect.Array && srcType.Kind() == reflect.Slice:
-		if dst.Len() < output.Type.SliceSize {
-			return fmt.Errorf("abi: cannot unmarshal src (len=%d) in to dst (len=%d)", output.Type.SliceSize, dst.Len())
-		}
-		reflect.Copy(dst, src)
 	case dstType.Kind() == reflect.Interface:
 		dst.Set(src)
 	case dstType.Kind() == reflect.Ptr:
@@ -91,3 +86,127 @@ func set(dst, src reflect.Value, output Argument) error {
 	}
 	return nil
 }
+
+// requireAssignable assures that `dest` is a pointer and it's not an interface.
+func requireAssignable(dst, src reflect.Value) error {
+	if dst.Kind() != reflect.Ptr && dst.Kind() != reflect.Interface {
+		return fmt.Errorf("abi: cannot unmarshal %v into %v", src.Type(), dst.Type())
+	}
+	return nil
+}
+
+// requireUnpackKind verifies preconditions for unpacking `args` into `kind`
+func requireUnpackKind(v reflect.Value, t reflect.Type, k reflect.Kind,
+	args Arguments) error {
+
+	switch k {
+	case reflect.Struct:
+	case reflect.Slice, reflect.Array:
+		if minLen := args.LengthNonIndexed(); v.Len() < minLen {
+			return fmt.Errorf("abi: insufficient number of elements in the list/array for unpack, want %d, got %d",
+				minLen, v.Len())
+		}
+	default:
+		return fmt.Errorf("abi: cannot unmarshal tuple into %v", t)
+	}
+	return nil
+}
+
+// mapAbiToStringField maps abi to struct fields.
+// first round: for each Exportable field that contains a `abi:""` tag
+//   and this field name exists in the arguments, pair them together.
+// second round: for each argument field that has not been already linked,
+//   find what variable is expected to be mapped into, if it exists and has not been
+//   used, pair them.
+func mapAbiToStructFields(args Arguments, value reflect.Value) (map[string]string, error) {
+
+	typ := value.Type()
+
+	abi2struct := make(map[string]string)
+	struct2abi := make(map[string]string)
+
+	// first round ~~~
+	for i := 0; i < typ.NumField(); i++ {
+		structFieldName := typ.Field(i).Name
+
+		// skip private struct fields.
+		if structFieldName[:1] != strings.ToUpper(structFieldName[:1]) {
+			continue
+		}
+
+		// skip fields that have no abi:"" tag.
+		var ok bool
+		var tagName string
+		if tagName, ok = typ.Field(i).Tag.Lookup("abi"); !ok {
+			continue
+		}
+
+		// check if tag is empty.
+		if tagName == "" {
+			return nil, fmt.Errorf("struct: abi tag in '%s' is empty", structFieldName)
+		}
+
+		// check which argument field matches with the abi tag.
+		found := false
+		for _, abiField := range args.NonIndexed() {
+			if abiField.Name == tagName {
+				if abi2struct[abiField.Name] != "" {
+					return nil, fmt.Errorf("struct: abi tag in '%s' already mapped", structFieldName)
+				}
+				// pair them
+				abi2struct[abiField.Name] = structFieldName
+				struct2abi[structFieldName] = abiField.Name
+				found = true
+			}
+		}
+
+		// check if this tag has been mapped.
+		if !found {
+			return nil, fmt.Errorf("struct: abi tag '%s' defined but not found in abi", tagName)
+		}
+
+	}
+
+	// second round ~~~
+	for _, arg := range args {
+
+		abiFieldName := arg.Name
+		structFieldName := capitalise(abiFieldName)
+
+		if structFieldName == "" {
+			return nil, fmt.Errorf("abi: purely underscored output cannot unpack to struct")
+		}
+
+		// this abi has already been paired, skip it... unless there exists another, yet unassigned
+		// struct field with the same field name. If so, raise an error:
+		//    abi: [ { "name": "value" } ]
+		//    struct { Value  *big.Int , Value1 *big.Int `abi:"value"`}
+		if abi2struct[abiFieldName] != "" {
+			if abi2struct[abiFieldName] != structFieldName &&
+				struct2abi[structFieldName] == "" &&
+				value.FieldByName(structFieldName).IsValid() {
+				return nil, fmt.Errorf("abi: multiple variables maps to the same abi field '%s'", abiFieldName)
+			}
+			continue
+		}
+
+		// return an error if this struct field has already been paired.
+		if struct2abi[structFieldName] != "" {
+			return nil, fmt.Errorf("abi: multiple outputs mapping to the same struct field '%s'", structFieldName)
+		}
+
+		if value.FieldByName(structFieldName).IsValid() {
+			// pair them
+			abi2struct[abiFieldName] = structFieldName
+			struct2abi[structFieldName] = abiFieldName
+		} else {
+			// not paired, but annotate as used, to detect cases like
+			//   abi : [ { "name": "value" }, { "name": "_value" } ]
+			//   struct { Value *big.Int }
+			struct2abi[structFieldName] = abiFieldName
+		}
+
+	}
+
+	return abi2struct, nil
+}

accounts/abi/type.go

@@ -21,14 +21,17 @@ import (
 	"reflect"
 	"regexp"
 	"strconv"
+	"strings"
 )
 
+// Type enumerator
 const (
 	IntTy byte = iota
 	UintTy
 	BoolTy
 	StringTy
 	SliceTy
+	ArrayTy
 	AddressTy
 	FixedBytesTy
 	BytesTy
@@ -39,9 +42,6 @@ const (
 
 // Type is the reflection of the supported argument type
 type Type struct {
-	IsSlice, IsArray bool
-	SliceSize        int
-
 	Elem *Type
 
 	Kind reflect.Kind
@@ -53,54 +53,57 @@ type Type struct {
 }
 
 var (
-	// fullTypeRegex parses the abi types
-	//
-	// Types can be in the format of:
-	//
-	// 	Input  = Type [ "[" [ Number ] "]" ] Name .
-	// 	Type   = [ "u" ] "int" [ Number ] [ x ] [ Number ].
-	//
-	// Examples:
-	//
-	//      string     int       uint       fixed
-	//      string32   int8      uint8      uint[]
-	//      address    int256    uint256    fixed128x128[2]
-	fullTypeRegex = regexp.MustCompile(`([a-zA-Z0-9]+)(\[([0-9]*)\])?`)
 	// typeRegex parses the abi sub types
 	typeRegex = regexp.MustCompile("([a-zA-Z]+)(([0-9]+)(x([0-9]+))?)?")
 )
 
 // NewType creates a new reflection type of abi type given in t.
 func NewType(t string) (typ Type, err error) {
-	res := fullTypeRegex.FindAllStringSubmatch(t, -1)[0]
-	// check if type is slice and parse type.
-	switch {
-	case res[3] != "":
-		// err is ignored. Already checked for number through the regexp
-		typ.SliceSize, _ = strconv.Atoi(res[3])
-		typ.IsArray = true
-	case res[2] != "":
-		typ.IsSlice, typ.SliceSize = true, -1
-	case res[0] == "":
-		return Type{}, fmt.Errorf("abi: type parse error: %s", t)
+	// check that array brackets are equal if they exist
+	if strings.Count(t, "[") != strings.Count(t, "]") {
+		return Type{}, fmt.Errorf("invalid arg type in abi")
 	}
-	if typ.IsArray || typ.IsSlice {
-		sliceType, err := NewType(res[1])
+
+	typ.stringKind = t
+
+	// if there are brackets, get ready to go into slice/array mode and
+	// recursively create the type
+	if strings.Count(t, "[") != 0 {
+		i := strings.LastIndex(t, "[")
+		// recursively embed the type
+		embeddedType, err := NewType(t[:i])
 		if err != nil {
 			return Type{}, err
 		}
-		typ.Elem = &sliceType
-		typ.stringKind = sliceType.stringKind + t[len(res[1]):]
-		// Although we know that this is an array, we cannot return
-		// as we don't know the type of the element, however, if it
-		// is still an array, then don't determine the type.
-		if typ.Elem.IsArray || typ.Elem.IsSlice {
-			return typ, nil
-		}
-	}
+		// grab the last cell and create a type from there
+		sliced := t[i:]
+		// grab the slice size with regexp
+		re := regexp.MustCompile("[0-9]+")
+		intz := re.FindAllString(sliced, -1)
 
+		if len(intz) == 0 {
+			// is a slice
+			typ.T = SliceTy
+			typ.Kind = reflect.Slice
+			typ.Elem = &embeddedType
+			typ.Type = reflect.SliceOf(embeddedType.Type)
+		} else if len(intz) == 1 {
+			// is a array
+			typ.T = ArrayTy
+			typ.Kind = reflect.Array
+			typ.Elem = &embeddedType
+			typ.Size, err = strconv.Atoi(intz[0])
+			if err != nil {
+				return Type{}, fmt.Errorf("abi: error parsing variable size: %v", err)
+			}
+			typ.Type = reflect.ArrayOf(typ.Size, embeddedType.Type)
+		} else {
+			return Type{}, fmt.Errorf("invalid formatting of array type")
+		}
+		return typ, err
+	}
 	// parse the type and size of the abi-type.
-	parsedType := typeRegex.FindAllStringSubmatch(res[1], -1)[0]
+	parsedType := typeRegex.FindAllStringSubmatch(t, -1)[0]
 	// varSize is the size of the variable
 	var varSize int
 	if len(parsedType[3]) > 0 {
@@ -109,22 +112,15 @@ func NewType(t string) (typ Type, err error) {
 		if err != nil {
 			return Type{}, fmt.Errorf("abi: error parsing variable size: %v", err)
 		}
+	} else {
+		if parsedType[0] == "uint" || parsedType[0] == "int" {
+			// this should fail because it means that there's something wrong with
+			// the abi type (the compiler should always format it to the size...always)
+			return Type{}, fmt.Errorf("unsupported arg type: %s", t)
+		}
 	}
 	// varType is the parsed abi type
-	varType := parsedType[1]
-	// substitute canonical integer
-	if varSize == 0 && (varType == "int" || varType == "uint") {
-		varSize = 256
-		t += "256"
-	}
-
-	// only set stringKind if not array or slice, as for those,
-	// the correct string type has been set
-	if !(typ.IsArray || typ.IsSlice) {
-		typ.stringKind = t
-	}
-
-	switch varType {
+	switch varType := parsedType[1]; varType {
 	case "int":
 		typ.Kind, typ.Type = reflectIntKindAndType(false, varSize)
 		typ.Size = varSize
@@ -136,33 +132,32 @@ func NewType(t string) (typ Type, err error) {
 	case "bool":
 		typ.Kind = reflect.Bool
 		typ.T = BoolTy
+		typ.Type = reflect.TypeOf(bool(false))
 	case "address":
 		typ.Kind = reflect.Array
-		typ.Type = address_t
+		typ.Type = addressT
 		typ.Size = 20
 		typ.T = AddressTy
 	case "string":
 		typ.Kind = reflect.String
-		typ.Size = -1
+		typ.Type = reflect.TypeOf("")
 		typ.T = StringTy
 	case "bytes":
-		sliceType, _ := NewType("uint8")
-		typ.Elem = &sliceType
 		if varSize == 0 {
-			typ.IsSlice = true
 			typ.T = BytesTy
-			typ.SliceSize = -1
+			typ.Kind = reflect.Slice
+			typ.Type = reflect.SliceOf(reflect.TypeOf(byte(0)))
 		} else {
-			typ.IsArray = true
 			typ.T = FixedBytesTy
-			typ.SliceSize = varSize
+			typ.Kind = reflect.Array
+			typ.Size = varSize
+			typ.Type = reflect.ArrayOf(varSize, reflect.TypeOf(byte(0)))
 		}
 	case "function":
-		sliceType, _ := NewType("uint8")
-		typ.Elem = &sliceType
-		typ.IsArray = true
+		typ.Kind = reflect.Array
 		typ.T = FunctionTy
-		typ.SliceSize = 24
+		typ.Size = 24
+		typ.Type = reflect.ArrayOf(24, reflect.TypeOf(byte(0)))
 	default:
 		return Type{}, fmt.Errorf("unsupported arg type: %s", t)
 	}
@@ -183,7 +178,7 @@ func (t Type) pack(v reflect.Value) ([]byte, error) {
 		return nil, err
 	}
 
-	if (t.IsSlice || t.IsArray) && t.T != BytesTy && t.T != FixedBytesTy && t.T != FunctionTy {
+	if t.T == SliceTy || t.T == ArrayTy {
 		var packed []byte
 
 		for i := 0; i < v.Len(); i++ {
@@ -193,18 +188,17 @@ func (t Type) pack(v reflect.Value) ([]byte, error) {
 			}
 			packed = append(packed, val...)
 		}
-		if t.IsSlice {
+		if t.T == SliceTy {
 			return packBytesSlice(packed, v.Len()), nil
-		} else if t.IsArray {
+		} else if t.T == ArrayTy {
 			return packed, nil
 		}
 	}
-
 	return packElement(t, v), nil
 }
 
 // requireLengthPrefix returns whether the type requires any sort of length
 // prefixing.
 func (t Type) requiresLengthPrefix() bool {
-	return t.T != FixedBytesTy && (t.T == StringTy || t.T == BytesTy || t.IsSlice)
+	return t.T == StringTy || t.T == BytesTy || t.T == SliceTy
 }

accounts/abi/type_test.go

@@ -21,6 +21,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/go-ethereum/common"
 )
 
@@ -34,51 +35,58 @@ func TestTypeRegexp(t *testing.T) {
 		blob string
 		kind Type
 	}{
-		{"bool", Type{Kind: reflect.Bool, T: BoolTy, stringKind: "bool"}},
-		{"bool[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Bool, T: BoolTy, Elem: &Type{Kind: reflect.Bool, T: BoolTy, stringKind: "bool"}, stringKind: "bool[]"}},
-		{"bool[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Bool, T: BoolTy, Elem: &Type{Kind: reflect.Bool, T: BoolTy, stringKind: "bool"}, stringKind: "bool[2]"}},
-		{"int8", Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}},
-		{"int16", Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}},
-		{"int32", Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}},
-		{"int64", Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}},
-		{"int256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}},
-		{"int8[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
-		{"int8[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
-		{"int16[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
-		{"int16[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
-		{"int32[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
-		{"int32[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
-		{"int64[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
-		{"int64[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
-		{"int256[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
-		{"int256[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
-		{"uint8", Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}},
-		{"uint16", Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}},
-		{"uint32", Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}},
-		{"uint64", Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}},
-		{"uint256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}},
-		{"uint8[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
-		{"uint8[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
-		{"uint16[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
-		{"uint16[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
-		{"uint32[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
-		{"uint32[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
-		{"uint64[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
-		{"uint64[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
-		{"uint256[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
-		{"uint256[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
-		{"bytes32", Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}},
-		{"bytes[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
-		{"bytes[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[2]"}},
-		{"bytes32[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[]"}},
-		{"bytes32[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[2]"}},
-		{"string", Type{Kind: reflect.String, Size: -1, T: StringTy, stringKind: "string"}},
-		{"string[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[]"}},
-		{"string[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[2]"}},
-		{"address", Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}},
-		{"address[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Array, Type: address_t, T: AddressTy, Size: 20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
-		{"address[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Array, Type: address_t, T: AddressTy, Size: 20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
-
+		{"bool", Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}},
+		{"bool[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool(nil)), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}},
+		{"bool[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}},
+		{"bool[2][]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}},
+		{"bool[][]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}},
+		{"bool[][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}},
+		{"bool[2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}},
+		{"bool[2][][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][][2]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}, stringKind: "bool[2][][2]"}},
+		{"bool[2][2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}, stringKind: "bool[2][2][2]"}},
+		{"bool[][][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}, stringKind: "bool[][][]"}},
+		{"bool[][2][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][2][]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}, stringKind: "bool[][2][]"}},
+		{"int8", Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}},
+		{"int16", Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}},
+		{"int32", Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}},
+		{"int64", Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}},
+		{"int256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}},
+		{"int8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
+		{"int8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
+		{"int16[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
+		{"int16[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
+		{"int32[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
+		{"int32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
+		{"int64[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
+		{"int64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
+		{"int256[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
+		{"int256[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
+		{"uint8", Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}},
+		{"uint16", Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}},
+		{"uint32", Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}},
+		{"uint64", Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}},
+		{"uint256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}},
+		{"uint8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
+		{"uint8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
+		{"uint16[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
+		{"uint16[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
+		{"uint32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
+		{"uint32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
+		{"uint64[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
+		{"uint64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
+		{"uint256[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
+		{"uint256[2]", Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
+		{"bytes32", Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}},
+		{"bytes[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]byte{}), Elem: &Type{Kind: reflect.Slice, Type: reflect.TypeOf([]byte{}), T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
+		{"bytes[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]byte{}), Elem: &Type{T: BytesTy, Type: reflect.TypeOf([]byte{}), Kind: reflect.Slice, stringKind: "bytes"}, stringKind: "bytes[2]"}},
+		{"bytes32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][32]byte{}), Elem: &Type{Kind: reflect.Array, Type: reflect.TypeOf([32]byte{}), T: FixedBytesTy, Size: 32, stringKind: "bytes32"}, stringKind: "bytes32[]"}},
+		{"bytes32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][32]byte{}), Elem: &Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}, stringKind: "bytes32[2]"}},
+		{"string", Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}},
+		{"string[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]string{}), Elem: &Type{Kind: reflect.String, Type: reflect.TypeOf(""), T: StringTy, stringKind: "string"}, stringKind: "string[]"}},
+		{"string[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]string{}), Elem: &Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}, stringKind: "string[2]"}},
+		{"address", Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}},
+		{"address[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
+		{"address[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
 		// TODO when fixed types are implemented properly
 		// {"fixed", Type{}},
 		// {"fixed128x128", Type{}},
@@ -87,13 +95,14 @@ func TestTypeRegexp(t *testing.T) {
 		// {"fixed128x128[]", Type{}},
 		// {"fixed128x128[2]", Type{}},
 	}
-	for i, tt := range tests {
+
+	for _, tt := range tests {
 		typ, err := NewType(tt.blob)
 		if err != nil {
-			t.Errorf("type %d: failed to parse type string: %v", i, err)
+			t.Errorf("type %q: failed to parse type string: %v", tt.blob, err)
 		}
 		if !reflect.DeepEqual(typ, tt.kind) {
-			t.Errorf("type %d: parsed type mismatch:\n  have %+v\n  want %+v", i, typeWithoutStringer(typ), typeWithoutStringer(tt.kind))
+			t.Errorf("type %q: parsed type mismatch:\nGOT %s\nWANT %s ", tt.blob, spew.Sdump(typeWithoutStringer(typ)), spew.Sdump(typeWithoutStringer(tt.kind)))
 		}
 	}
 }
@@ -104,15 +113,90 @@ func TestTypeCheck(t *testing.T) {
 		input interface{}
 		err   string
 	}{
-		{"uint", big.NewInt(1), ""},
-		{"int", big.NewInt(1), ""},
-		{"uint30", big.NewInt(1), ""},
+		{"uint", big.NewInt(1), "unsupported arg type: uint"},
+		{"int", big.NewInt(1), "unsupported arg type: int"},
+		{"uint256", big.NewInt(1), ""},
+		{"uint256[][3][]", [][3][]*big.Int{{{}}}, ""},
+		{"uint256[][][3]", [3][][]*big.Int{{{}}}, ""},
+		{"uint256[3][][]", [][][3]*big.Int{{{}}}, ""},
+		{"uint256[3][3][3]", [3][3][3]*big.Int{{{}}}, ""},
+		{"uint8[][]", [][]uint8{}, ""},
+		{"int256", big.NewInt(1), ""},
+		{"uint8", uint8(1), ""},
+		{"uint16", uint16(1), ""},
+		{"uint32", uint32(1), ""},
+		{"uint64", uint64(1), ""},
+		{"int8", int8(1), ""},
+		{"int16", int16(1), ""},
+		{"int32", int32(1), ""},
+		{"int64", int64(1), ""},
+		{"uint24", big.NewInt(1), ""},
+		{"uint40", big.NewInt(1), ""},
+		{"uint48", big.NewInt(1), ""},
+		{"uint56", big.NewInt(1), ""},
+		{"uint72", big.NewInt(1), ""},
+		{"uint80", big.NewInt(1), ""},
+		{"uint88", big.NewInt(1), ""},
+		{"uint96", big.NewInt(1), ""},
+		{"uint104", big.NewInt(1), ""},
+		{"uint112", big.NewInt(1), ""},
+		{"uint120", big.NewInt(1), ""},
+		{"uint128", big.NewInt(1), ""},
+		{"uint136", big.NewInt(1), ""},
+		{"uint144", big.NewInt(1), ""},
+		{"uint152", big.NewInt(1), ""},
+		{"uint160", big.NewInt(1), ""},
+		{"uint168", big.NewInt(1), ""},
+		{"uint176", big.NewInt(1), ""},
+		{"uint184", big.NewInt(1), ""},
+		{"uint192", big.NewInt(1), ""},
+		{"uint200", big.NewInt(1), ""},
+		{"uint208", big.NewInt(1), ""},
+		{"uint216", big.NewInt(1), ""},
+		{"uint224", big.NewInt(1), ""},
+		{"uint232", big.NewInt(1), ""},
+		{"uint240", big.NewInt(1), ""},
+		{"uint248", big.NewInt(1), ""},
+		{"int24", big.NewInt(1), ""},
+		{"int40", big.NewInt(1), ""},
+		{"int48", big.NewInt(1), ""},
+		{"int56", big.NewInt(1), ""},
+		{"int72", big.NewInt(1), ""},
+		{"int80", big.NewInt(1), ""},
+		{"int88", big.NewInt(1), ""},
+		{"int96", big.NewInt(1), ""},
+		{"int104", big.NewInt(1), ""},
+		{"int112", big.NewInt(1), ""},
+		{"int120", big.NewInt(1), ""},
+		{"int128", big.NewInt(1), ""},
+		{"int136", big.NewInt(1), ""},
+		{"int144", big.NewInt(1), ""},
+		{"int152", big.NewInt(1), ""},
+		{"int160", big.NewInt(1), ""},
+		{"int168", big.NewInt(1), ""},
+		{"int176", big.NewInt(1), ""},
+		{"int184", big.NewInt(1), ""},
+		{"int192", big.NewInt(1), ""},
+		{"int200", big.NewInt(1), ""},
+		{"int208", big.NewInt(1), ""},
+		{"int216", big.NewInt(1), ""},
+		{"int224", big.NewInt(1), ""},
+		{"int232", big.NewInt(1), ""},
+		{"int240", big.NewInt(1), ""},
+		{"int248", big.NewInt(1), ""},
 		{"uint30", uint8(1), "abi: cannot use uint8 as type ptr as argument"},
+		{"uint8", uint16(1), "abi: cannot use uint16 as type uint8 as argument"},
+		{"uint8", uint32(1), "abi: cannot use uint32 as type uint8 as argument"},
+		{"uint8", uint64(1), "abi: cannot use uint64 as type uint8 as argument"},
+		{"uint8", int8(1), "abi: cannot use int8 as type uint8 as argument"},
+		{"uint8", int16(1), "abi: cannot use int16 as type uint8 as argument"},
+		{"uint8", int32(1), "abi: cannot use int32 as type uint8 as argument"},
+		{"uint8", int64(1), "abi: cannot use int64 as type uint8 as argument"},
 		{"uint16", uint16(1), ""},
 		{"uint16", uint8(1), "abi: cannot use uint8 as type uint16 as argument"},
 		{"uint16[]", []uint16{1, 2, 3}, ""},
 		{"uint16[]", [3]uint16{1, 2, 3}, ""},
-		{"uint16[]", []uint32{1, 2, 3}, "abi: cannot use []uint32 as type []uint16 as argument"},
+		{"uint16[]", []uint32{1, 2, 3}, "abi: cannot use []uint32 as type [0]uint16 as argument"},
 		{"uint16[3]", [3]uint32{1, 2, 3}, "abi: cannot use [3]uint32 as type [3]uint16 as argument"},
 		{"uint16[3]", [4]uint16{1, 2, 3}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"},
 		{"uint16[3]", []uint16{1, 2, 3}, ""},
@@ -122,20 +206,64 @@ func TestTypeCheck(t *testing.T) {
 		{"address[1]", [1]common.Address{{1}}, ""},
 		{"address[2]", [1]common.Address{{1}}, "abi: cannot use [1]array as type [2]array as argument"},
 		{"bytes32", [32]byte{}, ""},
+		{"bytes31", [31]byte{}, ""},
+		{"bytes30", [30]byte{}, ""},
+		{"bytes29", [29]byte{}, ""},
+		{"bytes28", [28]byte{}, ""},
+		{"bytes27", [27]byte{}, ""},
+		{"bytes26", [26]byte{}, ""},
+		{"bytes25", [25]byte{}, ""},
+		{"bytes24", [24]byte{}, ""},
+		{"bytes23", [23]byte{}, ""},
+		{"bytes22", [22]byte{}, ""},
+		{"bytes21", [21]byte{}, ""},
+		{"bytes20", [20]byte{}, ""},
+		{"bytes19", [19]byte{}, ""},
+		{"bytes18", [18]byte{}, ""},
+		{"bytes17", [17]byte{}, ""},
+		{"bytes16", [16]byte{}, ""},
+		{"bytes15", [15]byte{}, ""},
+		{"bytes14", [14]byte{}, ""},
+		{"bytes13", [13]byte{}, ""},
+		{"bytes12", [12]byte{}, ""},
+		{"bytes11", [11]byte{}, ""},
+		{"bytes10", [10]byte{}, ""},
+		{"bytes9", [9]byte{}, ""},
+		{"bytes8", [8]byte{}, ""},
+		{"bytes7", [7]byte{}, ""},
+		{"bytes6", [6]byte{}, ""},
+		{"bytes5", [5]byte{}, ""},
+		{"bytes4", [4]byte{}, ""},
+		{"bytes3", [3]byte{}, ""},
+		{"bytes2", [2]byte{}, ""},
+		{"bytes1", [1]byte{}, ""},
 		{"bytes32", [33]byte{}, "abi: cannot use [33]uint8 as type [32]uint8 as argument"},
 		{"bytes32", common.Hash{1}, ""},
-		{"bytes31", [31]byte{}, ""},
+		{"bytes31", common.Hash{1}, "abi: cannot use common.Hash as type [31]uint8 as argument"},
 		{"bytes31", [32]byte{}, "abi: cannot use [32]uint8 as type [31]uint8 as argument"},
 		{"bytes", []byte{0, 1}, ""},
-		{"bytes", [2]byte{0, 1}, ""},
-		{"bytes", common.Hash{1}, ""},
+		{"bytes", [2]byte{0, 1}, "abi: cannot use array as type slice as argument"},
+		{"bytes", common.Hash{1}, "abi: cannot use array as type slice as argument"},
 		{"string", "hello world", ""},
+		{"string", string(""), ""},
+		{"string", []byte{}, "abi: cannot use slice as type string as argument"},
 		{"bytes32[]", [][32]byte{{}}, ""},
 		{"function", [24]byte{}, ""},
+		{"bytes20", common.Address{}, ""},
+		{"address", [20]byte{}, ""},
+		{"address", common.Address{}, ""},
+		{"bytes32[]]", "", "invalid arg type in abi"},
+		{"invalidType", "", "unsupported arg type: invalidType"},
+		{"invalidSlice[]", "", "unsupported arg type: invalidSlice"},
 	} {
 		typ, err := NewType(test.typ)
-		if err != nil {
+		if err != nil && len(test.err) == 0 {
 			t.Fatal("unexpected parse error:", err)
+		} else if err != nil && len(test.err) != 0 {
+			if err.Error() != test.err {
+				t.Errorf("%d failed. Expected err: '%v' got err: '%v'", i, test.err, err)
+			}
+			continue
 		}
 
 		err = typeCheck(typ, reflect.ValueOf(test.input))

accounts/abi/unpack.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -25,122 +25,11 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
-// toGoSliceType parses the input and casts it to the proper slice defined by the ABI
-// argument in T.
-func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
-	index := i * 32
-	// The slice must, at very least be large enough for the index+32 which is exactly the size required
-	// for the [offset in output, size of offset].
-	if index+32 > len(output) {
-		return nil, fmt.Errorf("abi: cannot marshal in to go slice: insufficient size output %d require %d", len(output), index+32)
-	}
-	elem := t.Type.Elem
-
-	// first we need to create a slice of the type
-	var refSlice reflect.Value
-	switch elem.T {
-	case IntTy, UintTy, BoolTy:
-		// create a new reference slice matching the element type
-		switch t.Type.Kind {
-		case reflect.Bool:
-			refSlice = reflect.ValueOf([]bool(nil))
-		case reflect.Uint8:
-			refSlice = reflect.ValueOf([]uint8(nil))
-		case reflect.Uint16:
-			refSlice = reflect.ValueOf([]uint16(nil))
-		case reflect.Uint32:
-			refSlice = reflect.ValueOf([]uint32(nil))
-		case reflect.Uint64:
-			refSlice = reflect.ValueOf([]uint64(nil))
-		case reflect.Int8:
-			refSlice = reflect.ValueOf([]int8(nil))
-		case reflect.Int16:
-			refSlice = reflect.ValueOf([]int16(nil))
-		case reflect.Int32:
-			refSlice = reflect.ValueOf([]int32(nil))
-		case reflect.Int64:
-			refSlice = reflect.ValueOf([]int64(nil))
-		default:
-			refSlice = reflect.ValueOf([]*big.Int(nil))
-		}
-	case AddressTy: // address must be of slice Address
-		refSlice = reflect.ValueOf([]common.Address(nil))
-	case HashTy: // hash must be of slice hash
-		refSlice = reflect.ValueOf([]common.Hash(nil))
-	case FixedBytesTy:
-		refSlice = reflect.ValueOf([][]byte(nil))
-	default: // no other types are supported
-		return nil, fmt.Errorf("abi: unsupported slice type %v", elem.T)
-	}
-
-	var slice []byte
-	var size int
-	var offset int
-	if t.Type.IsSlice {
-		// get the offset which determines the start of this array ...
-		offset = int(binary.BigEndian.Uint64(output[index+24 : index+32]))
-		if offset+32 > len(output) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go slice: offset %d would go over slice boundary (len=%d)", len(output), offset+32)
-		}
-
-		slice = output[offset:]
-		// ... starting with the size of the array in elements ...
-		size = int(binary.BigEndian.Uint64(slice[24:32]))
-		slice = slice[32:]
-		// ... and make sure that we've at the very least the amount of bytes
-		// available in the buffer.
-		if size*32 > len(slice) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go slice: insufficient size output %d require %d", len(output), offset+32+size*32)
-		}
-
-		// reslice to match the required size
-		slice = slice[:size*32]
-	} else if t.Type.IsArray {
-		//get the number of elements in the array
-		size = t.Type.SliceSize
-
-		//check to make sure array size matches up
-		if index+32*size > len(output) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go array: offset %d would go over slice boundary (len=%d)", len(output), index+32*size)
-		}
-		//slice is there for a fixed amount of times
-		slice = output[index : index+size*32]
-	}
-
-	for i := 0; i < size; i++ {
-		var (
-			inter        interface{}             // interface type
-			returnOutput = slice[i*32 : i*32+32] // the return output
-			err          error
-		)
-		// set inter to the correct type (cast)
-		switch elem.T {
-		case IntTy, UintTy:
-			inter = readInteger(t.Type.Kind, returnOutput)
-		case BoolTy:
-			inter, err = readBool(returnOutput)
-			if err != nil {
-				return nil, err
-			}
-		case AddressTy:
-			inter = common.BytesToAddress(returnOutput)
-		case HashTy:
-			inter = common.BytesToHash(returnOutput)
-		case FixedBytesTy:
-			inter = returnOutput
-		}
-		// append the item to our reflect slice
-		refSlice = reflect.Append(refSlice, reflect.ValueOf(inter))
-	}
-
-	// return the interface
-	return refSlice.Interface(), nil
-}
-
+// reads the integer based on its kind
 func readInteger(kind reflect.Kind, b []byte) interface{} {
 	switch kind {
 	case reflect.Uint8:
-		return uint8(b[len(b)-1])
+		return b[len(b)-1]
 	case reflect.Uint16:
 		return binary.BigEndian.Uint16(b[len(b)-2:])
 	case reflect.Uint32:
@@ -160,13 +49,10 @@ func readInteger(kind reflect.Kind, b []byte) interface{} {
 	}
 }
 
+// reads a bool
 func readBool(word []byte) (bool, error) {
-	if len(word) != 32 {
-		return false, fmt.Errorf("abi: fatal error: incorrect word length")
-	}
-
-	for i, b := range word {
-		if b != 0 && i != 31 {
+	for _, b := range word[:31] {
+		if b != 0 {
 			return false, errBadBool
 		}
 	}
@@ -178,58 +64,167 @@ func readBool(word []byte) (bool, error) {
 	default:
 		return false, errBadBool
 	}
+}
+
+// A function type is simply the address with the function selection signature at the end.
+// This enforces that standard by always presenting it as a 24-array (address + sig = 24 bytes)
+func readFunctionType(t Type, word []byte) (funcTy [24]byte, err error) {
+	if t.T != FunctionTy {
+		return [24]byte{}, fmt.Errorf("abi: invalid type in call to make function type byte array")
+	}
+	if garbage := binary.BigEndian.Uint64(word[24:32]); garbage != 0 {
+		err = fmt.Errorf("abi: got improperly encoded function type, got %v", word)
+	} else {
+		copy(funcTy[:], word[0:24])
+	}
+	return
+}
+
+// through reflection, creates a fixed array to be read from
+func readFixedBytes(t Type, word []byte) (interface{}, error) {
+	if t.T != FixedBytesTy {
+		return nil, fmt.Errorf("abi: invalid type in call to make fixed byte array")
+	}
+	// convert
+	array := reflect.New(t.Type).Elem()
+
+	reflect.Copy(array, reflect.ValueOf(word[0:t.Size]))
+	return array.Interface(), nil
 
 }
 
-// toGoType parses the input and casts it to the proper type defined by the ABI
-// argument in T.
-func toGoType(i int, t Argument, output []byte) (interface{}, error) {
-	// we need to treat slices differently
-	if (t.Type.IsSlice || t.Type.IsArray) && t.Type.T != BytesTy && t.Type.T != StringTy && t.Type.T != FixedBytesTy && t.Type.T != FunctionTy {
-		return toGoSlice(i, t, output)
+func getFullElemSize(elem *Type) int {
+	//all other should be counted as 32 (slices have pointers to respective elements)
+	size := 32
+	//arrays wrap it, each element being the same size
+	for elem.T == ArrayTy {
+		size *= elem.Size
+		elem = elem.Elem
+	}
+	return size
+}
+
+// iteratively unpack elements
+func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) {
+	if size < 0 {
+		return nil, fmt.Errorf("cannot marshal input to array, size is negative (%d)", size)
+	}
+	if start+32*size > len(output) {
+		return nil, fmt.Errorf("abi: cannot marshal in to go array: offset %d would go over slice boundary (len=%d)", len(output), start+32*size)
 	}
 
-	index := i * 32
+	// this value will become our slice or our array, depending on the type
+	var refSlice reflect.Value
+
+	if t.T == SliceTy {
+		// declare our slice
+		refSlice = reflect.MakeSlice(t.Type, size, size)
+	} else if t.T == ArrayTy {
+		// declare our array
+		refSlice = reflect.New(t.Type).Elem()
+	} else {
+		return nil, fmt.Errorf("abi: invalid type in array/slice unpacking stage")
+	}
+
+	// Arrays have packed elements, resulting in longer unpack steps.
+	// Slices have just 32 bytes per element (pointing to the contents).
+	elemSize := 32
+	if t.T == ArrayTy {
+		elemSize = getFullElemSize(t.Elem)
+	}
+
+	for i, j := start, 0; j < size; i, j = i+elemSize, j+1 {
+
+		inter, err := toGoType(i, *t.Elem, output)
+		if err != nil {
+			return nil, err
+		}
+
+		// append the item to our reflect slice
+		refSlice.Index(j).Set(reflect.ValueOf(inter))
+	}
+
+	// return the interface
+	return refSlice.Interface(), nil
+}
+
+// toGoType parses the output bytes and recursively assigns the value of these bytes
+// into a go type with accordance with the ABI spec.
+func toGoType(index int, t Type, output []byte) (interface{}, error) {
 	if index+32 > len(output) {
 		return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), index+32)
 	}
 
-	// Parse the given index output and check whether we need to read
-	// a different offset and length based on the type (i.e. string, bytes)
-	var returnOutput []byte
-	switch t.Type.T {
-	case StringTy, BytesTy: // variable arrays are written at the end of the return bytes
-		// parse offset from which we should start reading
-		offset := int(binary.BigEndian.Uint64(output[index+24 : index+32]))
-		if offset+32 > len(output) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32)
-		}
-		// parse the size up until we should be reading
-		size := int(binary.BigEndian.Uint64(output[offset+24 : offset+32]))
-		if offset+32+size > len(output) {
-			return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32+size)
-		}
+	var (
+		returnOutput []byte
+		begin, end   int
+		err          error
+	)
 
-		// get the bytes for this return value
-		returnOutput = output[offset+32 : offset+32+size]
-	default:
+	// if we require a length prefix, find the beginning word and size returned.
+	if t.requiresLengthPrefix() {
+		begin, end, err = lengthPrefixPointsTo(index, output)
+		if err != nil {
+			return nil, err
+		}
+	} else {
 		returnOutput = output[index : index+32]
 	}
 
-	// convert the bytes to whatever is specified by the ABI.
-	switch t.Type.T {
+	switch t.T {
+	case SliceTy:
+		return forEachUnpack(t, output, begin, end)
+	case ArrayTy:
+		return forEachUnpack(t, output, index, t.Size)
+	case StringTy: // variable arrays are written at the end of the return bytes
+		return string(output[begin : begin+end]), nil
 	case IntTy, UintTy:
-		return readInteger(t.Type.Kind, returnOutput), nil
+		return readInteger(t.Kind, returnOutput), nil
 	case BoolTy:
 		return readBool(returnOutput)
 	case AddressTy:
 		return common.BytesToAddress(returnOutput), nil
 	case HashTy:
 		return common.BytesToHash(returnOutput), nil
-	case BytesTy, FixedBytesTy, FunctionTy:
-		return returnOutput, nil
-	case StringTy:
-		return string(returnOutput), nil
+	case BytesTy:
+		return output[begin : begin+end], nil
+	case FixedBytesTy:
+		return readFixedBytes(t, returnOutput)
+	case FunctionTy:
+		return readFunctionType(t, returnOutput)
+	default:
+		return nil, fmt.Errorf("abi: unknown type %v", t.T)
 	}
-	return nil, fmt.Errorf("abi: unknown type %v", t.Type.T)
+}
+
+// interprets a 32 byte slice as an offset and then determines which indice to look to decode the type.
+func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err error) {
+	bigOffsetEnd := big.NewInt(0).SetBytes(output[index : index+32])
+	bigOffsetEnd.Add(bigOffsetEnd, common.Big32)
+	outputLength := big.NewInt(int64(len(output)))
+
+	if bigOffsetEnd.Cmp(outputLength) > 0 {
+		return 0, 0, fmt.Errorf("abi: cannot marshal in to go slice: offset %v would go over slice boundary (len=%v)", bigOffsetEnd, outputLength)
+	}
+
+	if bigOffsetEnd.BitLen() > 63 {
+		return 0, 0, fmt.Errorf("abi offset larger than int64: %v", bigOffsetEnd)
+	}
+
+	offsetEnd := int(bigOffsetEnd.Uint64())
+	lengthBig := big.NewInt(0).SetBytes(output[offsetEnd-32 : offsetEnd])
+
+	totalSize := big.NewInt(0)
+	totalSize.Add(totalSize, bigOffsetEnd)
+	totalSize.Add(totalSize, lengthBig)
+	if totalSize.BitLen() > 63 {
+		return 0, 0, fmt.Errorf("abi length larger than int64: %v", totalSize)
+	}
+
+	if totalSize.Cmp(outputLength) > 0 {
+		return 0, 0, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %v require %v", outputLength, totalSize)
+	}
+	start = int(bigOffsetEnd.Uint64())
+	length = int(lengthBig.Uint64())
+	return
 }

accounts/accounts.go

@@ -42,8 +42,9 @@ type Wallet interface {
 	URL() URL
 
 	// Status returns a textual status to aid the user in the current state of the
-	// wallet.
-	Status() string
+	// wallet. It also returns an error indicating any failure the wallet might have
+	// encountered.
+	Status() (string, error)
 
 	// Open initializes access to a wallet instance. It is not meant to unlock or
 	// decrypt account keys, rather simply to establish a connection to hardware
@@ -147,9 +148,26 @@ type Backend interface {
 	Subscribe(sink chan<- WalletEvent) event.Subscription
 }
 
+// WalletEventType represents the different event types that can be fired by
+// the wallet subscription subsystem.
+type WalletEventType int
+
+const (
+	// WalletArrived is fired when a new wallet is detected either via USB or via
+	// a filesystem event in the keystore.
+	WalletArrived WalletEventType = iota
+
+	// WalletOpened is fired when a wallet is successfully opened with the purpose
+	// of starting any background processes such as automatic key derivation.
+	WalletOpened
+
+	// WalletDropped
+	WalletDropped
+)
+
 // WalletEvent is an event fired by an account backend when a wallet arrival or
 // departure is detected.
 type WalletEvent struct {
-	Wallet Wallet // Wallet instance arrived or departed
-	Arrive bool   // Whether the wallet was added or removed
+	Wallet Wallet          // Wallet instance arrived or departed
+	Kind   WalletEventType // Event type that happened in the system
 }

accounts/errors.go

@@ -62,7 +62,7 @@ func NewAuthNeededError(needed string) error {
 	}
 }
 
-// Error implements the standard error interfacel.
+// Error implements the standard error interface.
 func (err *AuthNeededError) Error() string {
 	return fmt.Sprintf("authentication needed: %s", err.Needed)
 }

accounts/hd.go

@@ -27,12 +27,17 @@ import (
 // DefaultRootDerivationPath is the root path to which custom derivation endpoints
 // are appended. As such, the first account will be at m/44'/60'/0'/0, the second
 // at m/44'/60'/0'/1, etc.
-var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0}
+var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}
 
 // DefaultBaseDerivationPath is the base path from which custom derivation endpoints
 // are incremented. As such, the first account will be at m/44'/60'/0'/0, the second
 // at m/44'/60'/0'/1, etc.
-var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}
+var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0}
+
+// DefaultLedgerBaseDerivationPath is the base path from which custom derivation endpoints
+// are incremented. As such, the first account will be at m/44'/60'/0'/0, the second
+// at m/44'/60'/0'/1, etc.
+var DefaultLedgerBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}
 
 // DerivationPath represents the computer friendly version of a hierarchical
 // deterministic wallet account derivaion path.

accounts/hd_test.go

@@ -37,11 +37,11 @@ func TestHDPathParsing(t *testing.T) {
 		{"m/2147483692/2147483708/2147483648/2147483648", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
 
 		// Plain relative derivation paths
-		{"0", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
-		{"128", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 128}},
-		{"0'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
-		{"128'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 128}},
-		{"2147483648", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+		{"0", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0}},
+		{"128", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 128}},
+		{"0'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0x80000000 + 0}},
+		{"128'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0x80000000 + 128}},
+		{"2147483648", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0x80000000 + 0}},
 
 		// Hexadecimal absolute derivation paths
 		{"m/0x2C'/0x3c'/0x00'/0x00", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
@@ -52,11 +52,11 @@ func TestHDPathParsing(t *testing.T) {
 		{"m/0x8000002C/0x8000003c/0x80000000/0x80000000", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
 
 		// Hexadecimal relative derivation paths
-		{"0x00", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
-		{"0x80", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 128}},
-		{"0x00'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
-		{"0x80'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 128}},
-		{"0x80000000", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+		{"0x00", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0}},
+		{"0x80", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 128}},
+		{"0x00'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0x80000000 + 0}},
+		{"0x80'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0x80000000 + 128}},
+		{"0x80000000", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0x80000000 + 0}},
 
 		// Weird inputs just to ensure they work
 		{"	m  /   44			'\n/\n   60	\n\n\t'   /\n0 ' /\t\t	0", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},

accounts/keystore/account_cache.go

@@ -20,7 +20,6 @@ import (
 	"bufio"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -31,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
+	"gopkg.in/fatih/set.v0"
 )
 
 // Minimum amount of time between cache reloads. This limit applies if the platform does
@@ -71,6 +71,7 @@ type accountCache struct {
 	byAddr   map[common.Address][]accounts.Account
 	throttle *time.Timer
 	notify   chan struct{}
+	fileC    fileCache
 }
 
 func newAccountCache(keydir string) (*accountCache, chan struct{}) {
@@ -78,6 +79,7 @@ func newAccountCache(keydir string) (*accountCache, chan struct{}) {
 		keydir: keydir,
 		byAddr: make(map[common.Address][]accounts.Account),
 		notify: make(chan struct{}, 1),
+		fileC:  fileCache{all: set.NewNonTS()},
 	}
 	ac.watcher = newWatcher(ac)
 	return ac, ac.notify
@@ -127,6 +129,23 @@ func (ac *accountCache) delete(removed accounts.Account) {
 	}
 }
 
+// deleteByFile removes an account referenced by the given path.
+func (ac *accountCache) deleteByFile(path string) {
+	ac.mu.Lock()
+	defer ac.mu.Unlock()
+	i := sort.Search(len(ac.all), func(i int) bool { return ac.all[i].URL.Path >= path })
+
+	if i < len(ac.all) && ac.all[i].URL.Path == path {
+		removed := ac.all[i]
+		ac.all = append(ac.all[:i], ac.all[i+1:]...)
+		if ba := removeAccount(ac.byAddr[removed.Address], removed); len(ba) == 0 {
+			delete(ac.byAddr, removed.Address)
+		} else {
+			ac.byAddr[removed.Address] = ba
+		}
+	}
+}
+
 func removeAccount(slice []accounts.Account, elem accounts.Account) []accounts.Account {
 	for i := range slice {
 		if slice[i] == elem {
@@ -167,15 +186,16 @@ func (ac *accountCache) find(a accounts.Account) (accounts.Account, error) {
 	default:
 		err := &AmbiguousAddrError{Addr: a.Address, Matches: make([]accounts.Account, len(matches))}
 		copy(err.Matches, matches)
+		sort.Sort(accountsByURL(err.Matches))
 		return accounts.Account{}, err
 	}
 }
 
 func (ac *accountCache) maybeReload() {
 	ac.mu.Lock()
-	defer ac.mu.Unlock()
 
 	if ac.watcher.running {
+		ac.mu.Unlock()
 		return // A watcher is running and will keep the cache up-to-date.
 	}
 	if ac.throttle == nil {
@@ -184,12 +204,15 @@ func (ac *accountCache) maybeReload() {
 		select {
 		case <-ac.throttle.C:
 		default:
+			ac.mu.Unlock()
 			return // The cache was reloaded recently.
 		}
 	}
+	// No watcher running, start it.
 	ac.watcher.start()
-	ac.reload()
 	ac.throttle.Reset(minReloadInterval)
+	ac.mu.Unlock()
+	ac.scanAccounts()
 }
 
 func (ac *accountCache) close() {
@@ -205,80 +228,71 @@ func (ac *accountCache) close() {
 	ac.mu.Unlock()
 }
 
-// reload caches addresses of existing accounts.
-// Callers must hold ac.mu.
-func (ac *accountCache) reload() {
-	accounts, err := ac.scan()
+// scanAccounts checks if any changes have occurred on the filesystem, and
+// updates the account cache accordingly
+func (ac *accountCache) scanAccounts() error {
+	// Scan the entire folder metadata for file changes
+	creates, deletes, updates, err := ac.fileC.scan(ac.keydir)
 	if err != nil {
 		log.Debug("Failed to reload keystore contents", "err", err)
+		return err
 	}
-	ac.all = accounts
-	sort.Sort(ac.all)
-	for k := range ac.byAddr {
-		delete(ac.byAddr, k)
+	if creates.Size() == 0 && deletes.Size() == 0 && updates.Size() == 0 {
+		return nil
 	}
-	for _, a := range accounts {
-		ac.byAddr[a.Address] = append(ac.byAddr[a.Address], a)
+	// Create a helper method to scan the contents of the key files
+	var (
+		buf = new(bufio.Reader)
+		key struct {
+			Address string `json:"address"`
+		}
+	)
+	readAccount := func(path string) *accounts.Account {
+		fd, err := os.Open(path)
+		if err != nil {
+			log.Trace("Failed to open keystore file", "path", path, "err", err)
+			return nil
+		}
+		defer fd.Close()
+		buf.Reset(fd)
+		// Parse the address.
+		key.Address = ""
+		err = json.NewDecoder(buf).Decode(&key)
+		addr := common.HexToAddress(key.Address)
+		switch {
+		case err != nil:
+			log.Debug("Failed to decode keystore key", "path", path, "err", err)
+		case (addr == common.Address{}):
+			log.Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address")
+		default:
+			return &accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}}
+		}
+		return nil
 	}
+	// Process all the file diffs
+	start := time.Now()
+
+	for _, p := range creates.List() {
+		if a := readAccount(p.(string)); a != nil {
+			ac.add(*a)
+		}
+	}
+	for _, p := range deletes.List() {
+		ac.deleteByFile(p.(string))
+	}
+	for _, p := range updates.List() {
+		path := p.(string)
+		ac.deleteByFile(path)
+		if a := readAccount(path); a != nil {
+			ac.add(*a)
+		}
+	}
+	end := time.Now()
+
 	select {
 	case ac.notify <- struct{}{}:
 	default:
 	}
-	log.Debug("Reloaded keystore contents", "accounts", len(ac.all))
-}
-
-func (ac *accountCache) scan() ([]accounts.Account, error) {
-	files, err := ioutil.ReadDir(ac.keydir)
-	if err != nil {
-		return nil, err
-	}
-
-	var (
-		buf     = new(bufio.Reader)
-		addrs   []accounts.Account
-		keyJSON struct {
-			Address string `json:"address"`
-		}
-	)
-	for _, fi := range files {
-		path := filepath.Join(ac.keydir, fi.Name())
-		if skipKeyFile(fi) {
-			log.Trace("Ignoring file on account scan", "path", path)
-			continue
-		}
-		logger := log.New("path", path)
-
-		fd, err := os.Open(path)
-		if err != nil {
-			logger.Trace("Failed to open keystore file", "err", err)
-			continue
-		}
-		buf.Reset(fd)
-		// Parse the address.
-		keyJSON.Address = ""
-		err = json.NewDecoder(buf).Decode(&keyJSON)
-		addr := common.HexToAddress(keyJSON.Address)
-		switch {
-		case err != nil:
-			logger.Debug("Failed to decode keystore key", "err", err)
-		case (addr == common.Address{}):
-			logger.Debug("Failed to decode keystore key", "err", "missing or zero address")
-		default:
-			addrs = append(addrs, accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}})
-		}
-		fd.Close()
-	}
-	return addrs, err
-}
-
-func skipKeyFile(fi os.FileInfo) bool {
-	// Skip editor backups and UNIX-style hidden files.
-	if strings.HasSuffix(fi.Name(), "~") || strings.HasPrefix(fi.Name(), ".") {
-		return true
-	}
-	// Skip misc special files, directories (yes, symlinks too).
-	if fi.IsDir() || fi.Mode()&os.ModeType != 0 {
-		return true
-	}
-	return false
+	log.Trace("Handled keystore changes", "time", end.Sub(start))
+	return nil
 }

accounts/keystore/account_cache_test.go

@@ -18,6 +18,7 @@ package keystore
 
 import (
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -58,7 +59,7 @@ func TestWatchNewFile(t *testing.T) {
 
 	// Ensure the watcher is started before adding any files.
 	ks.Accounts()
-	time.Sleep(200 * time.Millisecond)
+	time.Sleep(1000 * time.Millisecond)
 
 	// Move in the files.
 	wantAccounts := make([]accounts.Account, len(cachetestAccounts))
@@ -295,3 +296,111 @@ func TestCacheFind(t *testing.T) {
 		}
 	}
 }
+
+func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error {
+	var list []accounts.Account
+	for d := 200 * time.Millisecond; d < 8*time.Second; d *= 2 {
+		list = ks.Accounts()
+		if reflect.DeepEqual(list, wantAccounts) {
+			// ks should have also received change notifications
+			select {
+			case <-ks.changes:
+			default:
+				return fmt.Errorf("wasn't notified of new accounts")
+			}
+			return nil
+		}
+		time.Sleep(d)
+	}
+	return fmt.Errorf("\ngot  %v\nwant %v", list, wantAccounts)
+}
+
+// TestUpdatedKeyfileContents tests that updating the contents of a keystore file
+// is noticed by the watcher, and the account cache is updated accordingly
+func TestUpdatedKeyfileContents(t *testing.T) {
+	t.Parallel()
+
+	// Create a temporary kesytore to test with
+	rand.Seed(time.Now().UnixNano())
+	dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-watch-test-%d-%d", os.Getpid(), rand.Int()))
+	ks := NewKeyStore(dir, LightScryptN, LightScryptP)
+
+	list := ks.Accounts()
+	if len(list) > 0 {
+		t.Error("initial account list not empty:", list)
+	}
+	time.Sleep(100 * time.Millisecond)
+
+	// Create the directory and copy a key file into it.
+	os.MkdirAll(dir, 0700)
+	defer os.RemoveAll(dir)
+	file := filepath.Join(dir, "aaa")
+
+	// Place one of our testfiles in there
+	if err := cp.CopyFile(file, cachetestAccounts[0].URL.Path); err != nil {
+		t.Fatal(err)
+	}
+
+	// ks should see the account.
+	wantAccounts := []accounts.Account{cachetestAccounts[0]}
+	wantAccounts[0].URL = accounts.URL{Scheme: KeyStoreScheme, Path: file}
+	if err := waitForAccounts(wantAccounts, ks); err != nil {
+		t.Error(err)
+		return
+	}
+
+	// needed so that modTime of `file` is different to its current value after forceCopyFile
+	time.Sleep(1000 * time.Millisecond)
+
+	// Now replace file contents
+	if err := forceCopyFile(file, cachetestAccounts[1].URL.Path); err != nil {
+		t.Fatal(err)
+		return
+	}
+	wantAccounts = []accounts.Account{cachetestAccounts[1]}
+	wantAccounts[0].URL = accounts.URL{Scheme: KeyStoreScheme, Path: file}
+	if err := waitForAccounts(wantAccounts, ks); err != nil {
+		t.Errorf("First replacement failed")
+		t.Error(err)
+		return
+	}
+
+	// needed so that modTime of `file` is different to its current value after forceCopyFile
+	time.Sleep(1000 * time.Millisecond)
+
+	// Now replace file contents again
+	if err := forceCopyFile(file, cachetestAccounts[2].URL.Path); err != nil {
+		t.Fatal(err)
+		return
+	}
+	wantAccounts = []accounts.Account{cachetestAccounts[2]}
+	wantAccounts[0].URL = accounts.URL{Scheme: KeyStoreScheme, Path: file}
+	if err := waitForAccounts(wantAccounts, ks); err != nil {
+		t.Errorf("Second replacement failed")
+		t.Error(err)
+		return
+	}
+
+	// needed so that modTime of `file` is different to its current value after ioutil.WriteFile
+	time.Sleep(1000 * time.Millisecond)
+
+	// Now replace file contents with crap
+	if err := ioutil.WriteFile(file, []byte("foo"), 0644); err != nil {
+		t.Fatal(err)
+		return
+	}
+	if err := waitForAccounts([]accounts.Account{}, ks); err != nil {
+		t.Errorf("Emptying account file failed")
+		t.Error(err)
+		return
+	}
+}
+
+// forceCopyFile is like cp.CopyFile, but doesn't complain if the destination exists.
+func forceCopyFile(dst, src string) error {
+	data, err := ioutil.ReadFile(src)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(dst, data, 0644)
+}

accounts/keystore/file_cache.go

@@ -0,0 +1,102 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package keystore
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	set "gopkg.in/fatih/set.v0"
+)
+
+// fileCache is a cache of files seen during scan of keystore.
+type fileCache struct {
+	all     *set.SetNonTS // Set of all files from the keystore folder
+	lastMod time.Time     // Last time instance when a file was modified
+	mu      sync.RWMutex
+}
+
+// scan performs a new scan on the given directory, compares against the already
+// cached filenames, and returns file sets: creates, deletes, updates.
+func (fc *fileCache) scan(keyDir string) (set.Interface, set.Interface, set.Interface, error) {
+	t0 := time.Now()
+
+	// List all the failes from the keystore folder
+	files, err := ioutil.ReadDir(keyDir)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	t1 := time.Now()
+
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	// Iterate all the files and gather their metadata
+	all := set.NewNonTS()
+	mods := set.NewNonTS()
+
+	var newLastMod time.Time
+	for _, fi := range files {
+		// Skip any non-key files from the folder
+		path := filepath.Join(keyDir, fi.Name())
+		if skipKeyFile(fi) {
+			log.Trace("Ignoring file on account scan", "path", path)
+			continue
+		}
+		// Gather the set of all and fresly modified files
+		all.Add(path)
+
+		modified := fi.ModTime()
+		if modified.After(fc.lastMod) {
+			mods.Add(path)
+		}
+		if modified.After(newLastMod) {
+			newLastMod = modified
+		}
+	}
+	t2 := time.Now()
+
+	// Update the tracked files and return the three sets
+	deletes := set.Difference(fc.all, all)   // Deletes = previous - current
+	creates := set.Difference(all, fc.all)   // Creates = current - previous
+	updates := set.Difference(mods, creates) // Updates = modified - creates
+
+	fc.all, fc.lastMod = all, newLastMod
+	t3 := time.Now()
+
+	// Report on the scanning stats and return
+	log.Debug("FS scan times", "list", t1.Sub(t0), "set", t2.Sub(t1), "diff", t3.Sub(t2))
+	return creates, deletes, updates, nil
+}
+
+// skipKeyFile ignores editor backups, hidden files and folders/symlinks.
+func skipKeyFile(fi os.FileInfo) bool {
+	// Skip editor backups and UNIX-style hidden files.
+	if strings.HasSuffix(fi.Name(), "~") || strings.HasPrefix(fi.Name(), ".") {
+		return true
+	}
+	// Skip misc special files, directories (yes, symlinks too).
+	if fi.IsDir() || fi.Mode()&os.ModeType != 0 {
+		return true
+	}
+	return false
+}

accounts/keystore/key.go

@@ -91,14 +91,6 @@ type cipherparamsJSON struct {
 	IV string `json:"iv"`
 }
 
-type scryptParamsJSON struct {
-	N     int    `json:"n"`
-	R     int    `json:"r"`
-	P     int    `json:"p"`
-	DkLen int    `json:"dklen"`
-	Salt  string `json:"salt"`
-}
-
 func (k *Key) MarshalJSON() (j []byte, err error) {
 	jStruct := plainKeyJSON{
 		hex.EncodeToString(k.Address[:]),

accounts/keystore/keystore.go

@@ -143,14 +143,14 @@ func (ks *KeyStore) refreshWallets() {
 	for _, account := range accs {
 		// Drop wallets while they were in front of the next account
 		for len(ks.wallets) > 0 && ks.wallets[0].URL().Cmp(account.URL) < 0 {
-			events = append(events, accounts.WalletEvent{Wallet: ks.wallets[0], Arrive: false})
+			events = append(events, accounts.WalletEvent{Wallet: ks.wallets[0], Kind: accounts.WalletDropped})
 			ks.wallets = ks.wallets[1:]
 		}
 		// If there are no more wallets or the account is before the next, wrap new wallet
 		if len(ks.wallets) == 0 || ks.wallets[0].URL().Cmp(account.URL) > 0 {
 			wallet := &keystoreWallet{account: account, keystore: ks}
 
-			events = append(events, accounts.WalletEvent{Wallet: wallet, Arrive: true})
+			events = append(events, accounts.WalletEvent{Wallet: wallet, Kind: accounts.WalletArrived})
 			wallets = append(wallets, wallet)
 			continue
 		}
@@ -163,7 +163,7 @@ func (ks *KeyStore) refreshWallets() {
 	}
 	// Drop any leftover wallets and set the new batch
 	for _, wallet := range ks.wallets {
-		events = append(events, accounts.WalletEvent{Wallet: wallet, Arrive: false})
+		events = append(events, accounts.WalletEvent{Wallet: wallet, Kind: accounts.WalletDropped})
 	}
 	ks.wallets = wallets
 	ks.mu.Unlock()

accounts/keystore/keystore_passphrase.go

@@ -28,6 +28,7 @@ package keystore
 import (
 	"bytes"
 	"crypto/aes"
+	crand "crypto/rand"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
@@ -90,6 +91,12 @@ func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string)
 	return key, nil
 }
 
+// StoreKey generates a key, encrypts with 'auth' and stores in the given directory
+func StoreKey(dir, auth string, scryptN, scryptP int) (common.Address, error) {
+	_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP}, crand.Reader, auth)
+	return a.Address, err
+}
+
 func (ks keyStorePassphrase) StoreKey(filename string, key *Key, auth string) error {
 	keyjson, err := EncryptKey(key, auth, ks.scryptN, ks.scryptP)
 	if err != nil {
@@ -101,9 +108,8 @@ func (ks keyStorePassphrase) StoreKey(filename string, key *Key, auth string) er
 func (ks keyStorePassphrase) JoinPath(filename string) string {
 	if filepath.IsAbs(filename) {
 		return filename
-	} else {
-		return filepath.Join(ks.keysDirPath, filename)
 	}
+	return filepath.Join(ks.keysDirPath, filename)
 }
 
 // EncryptKey encrypts a key using the specified scrypt parameters into a json
@@ -140,7 +146,7 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
 		Cipher:       "aes-128-ctr",
 		CipherText:   hex.EncodeToString(cipherText),
 		CipherParams: cipherParamsJSON,
-		KDF:          "scrypt",
+		KDF:          keyHeaderKDF,
 		KDFParams:    scryptParamsJSON,
 		MAC:          hex.EncodeToString(mac),
 	}
@@ -275,7 +281,7 @@ func getKDFKey(cryptoJSON cryptoJSON, auth string) ([]byte, error) {
 	}
 	dkLen := ensureInt(cryptoJSON.KDFParams["dklen"])
 
-	if cryptoJSON.KDF == "scrypt" {
+	if cryptoJSON.KDF == keyHeaderKDF {
 		n := ensureInt(cryptoJSON.KDFParams["n"])
 		r := ensureInt(cryptoJSON.KDFParams["r"])
 		p := ensureInt(cryptoJSON.KDFParams["p"])

accounts/keystore/keystore_plain.go

@@ -56,7 +56,6 @@ func (ks keyStorePlain) StoreKey(filename string, key *Key, auth string) error {
 func (ks keyStorePlain) JoinPath(filename string) string {
 	if filepath.IsAbs(filename) {
 		return filename
-	} else {
-		return filepath.Join(ks.keysDirPath, filename)
 	}
+	return filepath.Join(ks.keysDirPath, filename)
 }

accounts/keystore/keystore_test.go

@@ -272,82 +272,104 @@ func TestWalletNotifierLifecycle(t *testing.T) {
 	t.Errorf("wallet notifier didn't terminate after unsubscribe")
 }
 
+type walletEvent struct {
+	accounts.WalletEvent
+	a accounts.Account
+}
+
 // Tests that wallet notifications and correctly fired when accounts are added
 // or deleted from the keystore.
 func TestWalletNotifications(t *testing.T) {
-	// Create a temporary kesytore to test with
 	dir, ks := tmpKeyStore(t, false)
 	defer os.RemoveAll(dir)
 
-	// Subscribe to the wallet feed
-	updates := make(chan accounts.WalletEvent, 1)
-	sub := ks.Subscribe(updates)
+	// Subscribe to the wallet feed and collect events.
+	var (
+		events  []walletEvent
+		updates = make(chan accounts.WalletEvent)
+		sub     = ks.Subscribe(updates)
+	)
 	defer sub.Unsubscribe()
+	go func() {
+		for {
+			select {
+			case ev := <-updates:
+				events = append(events, walletEvent{ev, ev.Wallet.Accounts()[0]})
+			case <-sub.Err():
+				close(updates)
+				return
+			}
+		}
+	}()
 
-	// Randomly add and remove account and make sure events and wallets are in sync
-	live := make(map[common.Address]accounts.Account)
+	// Randomly add and remove accounts.
+	var (
+		live       = make(map[common.Address]accounts.Account)
+		wantEvents []walletEvent
+	)
 	for i := 0; i < 1024; i++ {
-		// Execute a creation or deletion and ensure event arrival
 		if create := len(live) == 0 || rand.Int()%4 > 0; create {
 			// Add a new account and ensure wallet notifications arrives
 			account, err := ks.NewAccount("")
 			if err != nil {
 				t.Fatalf("failed to create test account: %v", err)
 			}
-			select {
-			case event := <-updates:
-				if !event.Arrive {
-					t.Errorf("departure event on account creation")
-				}
-				if event.Wallet.Accounts()[0] != account {
-					t.Errorf("account mismatch on created wallet: have %v, want %v", event.Wallet.Accounts()[0], account)
-				}
-			default:
-				t.Errorf("wallet arrival event not fired on account creation")
-			}
 			live[account.Address] = account
+			wantEvents = append(wantEvents, walletEvent{accounts.WalletEvent{Kind: accounts.WalletArrived}, account})
 		} else {
-			// Select a random account to delete (crude, but works)
+			// Delete a random account.
 			var account accounts.Account
 			for _, a := range live {
 				account = a
 				break
 			}
-			// Remove an account and ensure wallet notifiaction arrives
 			if err := ks.Delete(account, ""); err != nil {
 				t.Fatalf("failed to delete test account: %v", err)
 			}
-			select {
-			case event := <-updates:
-				if event.Arrive {
-					t.Errorf("arrival event on account deletion")
-				}
-				if event.Wallet.Accounts()[0] != account {
-					t.Errorf("account mismatch on deleted wallet: have %v, want %v", event.Wallet.Accounts()[0], account)
-				}
-			default:
-				t.Errorf("wallet departure event not fired on account creation")
-			}
 			delete(live, account.Address)
+			wantEvents = append(wantEvents, walletEvent{accounts.WalletEvent{Kind: accounts.WalletDropped}, account})
 		}
-		// Retrieve the list of wallets and ensure it matches with our required live set
-		liveList := make([]accounts.Account, 0, len(live))
-		for _, account := range live {
-			liveList = append(liveList, account)
-		}
-		sort.Sort(accountsByURL(liveList))
+	}
 
-		wallets := ks.Wallets()
-		if len(liveList) != len(wallets) {
-			t.Errorf("wallet list doesn't match required accounts: have %v, want %v", wallets, liveList)
-		} else {
-			for j, wallet := range wallets {
-				if accs := wallet.Accounts(); len(accs) != 1 {
-					t.Errorf("wallet %d: contains invalid number of accounts: have %d, want 1", j, len(accs))
-				} else if accs[0] != liveList[j] {
-					t.Errorf("wallet %d: account mismatch: have %v, want %v", j, accs[0], liveList[j])
-				}
+	// Shut down the event collector and check events.
+	sub.Unsubscribe()
+	<-updates
+	checkAccounts(t, live, ks.Wallets())
+	checkEvents(t, wantEvents, events)
+}
+
+// checkAccounts checks that all known live accounts are present in the wallet list.
+func checkAccounts(t *testing.T, live map[common.Address]accounts.Account, wallets []accounts.Wallet) {
+	if len(live) != len(wallets) {
+		t.Errorf("wallet list doesn't match required accounts: have %d, want %d", len(wallets), len(live))
+		return
+	}
+	liveList := make([]accounts.Account, 0, len(live))
+	for _, account := range live {
+		liveList = append(liveList, account)
+	}
+	sort.Sort(accountsByURL(liveList))
+	for j, wallet := range wallets {
+		if accs := wallet.Accounts(); len(accs) != 1 {
+			t.Errorf("wallet %d: contains invalid number of accounts: have %d, want 1", j, len(accs))
+		} else if accs[0] != liveList[j] {
+			t.Errorf("wallet %d: account mismatch: have %v, want %v", j, accs[0], liveList[j])
+		}
+	}
+}
+
+// checkEvents checks that all events in 'want' are present in 'have'. Events may be present multiple times.
+func checkEvents(t *testing.T, want []walletEvent, have []walletEvent) {
+	for _, wantEv := range want {
+		nmatch := 0
+		for ; len(have) > 0; nmatch++ {
+			if have[0].Kind != wantEv.Kind || have[0].a != wantEv.a {
+				break
 			}
+			have = have[1:]
+		}
+		if nmatch == 0 {
+			t.Fatalf("can't find event with Kind=%v for %x", wantEv.Kind, wantEv.a.Address)
 		}
 	}
 }

accounts/keystore/keystore_wallet.go

@@ -36,16 +36,16 @@ func (w *keystoreWallet) URL() accounts.URL {
 	return w.account.URL
 }
 
-// Status implements accounts.Wallet, always returning "open", since there is no
-// concept of open/close for plain keystore accounts.
-func (w *keystoreWallet) Status() string {
+// Status implements accounts.Wallet, returning whether the account held by the
+// keystore wallet is unlocked or not.
+func (w *keystoreWallet) Status() (string, error) {
 	w.keystore.mu.RLock()
 	defer w.keystore.mu.RUnlock()
 
 	if _, ok := w.keystore.unlocked[w.account.Address]; ok {
-		return "Unlocked"
+		return "Unlocked", nil
 	}
-	return "Locked"
+	return "Locked", nil
 }
 
 // Open implements accounts.Wallet, but is a noop for plain wallets since there

accounts/keystore/presale.go

@@ -58,6 +58,9 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error
 	if err != nil {
 		return nil, errors.New("invalid hex in encSeed")
 	}
+	if len(encSeedBytes) < 16 {
+		return nil, errors.New("invalid encSeed, too short")
+	}
 	iv := encSeedBytes[:16]
 	cipherText := encSeedBytes[16:]
 	/*

accounts/keystore/watch.go

@@ -70,7 +70,6 @@ func (w *watcher) loop() {
 		return
 	}
 	defer notify.Stop(w.ev)
-
 	logger.Trace("Started watching keystore folder")
 	defer logger.Trace("Stopped watching keystore folder")
 
@@ -82,32 +81,28 @@ func (w *watcher) loop() {
 	// When an event occurs, the reload call is delayed a bit so that
 	// multiple events arriving quickly only cause a single reload.
 	var (
-		debounce          = time.NewTimer(0)
-		debounceDuration  = 500 * time.Millisecond
-		inCycle, hadEvent bool
+		debounceDuration = 500 * time.Millisecond
+		rescanTriggered  = false
+		debounce         = time.NewTimer(0)
 	)
+	// Ignore initial trigger
+	if !debounce.Stop() {
+		<-debounce.C
+	}
 	defer debounce.Stop()
 	for {
 		select {
 		case <-w.quit:
 			return
 		case <-w.ev:
-			if !inCycle {
+			// Trigger the scan (with delay), if not already triggered
+			if !rescanTriggered {
 				debounce.Reset(debounceDuration)
-				inCycle = true
-			} else {
-				hadEvent = true
+				rescanTriggered = true
 			}
 		case <-debounce.C:
-			w.ac.mu.Lock()
-			w.ac.reload()
-			w.ac.mu.Unlock()
-			if hadEvent {
-				debounce.Reset(debounceDuration)
-				inCycle, hadEvent = true, false
-			} else {
-				inCycle, hadEvent = false, false
-			}
+			w.ac.scanAccounts()
+			rescanTriggered = false
 		}
 	}
 }

accounts/manager.go

@@ -41,6 +41,11 @@ type Manager struct {
 // NewManager creates a generic account manager to sign transaction via various
 // supported backends.
 func NewManager(backends ...Backend) *Manager {
+	// Retrieve the initial list of wallets from the backends and sort by URL
+	var wallets []Wallet
+	for _, backend := range backends {
+		wallets = merge(wallets, backend.Wallets()...)
+	}
 	// Subscribe to wallet notifications from all backends
 	updates := make(chan WalletEvent, 4*len(backends))
 
@@ -48,11 +53,6 @@ func NewManager(backends ...Backend) *Manager {
 	for i, backend := range backends {
 		subs[i] = backend.Subscribe(updates)
 	}
-	// Retrieve the initial list of wallets from the backends and sort by URL
-	var wallets []Wallet
-	for _, backend := range backends {
-		wallets = merge(wallets, backend.Wallets()...)
-	}
 	// Assemble the account manager and return
 	am := &Manager{
 		backends: make(map[reflect.Type][]Backend),
@@ -96,9 +96,10 @@ func (am *Manager) update() {
 		case event := <-am.updates:
 			// Wallet event arrived, update local cache
 			am.lock.Lock()
-			if event.Arrive {
+			switch event.Kind {
+			case WalletArrived:
 				am.wallets = merge(am.wallets, event.Wallet)
-			} else {
+			case WalletDropped:
 				am.wallets = drop(am.wallets, event.Wallet)
 			}
 			am.lock.Unlock()

accounts/url.go

@@ -74,6 +74,22 @@ func (u URL) MarshalJSON() ([]byte, error) {
 	return json.Marshal(u.String())
 }
 
+// UnmarshalJSON parses url.
+func (u *URL) UnmarshalJSON(input []byte) error {
+	var textUrl string
+	err := json.Unmarshal(input, &textUrl)
+	if err != nil {
+		return err
+	}
+	url, err := parseURL(textUrl)
+	if err != nil {
+		return err
+	}
+	u.Scheme = url.Scheme
+	u.Path = url.Path
+	return nil
+}
+
 // Cmp compares x and y and returns:
 //
 //   -1 if x <  y

accounts/usbwallet/hub.go

@@ -14,10 +14,6 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// This file contains the implementation for interacting with the Ledger hardware
-// wallets. The wire protocol spec can be found in the Ledger Blue GitHub repo:
-// https://raw.githubusercontent.com/LedgerHQ/blue-app-eth/master/doc/ethapp.asc
-
 package usbwallet
 
 import (
@@ -33,24 +29,30 @@ import (
 )
 
 // LedgerScheme is the protocol scheme prefixing account and wallet URLs.
-var LedgerScheme = "ledger"
+const LedgerScheme = "ledger"
 
-// ledgerDeviceIDs are the known device IDs that Ledger wallets use.
-var ledgerDeviceIDs = []deviceID{
-	{Vendor: 0x2c97, Product: 0x0000}, // Ledger Blue
-	{Vendor: 0x2c97, Product: 0x0001}, // Ledger Nano S
-}
+// TrezorScheme is the protocol scheme prefixing account and wallet URLs.
+const TrezorScheme = "trezor"
 
-// Maximum time between wallet refreshes (if USB hotplug notifications don't work).
-const ledgerRefreshCycle = time.Second
+// refreshCycle is the maximum time between wallet refreshes (if USB hotplug
+// notifications don't work).
+const refreshCycle = time.Second
 
-// Minimum time between wallet refreshes to avoid USB trashing.
-const ledgerRefreshThrottling = 500 * time.Millisecond
+// refreshThrottling is the minimum time between wallet refreshes to avoid USB
+// trashing.
+const refreshThrottling = 500 * time.Millisecond
+
+// Hub is a accounts.Backend that can find and handle generic USB hardware wallets.
+type Hub struct {
+	scheme     string                  // Protocol scheme prefixing account and wallet URLs.
+	vendorID   uint16                  // USB vendor identifier used for device discovery
+	productIDs []uint16                // USB product identifiers used for device discovery
+	usageID    uint16                  // USB usage page identifier used for macOS device discovery
+	endpointID int                     // USB endpoint identifier used for non-macOS device discovery
+	makeDriver func(log.Logger) driver // Factory method to construct a vendor specific driver
 
-// LedgerHub is a accounts.Backend that can find and handle Ledger hardware wallets.
-type LedgerHub struct {
 	refreshed   time.Time               // Time instance when the list of wallets was last refreshed
-	wallets     []accounts.Wallet       // List of Ledger devices currently tracking
+	wallets     []accounts.Wallet       // List of USB wallet devices currently tracking
 	updateFeed  event.Feed              // Event feed to notify wallet additions/removals
 	updateScope event.SubscriptionScope // Subscription scope tracking current live listeners
 	updating    bool                    // Whether the event notification loop is running
@@ -65,20 +67,36 @@ type LedgerHub struct {
 }
 
 // NewLedgerHub creates a new hardware wallet manager for Ledger devices.
-func NewLedgerHub() (*LedgerHub, error) {
+func NewLedgerHub() (*Hub, error) {
+	return newHub(LedgerScheme, 0x2c97, []uint16{0x0000 /* Ledger Blue */, 0x0001 /* Ledger Nano S */}, 0xffa0, 0, newLedgerDriver)
+}
+
+// NewTrezorHub creates a new hardware wallet manager for Trezor devices.
+func NewTrezorHub() (*Hub, error) {
+	return newHub(TrezorScheme, 0x534c, []uint16{0x0001 /* Trezor 1 */}, 0xff00, 0, newTrezorDriver)
+}
+
+// newHub creates a new hardware wallet manager for generic USB devices.
+func newHub(scheme string, vendorID uint16, productIDs []uint16, usageID uint16, endpointID int, makeDriver func(log.Logger) driver) (*Hub, error) {
 	if !hid.Supported() {
 		return nil, errors.New("unsupported platform")
 	}
-	hub := &LedgerHub{
-		quit: make(chan chan error),
+	hub := &Hub{
+		scheme:     scheme,
+		vendorID:   vendorID,
+		productIDs: productIDs,
+		usageID:    usageID,
+		endpointID: endpointID,
+		makeDriver: makeDriver,
+		quit:       make(chan chan error),
 	}
 	hub.refreshWallets()
 	return hub, nil
 }
 
 // Wallets implements accounts.Backend, returning all the currently tracked USB
-// devices that appear to be Ledger hardware wallets.
-func (hub *LedgerHub) Wallets() []accounts.Wallet {
+// devices that appear to be hardware wallets.
+func (hub *Hub) Wallets() []accounts.Wallet {
 	// Make sure the list of wallets is up to date
 	hub.refreshWallets()
 
@@ -92,24 +110,24 @@ func (hub *LedgerHub) Wallets() []accounts.Wallet {
 
 // refreshWallets scans the USB devices attached to the machine and updates the
 // list of wallets based on the found devices.
-func (hub *LedgerHub) refreshWallets() {
+func (hub *Hub) refreshWallets() {
 	// Don't scan the USB like crazy it the user fetches wallets in a loop
 	hub.stateLock.RLock()
 	elapsed := time.Since(hub.refreshed)
 	hub.stateLock.RUnlock()
 
-	if elapsed < ledgerRefreshThrottling {
+	if elapsed < refreshThrottling {
 		return
 	}
-	// Retrieve the current list of Ledger devices
-	var ledgers []hid.DeviceInfo
+	// Retrieve the current list of USB wallet devices
+	var devices []hid.DeviceInfo
 
 	if runtime.GOOS == "linux" {
 		// hidapi on Linux opens the device during enumeration to retrieve some infos,
 		// breaking the Ledger protocol if that is waiting for user confirmation. This
 		// is a bug acknowledged at Ledger, but it won't be fixed on old devices so we
 		// need to prevent concurrent comms ourselves. The more elegant solution would
-		// be to ditch enumeration in favor of hutplug events, but that don't work yet
+		// be to ditch enumeration in favor of hotplug events, but that don't work yet
 		// on Windows so if we need to hack it anyway, this is more elegant for now.
 		hub.commsLock.Lock()
 		if hub.commsPend > 0 { // A confirmation is pending, don't refresh
@@ -117,10 +135,10 @@ func (hub *LedgerHub) refreshWallets() {
 			return
 		}
 	}
-	for _, info := range hid.Enumerate(0, 0) { // Can't enumerate directly, one valid ID is the 0 wildcard
-		for _, id := range ledgerDeviceIDs {
-			if info.VendorID == id.Vendor && info.ProductID == id.Product {
-				ledgers = append(ledgers, info)
+	for _, info := range hid.Enumerate(hub.vendorID, 0) {
+		for _, id := range hub.productIDs {
+			if info.ProductID == id && (info.UsagePage == hub.usageID || info.Interface == hub.endpointID) {
+				devices = append(devices, info)
 				break
 			}
 		}
@@ -132,22 +150,29 @@ func (hub *LedgerHub) refreshWallets() {
 	// Transform the current list of wallets into the new one
 	hub.stateLock.Lock()
 
-	wallets := make([]accounts.Wallet, 0, len(ledgers))
+	wallets := make([]accounts.Wallet, 0, len(devices))
 	events := []accounts.WalletEvent{}
 
-	for _, ledger := range ledgers {
-		url := accounts.URL{Scheme: LedgerScheme, Path: ledger.Path}
+	for _, device := range devices {
+		url := accounts.URL{Scheme: hub.scheme, Path: device.Path}
 
 		// Drop wallets in front of the next device or those that failed for some reason
-		for len(hub.wallets) > 0 && (hub.wallets[0].URL().Cmp(url) < 0 || hub.wallets[0].(*ledgerWallet).failed()) {
-			events = append(events, accounts.WalletEvent{Wallet: hub.wallets[0], Arrive: false})
+		for len(hub.wallets) > 0 {
+			// Abort if we're past the current device and found an operational one
+			_, failure := hub.wallets[0].Status()
+			if hub.wallets[0].URL().Cmp(url) >= 0 || failure == nil {
+				break
+			}
+			// Drop the stale and failed devices
+			events = append(events, accounts.WalletEvent{Wallet: hub.wallets[0], Kind: accounts.WalletDropped})
 			hub.wallets = hub.wallets[1:]
 		}
 		// If there are no more wallets or the device is before the next, wrap new wallet
 		if len(hub.wallets) == 0 || hub.wallets[0].URL().Cmp(url) > 0 {
-			wallet := &ledgerWallet{hub: hub, url: &url, info: ledger, log: log.New("url", url)}
+			logger := log.New("url", url)
+			wallet := &wallet{hub: hub, driver: hub.makeDriver(logger), url: &url, info: device, log: logger}
 
-			events = append(events, accounts.WalletEvent{Wallet: wallet, Arrive: true})
+			events = append(events, accounts.WalletEvent{Wallet: wallet, Kind: accounts.WalletArrived})
 			wallets = append(wallets, wallet)
 			continue
 		}
@@ -160,7 +185,7 @@ func (hub *LedgerHub) refreshWallets() {
 	}
 	// Drop any leftover wallets and set the new batch
 	for _, wallet := range hub.wallets {
-		events = append(events, accounts.WalletEvent{Wallet: wallet, Arrive: false})
+		events = append(events, accounts.WalletEvent{Wallet: wallet, Kind: accounts.WalletDropped})
 	}
 	hub.refreshed = time.Now()
 	hub.wallets = wallets
@@ -173,8 +198,8 @@ func (hub *LedgerHub) refreshWallets() {
 }
 
 // Subscribe implements accounts.Backend, creating an async subscription to
-// receive notifications on the addition or removal of Ledger wallets.
-func (hub *LedgerHub) Subscribe(sink chan<- accounts.WalletEvent) event.Subscription {
+// receive notifications on the addition or removal of USB wallets.
+func (hub *Hub) Subscribe(sink chan<- accounts.WalletEvent) event.Subscription {
 	// We need the mutex to reliably start/stop the update loop
 	hub.stateLock.Lock()
 	defer hub.stateLock.Unlock()
@@ -190,18 +215,14 @@ func (hub *LedgerHub) Subscribe(sink chan<- accounts.WalletEvent) event.Subscrip
 	return sub
 }
 
-// updater is responsible for maintaining an up-to-date list of wallets stored in
-// the keystore, and for firing wallet addition/removal events. It listens for
-// account change events from the underlying account cache, and also periodically
-// forces a manual refresh (only triggers for systems where the filesystem notifier
-// is not running).
-func (hub *LedgerHub) updater() {
+// updater is responsible for maintaining an up-to-date list of wallets managed
+// by the USB hub, and for firing wallet addition/removal events.
+func (hub *Hub) updater() {
 	for {
-		// Wait for a USB hotplug event (not supported yet) or a refresh timeout
-		select {
-		//case <-hub.changes: // reenable on hutplug implementation
-		case <-time.After(ledgerRefreshCycle):
-		}
+		// TODO: Wait for a USB hotplug event (not supported yet) or a refresh timeout
+		// <-hub.changes
+		time.Sleep(refreshCycle)
+
 		// Run the wallet refresher
 		hub.refreshWallets()
 

accounts/usbwallet/internal/trezor/messages.proto

@@ -0,0 +1,905 @@
+// This file originates from the SatoshiLabs Trezor `common` repository at:
+//   https://github.com/trezor/trezor-common/blob/master/protob/messages.proto
+// dated 28.07.2017, commit dd8ec3231fb5f7992360aff9bdfe30bb58130f4b.
+
+syntax = "proto2";
+
+/**
+ * Messages for TREZOR communication
+ */
+
+// Sugar for easier handling in Java
+option java_package = "com.satoshilabs.trezor.lib.protobuf";
+option java_outer_classname = "TrezorMessage";
+
+import "types.proto";
+
+/**
+ * Mapping between Trezor wire identifier (uint) and a protobuf message
+ */
+enum MessageType {
+	MessageType_Initialize = 0 [(wire_in) = true];
+	MessageType_Ping = 1 [(wire_in) = true];
+	MessageType_Success = 2 [(wire_out) = true];
+	MessageType_Failure = 3 [(wire_out) = true];
+	MessageType_ChangePin = 4 [(wire_in) = true];
+	MessageType_WipeDevice = 5 [(wire_in) = true];
+	MessageType_FirmwareErase = 6 [(wire_in) = true, (wire_bootloader) = true];
+	MessageType_FirmwareUpload = 7 [(wire_in) = true, (wire_bootloader) = true];
+	MessageType_FirmwareRequest = 8 [(wire_out) = true, (wire_bootloader) = true];
+	MessageType_GetEntropy = 9 [(wire_in) = true];
+	MessageType_Entropy = 10 [(wire_out) = true];
+	MessageType_GetPublicKey = 11 [(wire_in) = true];
+	MessageType_PublicKey = 12 [(wire_out) = true];
+	MessageType_LoadDevice = 13 [(wire_in) = true];
+	MessageType_ResetDevice = 14 [(wire_in) = true];
+	MessageType_SignTx = 15 [(wire_in) = true];
+	MessageType_SimpleSignTx = 16 [(wire_in) = true, deprecated = true];
+	MessageType_Features = 17 [(wire_out) = true];
+	MessageType_PinMatrixRequest = 18 [(wire_out) = true];
+	MessageType_PinMatrixAck = 19 [(wire_in) = true, (wire_tiny) = true];
+	MessageType_Cancel = 20 [(wire_in) = true];
+	MessageType_TxRequest = 21 [(wire_out) = true];
+	MessageType_TxAck = 22 [(wire_in) = true];
+	MessageType_CipherKeyValue = 23 [(wire_in) = true];
+	MessageType_ClearSession = 24 [(wire_in) = true];
+	MessageType_ApplySettings = 25 [(wire_in) = true];
+	MessageType_ButtonRequest = 26 [(wire_out) = true];
+	MessageType_ButtonAck = 27 [(wire_in) = true, (wire_tiny) = true];
+	MessageType_ApplyFlags = 28 [(wire_in) = true];
+	MessageType_GetAddress = 29 [(wire_in) = true];
+	MessageType_Address = 30 [(wire_out) = true];
+	MessageType_SelfTest = 32 [(wire_in) = true, (wire_bootloader) = true];
+	MessageType_BackupDevice = 34 [(wire_in) = true];
+	MessageType_EntropyRequest = 35 [(wire_out) = true];
+	MessageType_EntropyAck = 36 [(wire_in) = true];
+	MessageType_SignMessage = 38 [(wire_in) = true];
+	MessageType_VerifyMessage = 39 [(wire_in) = true];
+	MessageType_MessageSignature = 40 [(wire_out) = true];
+	MessageType_PassphraseRequest = 41 [(wire_out) = true];
+	MessageType_PassphraseAck = 42 [(wire_in) = true, (wire_tiny) = true];
+	MessageType_EstimateTxSize = 43 [(wire_in) = true, deprecated = true];
+	MessageType_TxSize = 44 [(wire_out) = true, deprecated = true];
+	MessageType_RecoveryDevice = 45 [(wire_in) = true];
+	MessageType_WordRequest = 46 [(wire_out) = true];
+	MessageType_WordAck = 47 [(wire_in) = true];
+	MessageType_CipheredKeyValue = 48 [(wire_out) = true];
+	MessageType_EncryptMessage = 49 [(wire_in) = true, deprecated = true];
+	MessageType_EncryptedMessage = 50 [(wire_out) = true, deprecated = true];
+	MessageType_DecryptMessage = 51 [(wire_in) = true, deprecated = true];
+	MessageType_DecryptedMessage = 52 [(wire_out) = true, deprecated = true];
+	MessageType_SignIdentity = 53 [(wire_in) = true];
+	MessageType_SignedIdentity = 54 [(wire_out) = true];
+	MessageType_GetFeatures = 55 [(wire_in) = true];
+	MessageType_EthereumGetAddress = 56 [(wire_in) = true];
+	MessageType_EthereumAddress = 57 [(wire_out) = true];
+	MessageType_EthereumSignTx = 58 [(wire_in) = true];
+	MessageType_EthereumTxRequest = 59 [(wire_out) = true];
+	MessageType_EthereumTxAck = 60 [(wire_in) = true];
+	MessageType_GetECDHSessionKey = 61 [(wire_in) = true];
+	MessageType_ECDHSessionKey = 62 [(wire_out) = true];
+	MessageType_SetU2FCounter = 63 [(wire_in) = true];
+	MessageType_EthereumSignMessage = 64 [(wire_in) = true];
+	MessageType_EthereumVerifyMessage = 65 [(wire_in) = true];
+	MessageType_EthereumMessageSignature = 66 [(wire_out) = true];
+	MessageType_DebugLinkDecision = 100 [(wire_debug_in) = true, (wire_tiny) = true];
+	MessageType_DebugLinkGetState = 101 [(wire_debug_in) = true];
+	MessageType_DebugLinkState = 102 [(wire_debug_out) = true];
+	MessageType_DebugLinkStop = 103 [(wire_debug_in) = true];
+	MessageType_DebugLinkLog = 104 [(wire_debug_out) = true];
+	MessageType_DebugLinkMemoryRead = 110 [(wire_debug_in) = true];
+	MessageType_DebugLinkMemory = 111 [(wire_debug_out) = true];
+	MessageType_DebugLinkMemoryWrite = 112 [(wire_debug_in) = true];
+	MessageType_DebugLinkFlashErase = 113 [(wire_debug_in) = true];
+}
+
+////////////////////
+// Basic messages //
+////////////////////
+
+/**
+ * Request: Reset device to default state and ask for device details
+ * @next Features
+ */
+message Initialize {
+}
+
+/**
+ * Request: Ask for device details (no device reset)
+ * @next Features
+ */
+message GetFeatures {
+}
+
+/**
+ * Response: Reports various information about the device
+ * @prev Initialize
+ * @prev GetFeatures
+ */
+message Features {
+	optional string vendor = 1;			// name of the manufacturer, e.g. "bitcointrezor.com"
+	optional uint32 major_version = 2;		// major version of the device, e.g. 1
+	optional uint32 minor_version = 3;		// minor version of the device, e.g. 0
+	optional uint32 patch_version = 4;		// patch version of the device, e.g. 0
+	optional bool bootloader_mode = 5;		// is device in bootloader mode?
+	optional string device_id = 6;			// device's unique identifier
+	optional bool pin_protection = 7;		// is device protected by PIN?
+	optional bool passphrase_protection = 8;	// is node/mnemonic encrypted using passphrase?
+	optional string language = 9;			// device language
+	optional string label = 10;			// device description label
+	repeated CoinType coins = 11;			// supported coins
+	optional bool initialized = 12;			// does device contain seed?
+	optional bytes revision = 13;			// SCM revision of firmware
+	optional bytes bootloader_hash = 14;		// hash of the bootloader
+	optional bool imported = 15;			// was storage imported from an external source?
+	optional bool pin_cached = 16;			// is PIN already cached in session?
+	optional bool passphrase_cached = 17;		// is passphrase already cached in session?
+	optional bool firmware_present = 18;		// is valid firmware loaded?
+	optional bool needs_backup = 19;		// does storage need backup? (equals to Storage.needs_backup)
+	optional uint32 flags = 20;			// device flags (equals to Storage.flags)
+}
+
+/**
+ * Request: clear session (removes cached PIN, passphrase, etc).
+ * @next Success
+ */
+message ClearSession {
+}
+
+/**
+ * Request: change language and/or label of the device
+ * @next Success
+ * @next Failure
+ * @next ButtonRequest
+ * @next PinMatrixRequest
+ */
+message ApplySettings {
+	optional string language = 1;
+	optional string label = 2;
+	optional bool use_passphrase = 3;
+	optional bytes homescreen = 4;
+}
+
+/**
+ * Request: set flags of the device
+ * @next Success
+ * @next Failure
+ */
+message ApplyFlags {
+	optional uint32 flags = 1;	// bitmask, can only set bits, not unset
+}
+
+/**
+ * Request: Starts workflow for setting/changing/removing the PIN
+ * @next ButtonRequest
+ * @next PinMatrixRequest
+ */
+message ChangePin {
+	optional bool remove = 1;	// is PIN removal requested?
+}
+
+/**
+ * Request: Test if the device is alive, device sends back the message in Success response
+ * @next Success
+ */
+message Ping {
+	optional string message = 1;			// message to send back in Success message
+	optional bool button_protection = 2;		// ask for button press
+	optional bool pin_protection = 3;		// ask for PIN if set in device
+	optional bool passphrase_protection = 4;	// ask for passphrase if set in device
+}
+
+/**
+ * Response: Success of the previous request
+ */
+message Success {
+	optional string message = 1;	// human readable description of action or request-specific payload
+}
+
+/**
+ * Response: Failure of the previous request
+ */
+message Failure {
+	optional FailureType code = 1;	// computer-readable definition of the error state
+	optional string message = 2;	// human-readable message of the error state
+}
+
+/**
+ * Response: Device is waiting for HW button press.
+ * @next ButtonAck
+ * @next Cancel
+ */
+message ButtonRequest {
+	optional ButtonRequestType code = 1;
+	optional string data = 2;
+}
+
+/**
+ * Request: Computer agrees to wait for HW button press
+ * @prev ButtonRequest
+ */
+message ButtonAck {
+}
+
+/**
+ * Response: Device is asking computer to show PIN matrix and awaits PIN encoded using this matrix scheme
+ * @next PinMatrixAck
+ * @next Cancel
+ */
+message PinMatrixRequest {
+	optional PinMatrixRequestType type = 1;
+}
+
+/**
+ * Request: Computer responds with encoded PIN
+ * @prev PinMatrixRequest
+ */
+message PinMatrixAck {
+	required string pin = 1;		// matrix encoded PIN entered by user
+}
+
+/**
+ * Request: Abort last operation that required user interaction
+ * @prev ButtonRequest
+ * @prev PinMatrixRequest
+ * @prev PassphraseRequest
+ */
+message Cancel {
+}
+
+/**
+ * Response: Device awaits encryption passphrase
+ * @next PassphraseAck
+ * @next Cancel
+ */
+message PassphraseRequest {
+}
+
+/**
+ * Request: Send passphrase back
+ * @prev PassphraseRequest
+ */
+message PassphraseAck {
+	required string passphrase = 1;
+}
+
+/**
+ * Request: Request a sample of random data generated by hardware RNG. May be used for testing.
+ * @next ButtonRequest
+ * @next Entropy
+ * @next Failure
+ */
+message GetEntropy {
+	required uint32 size = 1;		// size of requested entropy
+}
+
+/**
+ * Response: Reply with random data generated by internal RNG
+ * @prev GetEntropy
+ */
+message Entropy {
+	required bytes entropy = 1;		// stream of random generated bytes
+}
+
+/**
+ * Request: Ask device for public key corresponding to address_n path
+ * @next PassphraseRequest
+ * @next PublicKey
+ * @next Failure
+ */
+message GetPublicKey {
+	repeated uint32 address_n = 1;		// BIP-32 path to derive the key from master node
+	optional string ecdsa_curve_name = 2;	// ECDSA curve name to use
+	optional bool show_display = 3;		// optionally show on display before sending the result
+	optional string coin_name = 4 [default='Bitcoin'];
+}
+
+/**
+ * Response: Contains public key derived from device private seed
+ * @prev GetPublicKey
+ */
+message PublicKey {
+	required HDNodeType node = 1;		// BIP32 public node
+	optional string xpub = 2;		// serialized form of public node
+}
+
+/**
+ * Request: Ask device for address corresponding to address_n path
+ * @next PassphraseRequest
+ * @next Address
+ * @next Failure
+ */
+message GetAddress {
+	repeated uint32 address_n = 1;						// BIP-32 path to derive the key from master node
+	optional string coin_name = 2 [default='Bitcoin'];
+	optional bool show_display = 3			;			// optionally show on display before sending the result
+	optional MultisigRedeemScriptType multisig = 4;				// filled if we are showing a multisig address
+	optional InputScriptType script_type = 5 [default=SPENDADDRESS];	// used to distinguish between various address formats (non-segwit, segwit, etc.)
+}
+
+/**
+ * Request: Ask device for Ethereum address corresponding to address_n path
+ * @next PassphraseRequest
+ * @next EthereumAddress
+ * @next Failure
+ */
+message EthereumGetAddress {
+	repeated uint32 address_n = 1;			// BIP-32 path to derive the key from master node
+	optional bool show_display = 2;			// optionally show on display before sending the result
+}
+
+/**
+ * Response: Contains address derived from device private seed
+ * @prev GetAddress
+ */
+message Address {
+	required string address = 1;		// Coin address in Base58 encoding
+}
+
+/**
+ * Response: Contains an Ethereum address derived from device private seed
+ * @prev EthereumGetAddress
+ */
+message EthereumAddress {
+	required bytes address = 1;		// Coin address as an Ethereum 160 bit hash
+}
+
+/**
+ * Request: Request device to wipe all sensitive data and settings
+ * @next ButtonRequest
+ */
+message WipeDevice {
+}
+
+/**
+ * Request: Load seed and related internal settings from the computer
+ * @next ButtonRequest
+ * @next Success
+ * @next Failure
+ */
+message LoadDevice {
+	optional string mnemonic = 1;				// seed encoded as BIP-39 mnemonic (12, 18 or 24 words)
+	optional HDNodeType node = 2;				// BIP-32 node
+	optional string pin = 3;				// set PIN protection
+	optional bool passphrase_protection = 4;		// enable master node encryption using passphrase
+	optional string language = 5 [default='english'];	// device language
+	optional string label = 6;				// device label
+	optional bool skip_checksum = 7;			// do not test mnemonic for valid BIP-39 checksum
+	optional uint32 u2f_counter = 8;			// U2F counter
+}
+
+/**
+ * Request: Ask device to do initialization involving user interaction
+ * @next EntropyRequest
+ * @next Failure
+ */
+message ResetDevice {
+	optional bool display_random = 1;			// display entropy generated by the device before asking for additional entropy
+	optional uint32 strength = 2 [default=256];		// strength of seed in bits
+	optional bool passphrase_protection = 3;		// enable master node encryption using passphrase
+	optional bool pin_protection = 4;			// enable PIN protection
+	optional string language = 5 [default='english'];	// device language
+	optional string label = 6;				// device label
+	optional uint32 u2f_counter = 7;			// U2F counter
+	optional bool skip_backup = 8;				// postpone seed backup to BackupDevice workflow
+}
+
+/**
+ * Request: Perform backup of the device seed if not backed up using ResetDevice
+ * @next ButtonRequest
+ */
+message BackupDevice {
+}
+
+/**
+ * Response: Ask for additional entropy from host computer
+ * @prev ResetDevice
+ * @next EntropyAck
+ */
+message EntropyRequest {
+}
+
+/**
+ * Request: Provide additional entropy for seed generation function
+ * @prev EntropyRequest
+ * @next ButtonRequest
+ */
+message EntropyAck {
+	optional bytes entropy = 1;				// 256 bits (32 bytes) of random data
+}
+
+/**
+ * Request: Start recovery workflow asking user for specific words of mnemonic
+ * Used to recovery device safely even on untrusted computer.
+ * @next WordRequest
+ */
+message RecoveryDevice {
+	optional uint32 word_count = 1;				// number of words in BIP-39 mnemonic
+	optional bool passphrase_protection = 2;		// enable master node encryption using passphrase
+	optional bool pin_protection = 3;			// enable PIN protection
+	optional string language = 4 [default='english'];	// device language
+	optional string label = 5;				// device label
+	optional bool enforce_wordlist = 6;			// enforce BIP-39 wordlist during the process
+	// 7 reserved for unused recovery method
+	optional uint32 type = 8;				// supported recovery type (see RecoveryType)
+	optional uint32 u2f_counter = 9;			// U2F counter
+	optional bool dry_run = 10;				// perform dry-run recovery workflow (for safe mnemonic validation)
+}
+
+/**
+ * Response: Device is waiting for user to enter word of the mnemonic
+ * Its position is shown only on device's internal display.
+ * @prev RecoveryDevice
+ * @prev WordAck
+ */
+message WordRequest {
+	optional WordRequestType type = 1;
+}
+
+/**
+ * Request: Computer replies with word from the mnemonic
+ * @prev WordRequest
+ * @next WordRequest
+ * @next Success
+ * @next Failure
+ */
+message WordAck {
+	required string word = 1;				// one word of mnemonic on asked position
+}
+
+//////////////////////////////
+// Message signing messages //
+//////////////////////////////
+
+/**
+ * Request: Ask device to sign message
+ * @next MessageSignature
+ * @next Failure
+ */
+message SignMessage {
+	repeated uint32 address_n = 1;						// BIP-32 path to derive the key from master node
+	required bytes message = 2;						// message to be signed
+	optional string coin_name = 3 [default='Bitcoin'];			// coin to use for signing
+	optional InputScriptType script_type = 4 [default=SPENDADDRESS];	// used to distinguish between various address formats (non-segwit, segwit, etc.)
+}
+
+/**
+ * Request: Ask device to verify message
+ * @next Success
+ * @next Failure
+ */
+message VerifyMessage {
+	optional string address = 1;				// address to verify
+	optional bytes signature = 2;				// signature to verify
+	optional bytes message = 3;				// message to verify
+	optional string coin_name = 4 [default='Bitcoin'];	// coin to use for verifying
+}
+
+/**
+ * Response: Signed message
+ * @prev SignMessage
+ */
+message MessageSignature {
+	optional string address = 1;				// address used to sign the message
+	optional bytes signature = 2;				// signature of the message
+}
+
+///////////////////////////
+// Encryption/decryption //
+///////////////////////////
+
+/**
+ * Request: Ask device to encrypt message
+ * @next EncryptedMessage
+ * @next Failure
+ */
+message EncryptMessage {
+	optional bytes pubkey = 1;				// public key
+	optional bytes message = 2;				// message to encrypt
+	optional bool display_only = 3;				// show just on display? (don't send back via wire)
+	repeated uint32 address_n = 4;				// BIP-32 path to derive the signing key from master node
+	optional string coin_name = 5 [default='Bitcoin'];	// coin to use for signing
+}
+
+/**
+ * Response: Encrypted message
+ * @prev EncryptMessage
+ */
+message EncryptedMessage {
+	optional bytes nonce = 1;				// nonce used during encryption
+	optional bytes message = 2;				// encrypted message
+	optional bytes hmac = 3;				// message hmac
+}
+
+/**
+ * Request: Ask device to decrypt message
+ * @next Success
+ * @next Failure
+ */
+message DecryptMessage {
+	repeated uint32 address_n = 1;				// BIP-32 path to derive the decryption key from master node
+	optional bytes nonce = 2;				// nonce used during encryption
+	optional bytes message = 3;				// message to decrypt
+	optional bytes hmac = 4;				// message hmac
+}
+
+/**
+ * Response: Decrypted message
+ * @prev DecryptedMessage
+ */
+message DecryptedMessage {
+	optional bytes message = 1;				// decrypted message
+	optional string address = 2;				// address used to sign the message (if used)
+}
+
+/**
+ * Request: Ask device to encrypt or decrypt value of given key
+ * @next CipheredKeyValue
+ * @next Failure
+ */
+message CipherKeyValue {
+	repeated uint32 address_n = 1;		// BIP-32 path to derive the key from master node
+	optional string key = 2;		// key component of key:value
+	optional bytes value = 3;		// value component of key:value
+	optional bool encrypt = 4;		// are we encrypting (True) or decrypting (False)?
+	optional bool ask_on_encrypt = 5;	// should we ask on encrypt operation?
+	optional bool ask_on_decrypt = 6;	// should we ask on decrypt operation?
+	optional bytes iv = 7;			// initialization vector (will be computed if not set)
+}
+
+/**
+ * Response: Return ciphered/deciphered value
+ * @prev CipherKeyValue
+ */
+message CipheredKeyValue {
+	optional bytes value = 1;		// ciphered/deciphered value
+}
+
+//////////////////////////////////
+// Transaction signing messages //
+//////////////////////////////////
+
+/**
+ * Request: Estimated size of the transaction
+ * This behaves exactly like SignTx, which means that it can ask using TxRequest
+ * This call is non-blocking (except possible PassphraseRequest to unlock the seed)
+ * @next TxSize
+ * @next Failure
+ */
+message EstimateTxSize {
+	required uint32 outputs_count = 1;			// number of transaction outputs
+	required uint32 inputs_count = 2;			// number of transaction inputs
+	optional string coin_name = 3 [default='Bitcoin'];	// coin to use
+}
+
+/**
+ * Response: Estimated size of the transaction
+ * @prev EstimateTxSize
+ */
+message TxSize {
+	optional uint32 tx_size = 1;				// estimated size of transaction in bytes
+}
+
+/**
+ * Request: Ask device to sign transaction
+ * @next PassphraseRequest
+ * @next PinMatrixRequest
+ * @next TxRequest
+ * @next Failure
+ */
+message SignTx {
+	required uint32 outputs_count = 1;			// number of transaction outputs
+	required uint32 inputs_count = 2;			// number of transaction inputs
+	optional string coin_name = 3 [default='Bitcoin'];	// coin to use
+	optional uint32 version = 4 [default=1];		// transaction version
+	optional uint32 lock_time = 5 [default=0];		// transaction lock_time
+}
+
+/**
+ * Request: Simplified transaction signing
+ * This method doesn't support streaming, so there are hardware limits in number of inputs and outputs.
+ * In case of success, the result is returned using TxRequest message.
+ * @next PassphraseRequest
+ * @next PinMatrixRequest
+ * @next TxRequest
+ * @next Failure
+ */
+message SimpleSignTx {
+	repeated TxInputType inputs = 1;			// transaction inputs
+	repeated TxOutputType outputs = 2;			// transaction outputs
+	repeated TransactionType transactions = 3;		// transactions whose outputs are used to build current inputs
+	optional string coin_name = 4 [default='Bitcoin'];	// coin to use
+	optional uint32 version = 5 [default=1];		// transaction version
+	optional uint32 lock_time = 6 [default=0];		// transaction lock_time
+}
+
+/**
+ * Response: Device asks for information for signing transaction or returns the last result
+ * If request_index is set, device awaits TxAck message (with fields filled in according to request_type)
+ * If signature_index is set, 'signature' contains signed input of signature_index's input
+ * @prev SignTx
+ * @prev SimpleSignTx
+ * @prev TxAck
+ */
+message TxRequest {
+	optional RequestType request_type = 1;			// what should be filled in TxAck message?
+	optional TxRequestDetailsType details = 2;		// request for tx details
+	optional TxRequestSerializedType serialized = 3;	// serialized data and request for next
+}
+
+/**
+ * Request: Reported transaction data
+ * @prev TxRequest
+ * @next TxRequest
+ */
+message TxAck {
+	optional TransactionType tx = 1;
+}
+
+/**
+ * Request: Ask device to sign transaction
+ * All fields are optional from the protocol's point of view. Each field defaults to value `0` if missing.
+ * Note: the first at most 1024 bytes of data MUST be transmitted as part of this message.
+ * @next PassphraseRequest
+ * @next PinMatrixRequest
+ * @next EthereumTxRequest
+ * @next Failure
+ */
+message EthereumSignTx {
+	repeated uint32 address_n = 1;			// BIP-32 path to derive the key from master node
+	optional bytes nonce = 2;			// <=256 bit unsigned big endian
+	optional bytes gas_price = 3;			// <=256 bit unsigned big endian (in wei)
+	optional bytes gas_limit = 4;			// <=256 bit unsigned big endian
+	optional bytes to = 5;				// 160 bit address hash
+	optional bytes value = 6;			// <=256 bit unsigned big endian (in wei)
+	optional bytes data_initial_chunk = 7;		// The initial data chunk (<= 1024 bytes)
+	optional uint32 data_length = 8;		// Length of transaction payload
+	optional uint32 chain_id = 9;			// Chain Id for EIP 155
+}
+
+/**
+ * Response: Device asks for more data from transaction payload, or returns the signature.
+ * If data_length is set, device awaits that many more bytes of payload.
+ * Otherwise, the signature_* fields contain the computed transaction signature. All three fields will be present.
+ * @prev EthereumSignTx
+ * @next EthereumTxAck
+ */
+message EthereumTxRequest {
+	optional uint32 data_length = 1;		// Number of bytes being requested (<= 1024)
+	optional uint32 signature_v = 2;		// Computed signature (recovery parameter, limited to 27 or 28)
+	optional bytes signature_r = 3;			// Computed signature R component (256 bit)
+	optional bytes signature_s = 4;			// Computed signature S component (256 bit)
+}
+
+/**
+ * Request: Transaction payload data.
+ * @prev EthereumTxRequest
+ * @next EthereumTxRequest
+ */
+message EthereumTxAck {
+	optional bytes data_chunk = 1;			// Bytes from transaction payload (<= 1024 bytes)
+}
+
+////////////////////////////////////////
+// Ethereum: Message signing messages //
+////////////////////////////////////////
+
+/**
+ * Request: Ask device to sign message
+ * @next EthereumMessageSignature
+ * @next Failure
+ */
+message EthereumSignMessage {
+	repeated uint32 address_n = 1;				// BIP-32 path to derive the key from master node
+	required bytes message = 2;				// message to be signed
+}
+
+/**
+ * Request: Ask device to verify message
+ * @next Success
+ * @next Failure
+ */
+message EthereumVerifyMessage {
+	optional bytes address = 1;				// address to verify
+	optional bytes signature = 2;				// signature to verify
+	optional bytes message = 3;				// message to verify
+}
+
+/**
+ * Response: Signed message
+ * @prev EthereumSignMessage
+ */
+message EthereumMessageSignature {
+	optional bytes address = 1;				// address used to sign the message
+	optional bytes signature = 2;				// signature of the message
+}
+
+///////////////////////
+// Identity messages //
+///////////////////////
+
+/**
+ * Request: Ask device to sign identity
+ * @next SignedIdentity
+ * @next Failure
+ */
+message SignIdentity {
+	optional IdentityType identity = 1;		// identity
+	optional bytes challenge_hidden = 2;		// non-visible challenge
+	optional string challenge_visual = 3;		// challenge shown on display (e.g. date+time)
+	optional string ecdsa_curve_name = 4;		// ECDSA curve name to use
+}
+
+/**
+ * Response: Device provides signed identity
+ * @prev SignIdentity
+ */
+message SignedIdentity {
+	optional string address = 1;			// identity address
+	optional bytes public_key = 2;			// identity public key
+	optional bytes signature = 3;			// signature of the identity data
+}
+
+///////////////////
+// ECDH messages //
+///////////////////
+
+/**
+ * Request: Ask device to generate ECDH session key
+ * @next ECDHSessionKey
+ * @next Failure
+ */
+message GetECDHSessionKey {
+	optional IdentityType identity = 1;		// identity
+	optional bytes peer_public_key = 2;		// peer's public key
+	optional string ecdsa_curve_name = 3;		// ECDSA curve name to use
+}
+
+/**
+ * Response: Device provides ECDH session key
+ * @prev GetECDHSessionKey
+ */
+message ECDHSessionKey {
+	optional bytes session_key = 1;			// ECDH session key
+}
+
+///////////////////
+// U2F messages //
+///////////////////
+
+/**
+ * Request: Set U2F counter
+ * @next Success
+ */
+message SetU2FCounter {
+	optional uint32 u2f_counter = 1;		// counter
+}
+
+/////////////////////////
+// Bootloader messages //
+/////////////////////////
+
+/**
+ * Request: Ask device to erase its firmware (so it can be replaced via FirmwareUpload)
+ * @next Success
+ * @next FirmwareRequest
+ * @next Failure
+ */
+message FirmwareErase {
+	optional uint32 length = 1;			// length of new firmware
+}
+
+/**
+ * Response: Ask for firmware chunk
+ * @next FirmwareUpload
+ */
+message FirmwareRequest {
+	optional uint32 offset = 1;			// offset of requested firmware chunk
+	optional uint32 length = 2;			// length of requested firmware chunk
+}
+
+/**
+ * Request: Send firmware in binary form to the device
+ * @next Success
+ * @next Failure
+ */
+message FirmwareUpload {
+	required bytes payload = 1;			// firmware to be loaded into device
+	optional bytes hash = 2;			// hash of the payload
+}
+
+
+/**
+ * Request: Perform a device self-test
+ * @next Success
+ * @next Failure
+ */
+message SelfTest {
+	optional bytes payload = 1;			// payload to be used in self-test
+}
+
+/////////////////////////////////////////////////////////////
+// Debug messages (only available if DebugLink is enabled) //
+/////////////////////////////////////////////////////////////
+
+/**
+ * Request: "Press" the button on the device
+ * @next Success
+ */
+message DebugLinkDecision {
+	required bool yes_no = 1;			// true for "Confirm", false for "Cancel"
+}
+
+/**
+ * Request: Computer asks for device state
+ * @next DebugLinkState
+ */
+message DebugLinkGetState {
+}
+
+/**
+ * Response: Device current state
+ * @prev DebugLinkGetState
+ */
+message DebugLinkState {
+	optional bytes layout = 1;			// raw buffer of display
+	optional string pin = 2;			// current PIN, blank if PIN is not set/enabled
+	optional string matrix = 3;			// current PIN matrix
+	optional string mnemonic = 4;			// current BIP-39 mnemonic
+	optional HDNodeType node = 5;			// current BIP-32 node
+	optional bool passphrase_protection = 6;	// is node/mnemonic encrypted using passphrase?
+	optional string reset_word = 7;			// word on device display during ResetDevice workflow
+	optional bytes reset_entropy = 8;		// current entropy during ResetDevice workflow
+	optional string recovery_fake_word = 9;		// (fake) word on display during RecoveryDevice workflow
+	optional uint32 recovery_word_pos = 10;		// index of mnemonic word the device is expecting during RecoveryDevice workflow
+}
+
+/**
+ * Request: Ask device to restart
+ */
+message DebugLinkStop {
+}
+
+/**
+ * Response: Device wants host to log event
+ */
+message DebugLinkLog {
+	optional uint32 level = 1;
+	optional string bucket = 2;
+	optional string text = 3;
+}
+
+/**
+ * Request: Read memory from device
+ * @next DebugLinkMemory
+ */
+message DebugLinkMemoryRead {
+	optional uint32 address = 1;
+	optional uint32 length = 2;
+}
+
+/**
+ * Response: Device sends memory back
+ * @prev DebugLinkMemoryRead
+ */
+message DebugLinkMemory {
+	optional bytes memory = 1;
+}
+
+/**
+ * Request: Write memory to device.
+ * WARNING: Writing to the wrong location can irreparably break the device.
+ */
+message DebugLinkMemoryWrite {
+	optional uint32 address = 1;
+	optional bytes memory = 2;
+	optional bool flash = 3;
+}
+
+/**
+ * Request: Erase block of flash on device
+ * WARNING: Writing to the wrong location can irreparably break the device.
+ */
+message DebugLinkFlashErase {
+	optional uint32 sector = 1;
+}

accounts/usbwallet/internal/trezor/trezor.go

@@ -0,0 +1,46 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// This file contains the implementation for interacting with the Trezor hardware
+// wallets. The wire protocol spec can be found on the SatoshiLabs website:
+// https://doc.satoshilabs.com/trezor-tech/api-protobuf.html
+
+//go:generate protoc --go_out=import_path=trezor:. types.proto messages.proto
+
+// Package trezor contains the wire protocol wrapper in Go.
+package trezor
+
+import (
+	"reflect"
+
+	"github.com/golang/protobuf/proto"
+)
+
+// Type returns the protocol buffer type number of a specific message. If the
+// message is nil, this method panics!
+func Type(msg proto.Message) uint16 {
+	return uint16(MessageType_value["MessageType_"+reflect.TypeOf(msg).Elem().Name()])
+}
+
+// Name returns the friendly message type name of a specific protocol buffer
+// type numbers.
+func Name(kind uint16) string {
+	name := MessageType_name[int32(kind)]
+	if len(name) < 12 {
+		return name
+	}
+	return name[12:]
+}

accounts/usbwallet/internal/trezor/types.proto

@@ -0,0 +1,278 @@
+// This file originates from the SatoshiLabs Trezor `common` repository at:
+//   https://github.com/trezor/trezor-common/blob/master/protob/types.proto
+// dated 28.07.2017, commit dd8ec3231fb5f7992360aff9bdfe30bb58130f4b.
+
+syntax = "proto2";
+
+/**
+ * Types for TREZOR communication
+ *
+ * @author	Marek Palatinus <slush@satoshilabs.com>
+ * @version	1.2
+ */
+
+// Sugar for easier handling in Java
+option java_package = "com.satoshilabs.trezor.lib.protobuf";
+option java_outer_classname = "TrezorType";
+
+import "google/protobuf/descriptor.proto";
+
+/**
+ * Options for specifying message direction and type of wire (normal/debug)
+ */
+extend google.protobuf.EnumValueOptions {
+	optional bool wire_in = 50002;		// message can be transmitted via wire from PC to TREZOR
+	optional bool wire_out = 50003;		// message can be transmitted via wire from TREZOR to PC
+	optional bool wire_debug_in = 50004;	// message can be transmitted via debug wire from PC to TREZOR
+	optional bool wire_debug_out = 50005;	// message can be transmitted via debug wire from TREZOR to PC
+	optional bool wire_tiny = 50006;	// message is handled by TREZOR when the USB stack is in tiny mode
+	optional bool wire_bootloader = 50007;  // message is only handled by TREZOR Bootloader
+}
+
+/**
+ * Type of failures returned by Failure message
+ * @used_in Failure
+ */
+enum FailureType {
+	Failure_UnexpectedMessage = 1;
+	Failure_ButtonExpected = 2;
+	Failure_DataError = 3;
+	Failure_ActionCancelled = 4;
+	Failure_PinExpected = 5;
+	Failure_PinCancelled = 6;
+	Failure_PinInvalid = 7;
+	Failure_InvalidSignature = 8;
+	Failure_ProcessError = 9;
+	Failure_NotEnoughFunds = 10;
+	Failure_NotInitialized = 11;
+	Failure_FirmwareError = 99;
+}
+
+/**
+ * Type of script which will be used for transaction output
+ * @used_in TxOutputType
+ */
+enum OutputScriptType {
+	PAYTOADDRESS = 0;	// used for all addresses (bitcoin, p2sh, witness)
+	PAYTOSCRIPTHASH = 1;	// p2sh address (deprecated; use PAYTOADDRESS)
+	PAYTOMULTISIG = 2;	// only for change output
+	PAYTOOPRETURN = 3;	// op_return
+	PAYTOWITNESS = 4;	// only for change output
+	PAYTOP2SHWITNESS = 5;	// only for change output
+}
+
+/**
+ * Type of script which will be used for transaction output
+ * @used_in TxInputType
+ */
+enum InputScriptType {
+	SPENDADDRESS = 0;		// standard p2pkh address
+	SPENDMULTISIG = 1;		// p2sh multisig address
+	EXTERNAL = 2;			// reserved for external inputs (coinjoin)
+	SPENDWITNESS = 3;		// native segwit
+	SPENDP2SHWITNESS = 4;		// segwit over p2sh (backward compatible)
+}
+
+/**
+ * Type of information required by transaction signing process
+ * @used_in TxRequest
+ */
+enum RequestType {
+	TXINPUT = 0;
+	TXOUTPUT = 1;
+	TXMETA = 2;
+	TXFINISHED = 3;
+	TXEXTRADATA = 4;
+}
+
+/**
+ * Type of button request
+ * @used_in ButtonRequest
+ */
+enum ButtonRequestType {
+	ButtonRequest_Other = 1;
+	ButtonRequest_FeeOverThreshold = 2;
+	ButtonRequest_ConfirmOutput = 3;
+	ButtonRequest_ResetDevice = 4;
+	ButtonRequest_ConfirmWord = 5;
+	ButtonRequest_WipeDevice = 6;
+	ButtonRequest_ProtectCall = 7;
+	ButtonRequest_SignTx = 8;
+	ButtonRequest_FirmwareCheck = 9;
+	ButtonRequest_Address = 10;
+	ButtonRequest_PublicKey = 11;
+}
+
+/**
+ * Type of PIN request
+ * @used_in PinMatrixRequest
+ */
+enum PinMatrixRequestType {
+	PinMatrixRequestType_Current = 1;
+	PinMatrixRequestType_NewFirst = 2;
+	PinMatrixRequestType_NewSecond = 3;
+}
+
+/**
+ * Type of recovery procedure. These should be used as bitmask, e.g.,
+ * `RecoveryDeviceType_ScrambledWords | RecoveryDeviceType_Matrix`
+ * listing every method supported by the host computer.
+ *
+ * Note that ScrambledWords must be supported by every implementation
+ * for backward compatibility; there is no way to not support it.
+ *
+ * @used_in RecoveryDevice
+ */
+enum RecoveryDeviceType {
+	// use powers of two when extending this field
+	RecoveryDeviceType_ScrambledWords = 0;		// words in scrambled order
+	RecoveryDeviceType_Matrix = 1;				// matrix recovery type
+}
+
+/**
+ * Type of Recovery Word request
+ * @used_in WordRequest
+ */
+enum WordRequestType {
+	WordRequestType_Plain = 0;
+	WordRequestType_Matrix9 = 1;
+	WordRequestType_Matrix6 = 2;
+}
+
+/**
+ * Structure representing BIP32 (hierarchical deterministic) node
+ * Used for imports of private key into the device and exporting public key out of device
+ * @used_in PublicKey
+ * @used_in LoadDevice
+ * @used_in DebugLinkState
+ * @used_in Storage
+ */
+message HDNodeType {
+	required uint32 depth = 1;
+	required uint32 fingerprint = 2;
+	required uint32 child_num = 3;
+	required bytes chain_code = 4;
+	optional bytes private_key = 5;
+	optional bytes public_key = 6;
+}
+
+message HDNodePathType {
+	required HDNodeType node = 1;						// BIP-32 node in deserialized form
+	repeated uint32 address_n = 2;						// BIP-32 path to derive the key from node
+}
+
+/**
+ * Structure representing Coin
+ * @used_in Features
+ */
+message CoinType {
+	optional string coin_name = 1;
+	optional string coin_shortcut = 2;
+	optional uint32 address_type = 3 [default=0];
+	optional uint64 maxfee_kb = 4;
+	optional uint32 address_type_p2sh = 5 [default=5];
+	optional string signed_message_header = 8;
+	optional uint32 xpub_magic = 9 [default=76067358];	// default=0x0488b21e
+	optional uint32 xprv_magic = 10 [default=76066276];	// default=0x0488ade4
+	optional bool segwit = 11;
+	optional uint32 forkid = 12;
+}
+
+/**
+ * Type of redeem script used in input
+ * @used_in TxInputType
+ */
+message MultisigRedeemScriptType {
+	repeated HDNodePathType pubkeys = 1;					// pubkeys from multisig address (sorted lexicographically)
+	repeated bytes signatures = 2;						// existing signatures for partially signed input
+	optional uint32 m = 3;							// "m" from n, how many valid signatures is necessary for spending
+}
+
+/**
+ * Structure representing transaction input
+ * @used_in SimpleSignTx
+ * @used_in TransactionType
+ */
+message TxInputType {
+	repeated uint32 address_n = 1;						// BIP-32 path to derive the key from master node
+	required bytes prev_hash = 2;						// hash of previous transaction output to spend by this input
+	required uint32 prev_index = 3;						// index of previous output to spend
+	optional bytes script_sig = 4;						// script signature, unset for tx to sign
+	optional uint32 sequence = 5 [default=4294967295];			// sequence (default=0xffffffff)
+	optional InputScriptType script_type = 6 [default=SPENDADDRESS];	// defines template of input script
+	optional MultisigRedeemScriptType multisig = 7;				// Filled if input is going to spend multisig tx
+	optional uint64 amount = 8;						// amount of previous transaction output (for segwit only)
+}
+
+/**
+ * Structure representing transaction output
+ * @used_in SimpleSignTx
+ * @used_in TransactionType
+ */
+message TxOutputType {
+	optional string address = 1;			// target coin address in Base58 encoding
+	repeated uint32 address_n = 2;			// BIP-32 path to derive the key from master node; has higher priority than "address"
+	required uint64 amount = 3;			// amount to spend in satoshis
+	required OutputScriptType script_type = 4;	// output script type
+	optional MultisigRedeemScriptType multisig = 5; // defines multisig address; script_type must be PAYTOMULTISIG
+	optional bytes op_return_data = 6;		// defines op_return data; script_type must be PAYTOOPRETURN, amount must be 0
+}
+
+/**
+ * Structure representing compiled transaction output
+ * @used_in TransactionType
+ */
+message TxOutputBinType {
+	required uint64 amount = 1;
+	required bytes script_pubkey = 2;
+}
+
+/**
+ * Structure representing transaction
+ * @used_in SimpleSignTx
+ */
+message TransactionType {
+	optional uint32 version = 1;
+	repeated TxInputType inputs = 2;
+	repeated TxOutputBinType bin_outputs = 3;
+	repeated TxOutputType outputs = 5;
+	optional uint32 lock_time = 4;
+	optional uint32 inputs_cnt = 6;
+	optional uint32 outputs_cnt = 7;
+	optional bytes extra_data = 8;
+	optional uint32 extra_data_len = 9;
+}
+
+/**
+ * Structure representing request details
+ * @used_in TxRequest
+ */
+message TxRequestDetailsType {
+	optional uint32 request_index = 1;	// device expects TxAck message from the computer
+	optional bytes tx_hash = 2;		// tx_hash of requested transaction
+	optional uint32 extra_data_len = 3;	// length of requested extra data
+	optional uint32 extra_data_offset = 4;	// offset of requested extra data
+}
+
+/**
+ * Structure representing serialized data
+ * @used_in TxRequest
+ */
+message TxRequestSerializedType {
+	optional uint32 signature_index = 1;	// 'signature' field contains signed input of this index
+	optional bytes signature = 2;		// signature of the signature_index input
+	optional bytes serialized_tx = 3;	// part of serialized and signed transaction
+}
+
+/**
+ * Structure representing identity data
+ * @used_in IdentityType
+ */
+message IdentityType {
+	optional string proto = 1;			// proto part of URI
+	optional string user = 2;			// user part of URI
+	optional string host = 3;			// host part of URI
+	optional string port = 4;			// port part of URI
+	optional string path = 5;			// path part of URI
+	optional uint32 index = 6 [default=0];		// identity index
+}

accounts/usbwallet/ledger.go

@@ -0,0 +1,462 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// This file contains the implementation for interacting with the Ledger hardware
+// wallets. The wire protocol spec can be found in the Ledger Blue GitHub repo:
+// https://raw.githubusercontent.com/LedgerHQ/blue-app-eth/master/doc/ethapp.asc
+
+package usbwallet
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+// ledgerOpcode is an enumeration encoding the supported Ledger opcodes.
+type ledgerOpcode byte
+
+// ledgerParam1 is an enumeration encoding the supported Ledger parameters for
+// specific opcodes. The same parameter values may be reused between opcodes.
+type ledgerParam1 byte
+
+// ledgerParam2 is an enumeration encoding the supported Ledger parameters for
+// specific opcodes. The same parameter values may be reused between opcodes.
+type ledgerParam2 byte
+
+const (
+	ledgerOpRetrieveAddress  ledgerOpcode = 0x02 // Returns the public key and Ethereum address for a given BIP 32 path
+	ledgerOpSignTransaction  ledgerOpcode = 0x04 // Signs an Ethereum transaction after having the user validate the parameters
+	ledgerOpGetConfiguration ledgerOpcode = 0x06 // Returns specific wallet application configuration
+
+	ledgerP1DirectlyFetchAddress    ledgerParam1 = 0x00 // Return address directly from the wallet
+	ledgerP1InitTransactionData     ledgerParam1 = 0x00 // First transaction data block for signing
+	ledgerP1ContTransactionData     ledgerParam1 = 0x80 // Subsequent transaction data block for signing
+	ledgerP2DiscardAddressChainCode ledgerParam2 = 0x00 // Do not return the chain code along with the address
+)
+
+// errLedgerReplyInvalidHeader is the error message returned by a Ledger data exchange
+// if the device replies with a mismatching header. This usually means the device
+// is in browser mode.
+var errLedgerReplyInvalidHeader = errors.New("ledger: invalid reply header")
+
+// errLedgerInvalidVersionReply is the error message returned by a Ledger version retrieval
+// when a response does arrive, but it does not contain the expected data.
+var errLedgerInvalidVersionReply = errors.New("ledger: invalid version reply")
+
+// ledgerDriver implements the communication with a Ledger hardware wallet.
+type ledgerDriver struct {
+	device  io.ReadWriter // USB device connection to communicate through
+	version [3]byte       // Current version of the Ledger firmware (zero if app is offline)
+	browser bool          // Flag whether the Ledger is in browser mode (reply channel mismatch)
+	failure error         // Any failure that would make the device unusable
+	log     log.Logger    // Contextual logger to tag the ledger with its id
+}
+
+// newLedgerDriver creates a new instance of a Ledger USB protocol driver.
+func newLedgerDriver(logger log.Logger) driver {
+	return &ledgerDriver{
+		log: logger,
+	}
+}
+
+// Status implements usbwallet.driver, returning various states the Ledger can
+// currently be in.
+func (w *ledgerDriver) Status() (string, error) {
+	if w.failure != nil {
+		return fmt.Sprintf("Failed: %v", w.failure), w.failure
+	}
+	if w.browser {
+		return "Ethereum app in browser mode", w.failure
+	}
+	if w.offline() {
+		return "Ethereum app offline", w.failure
+	}
+	return fmt.Sprintf("Ethereum app v%d.%d.%d online", w.version[0], w.version[1], w.version[2]), w.failure
+}
+
+// offline returns whether the wallet and the Ethereum app is offline or not.
+//
+// The method assumes that the state lock is held!
+func (w *ledgerDriver) offline() bool {
+	return w.version == [3]byte{0, 0, 0}
+}
+
+// Open implements usbwallet.driver, attempting to initialize the connection to the
+// Ledger hardware wallet. The Ledger does not require a user passphrase, so that
+// parameter is silently discarded.
+func (w *ledgerDriver) Open(device io.ReadWriter, passphrase string) error {
+	w.device, w.failure = device, nil
+
+	_, err := w.ledgerDerive(accounts.DefaultBaseDerivationPath)
+	if err != nil {
+		// Ethereum app is not running or in browser mode, nothing more to do, return
+		if err == errLedgerReplyInvalidHeader {
+			w.browser = true
+		}
+		return nil
+	}
+	// Try to resolve the Ethereum app's version, will fail prior to v1.0.2
+	if w.version, err = w.ledgerVersion(); err != nil {
+		w.version = [3]byte{1, 0, 0} // Assume worst case, can't verify if v1.0.0 or v1.0.1
+	}
+	return nil
+}
+
+// Close implements usbwallet.driver, cleaning up and metadata maintained within
+// the Ledger driver.
+func (w *ledgerDriver) Close() error {
+	w.browser, w.version = false, [3]byte{}
+	return nil
+}
+
+// Heartbeat implements usbwallet.driver, performing a sanity check against the
+// Ledger to see if it's still online.
+func (w *ledgerDriver) Heartbeat() error {
+	if _, err := w.ledgerVersion(); err != nil && err != errLedgerInvalidVersionReply {
+		w.failure = err
+		return err
+	}
+	return nil
+}
+
+// Derive implements usbwallet.driver, sending a derivation request to the Ledger
+// and returning the Ethereum address located on that derivation path.
+func (w *ledgerDriver) Derive(path accounts.DerivationPath) (common.Address, error) {
+	return w.ledgerDerive(path)
+}
+
+// SignTx implements usbwallet.driver, sending the transaction to the Ledger and
+// waiting for the user to confirm or deny the transaction.
+//
+// Note, if the version of the Ethereum application running on the Ledger wallet is
+// too old to sign EIP-155 transactions, but such is requested nonetheless, an error
+// will be returned opposed to silently signing in Homestead mode.
+func (w *ledgerDriver) SignTx(path accounts.DerivationPath, tx *types.Transaction, chainID *big.Int) (common.Address, *types.Transaction, error) {
+	// If the Ethereum app doesn't run, abort
+	if w.offline() {
+		return common.Address{}, nil, accounts.ErrWalletClosed
+	}
+	// Ensure the wallet is capable of signing the given transaction
+	if chainID != nil && w.version[0] <= 1 && w.version[1] <= 0 && w.version[2] <= 2 {
+		return common.Address{}, nil, fmt.Errorf("Ledger v%d.%d.%d doesn't support signing this transaction, please update to v1.0.3 at least", w.version[0], w.version[1], w.version[2])
+	}
+	// All infos gathered and metadata checks out, request signing
+	return w.ledgerSign(path, tx, chainID)
+}
+
+// ledgerVersion retrieves the current version of the Ethereum wallet app running
+// on the Ledger wallet.
+//
+// The version retrieval protocol is defined as follows:
+//
+//   CLA | INS | P1 | P2 | Lc | Le
+//   ----+-----+----+----+----+---
+//    E0 | 06  | 00 | 00 | 00 | 04
+//
+// With no input data, and the output data being:
+//
+//   Description                                        | Length
+//   ---------------------------------------------------+--------
+//   Flags 01: arbitrary data signature enabled by user | 1 byte
+//   Application major version                          | 1 byte
+//   Application minor version                          | 1 byte
+//   Application patch version                          | 1 byte
+func (w *ledgerDriver) ledgerVersion() ([3]byte, error) {
+	// Send the request and wait for the response
+	reply, err := w.ledgerExchange(ledgerOpGetConfiguration, 0, 0, nil)
+	if err != nil {
+		return [3]byte{}, err
+	}
+	if len(reply) != 4 {
+		return [3]byte{}, errLedgerInvalidVersionReply
+	}
+	// Cache the version for future reference
+	var version [3]byte
+	copy(version[:], reply[1:])
+	return version, nil
+}
+
+// ledgerDerive retrieves the currently active Ethereum address from a Ledger
+// wallet at the specified derivation path.
+//
+// The address derivation protocol is defined as follows:
+//
+//   CLA | INS | P1 | P2 | Lc  | Le
+//   ----+-----+----+----+-----+---
+//    E0 | 02  | 00 return address
+//               01 display address and confirm before returning
+//                  | 00: do not return the chain code
+//                  | 01: return the chain code
+//                       | var | 00
+//
+// Where the input data is:
+//
+//   Description                                      | Length
+//   -------------------------------------------------+--------
+//   Number of BIP 32 derivations to perform (max 10) | 1 byte
+//   First derivation index (big endian)              | 4 bytes
+//   ...                                              | 4 bytes
+//   Last derivation index (big endian)               | 4 bytes
+//
+// And the output data is:
+//
+//   Description             | Length
+//   ------------------------+-------------------
+//   Public Key length       | 1 byte
+//   Uncompressed Public Key | arbitrary
+//   Ethereum address length | 1 byte
+//   Ethereum address        | 40 bytes hex ascii
+//   Chain code if requested | 32 bytes
+func (w *ledgerDriver) ledgerDerive(derivationPath []uint32) (common.Address, error) {
+	// Flatten the derivation path into the Ledger request
+	path := make([]byte, 1+4*len(derivationPath))
+	path[0] = byte(len(derivationPath))
+	for i, component := range derivationPath {
+		binary.BigEndian.PutUint32(path[1+4*i:], component)
+	}
+	// Send the request and wait for the response
+	reply, err := w.ledgerExchange(ledgerOpRetrieveAddress, ledgerP1DirectlyFetchAddress, ledgerP2DiscardAddressChainCode, path)
+	if err != nil {
+		return common.Address{}, err
+	}
+	// Discard the public key, we don't need that for now
+	if len(reply) < 1 || len(reply) < 1+int(reply[0]) {
+		return common.Address{}, errors.New("reply lacks public key entry")
+	}
+	reply = reply[1+int(reply[0]):]
+
+	// Extract the Ethereum hex address string
+	if len(reply) < 1 || len(reply) < 1+int(reply[0]) {
+		return common.Address{}, errors.New("reply lacks address entry")
+	}
+	hexstr := reply[1 : 1+int(reply[0])]
+
+	// Decode the hex sting into an Ethereum address and return
+	var address common.Address
+	hex.Decode(address[:], hexstr)
+	return address, nil
+}
+
+// ledgerSign sends the transaction to the Ledger wallet, and waits for the user
+// to confirm or deny the transaction.
+//
+// The transaction signing protocol is defined as follows:
+//
+//   CLA | INS | P1 | P2 | Lc  | Le
+//   ----+-----+----+----+-----+---
+//    E0 | 04  | 00: first transaction data block
+//               80: subsequent transaction data block
+//                  | 00 | variable | variable
+//
+// Where the input for the first transaction block (first 255 bytes) is:
+//
+//   Description                                      | Length
+//   -------------------------------------------------+----------
+//   Number of BIP 32 derivations to perform (max 10) | 1 byte
+//   First derivation index (big endian)              | 4 bytes
+//   ...                                              | 4 bytes
+//   Last derivation index (big endian)               | 4 bytes
+//   RLP transaction chunk                            | arbitrary
+//
+// And the input for subsequent transaction blocks (first 255 bytes) are:
+//
+//   Description           | Length
+//   ----------------------+----------
+//   RLP transaction chunk | arbitrary
+//
+// And the output data is:
+//
+//   Description | Length
+//   ------------+---------
+//   signature V | 1 byte
+//   signature R | 32 bytes
+//   signature S | 32 bytes
+func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction, chainID *big.Int) (common.Address, *types.Transaction, error) {
+	// Flatten the derivation path into the Ledger request
+	path := make([]byte, 1+4*len(derivationPath))
+	path[0] = byte(len(derivationPath))
+	for i, component := range derivationPath {
+		binary.BigEndian.PutUint32(path[1+4*i:], component)
+	}
+	// Create the transaction RLP based on whether legacy or EIP155 signing was requeste
+	var (
+		txrlp []byte
+		err   error
+	)
+	if chainID == nil {
+		if txrlp, err = rlp.EncodeToBytes([]interface{}{tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data()}); err != nil {
+			return common.Address{}, nil, err
+		}
+	} else {
+		if txrlp, err = rlp.EncodeToBytes([]interface{}{tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data(), chainID, big.NewInt(0), big.NewInt(0)}); err != nil {
+			return common.Address{}, nil, err
+		}
+	}
+	payload := append(path, txrlp...)
+
+	// Send the request and wait for the response
+	var (
+		op    = ledgerP1InitTransactionData
+		reply []byte
+	)
+	for len(payload) > 0 {
+		// Calculate the size of the next data chunk
+		chunk := 255
+		if chunk > len(payload) {
+			chunk = len(payload)
+		}
+		// Send the chunk over, ensuring it's processed correctly
+		reply, err = w.ledgerExchange(ledgerOpSignTransaction, op, 0, payload[:chunk])
+		if err != nil {
+			return common.Address{}, nil, err
+		}
+		// Shift the payload and ensure subsequent chunks are marked as such
+		payload = payload[chunk:]
+		op = ledgerP1ContTransactionData
+	}
+	// Extract the Ethereum signature and do a sanity validation
+	if len(reply) != 65 {
+		return common.Address{}, nil, errors.New("reply lacks signature")
+	}
+	signature := append(reply[1:], reply[0])
+
+	// Create the correct signer and signature transform based on the chain ID
+	var signer types.Signer
+	if chainID == nil {
+		signer = new(types.HomesteadSigner)
+	} else {
+		signer = types.NewEIP155Signer(chainID)
+		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+	}
+	signed, err := tx.WithSignature(signer, signature)
+	if err != nil {
+		return common.Address{}, nil, err
+	}
+	sender, err := types.Sender(signer, signed)
+	if err != nil {
+		return common.Address{}, nil, err
+	}
+	return sender, signed, nil
+}
+
+// ledgerExchange performs a data exchange with the Ledger wallet, sending it a
+// message and retrieving the response.
+//
+// The common transport header is defined as follows:
+//
+//  Description                           | Length
+//  --------------------------------------+----------
+//  Communication channel ID (big endian) | 2 bytes
+//  Command tag                           | 1 byte
+//  Packet sequence index (big endian)    | 2 bytes
+//  Payload                               | arbitrary
+//
+// The Communication channel ID allows commands multiplexing over the same
+// physical link. It is not used for the time being, and should be set to 0101
+// to avoid compatibility issues with implementations ignoring a leading 00 byte.
+//
+// The Command tag describes the message content. Use TAG_APDU (0x05) for standard
+// APDU payloads, or TAG_PING (0x02) for a simple link test.
+//
+// The Packet sequence index describes the current sequence for fragmented payloads.
+// The first fragment index is 0x00.
+//
+// APDU Command payloads are encoded as follows:
+//
+//  Description              | Length
+//  -----------------------------------
+//  APDU length (big endian) | 2 bytes
+//  APDU CLA                 | 1 byte
+//  APDU INS                 | 1 byte
+//  APDU P1                  | 1 byte
+//  APDU P2                  | 1 byte
+//  APDU length              | 1 byte
+//  Optional APDU data       | arbitrary
+func (w *ledgerDriver) ledgerExchange(opcode ledgerOpcode, p1 ledgerParam1, p2 ledgerParam2, data []byte) ([]byte, error) {
+	// Construct the message payload, possibly split into multiple chunks
+	apdu := make([]byte, 2, 7+len(data))
+
+	binary.BigEndian.PutUint16(apdu, uint16(5+len(data)))
+	apdu = append(apdu, []byte{0xe0, byte(opcode), byte(p1), byte(p2), byte(len(data))}...)
+	apdu = append(apdu, data...)
+
+	// Stream all the chunks to the device
+	header := []byte{0x01, 0x01, 0x05, 0x00, 0x00} // Channel ID and command tag appended
+	chunk := make([]byte, 64)
+	space := len(chunk) - len(header)
+
+	for i := 0; len(apdu) > 0; i++ {
+		// Construct the new message to stream
+		chunk = append(chunk[:0], header...)
+		binary.BigEndian.PutUint16(chunk[3:], uint16(i))
+
+		if len(apdu) > space {
+			chunk = append(chunk, apdu[:space]...)
+			apdu = apdu[space:]
+		} else {
+			chunk = append(chunk, apdu...)
+			apdu = nil
+		}
+		// Send over to the device
+		w.log.Trace("Data chunk sent to the Ledger", "chunk", hexutil.Bytes(chunk))
+		if _, err := w.device.Write(chunk); err != nil {
+			return nil, err
+		}
+	}
+	// Stream the reply back from the wallet in 64 byte chunks
+	var reply []byte
+	chunk = chunk[:64] // Yeah, we surely have enough space
+	for {
+		// Read the next chunk from the Ledger wallet
+		if _, err := io.ReadFull(w.device, chunk); err != nil {
+			return nil, err
+		}
+		w.log.Trace("Data chunk received from the Ledger", "chunk", hexutil.Bytes(chunk))
+
+		// Make sure the transport header matches
+		if chunk[0] != 0x01 || chunk[1] != 0x01 || chunk[2] != 0x05 {
+			return nil, errLedgerReplyInvalidHeader
+		}
+		// If it's the first chunk, retrieve the total message length
+		var payload []byte
+
+		if chunk[3] == 0x00 && chunk[4] == 0x00 {
+			reply = make([]byte, 0, int(binary.BigEndian.Uint16(chunk[5:7])))
+			payload = chunk[7:]
+		} else {
+			payload = chunk[5:]
+		}
+		// Append to the reply and stop when filled up
+		if left := cap(reply) - len(reply); left > len(payload) {
+			reply = append(reply, payload...)
+		} else {
+			reply = append(reply, payload[:left]...)
+			break
+		}
+	}
+	return reply[:len(reply)-2], nil
+}

accounts/usbwallet/ledger_wallet.go

@@ -1,903 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// This file contains the implementation for interacting with the Ledger hardware
-// wallets. The wire protocol spec can be found in the Ledger Blue GitHub repo:
-// https://raw.githubusercontent.com/LedgerHQ/blue-app-eth/master/doc/ethapp.asc
-
-package usbwallet
-
-import (
-	"context"
-	"encoding/binary"
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"io"
-	"math/big"
-	"sync"
-	"time"
-
-	ethereum "github.com/ethereum/go-ethereum"
-	"github.com/ethereum/go-ethereum/accounts"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/karalabe/hid"
-)
-
-// Maximum time between wallet health checks to detect USB unplugs.
-const ledgerHeartbeatCycle = time.Second
-
-// Minimum time to wait between self derivation attempts, even it the user is
-// requesting accounts like crazy.
-const ledgerSelfDeriveThrottling = time.Second
-
-// ledgerOpcode is an enumeration encoding the supported Ledger opcodes.
-type ledgerOpcode byte
-
-// ledgerParam1 is an enumeration encoding the supported Ledger parameters for
-// specific opcodes. The same parameter values may be reused between opcodes.
-type ledgerParam1 byte
-
-// ledgerParam2 is an enumeration encoding the supported Ledger parameters for
-// specific opcodes. The same parameter values may be reused between opcodes.
-type ledgerParam2 byte
-
-const (
-	ledgerOpRetrieveAddress  ledgerOpcode = 0x02 // Returns the public key and Ethereum address for a given BIP 32 path
-	ledgerOpSignTransaction  ledgerOpcode = 0x04 // Signs an Ethereum transaction after having the user validate the parameters
-	ledgerOpGetConfiguration ledgerOpcode = 0x06 // Returns specific wallet application configuration
-
-	ledgerP1DirectlyFetchAddress    ledgerParam1 = 0x00 // Return address directly from the wallet
-	ledgerP1ConfirmFetchAddress     ledgerParam1 = 0x01 // Require a user confirmation before returning the address
-	ledgerP1InitTransactionData     ledgerParam1 = 0x00 // First transaction data block for signing
-	ledgerP1ContTransactionData     ledgerParam1 = 0x80 // Subsequent transaction data block for signing
-	ledgerP2DiscardAddressChainCode ledgerParam2 = 0x00 // Do not return the chain code along with the address
-	ledgerP2ReturnAddressChainCode  ledgerParam2 = 0x01 // Require a user confirmation before returning the address
-)
-
-// errReplyInvalidHeader is the error message returned by a Ledger data exchange
-// if the device replies with a mismatching header. This usually means the device
-// is in browser mode.
-var errReplyInvalidHeader = errors.New("invalid reply header")
-
-// errInvalidVersionReply is the error message returned by a Ledger version retrieval
-// when a response does arrive, but it does not contain the expected data.
-var errInvalidVersionReply = errors.New("invalid version reply")
-
-// ledgerWallet represents a live USB Ledger hardware wallet.
-type ledgerWallet struct {
-	hub *LedgerHub    // USB hub the device originates from (TODO(karalabe): remove if hotplug lands on Windows)
-	url *accounts.URL // Textual URL uniquely identifying this wallet
-
-	info    hid.DeviceInfo // Known USB device infos about the wallet
-	device  *hid.Device    // USB device advertising itself as a Ledger wallet
-	failure error          // Any failure that would make the device unusable
-
-	version  [3]byte                                    // Current version of the Ledger Ethereum app (zero if app is offline)
-	browser  bool                                       // Flag whether the Ledger is in browser mode (reply channel mismatch)
-	accounts []accounts.Account                         // List of derive accounts pinned on the Ledger
-	paths    map[common.Address]accounts.DerivationPath // Known derivation paths for signing operations
-
-	deriveNextPath accounts.DerivationPath   // Next derivation path for account auto-discovery
-	deriveNextAddr common.Address            // Next derived account address for auto-discovery
-	deriveChain    ethereum.ChainStateReader // Blockchain state reader to discover used account with
-	deriveReq      chan chan struct{}        // Channel to request a self-derivation on
-	deriveQuit     chan chan error           // Channel to terminate the self-deriver with
-
-	healthQuit chan chan error
-
-	// Locking a hardware wallet is a bit special. Since hardware devices are lower
-	// performing, any communication with them might take a non negligible amount of
-	// time. Worse still, waiting for user confirmation can take arbitrarily long,
-	// but exclusive communication must be upheld during. Locking the entire wallet
-	// in the mean time however would stall any parts of the system that don't want
-	// to communicate, just read some state (e.g. list the accounts).
-	//
-	// As such, a hardware wallet needs two locks to function correctly. A state
-	// lock can be used to protect the wallet's software-side internal state, which
-	// must not be held exlusively during hardware communication. A communication
-	// lock can be used to achieve exclusive access to the device itself, this one
-	// however should allow "skipping" waiting for operations that might want to
-	// use the device, but can live without too (e.g. account self-derivation).
-	//
-	// Since we have two locks, it's important to know how to properly use them:
-	//   - Communication requires the `device` to not change, so obtaining the
-	//     commsLock should be done after having a stateLock.
-	//   - Communication must not disable read access to the wallet state, so it
-	//     must only ever hold a *read* lock to stateLock.
-	commsLock chan struct{} // Mutex (buf=1) for the USB comms without keeping the state locked
-	stateLock sync.RWMutex  // Protects read and write access to the wallet struct fields
-
-	log log.Logger // Contextual logger to tag the ledger with its id
-}
-
-// URL implements accounts.Wallet, returning the URL of the Ledger device.
-func (w *ledgerWallet) URL() accounts.URL {
-	return *w.url // Immutable, no need for a lock
-}
-
-// Status implements accounts.Wallet, always whether the Ledger is opened, closed
-// or whether the Ethereum app was not started on it.
-func (w *ledgerWallet) Status() string {
-	w.stateLock.RLock() // No device communication, state lock is enough
-	defer w.stateLock.RUnlock()
-
-	if w.failure != nil {
-		return fmt.Sprintf("Failed: %v", w.failure)
-	}
-	if w.device == nil {
-		return "Closed"
-	}
-	if w.browser {
-		return "Ethereum app in browser mode"
-	}
-	if w.offline() {
-		return "Ethereum app offline"
-	}
-	return fmt.Sprintf("Ethereum app v%d.%d.%d online", w.version[0], w.version[1], w.version[2])
-}
-
-// offline returns whether the wallet and the Ethereum app is offline or not.
-//
-// The method assumes that the state lock is held!
-func (w *ledgerWallet) offline() bool {
-	return w.version == [3]byte{0, 0, 0}
-}
-
-// failed returns if the USB device wrapped by the wallet failed for some reason.
-// This is used by the device scanner to report failed wallets as departed.
-//
-// The method assumes that the state lock is *not* held!
-func (w *ledgerWallet) failed() bool {
-	w.stateLock.RLock() // No device communication, state lock is enough
-	defer w.stateLock.RUnlock()
-
-	return w.failure != nil
-}
-
-// Open implements accounts.Wallet, attempting to open a USB connection to the
-// Ledger hardware wallet. The Ledger does not require a user passphrase, so that
-// parameter is silently discarded.
-func (w *ledgerWallet) Open(passphrase string) error {
-	w.stateLock.Lock() // State lock is enough since there's no connection yet at this point
-	defer w.stateLock.Unlock()
-
-	// If the wallet was already opened, don't try to open again
-	if w.device != nil {
-		return accounts.ErrWalletAlreadyOpen
-	}
-	// Otherwise iterate over all USB devices and find this again (no way to directly do this)
-	device, err := w.info.Open()
-	if err != nil {
-		return err
-	}
-	// Wallet seems to be successfully opened, guess if the Ethereum app is running
-	w.device = device
-	w.commsLock = make(chan struct{}, 1)
-	w.commsLock <- struct{}{} // Enable lock
-
-	w.paths = make(map[common.Address]accounts.DerivationPath)
-
-	w.deriveReq = make(chan chan struct{})
-	w.deriveQuit = make(chan chan error)
-	w.healthQuit = make(chan chan error)
-
-	defer func() {
-		go w.heartbeat()
-		go w.selfDerive()
-	}()
-
-	if _, err = w.ledgerDerive(accounts.DefaultBaseDerivationPath); err != nil {
-		// Ethereum app is not running or in browser mode, nothing more to do, return
-		if err == errReplyInvalidHeader {
-			w.browser = true
-		}
-		return nil
-	}
-	// Try to resolve the Ethereum app's version, will fail prior to v1.0.2
-	if w.version, err = w.ledgerVersion(); err != nil {
-		w.version = [3]byte{1, 0, 0} // Assume worst case, can't verify if v1.0.0 or v1.0.1
-	}
-	return nil
-}
-
-// heartbeat is a health check loop for the Ledger wallets to periodically verify
-// whether they are still present or if they malfunctioned. It is needed because:
-//  - libusb on Windows doesn't support hotplug, so we can't detect USB unplugs
-//  - communication timeout on the Ledger requires a device power cycle to fix
-func (w *ledgerWallet) heartbeat() {
-	w.log.Debug("Ledger health-check started")
-	defer w.log.Debug("Ledger health-check stopped")
-
-	// Execute heartbeat checks until termination or error
-	var (
-		errc chan error
-		err  error
-	)
-	for errc == nil && err == nil {
-		// Wait until termination is requested or the heartbeat cycle arrives
-		select {
-		case errc = <-w.healthQuit:
-			// Termination requested
-			continue
-		case <-time.After(ledgerHeartbeatCycle):
-			// Heartbeat time
-		}
-		// Execute a tiny data exchange to see responsiveness
-		w.stateLock.RLock()
-		if w.device == nil {
-			// Terminated while waiting for the lock
-			w.stateLock.RUnlock()
-			continue
-		}
-		<-w.commsLock // Don't lock state while resolving version
-		_, err = w.ledgerVersion()
-		w.commsLock <- struct{}{}
-		w.stateLock.RUnlock()
-
-		if err != nil && err != errInvalidVersionReply {
-			w.stateLock.Lock() // Lock state to tear the wallet down
-			w.failure = err
-			w.close()
-			w.stateLock.Unlock()
-		}
-		// Ignore non hardware related errors
-		err = nil
-	}
-	// In case of error, wait for termination
-	if err != nil {
-		w.log.Debug("Ledger health-check failed", "err", err)
-		errc = <-w.healthQuit
-	}
-	errc <- err
-}
-
-// Close implements accounts.Wallet, closing the USB connection to the Ledger.
-func (w *ledgerWallet) Close() error {
-	// Ensure the wallet was opened
-	w.stateLock.RLock()
-	hQuit, dQuit := w.healthQuit, w.deriveQuit
-	w.stateLock.RUnlock()
-
-	// Terminate the health checks
-	var herr error
-	if hQuit != nil {
-		errc := make(chan error)
-		hQuit <- errc
-		herr = <-errc // Save for later, we *must* close the USB
-	}
-	// Terminate the self-derivations
-	var derr error
-	if dQuit != nil {
-		errc := make(chan error)
-		dQuit <- errc
-		derr = <-errc // Save for later, we *must* close the USB
-	}
-	// Terminate the device connection
-	w.stateLock.Lock()
-	defer w.stateLock.Unlock()
-
-	w.healthQuit = nil
-	w.deriveQuit = nil
-	w.deriveReq = nil
-
-	if err := w.close(); err != nil {
-		return err
-	}
-	if herr != nil {
-		return herr
-	}
-	return derr
-}
-
-// close is the internal wallet closer that terminates the USB connection and
-// resets all the fields to their defaults.
-//
-// Note, close assumes the state lock is held!
-func (w *ledgerWallet) close() error {
-	// Allow duplicate closes, especially for health-check failures
-	if w.device == nil {
-		return nil
-	}
-	// Close the device, clear everything, then return
-	w.device.Close()
-	w.device = nil
-
-	w.browser, w.version = false, [3]byte{}
-	w.accounts, w.paths = nil, nil
-
-	return nil
-}
-
-// Accounts implements accounts.Wallet, returning the list of accounts pinned to
-// the Ledger hardware wallet. If self-derivation was enabled, the account list
-// is periodically expanded based on current chain state.
-func (w *ledgerWallet) Accounts() []accounts.Account {
-	// Attempt self-derivation if it's running
-	reqc := make(chan struct{}, 1)
-	select {
-	case w.deriveReq <- reqc:
-		// Self-derivation request accepted, wait for it
-		<-reqc
-	default:
-		// Self-derivation offline, throttled or busy, skip
-	}
-	// Return whatever account list we ended up with
-	w.stateLock.RLock()
-	defer w.stateLock.RUnlock()
-
-	cpy := make([]accounts.Account, len(w.accounts))
-	copy(cpy, w.accounts)
-	return cpy
-}
-
-// selfDerive is an account derivation loop that upon request attempts to find
-// new non-zero accounts.
-func (w *ledgerWallet) selfDerive() {
-	w.log.Debug("Ledger self-derivation started")
-	defer w.log.Debug("Ledger self-derivation stopped")
-
-	// Execute self-derivations until termination or error
-	var (
-		reqc chan struct{}
-		errc chan error
-		err  error
-	)
-	for errc == nil && err == nil {
-		// Wait until either derivation or termination is requested
-		select {
-		case errc = <-w.deriveQuit:
-			// Termination requested
-			continue
-		case reqc = <-w.deriveReq:
-			// Account discovery requested
-		}
-		// Derivation needs a chain and device access, skip if either unavailable
-		w.stateLock.RLock()
-		if w.device == nil || w.deriveChain == nil || w.offline() {
-			w.stateLock.RUnlock()
-			reqc <- struct{}{}
-			continue
-		}
-		select {
-		case <-w.commsLock:
-		default:
-			w.stateLock.RUnlock()
-			reqc <- struct{}{}
-			continue
-		}
-		// Device lock obtained, derive the next batch of accounts
-		var (
-			accs  []accounts.Account
-			paths []accounts.DerivationPath
-
-			nextAddr = w.deriveNextAddr
-			nextPath = w.deriveNextPath
-
-			context = context.Background()
-		)
-		for empty := false; !empty; {
-			// Retrieve the next derived Ethereum account
-			if nextAddr == (common.Address{}) {
-				if nextAddr, err = w.ledgerDerive(nextPath); err != nil {
-					w.log.Warn("Ledger account derivation failed", "err", err)
-					break
-				}
-			}
-			// Check the account's status against the current chain state
-			var (
-				balance *big.Int
-				nonce   uint64
-			)
-			balance, err = w.deriveChain.BalanceAt(context, nextAddr, nil)
-			if err != nil {
-				w.log.Warn("Ledger balance retrieval failed", "err", err)
-				break
-			}
-			nonce, err = w.deriveChain.NonceAt(context, nextAddr, nil)
-			if err != nil {
-				w.log.Warn("Ledger nonce retrieval failed", "err", err)
-				break
-			}
-			// If the next account is empty, stop self-derivation, but add it nonetheless
-			if balance.Sign() == 0 && nonce == 0 {
-				empty = true
-			}
-			// We've just self-derived a new account, start tracking it locally
-			path := make(accounts.DerivationPath, len(nextPath))
-			copy(path[:], nextPath[:])
-			paths = append(paths, path)
-
-			account := accounts.Account{
-				Address: nextAddr,
-				URL:     accounts.URL{Scheme: w.url.Scheme, Path: fmt.Sprintf("%s/%s", w.url.Path, path)},
-			}
-			accs = append(accs, account)
-
-			// Display a log message to the user for new (or previously empty accounts)
-			if _, known := w.paths[nextAddr]; !known || (!empty && nextAddr == w.deriveNextAddr) {
-				w.log.Info("Ledger discovered new account", "address", nextAddr, "path", path, "balance", balance, "nonce", nonce)
-			}
-			// Fetch the next potential account
-			if !empty {
-				nextAddr = common.Address{}
-				nextPath[len(nextPath)-1]++
-			}
-		}
-		// Self derivation complete, release device lock
-		w.commsLock <- struct{}{}
-		w.stateLock.RUnlock()
-
-		// Insert any accounts successfully derived
-		w.stateLock.Lock()
-		for i := 0; i < len(accs); i++ {
-			if _, ok := w.paths[accs[i].Address]; !ok {
-				w.accounts = append(w.accounts, accs[i])
-				w.paths[accs[i].Address] = paths[i]
-			}
-		}
-		// Shift the self-derivation forward
-		// TODO(karalabe): don't overwrite changes from wallet.SelfDerive
-		w.deriveNextAddr = nextAddr
-		w.deriveNextPath = nextPath
-		w.stateLock.Unlock()
-
-		// Notify the user of termination and loop after a bit of time (to avoid trashing)
-		reqc <- struct{}{}
-		if err == nil {
-			select {
-			case errc = <-w.deriveQuit:
-				// Termination requested, abort
-			case <-time.After(ledgerSelfDeriveThrottling):
-				// Waited enough, willing to self-derive again
-			}
-		}
-	}
-	// In case of error, wait for termination
-	if err != nil {
-		w.log.Debug("Ledger self-derivation failed", "err", err)
-		errc = <-w.deriveQuit
-	}
-	errc <- err
-}
-
-// Contains implements accounts.Wallet, returning whether a particular account is
-// or is not pinned into this Ledger instance. Although we could attempt to resolve
-// unpinned accounts, that would be an non-negligible hardware operation.
-func (w *ledgerWallet) Contains(account accounts.Account) bool {
-	w.stateLock.RLock()
-	defer w.stateLock.RUnlock()
-
-	_, exists := w.paths[account.Address]
-	return exists
-}
-
-// Derive implements accounts.Wallet, deriving a new account at the specific
-// derivation path. If pin is set to true, the account will be added to the list
-// of tracked accounts.
-func (w *ledgerWallet) Derive(path accounts.DerivationPath, pin bool) (accounts.Account, error) {
-	// Try to derive the actual account and update its URL if successful
-	w.stateLock.RLock() // Avoid device disappearing during derivation
-
-	if w.device == nil || w.offline() {
-		w.stateLock.RUnlock()
-		return accounts.Account{}, accounts.ErrWalletClosed
-	}
-	<-w.commsLock // Avoid concurrent hardware access
-	address, err := w.ledgerDerive(path)
-	w.commsLock <- struct{}{}
-
-	w.stateLock.RUnlock()
-
-	// If an error occurred or no pinning was requested, return
-	if err != nil {
-		return accounts.Account{}, err
-	}
-	account := accounts.Account{
-		Address: address,
-		URL:     accounts.URL{Scheme: w.url.Scheme, Path: fmt.Sprintf("%s/%s", w.url.Path, path)},
-	}
-	if !pin {
-		return account, nil
-	}
-	// Pinning needs to modify the state
-	w.stateLock.Lock()
-	defer w.stateLock.Unlock()
-
-	if _, ok := w.paths[address]; !ok {
-		w.accounts = append(w.accounts, account)
-		w.paths[address] = path
-	}
-	return account, nil
-}
-
-// SelfDerive implements accounts.Wallet, trying to discover accounts that the
-// user used previously (based on the chain state), but ones that he/she did not
-// explicitly pin to the wallet manually. To avoid chain head monitoring, self
-// derivation only runs during account listing (and even then throttled).
-func (w *ledgerWallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {
-	w.stateLock.Lock()
-	defer w.stateLock.Unlock()
-
-	w.deriveNextPath = make(accounts.DerivationPath, len(base))
-	copy(w.deriveNextPath[:], base[:])
-
-	w.deriveNextAddr = common.Address{}
-	w.deriveChain = chain
-}
-
-// SignHash implements accounts.Wallet, however signing arbitrary data is not
-// supported for Ledger wallets, so this method will always return an error.
-func (w *ledgerWallet) SignHash(acc accounts.Account, hash []byte) ([]byte, error) {
-	return nil, accounts.ErrNotSupported
-}
-
-// SignTx implements accounts.Wallet. It sends the transaction over to the Ledger
-// wallet to request a confirmation from the user. It returns either the signed
-// transaction or a failure if the user denied the transaction.
-//
-// Note, if the version of the Ethereum application running on the Ledger wallet is
-// too old to sign EIP-155 transactions, but such is requested nonetheless, an error
-// will be returned opposed to silently signing in Homestead mode.
-func (w *ledgerWallet) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
-	w.stateLock.RLock() // Comms have own mutex, this is for the state fields
-	defer w.stateLock.RUnlock()
-
-	// If the wallet is closed, or the Ethereum app doesn't run, abort
-	if w.device == nil || w.offline() {
-		return nil, accounts.ErrWalletClosed
-	}
-	// Make sure the requested account is contained within
-	path, ok := w.paths[account.Address]
-	if !ok {
-		return nil, accounts.ErrUnknownAccount
-	}
-	// Ensure the wallet is capable of signing the given transaction
-	if chainID != nil && w.version[0] <= 1 && w.version[1] <= 0 && w.version[2] <= 2 {
-		return nil, fmt.Errorf("Ledger v%d.%d.%d doesn't support signing this transaction, please update to v1.0.3 at least", w.version[0], w.version[1], w.version[2])
-	}
-	// All infos gathered and metadata checks out, request signing
-	<-w.commsLock
-	defer func() { w.commsLock <- struct{}{} }()
-
-	// Ensure the device isn't screwed with while user confirmation is pending
-	// TODO(karalabe): remove if hotplug lands on Windows
-	w.hub.commsLock.Lock()
-	w.hub.commsPend++
-	w.hub.commsLock.Unlock()
-
-	defer func() {
-		w.hub.commsLock.Lock()
-		w.hub.commsPend--
-		w.hub.commsLock.Unlock()
-	}()
-	return w.ledgerSign(path, account.Address, tx, chainID)
-}
-
-// SignHashWithPassphrase implements accounts.Wallet, however signing arbitrary
-// data is not supported for Ledger wallets, so this method will always return
-// an error.
-func (w *ledgerWallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
-	return nil, accounts.ErrNotSupported
-}
-
-// SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given
-// transaction with the given account using passphrase as extra authentication.
-// Since the Ledger does not support extra passphrases, it is silently ignored.
-func (w *ledgerWallet) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
-	return w.SignTx(account, tx, chainID)
-}
-
-// ledgerVersion retrieves the current version of the Ethereum wallet app running
-// on the Ledger wallet.
-//
-// The version retrieval protocol is defined as follows:
-//
-//   CLA | INS | P1 | P2 | Lc | Le
-//   ----+-----+----+----+----+---
-//    E0 | 06  | 00 | 00 | 00 | 04
-//
-// With no input data, and the output data being:
-//
-//   Description                                        | Length
-//   ---------------------------------------------------+--------
-//   Flags 01: arbitrary data signature enabled by user | 1 byte
-//   Application major version                          | 1 byte
-//   Application minor version                          | 1 byte
-//   Application patch version                          | 1 byte
-func (w *ledgerWallet) ledgerVersion() ([3]byte, error) {
-	// Send the request and wait for the response
-	reply, err := w.ledgerExchange(ledgerOpGetConfiguration, 0, 0, nil)
-	if err != nil {
-		return [3]byte{}, err
-	}
-	if len(reply) != 4 {
-		return [3]byte{}, errInvalidVersionReply
-	}
-	// Cache the version for future reference
-	var version [3]byte
-	copy(version[:], reply[1:])
-	return version, nil
-}
-
-// ledgerDerive retrieves the currently active Ethereum address from a Ledger
-// wallet at the specified derivation path.
-//
-// The address derivation protocol is defined as follows:
-//
-//   CLA | INS | P1 | P2 | Lc  | Le
-//   ----+-----+----+----+-----+---
-//    E0 | 02  | 00 return address
-//               01 display address and confirm before returning
-//                  | 00: do not return the chain code
-//                  | 01: return the chain code
-//                       | var | 00
-//
-// Where the input data is:
-//
-//   Description                                      | Length
-//   -------------------------------------------------+--------
-//   Number of BIP 32 derivations to perform (max 10) | 1 byte
-//   First derivation index (big endian)              | 4 bytes
-//   ...                                              | 4 bytes
-//   Last derivation index (big endian)               | 4 bytes
-//
-// And the output data is:
-//
-//   Description             | Length
-//   ------------------------+-------------------
-//   Public Key length       | 1 byte
-//   Uncompressed Public Key | arbitrary
-//   Ethereum address length | 1 byte
-//   Ethereum address        | 40 bytes hex ascii
-//   Chain code if requested | 32 bytes
-func (w *ledgerWallet) ledgerDerive(derivationPath []uint32) (common.Address, error) {
-	// Flatten the derivation path into the Ledger request
-	path := make([]byte, 1+4*len(derivationPath))
-	path[0] = byte(len(derivationPath))
-	for i, component := range derivationPath {
-		binary.BigEndian.PutUint32(path[1+4*i:], component)
-	}
-	// Send the request and wait for the response
-	reply, err := w.ledgerExchange(ledgerOpRetrieveAddress, ledgerP1DirectlyFetchAddress, ledgerP2DiscardAddressChainCode, path)
-	if err != nil {
-		return common.Address{}, err
-	}
-	// Discard the public key, we don't need that for now
-	if len(reply) < 1 || len(reply) < 1+int(reply[0]) {
-		return common.Address{}, errors.New("reply lacks public key entry")
-	}
-	reply = reply[1+int(reply[0]):]
-
-	// Extract the Ethereum hex address string
-	if len(reply) < 1 || len(reply) < 1+int(reply[0]) {
-		return common.Address{}, errors.New("reply lacks address entry")
-	}
-	hexstr := reply[1 : 1+int(reply[0])]
-
-	// Decode the hex sting into an Ethereum address and return
-	var address common.Address
-	hex.Decode(address[:], hexstr)
-	return address, nil
-}
-
-// ledgerSign sends the transaction to the Ledger wallet, and waits for the user
-// to confirm or deny the transaction.
-//
-// The transaction signing protocol is defined as follows:
-//
-//   CLA | INS | P1 | P2 | Lc  | Le
-//   ----+-----+----+----+-----+---
-//    E0 | 04  | 00: first transaction data block
-//               80: subsequent transaction data block
-//                  | 00 | variable | variable
-//
-// Where the input for the first transaction block (first 255 bytes) is:
-//
-//   Description                                      | Length
-//   -------------------------------------------------+----------
-//   Number of BIP 32 derivations to perform (max 10) | 1 byte
-//   First derivation index (big endian)              | 4 bytes
-//   ...                                              | 4 bytes
-//   Last derivation index (big endian)               | 4 bytes
-//   RLP transaction chunk                            | arbitrary
-//
-// And the input for subsequent transaction blocks (first 255 bytes) are:
-//
-//   Description           | Length
-//   ----------------------+----------
-//   RLP transaction chunk | arbitrary
-//
-// And the output data is:
-//
-//   Description | Length
-//   ------------+---------
-//   signature V | 1 byte
-//   signature R | 32 bytes
-//   signature S | 32 bytes
-func (w *ledgerWallet) ledgerSign(derivationPath []uint32, address common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
-	// Flatten the derivation path into the Ledger request
-	path := make([]byte, 1+4*len(derivationPath))
-	path[0] = byte(len(derivationPath))
-	for i, component := range derivationPath {
-		binary.BigEndian.PutUint32(path[1+4*i:], component)
-	}
-	// Create the transaction RLP based on whether legacy or EIP155 signing was requeste
-	var (
-		txrlp []byte
-		err   error
-	)
-	if chainID == nil {
-		if txrlp, err = rlp.EncodeToBytes([]interface{}{tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data()}); err != nil {
-			return nil, err
-		}
-	} else {
-		if txrlp, err = rlp.EncodeToBytes([]interface{}{tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data(), chainID, big.NewInt(0), big.NewInt(0)}); err != nil {
-			return nil, err
-		}
-	}
-	payload := append(path, txrlp...)
-
-	// Send the request and wait for the response
-	var (
-		op    = ledgerP1InitTransactionData
-		reply []byte
-	)
-	for len(payload) > 0 {
-		// Calculate the size of the next data chunk
-		chunk := 255
-		if chunk > len(payload) {
-			chunk = len(payload)
-		}
-		// Send the chunk over, ensuring it's processed correctly
-		reply, err = w.ledgerExchange(ledgerOpSignTransaction, op, 0, payload[:chunk])
-		if err != nil {
-			return nil, err
-		}
-		// Shift the payload and ensure subsequent chunks are marked as such
-		payload = payload[chunk:]
-		op = ledgerP1ContTransactionData
-	}
-	// Extract the Ethereum signature and do a sanity validation
-	if len(reply) != 65 {
-		return nil, errors.New("reply lacks signature")
-	}
-	signature := append(reply[1:], reply[0])
-
-	// Create the correct signer and signature transform based on the chain ID
-	var signer types.Signer
-	if chainID == nil {
-		signer = new(types.HomesteadSigner)
-	} else {
-		signer = types.NewEIP155Signer(chainID)
-		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
-	}
-	// Inject the final signature into the transaction and sanity check the sender
-	signed, err := tx.WithSignature(signer, signature)
-	if err != nil {
-		return nil, err
-	}
-	sender, err := types.Sender(signer, signed)
-	if err != nil {
-		return nil, err
-	}
-	if sender != address {
-		return nil, fmt.Errorf("signer mismatch: expected %s, got %s", address.Hex(), sender.Hex())
-	}
-	return signed, nil
-}
-
-// ledgerExchange performs a data exchange with the Ledger wallet, sending it a
-// message and retrieving the response.
-//
-// The common transport header is defined as follows:
-//
-//  Description                           | Length
-//  --------------------------------------+----------
-//  Communication channel ID (big endian) | 2 bytes
-//  Command tag                           | 1 byte
-//  Packet sequence index (big endian)    | 2 bytes
-//  Payload                               | arbitrary
-//
-// The Communication channel ID allows commands multiplexing over the same
-// physical link. It is not used for the time being, and should be set to 0101
-// to avoid compatibility issues with implementations ignoring a leading 00 byte.
-//
-// The Command tag describes the message content. Use TAG_APDU (0x05) for standard
-// APDU payloads, or TAG_PING (0x02) for a simple link test.
-//
-// The Packet sequence index describes the current sequence for fragmented payloads.
-// The first fragment index is 0x00.
-//
-// APDU Command payloads are encoded as follows:
-//
-//  Description              | Length
-//  -----------------------------------
-//  APDU length (big endian) | 2 bytes
-//  APDU CLA                 | 1 byte
-//  APDU INS                 | 1 byte
-//  APDU P1                  | 1 byte
-//  APDU P2                  | 1 byte
-//  APDU length              | 1 byte
-//  Optional APDU data       | arbitrary
-func (w *ledgerWallet) ledgerExchange(opcode ledgerOpcode, p1 ledgerParam1, p2 ledgerParam2, data []byte) ([]byte, error) {
-	// Construct the message payload, possibly split into multiple chunks
-	apdu := make([]byte, 2, 7+len(data))
-
-	binary.BigEndian.PutUint16(apdu, uint16(5+len(data)))
-	apdu = append(apdu, []byte{0xe0, byte(opcode), byte(p1), byte(p2), byte(len(data))}...)
-	apdu = append(apdu, data...)
-
-	// Stream all the chunks to the device
-	header := []byte{0x01, 0x01, 0x05, 0x00, 0x00} // Channel ID and command tag appended
-	chunk := make([]byte, 64)
-	space := len(chunk) - len(header)
-
-	for i := 0; len(apdu) > 0; i++ {
-		// Construct the new message to stream
-		chunk = append(chunk[:0], header...)
-		binary.BigEndian.PutUint16(chunk[3:], uint16(i))
-
-		if len(apdu) > space {
-			chunk = append(chunk, apdu[:space]...)
-			apdu = apdu[space:]
-		} else {
-			chunk = append(chunk, apdu...)
-			apdu = nil
-		}
-		// Send over to the device
-		w.log.Trace("Data chunk sent to the Ledger", "chunk", hexutil.Bytes(chunk))
-		if _, err := w.device.Write(chunk); err != nil {
-			return nil, err
-		}
-	}
-	// Stream the reply back from the wallet in 64 byte chunks
-	var reply []byte
-	chunk = chunk[:64] // Yeah, we surely have enough space
-	for {
-		// Read the next chunk from the Ledger wallet
-		if _, err := io.ReadFull(w.device, chunk); err != nil {
-			return nil, err
-		}
-		w.log.Trace("Data chunk received from the Ledger", "chunk", hexutil.Bytes(chunk))
-
-		// Make sure the transport header matches
-		if chunk[0] != 0x01 || chunk[1] != 0x01 || chunk[2] != 0x05 {
-			return nil, errReplyInvalidHeader
-		}
-		// If it's the first chunk, retrieve the total message length
-		var payload []byte
-
-		if chunk[3] == 0x00 && chunk[4] == 0x00 {
-			reply = make([]byte, 0, int(binary.BigEndian.Uint16(chunk[5:7])))
-			payload = chunk[7:]
-		} else {
-			payload = chunk[5:]
-		}
-		// Append to the reply and stop when filled up
-		if left := cap(reply) - len(reply); left > len(payload) {
-			reply = append(reply, payload...)
-		} else {
-			reply = append(reply, payload[:left]...)
-			break
-		}
-	}
-	return reply[:len(reply)-2], nil
-}

accounts/usbwallet/trezor.go

@@ -0,0 +1,330 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// This file contains the implementation for interacting with the Trezor hardware
+// wallets. The wire protocol spec can be found on the SatoshiLabs website:
+// https://doc.satoshilabs.com/trezor-tech/api-protobuf.html
+
+package usbwallet
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/usbwallet/internal/trezor"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/golang/protobuf/proto"
+)
+
+// ErrTrezorPINNeeded is returned if opening the trezor requires a PIN code. In
+// this case, the calling application should display a pinpad and send back the
+// encoded passphrase.
+var ErrTrezorPINNeeded = errors.New("trezor: pin needed")
+
+// errTrezorReplyInvalidHeader is the error message returned by a Trezor data exchange
+// if the device replies with a mismatching header. This usually means the device
+// is in browser mode.
+var errTrezorReplyInvalidHeader = errors.New("trezor: invalid reply header")
+
+// trezorDriver implements the communication with a Trezor hardware wallet.
+type trezorDriver struct {
+	device  io.ReadWriter // USB device connection to communicate through
+	version [3]uint32     // Current version of the Trezor firmware
+	label   string        // Current textual label of the Trezor device
+	pinwait bool          // Flags whether the device is waiting for PIN entry
+	failure error         // Any failure that would make the device unusable
+	log     log.Logger    // Contextual logger to tag the trezor with its id
+}
+
+// newTrezorDriver creates a new instance of a Trezor USB protocol driver.
+func newTrezorDriver(logger log.Logger) driver {
+	return &trezorDriver{
+		log: logger,
+	}
+}
+
+// Status implements accounts.Wallet, always whether the Trezor is opened, closed
+// or whether the Ethereum app was not started on it.
+func (w *trezorDriver) Status() (string, error) {
+	if w.failure != nil {
+		return fmt.Sprintf("Failed: %v", w.failure), w.failure
+	}
+	if w.device == nil {
+		return "Closed", w.failure
+	}
+	if w.pinwait {
+		return fmt.Sprintf("Trezor v%d.%d.%d '%s' waiting for PIN", w.version[0], w.version[1], w.version[2], w.label), w.failure
+	}
+	return fmt.Sprintf("Trezor v%d.%d.%d '%s' online", w.version[0], w.version[1], w.version[2], w.label), w.failure
+}
+
+// Open implements usbwallet.driver, attempting to initialize the connection to
+// the Trezor hardware wallet. Initializing the Trezor is a two phase operation:
+//  * The first phase is to initialize the connection and read the wallet's
+//    features. This phase is invoked is the provided passphrase is empty. The
+//    device will display the pinpad as a result and will return an appropriate
+//    error to notify the user that a second open phase is needed.
+//  * The second phase is to unlock access to the Trezor, which is done by the
+//    user actually providing a passphrase mapping a keyboard keypad to the pin
+//    number of the user (shuffled according to the pinpad displayed).
+func (w *trezorDriver) Open(device io.ReadWriter, passphrase string) error {
+	w.device, w.failure = device, nil
+
+	// If phase 1 is requested, init the connection and wait for user callback
+	if passphrase == "" {
+		// If we're already waiting for a PIN entry, insta-return
+		if w.pinwait {
+			return ErrTrezorPINNeeded
+		}
+		// Initialize a connection to the device
+		features := new(trezor.Features)
+		if _, err := w.trezorExchange(&trezor.Initialize{}, features); err != nil {
+			return err
+		}
+		w.version = [3]uint32{features.GetMajorVersion(), features.GetMinorVersion(), features.GetPatchVersion()}
+		w.label = features.GetLabel()
+
+		// Do a manual ping, forcing the device to ask for its PIN
+		askPin := true
+		res, err := w.trezorExchange(&trezor.Ping{PinProtection: &askPin}, new(trezor.PinMatrixRequest), new(trezor.Success))
+		if err != nil {
+			return err
+		}
+		// Only return the PIN request if the device wasn't unlocked until now
+		if res == 1 {
+			return nil // Device responded with trezor.Success
+		}
+		w.pinwait = true
+		return ErrTrezorPINNeeded
+	}
+	// Phase 2 requested with actual PIN entry
+	w.pinwait = false
+
+	if _, err := w.trezorExchange(&trezor.PinMatrixAck{Pin: &passphrase}, new(trezor.Success)); err != nil {
+		w.failure = err
+		return err
+	}
+	return nil
+}
+
+// Close implements usbwallet.driver, cleaning up and metadata maintained within
+// the Trezor driver.
+func (w *trezorDriver) Close() error {
+	w.version, w.label, w.pinwait = [3]uint32{}, "", false
+	return nil
+}
+
+// Heartbeat implements usbwallet.driver, performing a sanity check against the
+// Trezor to see if it's still online.
+func (w *trezorDriver) Heartbeat() error {
+	if _, err := w.trezorExchange(&trezor.Ping{}, new(trezor.Success)); err != nil {
+		w.failure = err
+		return err
+	}
+	return nil
+}
+
+// Derive implements usbwallet.driver, sending a derivation request to the Trezor
+// and returning the Ethereum address located on that derivation path.
+func (w *trezorDriver) Derive(path accounts.DerivationPath) (common.Address, error) {
+	return w.trezorDerive(path)
+}
+
+// SignTx implements usbwallet.driver, sending the transaction to the Trezor and
+// waiting for the user to confirm or deny the transaction.
+func (w *trezorDriver) SignTx(path accounts.DerivationPath, tx *types.Transaction, chainID *big.Int) (common.Address, *types.Transaction, error) {
+	if w.device == nil {
+		return common.Address{}, nil, accounts.ErrWalletClosed
+	}
+	return w.trezorSign(path, tx, chainID)
+}
+
+// trezorDerive sends a derivation request to the Trezor device and returns the
+// Ethereum address located on that path.
+func (w *trezorDriver) trezorDerive(derivationPath []uint32) (common.Address, error) {
+	address := new(trezor.EthereumAddress)
+	if _, err := w.trezorExchange(&trezor.EthereumGetAddress{AddressN: derivationPath}, address); err != nil {
+		return common.Address{}, err
+	}
+	return common.BytesToAddress(address.GetAddress()), nil
+}
+
+// trezorSign sends the transaction to the Trezor wallet, and waits for the user
+// to confirm or deny the transaction.
+func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction, chainID *big.Int) (common.Address, *types.Transaction, error) {
+	// Create the transaction initiation message
+	data := tx.Data()
+	length := uint32(len(data))
+
+	request := &trezor.EthereumSignTx{
+		AddressN:   derivationPath,
+		Nonce:      new(big.Int).SetUint64(tx.Nonce()).Bytes(),
+		GasPrice:   tx.GasPrice().Bytes(),
+		GasLimit:   new(big.Int).SetUint64(tx.Gas()).Bytes(),
+		Value:      tx.Value().Bytes(),
+		DataLength: &length,
+	}
+	if to := tx.To(); to != nil {
+		request.To = (*to)[:] // Non contract deploy, set recipient explicitly
+	}
+	if length > 1024 { // Send the data chunked if that was requested
+		request.DataInitialChunk, data = data[:1024], data[1024:]
+	} else {
+		request.DataInitialChunk, data = data, nil
+	}
+	if chainID != nil { // EIP-155 transaction, set chain ID explicitly (only 32 bit is supported!?)
+		id := uint32(chainID.Int64())
+		request.ChainId = &id
+	}
+	// Send the initiation message and stream content until a signature is returned
+	response := new(trezor.EthereumTxRequest)
+	if _, err := w.trezorExchange(request, response); err != nil {
+		return common.Address{}, nil, err
+	}
+	for response.DataLength != nil && int(*response.DataLength) <= len(data) {
+		chunk := data[:*response.DataLength]
+		data = data[*response.DataLength:]
+
+		if _, err := w.trezorExchange(&trezor.EthereumTxAck{DataChunk: chunk}, response); err != nil {
+			return common.Address{}, nil, err
+		}
+	}
+	// Extract the Ethereum signature and do a sanity validation
+	if len(response.GetSignatureR()) == 0 || len(response.GetSignatureS()) == 0 || response.GetSignatureV() == 0 {
+		return common.Address{}, nil, errors.New("reply lacks signature")
+	}
+	signature := append(append(response.GetSignatureR(), response.GetSignatureS()...), byte(response.GetSignatureV()))
+
+	// Create the correct signer and signature transform based on the chain ID
+	var signer types.Signer
+	if chainID == nil {
+		signer = new(types.HomesteadSigner)
+	} else {
+		signer = types.NewEIP155Signer(chainID)
+		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+	}
+	// Inject the final signature into the transaction and sanity check the sender
+	signed, err := tx.WithSignature(signer, signature)
+	if err != nil {
+		return common.Address{}, nil, err
+	}
+	sender, err := types.Sender(signer, signed)
+	if err != nil {
+		return common.Address{}, nil, err
+	}
+	return sender, signed, nil
+}
+
+// trezorExchange performs a data exchange with the Trezor wallet, sending it a
+// message and retrieving the response. If multiple responses are possible, the
+// method will also return the index of the destination object used.
+func (w *trezorDriver) trezorExchange(req proto.Message, results ...proto.Message) (int, error) {
+	// Construct the original message payload to chunk up
+	data, err := proto.Marshal(req)
+	if err != nil {
+		return 0, err
+	}
+	payload := make([]byte, 8+len(data))
+	copy(payload, []byte{0x23, 0x23})
+	binary.BigEndian.PutUint16(payload[2:], trezor.Type(req))
+	binary.BigEndian.PutUint32(payload[4:], uint32(len(data)))
+	copy(payload[8:], data)
+
+	// Stream all the chunks to the device
+	chunk := make([]byte, 64)
+	chunk[0] = 0x3f // Report ID magic number
+
+	for len(payload) > 0 {
+		// Construct the new message to stream, padding with zeroes if needed
+		if len(payload) > 63 {
+			copy(chunk[1:], payload[:63])
+			payload = payload[63:]
+		} else {
+			copy(chunk[1:], payload)
+			copy(chunk[1+len(payload):], make([]byte, 63-len(payload)))
+			payload = nil
+		}
+		// Send over to the device
+		w.log.Trace("Data chunk sent to the Trezor", "chunk", hexutil.Bytes(chunk))
+		if _, err := w.device.Write(chunk); err != nil {
+			return 0, err
+		}
+	}
+	// Stream the reply back from the wallet in 64 byte chunks
+	var (
+		kind  uint16
+		reply []byte
+	)
+	for {
+		// Read the next chunk from the Trezor wallet
+		if _, err := io.ReadFull(w.device, chunk); err != nil {
+			return 0, err
+		}
+		w.log.Trace("Data chunk received from the Trezor", "chunk", hexutil.Bytes(chunk))
+
+		// Make sure the transport header matches
+		if chunk[0] != 0x3f || (len(reply) == 0 && (chunk[1] != 0x23 || chunk[2] != 0x23)) {
+			return 0, errTrezorReplyInvalidHeader
+		}
+		// If it's the first chunk, retrieve the reply message type and total message length
+		var payload []byte
+
+		if len(reply) == 0 {
+			kind = binary.BigEndian.Uint16(chunk[3:5])
+			reply = make([]byte, 0, int(binary.BigEndian.Uint32(chunk[5:9])))
+			payload = chunk[9:]
+		} else {
+			payload = chunk[1:]
+		}
+		// Append to the reply and stop when filled up
+		if left := cap(reply) - len(reply); left > len(payload) {
+			reply = append(reply, payload...)
+		} else {
+			reply = append(reply, payload[:left]...)
+			break
+		}
+	}
+	// Try to parse the reply into the requested reply message
+	if kind == uint16(trezor.MessageType_MessageType_Failure) {
+		// Trezor returned a failure, extract and return the message
+		failure := new(trezor.Failure)
+		if err := proto.Unmarshal(reply, failure); err != nil {
+			return 0, err
+		}
+		return 0, errors.New("trezor: " + failure.GetMessage())
+	}
+	if kind == uint16(trezor.MessageType_MessageType_ButtonRequest) {
+		// Trezor is waiting for user confirmation, ack and wait for the next message
+		return w.trezorExchange(&trezor.ButtonAck{}, results...)
+	}
+	for i, res := range results {
+		if trezor.Type(res) == kind {
+			return i, proto.Unmarshal(reply, res)
+		}
+	}
+	expected := make([]string, len(results))
+	for i, res := range results {
+		expected[i] = trezor.Name(trezor.Type(res))
+	}
+	return 0, fmt.Errorf("trezor: expected reply types %s, got %s", expected, trezor.Name(kind))
+}

accounts/usbwallet/wallet.go

@@ -0,0 +1,562 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package usbwallet implements support for USB hardware wallets.
+package usbwallet
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math/big"
+	"sync"
+	"time"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/karalabe/hid"
+)
+
+// Maximum time between wallet health checks to detect USB unplugs.
+const heartbeatCycle = time.Second
+
+// Minimum time to wait between self derivation attempts, even it the user is
+// requesting accounts like crazy.
+const selfDeriveThrottling = time.Second
+
+// driver defines the vendor specific functionality hardware wallets instances
+// must implement to allow using them with the wallet lifecycle management.
+type driver interface {
+	// Status returns a textual status to aid the user in the current state of the
+	// wallet. It also returns an error indicating any failure the wallet might have
+	// encountered.
+	Status() (string, error)
+
+	// Open initializes access to a wallet instance. The passphrase parameter may
+	// or may not be used by the implementation of a particular wallet instance.
+	Open(device io.ReadWriter, passphrase string) error
+
+	// Close releases any resources held by an open wallet instance.
+	Close() error
+
+	// Heartbeat performs a sanity check against the hardware wallet to see if it
+	// is still online and healthy.
+	Heartbeat() error
+
+	// Derive sends a derivation request to the USB device and returns the Ethereum
+	// address located on that path.
+	Derive(path accounts.DerivationPath) (common.Address, error)
+
+	// SignTx sends the transaction to the USB device and waits for the user to confirm
+	// or deny the transaction.
+	SignTx(path accounts.DerivationPath, tx *types.Transaction, chainID *big.Int) (common.Address, *types.Transaction, error)
+}
+
+// wallet represents the common functionality shared by all USB hardware
+// wallets to prevent reimplementing the same complex maintenance mechanisms
+// for different vendors.
+type wallet struct {
+	hub    *Hub          // USB hub scanning
+	driver driver        // Hardware implementation of the low level device operations
+	url    *accounts.URL // Textual URL uniquely identifying this wallet
+
+	info   hid.DeviceInfo // Known USB device infos about the wallet
+	device *hid.Device    // USB device advertising itself as a hardware wallet
+
+	accounts []accounts.Account                         // List of derive accounts pinned on the hardware wallet
+	paths    map[common.Address]accounts.DerivationPath // Known derivation paths for signing operations
+
+	deriveNextPath accounts.DerivationPath   // Next derivation path for account auto-discovery
+	deriveNextAddr common.Address            // Next derived account address for auto-discovery
+	deriveChain    ethereum.ChainStateReader // Blockchain state reader to discover used account with
+	deriveReq      chan chan struct{}        // Channel to request a self-derivation on
+	deriveQuit     chan chan error           // Channel to terminate the self-deriver with
+
+	healthQuit chan chan error
+
+	// Locking a hardware wallet is a bit special. Since hardware devices are lower
+	// performing, any communication with them might take a non negligible amount of
+	// time. Worse still, waiting for user confirmation can take arbitrarily long,
+	// but exclusive communication must be upheld during. Locking the entire wallet
+	// in the mean time however would stall any parts of the system that don't want
+	// to communicate, just read some state (e.g. list the accounts).
+	//
+	// As such, a hardware wallet needs two locks to function correctly. A state
+	// lock can be used to protect the wallet's software-side internal state, which
+	// must not be held exclusively during hardware communication. A communication
+	// lock can be used to achieve exclusive access to the device itself, this one
+	// however should allow "skipping" waiting for operations that might want to
+	// use the device, but can live without too (e.g. account self-derivation).
+	//
+	// Since we have two locks, it's important to know how to properly use them:
+	//   - Communication requires the `device` to not change, so obtaining the
+	//     commsLock should be done after having a stateLock.
+	//   - Communication must not disable read access to the wallet state, so it
+	//     must only ever hold a *read* lock to stateLock.
+	commsLock chan struct{} // Mutex (buf=1) for the USB comms without keeping the state locked
+	stateLock sync.RWMutex  // Protects read and write access to the wallet struct fields
+
+	log log.Logger // Contextual logger to tag the base with its id
+}
+
+// URL implements accounts.Wallet, returning the URL of the USB hardware device.
+func (w *wallet) URL() accounts.URL {
+	return *w.url // Immutable, no need for a lock
+}
+
+// Status implements accounts.Wallet, returning a custom status message from the
+// underlying vendor-specific hardware wallet implementation.
+func (w *wallet) Status() (string, error) {
+	w.stateLock.RLock() // No device communication, state lock is enough
+	defer w.stateLock.RUnlock()
+
+	status, failure := w.driver.Status()
+	if w.device == nil {
+		return "Closed", failure
+	}
+	return status, failure
+}
+
+// Open implements accounts.Wallet, attempting to open a USB connection to the
+// hardware wallet.
+func (w *wallet) Open(passphrase string) error {
+	w.stateLock.Lock() // State lock is enough since there's no connection yet at this point
+	defer w.stateLock.Unlock()
+
+	// If the device was already opened once, refuse to try again
+	if w.paths != nil {
+		return accounts.ErrWalletAlreadyOpen
+	}
+	// Make sure the actual device connection is done only once
+	if w.device == nil {
+		device, err := w.info.Open()
+		if err != nil {
+			return err
+		}
+		w.device = device
+		w.commsLock = make(chan struct{}, 1)
+		w.commsLock <- struct{}{} // Enable lock
+	}
+	// Delegate device initialization to the underlying driver
+	if err := w.driver.Open(w.device, passphrase); err != nil {
+		return err
+	}
+	// Connection successful, start life-cycle management
+	w.paths = make(map[common.Address]accounts.DerivationPath)
+
+	w.deriveReq = make(chan chan struct{})
+	w.deriveQuit = make(chan chan error)
+	w.healthQuit = make(chan chan error)
+
+	go w.heartbeat()
+	go w.selfDerive()
+
+	// Notify anyone listening for wallet events that a new device is accessible
+	go w.hub.updateFeed.Send(accounts.WalletEvent{Wallet: w, Kind: accounts.WalletOpened})
+
+	return nil
+}
+
+// heartbeat is a health check loop for the USB wallets to periodically verify
+// whether they are still present or if they malfunctioned.
+func (w *wallet) heartbeat() {
+	w.log.Debug("USB wallet health-check started")
+	defer w.log.Debug("USB wallet health-check stopped")
+
+	// Execute heartbeat checks until termination or error
+	var (
+		errc chan error
+		err  error
+	)
+	for errc == nil && err == nil {
+		// Wait until termination is requested or the heartbeat cycle arrives
+		select {
+		case errc = <-w.healthQuit:
+			// Termination requested
+			continue
+		case <-time.After(heartbeatCycle):
+			// Heartbeat time
+		}
+		// Execute a tiny data exchange to see responsiveness
+		w.stateLock.RLock()
+		if w.device == nil {
+			// Terminated while waiting for the lock
+			w.stateLock.RUnlock()
+			continue
+		}
+		<-w.commsLock // Don't lock state while resolving version
+		err = w.driver.Heartbeat()
+		w.commsLock <- struct{}{}
+		w.stateLock.RUnlock()
+
+		if err != nil {
+			w.stateLock.Lock() // Lock state to tear the wallet down
+			w.close()
+			w.stateLock.Unlock()
+		}
+		// Ignore non hardware related errors
+		err = nil
+	}
+	// In case of error, wait for termination
+	if err != nil {
+		w.log.Debug("USB wallet health-check failed", "err", err)
+		errc = <-w.healthQuit
+	}
+	errc <- err
+}
+
+// Close implements accounts.Wallet, closing the USB connection to the device.
+func (w *wallet) Close() error {
+	// Ensure the wallet was opened
+	w.stateLock.RLock()
+	hQuit, dQuit := w.healthQuit, w.deriveQuit
+	w.stateLock.RUnlock()
+
+	// Terminate the health checks
+	var herr error
+	if hQuit != nil {
+		errc := make(chan error)
+		hQuit <- errc
+		herr = <-errc // Save for later, we *must* close the USB
+	}
+	// Terminate the self-derivations
+	var derr error
+	if dQuit != nil {
+		errc := make(chan error)
+		dQuit <- errc
+		derr = <-errc // Save for later, we *must* close the USB
+	}
+	// Terminate the device connection
+	w.stateLock.Lock()
+	defer w.stateLock.Unlock()
+
+	w.healthQuit = nil
+	w.deriveQuit = nil
+	w.deriveReq = nil
+
+	if err := w.close(); err != nil {
+		return err
+	}
+	if herr != nil {
+		return herr
+	}
+	return derr
+}
+
+// close is the internal wallet closer that terminates the USB connection and
+// resets all the fields to their defaults.
+//
+// Note, close assumes the state lock is held!
+func (w *wallet) close() error {
+	// Allow duplicate closes, especially for health-check failures
+	if w.device == nil {
+		return nil
+	}
+	// Close the device, clear everything, then return
+	w.device.Close()
+	w.device = nil
+
+	w.accounts, w.paths = nil, nil
+	w.driver.Close()
+
+	return nil
+}
+
+// Accounts implements accounts.Wallet, returning the list of accounts pinned to
+// the USB hardware wallet. If self-derivation was enabled, the account list is
+// periodically expanded based on current chain state.
+func (w *wallet) Accounts() []accounts.Account {
+	// Attempt self-derivation if it's running
+	reqc := make(chan struct{}, 1)
+	select {
+	case w.deriveReq <- reqc:
+		// Self-derivation request accepted, wait for it
+		<-reqc
+	default:
+		// Self-derivation offline, throttled or busy, skip
+	}
+	// Return whatever account list we ended up with
+	w.stateLock.RLock()
+	defer w.stateLock.RUnlock()
+
+	cpy := make([]accounts.Account, len(w.accounts))
+	copy(cpy, w.accounts)
+	return cpy
+}
+
+// selfDerive is an account derivation loop that upon request attempts to find
+// new non-zero accounts.
+func (w *wallet) selfDerive() {
+	w.log.Debug("USB wallet self-derivation started")
+	defer w.log.Debug("USB wallet self-derivation stopped")
+
+	// Execute self-derivations until termination or error
+	var (
+		reqc chan struct{}
+		errc chan error
+		err  error
+	)
+	for errc == nil && err == nil {
+		// Wait until either derivation or termination is requested
+		select {
+		case errc = <-w.deriveQuit:
+			// Termination requested
+			continue
+		case reqc = <-w.deriveReq:
+			// Account discovery requested
+		}
+		// Derivation needs a chain and device access, skip if either unavailable
+		w.stateLock.RLock()
+		if w.device == nil || w.deriveChain == nil {
+			w.stateLock.RUnlock()
+			reqc <- struct{}{}
+			continue
+		}
+		select {
+		case <-w.commsLock:
+		default:
+			w.stateLock.RUnlock()
+			reqc <- struct{}{}
+			continue
+		}
+		// Device lock obtained, derive the next batch of accounts
+		var (
+			accs  []accounts.Account
+			paths []accounts.DerivationPath
+
+			nextAddr = w.deriveNextAddr
+			nextPath = w.deriveNextPath
+
+			context = context.Background()
+		)
+		for empty := false; !empty; {
+			// Retrieve the next derived Ethereum account
+			if nextAddr == (common.Address{}) {
+				if nextAddr, err = w.driver.Derive(nextPath); err != nil {
+					w.log.Warn("USB wallet account derivation failed", "err", err)
+					break
+				}
+			}
+			// Check the account's status against the current chain state
+			var (
+				balance *big.Int
+				nonce   uint64
+			)
+			balance, err = w.deriveChain.BalanceAt(context, nextAddr, nil)
+			if err != nil {
+				w.log.Warn("USB wallet balance retrieval failed", "err", err)
+				break
+			}
+			nonce, err = w.deriveChain.NonceAt(context, nextAddr, nil)
+			if err != nil {
+				w.log.Warn("USB wallet nonce retrieval failed", "err", err)
+				break
+			}
+			// If the next account is empty, stop self-derivation, but add it nonetheless
+			if balance.Sign() == 0 && nonce == 0 {
+				empty = true
+			}
+			// We've just self-derived a new account, start tracking it locally
+			path := make(accounts.DerivationPath, len(nextPath))
+			copy(path[:], nextPath[:])
+			paths = append(paths, path)
+
+			account := accounts.Account{
+				Address: nextAddr,
+				URL:     accounts.URL{Scheme: w.url.Scheme, Path: fmt.Sprintf("%s/%s", w.url.Path, path)},
+			}
+			accs = append(accs, account)
+
+			// Display a log message to the user for new (or previously empty accounts)
+			if _, known := w.paths[nextAddr]; !known || (!empty && nextAddr == w.deriveNextAddr) {
+				w.log.Info("USB wallet discovered new account", "address", nextAddr, "path", path, "balance", balance, "nonce", nonce)
+			}
+			// Fetch the next potential account
+			if !empty {
+				nextAddr = common.Address{}
+				nextPath[len(nextPath)-1]++
+			}
+		}
+		// Self derivation complete, release device lock
+		w.commsLock <- struct{}{}
+		w.stateLock.RUnlock()
+
+		// Insert any accounts successfully derived
+		w.stateLock.Lock()
+		for i := 0; i < len(accs); i++ {
+			if _, ok := w.paths[accs[i].Address]; !ok {
+				w.accounts = append(w.accounts, accs[i])
+				w.paths[accs[i].Address] = paths[i]
+			}
+		}
+		// Shift the self-derivation forward
+		// TODO(karalabe): don't overwrite changes from wallet.SelfDerive
+		w.deriveNextAddr = nextAddr
+		w.deriveNextPath = nextPath
+		w.stateLock.Unlock()
+
+		// Notify the user of termination and loop after a bit of time (to avoid trashing)
+		reqc <- struct{}{}
+		if err == nil {
+			select {
+			case errc = <-w.deriveQuit:
+				// Termination requested, abort
+			case <-time.After(selfDeriveThrottling):
+				// Waited enough, willing to self-derive again
+			}
+		}
+	}
+	// In case of error, wait for termination
+	if err != nil {
+		w.log.Debug("USB wallet self-derivation failed", "err", err)
+		errc = <-w.deriveQuit
+	}
+	errc <- err
+}
+
+// Contains implements accounts.Wallet, returning whether a particular account is
+// or is not pinned into this wallet instance. Although we could attempt to resolve
+// unpinned accounts, that would be an non-negligible hardware operation.
+func (w *wallet) Contains(account accounts.Account) bool {
+	w.stateLock.RLock()
+	defer w.stateLock.RUnlock()
+
+	_, exists := w.paths[account.Address]
+	return exists
+}
+
+// Derive implements accounts.Wallet, deriving a new account at the specific
+// derivation path. If pin is set to true, the account will be added to the list
+// of tracked accounts.
+func (w *wallet) Derive(path accounts.DerivationPath, pin bool) (accounts.Account, error) {
+	// Try to derive the actual account and update its URL if successful
+	w.stateLock.RLock() // Avoid device disappearing during derivation
+
+	if w.device == nil {
+		w.stateLock.RUnlock()
+		return accounts.Account{}, accounts.ErrWalletClosed
+	}
+	<-w.commsLock // Avoid concurrent hardware access
+	address, err := w.driver.Derive(path)
+	w.commsLock <- struct{}{}
+
+	w.stateLock.RUnlock()
+
+	// If an error occurred or no pinning was requested, return
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	account := accounts.Account{
+		Address: address,
+		URL:     accounts.URL{Scheme: w.url.Scheme, Path: fmt.Sprintf("%s/%s", w.url.Path, path)},
+	}
+	if !pin {
+		return account, nil
+	}
+	// Pinning needs to modify the state
+	w.stateLock.Lock()
+	defer w.stateLock.Unlock()
+
+	if _, ok := w.paths[address]; !ok {
+		w.accounts = append(w.accounts, account)
+		w.paths[address] = path
+	}
+	return account, nil
+}
+
+// SelfDerive implements accounts.Wallet, trying to discover accounts that the
+// user used previously (based on the chain state), but ones that he/she did not
+// explicitly pin to the wallet manually. To avoid chain head monitoring, self
+// derivation only runs during account listing (and even then throttled).
+func (w *wallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {
+	w.stateLock.Lock()
+	defer w.stateLock.Unlock()
+
+	w.deriveNextPath = make(accounts.DerivationPath, len(base))
+	copy(w.deriveNextPath[:], base[:])
+
+	w.deriveNextAddr = common.Address{}
+	w.deriveChain = chain
+}
+
+// SignHash implements accounts.Wallet, however signing arbitrary data is not
+// supported for hardware wallets, so this method will always return an error.
+func (w *wallet) SignHash(account accounts.Account, hash []byte) ([]byte, error) {
+	return nil, accounts.ErrNotSupported
+}
+
+// SignTx implements accounts.Wallet. It sends the transaction over to the Ledger
+// wallet to request a confirmation from the user. It returns either the signed
+// transaction or a failure if the user denied the transaction.
+//
+// Note, if the version of the Ethereum application running on the Ledger wallet is
+// too old to sign EIP-155 transactions, but such is requested nonetheless, an error
+// will be returned opposed to silently signing in Homestead mode.
+func (w *wallet) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	w.stateLock.RLock() // Comms have own mutex, this is for the state fields
+	defer w.stateLock.RUnlock()
+
+	// If the wallet is closed, abort
+	if w.device == nil {
+		return nil, accounts.ErrWalletClosed
+	}
+	// Make sure the requested account is contained within
+	path, ok := w.paths[account.Address]
+	if !ok {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// All infos gathered and metadata checks out, request signing
+	<-w.commsLock
+	defer func() { w.commsLock <- struct{}{} }()
+
+	// Ensure the device isn't screwed with while user confirmation is pending
+	// TODO(karalabe): remove if hotplug lands on Windows
+	w.hub.commsLock.Lock()
+	w.hub.commsPend++
+	w.hub.commsLock.Unlock()
+
+	defer func() {
+		w.hub.commsLock.Lock()
+		w.hub.commsPend--
+		w.hub.commsLock.Unlock()
+	}()
+	// Sign the transaction and verify the sender to avoid hardware fault surprises
+	sender, signed, err := w.driver.SignTx(path, tx, chainID)
+	if err != nil {
+		return nil, err
+	}
+	if sender != account.Address {
+		return nil, fmt.Errorf("signer mismatch: expected %s, got %s", account.Address.Hex(), sender.Hex())
+	}
+	return signed, nil
+}
+
+// SignHashWithPassphrase implements accounts.Wallet, however signing arbitrary
+// data is not supported for Ledger wallets, so this method will always return
+// an error.
+func (w *wallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
+	return w.SignHash(account, hash)
+}
+
+// SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given
+// transaction with the given account using passphrase as extra authentication.
+// Since USB wallets don't rely on passphrases, these are silently ignored.
+func (w *wallet) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	return w.SignTx(account, tx, chainID)
+}

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.3.windows-%GETH_ARCH%.zip
-  - 7z x go1.8.3.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.2.windows-%GETH_ARCH%.zip
+  - 7z x go1.10.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

bmt/bmt.go

@@ -0,0 +1,560 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package bmt provides a binary merkle tree implementation
+package bmt
+
+import (
+	"fmt"
+	"hash"
+	"io"
+	"strings"
+	"sync"
+	"sync/atomic"
+)
+
+/*
+Binary Merkle Tree Hash is a hash function over arbitrary datachunks of limited size
+It is defined as the root hash of the binary merkle tree built over fixed size segments
+of the underlying chunk using any base hash function (e.g keccak 256 SHA3)
+
+It is used as the chunk hash function in swarm which in turn is the basis for the
+128 branching swarm hash http://swarm-guide.readthedocs.io/en/latest/architecture.html#swarm-hash
+
+The BMT is optimal for providing compact inclusion proofs, i.e. prove that a
+segment is a substring of a chunk starting at a particular offset
+The size of the underlying segments is fixed at 32 bytes (called the resolution
+of the BMT hash), the EVM word size to optimize for on-chain BMT verification
+as well as the hash size optimal for inclusion proofs in the merkle tree of the swarm hash.
+
+Two implementations are provided:
+
+* RefHasher is optimized for code simplicity and meant as a reference implementation
+* Hasher is optimized for speed taking advantage of concurrency with minimalistic
+  control structure to coordinate the concurrent routines
+  It implements the ChunkHash interface as well as the go standard hash.Hash interface
+
+*/
+
+const (
+	// DefaultSegmentCount is the maximum number of segments of the underlying chunk
+	DefaultSegmentCount = 128 // Should be equal to storage.DefaultBranches
+	// DefaultPoolSize is the maximum number of bmt trees used by the hashers, i.e,
+	// the maximum number of concurrent BMT hashing operations performed by the same hasher
+	DefaultPoolSize = 8
+)
+
+// BaseHasher is a hash.Hash constructor function used for the base hash of the  BMT.
+type BaseHasher func() hash.Hash
+
+// Hasher a reusable hasher for fixed maximum size chunks representing a BMT
+// implements the hash.Hash interface
+// reuse pool of Tree-s for amortised memory allocation and resource control
+// supports order-agnostic concurrent segment writes
+// as well as sequential read and write
+// can not be called concurrently on more than one chunk
+// can be further appended after Sum
+// Reset gives back the Tree to the pool and guaranteed to leave
+// the tree and itself in a state reusable for hashing a new chunk
+type Hasher struct {
+	pool        *TreePool   // BMT resource pool
+	bmt         *Tree       // prebuilt BMT resource for flowcontrol and proofs
+	blocksize   int         // segment size (size of hash) also for hash.Hash
+	count       int         // segment count
+	size        int         // for hash.Hash same as hashsize
+	cur         int         // cursor position for rightmost currently open chunk
+	segment     []byte      // the rightmost open segment (not complete)
+	depth       int         // index of last level
+	result      chan []byte // result channel
+	hash        []byte      // to record the result
+	max         int32       // max segments for SegmentWriter interface
+	blockLength []byte      // The block length that needes to be added in Sum
+}
+
+// New creates a reusable Hasher
+// implements the hash.Hash interface
+// pulls a new Tree from a resource pool for hashing each chunk
+func New(p *TreePool) *Hasher {
+	return &Hasher{
+		pool:      p,
+		depth:     depth(p.SegmentCount),
+		size:      p.SegmentSize,
+		blocksize: p.SegmentSize,
+		count:     p.SegmentCount,
+		result:    make(chan []byte),
+	}
+}
+
+// Node is a reuseable segment hasher representing a node in a BMT
+// it allows for continued writes after a Sum
+// and is left in completely reusable state after Reset
+type Node struct {
+	level, index int   // position of node for information/logging only
+	initial      bool  // first and last node
+	root         bool  // whether the node is root to a smaller BMT
+	isLeft       bool  // whether it is left side of the parent double segment
+	unbalanced   bool  // indicates if a node has only the left segment
+	parent       *Node // BMT connections
+	state        int32 // atomic increment impl concurrent boolean toggle
+	left, right  []byte
+}
+
+// NewNode constructor for segment hasher nodes in the BMT
+func NewNode(level, index int, parent *Node) *Node {
+	return &Node{
+		parent:  parent,
+		level:   level,
+		index:   index,
+		initial: index == 0,
+		isLeft:  index%2 == 0,
+	}
+}
+
+// TreePool provides a pool of Trees used as resources by Hasher
+// a Tree popped from the pool is guaranteed to have clean state
+// for hashing a new chunk
+// Hasher Reset releases the Tree to the pool
+type TreePool struct {
+	lock         sync.Mutex
+	c            chan *Tree
+	hasher       BaseHasher
+	SegmentSize  int
+	SegmentCount int
+	Capacity     int
+	count        int
+}
+
+// NewTreePool creates a Tree pool with hasher, segment size, segment count and capacity
+// on GetTree it reuses free Trees or creates a new one if size is not reached
+func NewTreePool(hasher BaseHasher, segmentCount, capacity int) *TreePool {
+	return &TreePool{
+		c:            make(chan *Tree, capacity),
+		hasher:       hasher,
+		SegmentSize:  hasher().Size(),
+		SegmentCount: segmentCount,
+		Capacity:     capacity,
+	}
+}
+
+// Drain drains the pool until it has no more than n resources
+func (p *TreePool) Drain(n int) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	for len(p.c) > n {
+		<-p.c
+		p.count--
+	}
+}
+
+// Reserve is blocking until it returns an available Tree
+// it reuses free Trees or creates a new one if size is not reached
+func (p *TreePool) Reserve() *Tree {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+	var t *Tree
+	if p.count == p.Capacity {
+		return <-p.c
+	}
+	select {
+	case t = <-p.c:
+	default:
+		t = NewTree(p.hasher, p.SegmentSize, p.SegmentCount)
+		p.count++
+	}
+	return t
+}
+
+// Release gives back a Tree to the pool.
+// This Tree is guaranteed to be in reusable state
+// does not need locking
+func (p *TreePool) Release(t *Tree) {
+	p.c <- t // can never fail but...
+}
+
+// Tree is a reusable control structure representing a BMT
+// organised in a binary tree
+// Hasher uses a TreePool to pick one for each chunk hash
+// the Tree is 'locked' while not in the pool
+type Tree struct {
+	leaves []*Node
+}
+
+// Draw draws the BMT (badly)
+func (t *Tree) Draw(hash []byte, d int) string {
+	var left, right []string
+	var anc []*Node
+	for i, n := range t.leaves {
+		left = append(left, fmt.Sprintf("%v", hashstr(n.left)))
+		if i%2 == 0 {
+			anc = append(anc, n.parent)
+		}
+		right = append(right, fmt.Sprintf("%v", hashstr(n.right)))
+	}
+	anc = t.leaves
+	var hashes [][]string
+	for l := 0; len(anc) > 0; l++ {
+		var nodes []*Node
+		hash := []string{""}
+		for i, n := range anc {
+			hash = append(hash, fmt.Sprintf("%v|%v", hashstr(n.left), hashstr(n.right)))
+			if i%2 == 0 && n.parent != nil {
+				nodes = append(nodes, n.parent)
+			}
+		}
+		hash = append(hash, "")
+		hashes = append(hashes, hash)
+		anc = nodes
+	}
+	hashes = append(hashes, []string{"", fmt.Sprintf("%v", hashstr(hash)), ""})
+	total := 60
+	del := "                             "
+	var rows []string
+	for i := len(hashes) - 1; i >= 0; i-- {
+		var textlen int
+		hash := hashes[i]
+		for _, s := range hash {
+			textlen += len(s)
+		}
+		if total < textlen {
+			total = textlen + len(hash)
+		}
+		delsize := (total - textlen) / (len(hash) - 1)
+		if delsize > len(del) {
+			delsize = len(del)
+		}
+		row := fmt.Sprintf("%v: %v", len(hashes)-i-1, strings.Join(hash, del[:delsize]))
+		rows = append(rows, row)
+
+	}
+	rows = append(rows, strings.Join(left, "  "))
+	rows = append(rows, strings.Join(right, "  "))
+	return strings.Join(rows, "\n") + "\n"
+}
+
+// NewTree initialises the Tree by building up the nodes of a BMT
+// segment size is stipulated to be the size of the hash
+// segmentCount needs to be positive integer and does not need to be
+// a power of two and can even be an odd number
+// segmentSize * segmentCount determines the maximum chunk size
+// hashed using the tree
+func NewTree(hasher BaseHasher, segmentSize, segmentCount int) *Tree {
+	n := NewNode(0, 0, nil)
+	n.root = true
+	prevlevel := []*Node{n}
+	// iterate over levels and creates 2^level nodes
+	level := 1
+	count := 2
+	for d := 1; d <= depth(segmentCount); d++ {
+		nodes := make([]*Node, count)
+		for i := 0; i < len(nodes); i++ {
+			parent := prevlevel[i/2]
+			t := NewNode(level, i, parent)
+			nodes[i] = t
+		}
+		prevlevel = nodes
+		level++
+		count *= 2
+	}
+	// the datanode level is the nodes on the last level where
+	return &Tree{
+		leaves: prevlevel,
+	}
+}
+
+// methods needed by hash.Hash
+
+// Size returns the size
+func (h *Hasher) Size() int {
+	return h.size
+}
+
+// BlockSize returns the block size
+func (h *Hasher) BlockSize() int {
+	return h.blocksize
+}
+
+// Sum returns the hash of the buffer
+// hash.Hash interface Sum method appends the byte slice to the underlying
+// data before it calculates and returns the hash of the chunk
+func (h *Hasher) Sum(b []byte) (r []byte) {
+	t := h.bmt
+	i := h.cur
+	n := t.leaves[i]
+	j := i
+	// must run strictly before all nodes calculate
+	// datanodes are guaranteed to have a parent
+	if len(h.segment) > h.size && i > 0 && n.parent != nil {
+		n = n.parent
+	} else {
+		i *= 2
+	}
+	d := h.finalise(n, i)
+	h.writeSegment(j, h.segment, d)
+	c := <-h.result
+	h.releaseTree()
+
+	// sha3(length + BMT(pure_chunk))
+	if h.blockLength == nil {
+		return c
+	}
+	res := h.pool.hasher()
+	res.Reset()
+	res.Write(h.blockLength)
+	res.Write(c)
+	return res.Sum(nil)
+}
+
+// Hasher implements the SwarmHash interface
+
+// Hash waits for the hasher result and returns it
+// caller must call this on a BMT Hasher being written to
+func (h *Hasher) Hash() []byte {
+	return <-h.result
+}
+
+// Hasher implements the io.Writer interface
+
+// Write fills the buffer to hash
+// with every full segment complete launches a hasher go routine
+// that shoots up the BMT
+func (h *Hasher) Write(b []byte) (int, error) {
+	l := len(b)
+	if l <= 0 {
+		return 0, nil
+	}
+	s := h.segment
+	i := h.cur
+	count := (h.count + 1) / 2
+	need := h.count*h.size - h.cur*2*h.size
+	size := h.size
+	if need > size {
+		size *= 2
+	}
+	if l < need {
+		need = l
+	}
+	// calculate missing bit to complete current open segment
+	rest := size - len(s)
+	if need < rest {
+		rest = need
+	}
+	s = append(s, b[:rest]...)
+	need -= rest
+	// read full segments and the last possibly partial segment
+	for need > 0 && i < count-1 {
+		// push all finished chunks we read
+		h.writeSegment(i, s, h.depth)
+		need -= size
+		if need < 0 {
+			size += need
+		}
+		s = b[rest : rest+size]
+		rest += size
+		i++
+	}
+	h.segment = s
+	h.cur = i
+	// otherwise, we can assume len(s) == 0, so all buffer is read and chunk is not yet full
+	return l, nil
+}
+
+// Hasher implements the io.ReaderFrom interface
+
+// ReadFrom reads from io.Reader and appends to the data to hash using Write
+// it reads so that chunk to hash is maximum length or reader reaches EOF
+// caller must Reset the hasher prior to call
+func (h *Hasher) ReadFrom(r io.Reader) (m int64, err error) {
+	bufsize := h.size*h.count - h.size*h.cur - len(h.segment)
+	buf := make([]byte, bufsize)
+	var read int
+	for {
+		var n int
+		n, err = r.Read(buf)
+		read += n
+		if err == io.EOF || read == len(buf) {
+			hash := h.Sum(buf[:n])
+			if read == len(buf) {
+				err = NewEOC(hash)
+			}
+			break
+		}
+		if err != nil {
+			break
+		}
+		n, err = h.Write(buf[:n])
+		if err != nil {
+			break
+		}
+	}
+	return int64(read), err
+}
+
+// Reset needs to be called before writing to the hasher
+func (h *Hasher) Reset() {
+	h.getTree()
+	h.blockLength = nil
+}
+
+// Hasher implements the SwarmHash interface
+
+// ResetWithLength needs to be called before writing to the hasher
+// the argument is supposed to be the byte slice binary representation of
+// the length of the data subsumed under the hash
+func (h *Hasher) ResetWithLength(l []byte) {
+	h.Reset()
+	h.blockLength = l
+}
+
+// Release gives back the Tree to the pool whereby it unlocks
+// it resets tree, segment and index
+func (h *Hasher) releaseTree() {
+	if h.bmt != nil {
+		n := h.bmt.leaves[h.cur]
+		for ; n != nil; n = n.parent {
+			n.unbalanced = false
+			if n.parent != nil {
+				n.root = false
+			}
+		}
+		h.pool.Release(h.bmt)
+		h.bmt = nil
+
+	}
+	h.cur = 0
+	h.segment = nil
+}
+
+func (h *Hasher) writeSegment(i int, s []byte, d int) {
+	hash := h.pool.hasher()
+	n := h.bmt.leaves[i]
+
+	if len(s) > h.size && n.parent != nil {
+		go func() {
+			hash.Reset()
+			hash.Write(s)
+			s = hash.Sum(nil)
+
+			if n.root {
+				h.result <- s
+				return
+			}
+			h.run(n.parent, hash, d, n.index, s)
+		}()
+		return
+	}
+	go h.run(n, hash, d, i*2, s)
+}
+
+func (h *Hasher) run(n *Node, hash hash.Hash, d int, i int, s []byte) {
+	isLeft := i%2 == 0
+	for {
+		if isLeft {
+			n.left = s
+		} else {
+			n.right = s
+		}
+		if !n.unbalanced && n.toggle() {
+			return
+		}
+		if !n.unbalanced || !isLeft || i == 0 && d == 0 {
+			hash.Reset()
+			hash.Write(n.left)
+			hash.Write(n.right)
+			s = hash.Sum(nil)
+
+		} else {
+			s = append(n.left, n.right...)
+		}
+
+		h.hash = s
+		if n.root {
+			h.result <- s
+			return
+		}
+
+		isLeft = n.isLeft
+		n = n.parent
+		i++
+	}
+}
+
+// getTree obtains a BMT resource by reserving one from the pool
+func (h *Hasher) getTree() *Tree {
+	if h.bmt != nil {
+		return h.bmt
+	}
+	t := h.pool.Reserve()
+	h.bmt = t
+	return t
+}
+
+// atomic bool toggle implementing a concurrent reusable 2-state object
+// atomic addint with %2 implements atomic bool toggle
+// it returns true if the toggler just put it in the active/waiting state
+func (n *Node) toggle() bool {
+	return atomic.AddInt32(&n.state, 1)%2 == 1
+}
+
+func hashstr(b []byte) string {
+	end := len(b)
+	if end > 4 {
+		end = 4
+	}
+	return fmt.Sprintf("%x", b[:end])
+}
+
+func depth(n int) (d int) {
+	for l := (n - 1) / 2; l > 0; l /= 2 {
+		d++
+	}
+	return d
+}
+
+// finalise is following the zigzags on the tree belonging
+// to the final datasegment
+func (h *Hasher) finalise(n *Node, i int) (d int) {
+	isLeft := i%2 == 0
+	for {
+		// when the final segment's path is going via left segments
+		// the incoming data is pushed to the parent upon pulling the left
+		// we do not need toggle the state since this condition is
+		// detectable
+		n.unbalanced = isLeft
+		n.right = nil
+		if n.initial {
+			n.root = true
+			return d
+		}
+		isLeft = n.isLeft
+		n = n.parent
+		d++
+	}
+}
+
+// EOC (end of chunk) implements the error interface
+type EOC struct {
+	Hash []byte // read the hash of the chunk off the error
+}
+
+// Error returns the error string
+func (e *EOC) Error() string {
+	return fmt.Sprintf("hasher limit reached, chunk hash: %x", e.Hash)
+}
+
+// NewEOC creates new end of chunk error with the hash
+func NewEOC(hash []byte) *EOC {
+	return &EOC{hash}
+}

bmt/bmt_r.go

@@ -0,0 +1,85 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// simple nonconcurrent reference implementation for hashsize segment based
+// Binary Merkle tree hash on arbitrary but fixed maximum chunksize
+//
+// This implementation does not take advantage of any paralellisms and uses
+// far more memory than necessary, but it is easy to see that it is correct.
+// It can be used for generating test cases for optimized implementations.
+// see testBMTHasherCorrectness function in bmt_test.go
+package bmt
+
+import (
+	"hash"
+)
+
+// RefHasher is the non-optimized easy to read reference implementation of BMT
+type RefHasher struct {
+	span    int
+	section int
+	cap     int
+	h       hash.Hash
+}
+
+// NewRefHasher returns a new RefHasher
+func NewRefHasher(hasher BaseHasher, count int) *RefHasher {
+	h := hasher()
+	hashsize := h.Size()
+	maxsize := hashsize * count
+	c := 2
+	for ; c < count; c *= 2 {
+	}
+	if c > 2 {
+		c /= 2
+	}
+	return &RefHasher{
+		section: 2 * hashsize,
+		span:    c * hashsize,
+		cap:     maxsize,
+		h:       h,
+	}
+}
+
+// Hash returns the BMT hash of the byte slice
+// implements the SwarmHash interface
+func (rh *RefHasher) Hash(d []byte) []byte {
+	if len(d) > rh.cap {
+		d = d[:rh.cap]
+	}
+
+	return rh.hash(d, rh.span)
+}
+
+func (rh *RefHasher) hash(d []byte, s int) []byte {
+	l := len(d)
+	left := d
+	var right []byte
+	if l > rh.section {
+		for ; s >= l; s /= 2 {
+		}
+		left = rh.hash(d[:s], s)
+		right = d[s:]
+		if l-s > rh.section/2 {
+			right = rh.hash(right, s)
+		}
+	}
+	defer rh.h.Reset()
+	rh.h.Write(left)
+	rh.h.Write(right)
+	h := rh.h.Sum(nil)
+	return h
+}

bmt/bmt_test.go

@@ -0,0 +1,481 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package bmt
+
+import (
+	"bytes"
+	crand "crypto/rand"
+	"fmt"
+	"hash"
+	"io"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/crypto/sha3"
+)
+
+const (
+	maxproccnt = 8
+)
+
+// TestRefHasher tests that the RefHasher computes the expected BMT hash for
+// all data lengths between 0 and 256 bytes
+func TestRefHasher(t *testing.T) {
+	hashFunc := sha3.NewKeccak256
+
+	sha3 := func(data ...[]byte) []byte {
+		h := hashFunc()
+		for _, v := range data {
+			h.Write(v)
+		}
+		return h.Sum(nil)
+	}
+
+	// the test struct is used to specify the expected BMT hash for data
+	// lengths between "from" and "to"
+	type test struct {
+		from     int64
+		to       int64
+		expected func([]byte) []byte
+	}
+
+	var tests []*test
+
+	// all lengths in [0,64] should be:
+	//
+	//   sha3(data)
+	//
+	tests = append(tests, &test{
+		from: 0,
+		to:   64,
+		expected: func(data []byte) []byte {
+			return sha3(data)
+		},
+	})
+
+	// all lengths in [65,96] should be:
+	//
+	//   sha3(
+	//     sha3(data[:64])
+	//     data[64:]
+	//   )
+	//
+	tests = append(tests, &test{
+		from: 65,
+		to:   96,
+		expected: func(data []byte) []byte {
+			return sha3(sha3(data[:64]), data[64:])
+		},
+	})
+
+	// all lengths in [97,128] should be:
+	//
+	//   sha3(
+	//     sha3(data[:64])
+	//     sha3(data[64:])
+	//   )
+	//
+	tests = append(tests, &test{
+		from: 97,
+		to:   128,
+		expected: func(data []byte) []byte {
+			return sha3(sha3(data[:64]), sha3(data[64:]))
+		},
+	})
+
+	// all lengths in [129,160] should be:
+	//
+	//   sha3(
+	//     sha3(
+	//       sha3(data[:64])
+	//       sha3(data[64:128])
+	//     )
+	//     data[128:]
+	//   )
+	//
+	tests = append(tests, &test{
+		from: 129,
+		to:   160,
+		expected: func(data []byte) []byte {
+			return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), data[128:])
+		},
+	})
+
+	// all lengths in [161,192] should be:
+	//
+	//   sha3(
+	//     sha3(
+	//       sha3(data[:64])
+	//       sha3(data[64:128])
+	//     )
+	//     sha3(data[128:])
+	//   )
+	//
+	tests = append(tests, &test{
+		from: 161,
+		to:   192,
+		expected: func(data []byte) []byte {
+			return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), sha3(data[128:]))
+		},
+	})
+
+	// all lengths in [193,224] should be:
+	//
+	//   sha3(
+	//     sha3(
+	//       sha3(data[:64])
+	//       sha3(data[64:128])
+	//     )
+	//     sha3(
+	//       sha3(data[128:192])
+	//       data[192:]
+	//     )
+	//   )
+	//
+	tests = append(tests, &test{
+		from: 193,
+		to:   224,
+		expected: func(data []byte) []byte {
+			return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), sha3(sha3(data[128:192]), data[192:]))
+		},
+	})
+
+	// all lengths in [225,256] should be:
+	//
+	//   sha3(
+	//     sha3(
+	//       sha3(data[:64])
+	//       sha3(data[64:128])
+	//     )
+	//     sha3(
+	//       sha3(data[128:192])
+	//       sha3(data[192:])
+	//     )
+	//   )
+	//
+	tests = append(tests, &test{
+		from: 225,
+		to:   256,
+		expected: func(data []byte) []byte {
+			return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), sha3(sha3(data[128:192]), sha3(data[192:])))
+		},
+	})
+
+	// run the tests
+	for _, x := range tests {
+		for length := x.from; length <= x.to; length++ {
+			t.Run(fmt.Sprintf("%d_bytes", length), func(t *testing.T) {
+				data := make([]byte, length)
+				if _, err := io.ReadFull(crand.Reader, data); err != nil && err != io.EOF {
+					t.Fatal(err)
+				}
+				expected := x.expected(data)
+				actual := NewRefHasher(hashFunc, 128).Hash(data)
+				if !bytes.Equal(actual, expected) {
+					t.Fatalf("expected %x, got %x", expected, actual)
+				}
+			})
+		}
+	}
+}
+
+func testDataReader(l int) (r io.Reader) {
+	return io.LimitReader(crand.Reader, int64(l))
+}
+
+func TestHasherCorrectness(t *testing.T) {
+	err := testHasher(testBaseHasher)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testHasher(f func(BaseHasher, []byte, int, int) error) error {
+	tdata := testDataReader(4128)
+	data := make([]byte, 4128)
+	tdata.Read(data)
+	hasher := sha3.NewKeccak256
+	size := hasher().Size()
+	counts := []int{1, 2, 3, 4, 5, 8, 16, 32, 64, 128}
+
+	var err error
+	for _, count := range counts {
+		max := count * size
+		incr := 1
+		for n := 0; n <= max+incr; n += incr {
+			err = f(hasher, data, n, count)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func TestHasherReuseWithoutRelease(t *testing.T) {
+	testHasherReuse(1, t)
+}
+
+func TestHasherReuseWithRelease(t *testing.T) {
+	testHasherReuse(maxproccnt, t)
+}
+
+func testHasherReuse(i int, t *testing.T) {
+	hasher := sha3.NewKeccak256
+	pool := NewTreePool(hasher, 128, i)
+	defer pool.Drain(0)
+	bmt := New(pool)
+
+	for i := 0; i < 500; i++ {
+		n := rand.Intn(4096)
+		tdata := testDataReader(n)
+		data := make([]byte, n)
+		tdata.Read(data)
+
+		err := testHasherCorrectness(bmt, hasher, data, n, 128)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestHasherConcurrency(t *testing.T) {
+	hasher := sha3.NewKeccak256
+	pool := NewTreePool(hasher, 128, maxproccnt)
+	defer pool.Drain(0)
+	wg := sync.WaitGroup{}
+	cycles := 100
+	wg.Add(maxproccnt * cycles)
+	errc := make(chan error)
+
+	for p := 0; p < maxproccnt; p++ {
+		for i := 0; i < cycles; i++ {
+			go func() {
+				bmt := New(pool)
+				n := rand.Intn(4096)
+				tdata := testDataReader(n)
+				data := make([]byte, n)
+				tdata.Read(data)
+				err := testHasherCorrectness(bmt, hasher, data, n, 128)
+				wg.Done()
+				if err != nil {
+					errc <- err
+				}
+			}()
+		}
+	}
+	go func() {
+		wg.Wait()
+		close(errc)
+	}()
+	var err error
+	select {
+	case <-time.NewTimer(5 * time.Second).C:
+		err = fmt.Errorf("timed out")
+	case err = <-errc:
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func testBaseHasher(hasher BaseHasher, d []byte, n, count int) error {
+	pool := NewTreePool(hasher, count, 1)
+	defer pool.Drain(0)
+	bmt := New(pool)
+	return testHasherCorrectness(bmt, hasher, d, n, count)
+}
+
+func testHasherCorrectness(bmt hash.Hash, hasher BaseHasher, d []byte, n, count int) (err error) {
+	data := d[:n]
+	rbmt := NewRefHasher(hasher, count)
+	exp := rbmt.Hash(data)
+	timeout := time.NewTimer(time.Second)
+	c := make(chan error)
+
+	go func() {
+		bmt.Reset()
+		bmt.Write(data)
+		got := bmt.Sum(nil)
+		if !bytes.Equal(got, exp) {
+			c <- fmt.Errorf("wrong hash: expected %x, got %x", exp, got)
+		}
+		close(c)
+	}()
+	select {
+	case <-timeout.C:
+		err = fmt.Errorf("BMT hash calculation timed out")
+	case err = <-c:
+	}
+	return err
+}
+
+func BenchmarkSHA3_4k(t *testing.B)   { benchmarkSHA3(4096, t) }
+func BenchmarkSHA3_2k(t *testing.B)   { benchmarkSHA3(4096/2, t) }
+func BenchmarkSHA3_1k(t *testing.B)   { benchmarkSHA3(4096/4, t) }
+func BenchmarkSHA3_512b(t *testing.B) { benchmarkSHA3(4096/8, t) }
+func BenchmarkSHA3_256b(t *testing.B) { benchmarkSHA3(4096/16, t) }
+func BenchmarkSHA3_128b(t *testing.B) { benchmarkSHA3(4096/32, t) }
+
+func BenchmarkBMTBaseline_4k(t *testing.B)   { benchmarkBMTBaseline(4096, t) }
+func BenchmarkBMTBaseline_2k(t *testing.B)   { benchmarkBMTBaseline(4096/2, t) }
+func BenchmarkBMTBaseline_1k(t *testing.B)   { benchmarkBMTBaseline(4096/4, t) }
+func BenchmarkBMTBaseline_512b(t *testing.B) { benchmarkBMTBaseline(4096/8, t) }
+func BenchmarkBMTBaseline_256b(t *testing.B) { benchmarkBMTBaseline(4096/16, t) }
+func BenchmarkBMTBaseline_128b(t *testing.B) { benchmarkBMTBaseline(4096/32, t) }
+
+func BenchmarkRefHasher_4k(t *testing.B)   { benchmarkRefHasher(4096, t) }
+func BenchmarkRefHasher_2k(t *testing.B)   { benchmarkRefHasher(4096/2, t) }
+func BenchmarkRefHasher_1k(t *testing.B)   { benchmarkRefHasher(4096/4, t) }
+func BenchmarkRefHasher_512b(t *testing.B) { benchmarkRefHasher(4096/8, t) }
+func BenchmarkRefHasher_256b(t *testing.B) { benchmarkRefHasher(4096/16, t) }
+func BenchmarkRefHasher_128b(t *testing.B) { benchmarkRefHasher(4096/32, t) }
+
+func BenchmarkHasher_4k(t *testing.B)   { benchmarkHasher(4096, t) }
+func BenchmarkHasher_2k(t *testing.B)   { benchmarkHasher(4096/2, t) }
+func BenchmarkHasher_1k(t *testing.B)   { benchmarkHasher(4096/4, t) }
+func BenchmarkHasher_512b(t *testing.B) { benchmarkHasher(4096/8, t) }
+func BenchmarkHasher_256b(t *testing.B) { benchmarkHasher(4096/16, t) }
+func BenchmarkHasher_128b(t *testing.B) { benchmarkHasher(4096/32, t) }
+
+func BenchmarkHasherNoReuse_4k(t *testing.B)   { benchmarkHasherReuse(1, 4096, t) }
+func BenchmarkHasherNoReuse_2k(t *testing.B)   { benchmarkHasherReuse(1, 4096/2, t) }
+func BenchmarkHasherNoReuse_1k(t *testing.B)   { benchmarkHasherReuse(1, 4096/4, t) }
+func BenchmarkHasherNoReuse_512b(t *testing.B) { benchmarkHasherReuse(1, 4096/8, t) }
+func BenchmarkHasherNoReuse_256b(t *testing.B) { benchmarkHasherReuse(1, 4096/16, t) }
+func BenchmarkHasherNoReuse_128b(t *testing.B) { benchmarkHasherReuse(1, 4096/32, t) }
+
+func BenchmarkHasherReuse_4k(t *testing.B)   { benchmarkHasherReuse(16, 4096, t) }
+func BenchmarkHasherReuse_2k(t *testing.B)   { benchmarkHasherReuse(16, 4096/2, t) }
+func BenchmarkHasherReuse_1k(t *testing.B)   { benchmarkHasherReuse(16, 4096/4, t) }
+func BenchmarkHasherReuse_512b(t *testing.B) { benchmarkHasherReuse(16, 4096/8, t) }
+func BenchmarkHasherReuse_256b(t *testing.B) { benchmarkHasherReuse(16, 4096/16, t) }
+func BenchmarkHasherReuse_128b(t *testing.B) { benchmarkHasherReuse(16, 4096/32, t) }
+
+// benchmarks the minimum hashing time for a balanced (for simplicity) BMT
+// by doing count/segmentsize parallel hashings of 2*segmentsize bytes
+// doing it on n maxproccnt each reusing the base hasher
+// the premise is that this is the minimum computation needed for a BMT
+// therefore this serves as a theoretical optimum for concurrent implementations
+func benchmarkBMTBaseline(n int, t *testing.B) {
+	tdata := testDataReader(64)
+	data := make([]byte, 64)
+	tdata.Read(data)
+	hasher := sha3.NewKeccak256
+
+	t.ReportAllocs()
+	t.ResetTimer()
+	for i := 0; i < t.N; i++ {
+		count := int32((n-1)/hasher().Size() + 1)
+		wg := sync.WaitGroup{}
+		wg.Add(maxproccnt)
+		var i int32
+		for j := 0; j < maxproccnt; j++ {
+			go func() {
+				defer wg.Done()
+				h := hasher()
+				for atomic.AddInt32(&i, 1) < count {
+					h.Reset()
+					h.Write(data)
+					h.Sum(nil)
+				}
+			}()
+		}
+		wg.Wait()
+	}
+}
+
+func benchmarkHasher(n int, t *testing.B) {
+	tdata := testDataReader(n)
+	data := make([]byte, n)
+	tdata.Read(data)
+
+	size := 1
+	hasher := sha3.NewKeccak256
+	segmentCount := 128
+	pool := NewTreePool(hasher, segmentCount, size)
+	bmt := New(pool)
+
+	t.ReportAllocs()
+	t.ResetTimer()
+	for i := 0; i < t.N; i++ {
+		bmt.Reset()
+		bmt.Write(data)
+		bmt.Sum(nil)
+	}
+}
+
+func benchmarkHasherReuse(poolsize, n int, t *testing.B) {
+	tdata := testDataReader(n)
+	data := make([]byte, n)
+	tdata.Read(data)
+
+	hasher := sha3.NewKeccak256
+	segmentCount := 128
+	pool := NewTreePool(hasher, segmentCount, poolsize)
+	cycles := 200
+
+	t.ReportAllocs()
+	t.ResetTimer()
+	for i := 0; i < t.N; i++ {
+		wg := sync.WaitGroup{}
+		wg.Add(cycles)
+		for j := 0; j < cycles; j++ {
+			bmt := New(pool)
+			go func() {
+				defer wg.Done()
+				bmt.Reset()
+				bmt.Write(data)
+				bmt.Sum(nil)
+			}()
+		}
+		wg.Wait()
+	}
+}
+
+func benchmarkSHA3(n int, t *testing.B) {
+	data := make([]byte, n)
+	tdata := testDataReader(n)
+	tdata.Read(data)
+	hasher := sha3.NewKeccak256
+	h := hasher()
+
+	t.ReportAllocs()
+	t.ResetTimer()
+	for i := 0; i < t.N; i++ {
+		h.Reset()
+		h.Write(data)
+		h.Sum(nil)
+	}
+}
+
+func benchmarkRefHasher(n int, t *testing.B) {
+	data := make([]byte, n)
+	tdata := testDataReader(n)
+	tdata.Read(data)
+	hasher := sha3.NewKeccak256
+	rbmt := NewRefHasher(hasher, 128)
+
+	t.ReportAllocs()
+	t.ResetTimer()
+	for i := 0; i < t.N; i++ {
+		rbmt.Hash(data)
+	}
+}

build/ci-notes.md

@@ -2,12 +2,7 @@
 
 Tagged releases and develop branch commits are available as installable Debian packages
 for Ubuntu. Packages are built for the all Ubuntu versions which are supported by
-Canonical:
-
-- Trusty Tahr (14.04 LTS)
-- Xenial Xerus (16.04 LTS)
-- Yakkety Yak (16.10)
-- Zesty Zapus (17.04)
+Canonical.
 
 Packages of develop branch commits have suffix -unstable and cannot be installed alongside
 the stable version. Switching between release streams requires user intervention.
@@ -21,18 +16,18 @@ variable which Travis CI makes available to certain builds.
 We want to build go-ethereum with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
 our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
-golang-1.8, which is co-installable alongside the regular golang package. PPA dependencies
+golang-1.10, which is co-installable alongside the regular golang package. PPA dependencies
 can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies
 
 ## Building Packages Locally (for testing)
 
 You need to run Ubuntu to do test packaging.
 
-Add the gophers PPA and install Go 1.8 and Debian packaging tools:
+Add the gophers PPA and install Go 1.10 and Debian packaging tools:
 
     $ sudo apt-add-repository ppa:gophers/ubuntu/archive
     $ sudo apt-get update
-    $ sudo apt-get install build-essential golang-1.8 devscripts debhelper
+    $ sudo apt-get install build-essential golang-1.10 devscripts debhelper
 
 Create the source packages:
 

build/ci.go

@@ -19,12 +19,13 @@
 /*
 The ci command is called from Continuous Integration scripts.
 
-Usage: go run ci.go <command> <command flags/arguments>
+Usage: go run build/ci.go <command> <command flags/arguments>
 
 Available commands are:
 
-   install    [ -arch architecture ] [ packages... ]                                           -- builds packages and executables
-   test       [ -coverage ] [ -misspell ] [ packages... ]                                      -- runs the tests
+   install    [ -arch architecture ] [ -cc compiler ] [ packages... ]                          -- builds packages and executables
+   test       [ -coverage ] [ packages... ]                                                    -- runs the tests
+   lint                                                                                        -- runs certain pre-selected linters
    archive    [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
    importkeys                                                                                  -- imports signing keys from env
    debsrc     [ -signer key-id ] [ -upload dest ]                                              -- creates a debian source package
@@ -119,7 +120,9 @@ var (
 	// Distros for which packages are created.
 	// Note: vivid is unsupported because there is no golang-1.6 package for it.
 	// Note: wily is unsupported because it was officially deprecated on lanchpad.
-	debDistros = []string{"trusty", "xenial", "yakkety", "zesty"}
+	// Note: yakkety is unsupported because it was officially deprecated on lanchpad.
+	// Note: zesty is unsupported because it was officially deprecated on lanchpad.
+	debDistros = []string{"trusty", "xenial", "artful", "bionic"}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -145,6 +148,8 @@ func main() {
 		doInstall(os.Args[2:])
 	case "test":
 		doTest(os.Args[2:])
+	case "lint":
+		doLint(os.Args[2:])
 	case "archive":
 		doArchive(os.Args[2:])
 	case "debsrc":
@@ -169,17 +174,24 @@ func main() {
 func doInstall(cmdline []string) {
 	var (
 		arch = flag.String("arch", "", "Architecture to cross build for")
+		cc   = flag.String("cc", "", "C compiler to cross build with")
 	)
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
 
 	// Check Go version. People regularly open issues about compilation
 	// failure with outdated Go. This should save them the trouble.
-	if runtime.Version() < "go1.7" && !strings.Contains(runtime.Version(), "devel") {
-		log.Println("You have Go version", runtime.Version())
-		log.Println("go-ethereum requires at least Go version 1.7 and cannot")
-		log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
-		os.Exit(1)
+	if !strings.Contains(runtime.Version(), "devel") {
+		// Figure out the minor version number since we can't textually compare (1.10 < 1.9)
+		var minor int
+		fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor)
+
+		if minor < 9 {
+			log.Println("You have Go version", runtime.Version())
+			log.Println("go-ethereum requires at least Go version 1.9 and cannot")
+			log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
+			os.Exit(1)
+		}
 	}
 	// Compile packages given as arguments, or everything if there are no arguments.
 	packages := []string{"./..."}
@@ -195,7 +207,7 @@ func doInstall(cmdline []string) {
 		build.MustRun(goinstall)
 		return
 	}
-	// If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any prvious builds
+	// If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any previous builds
 	if *arch == "arm" {
 		os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm"))
 		for _, path := range filepath.SplitList(build.GOPATH()) {
@@ -203,7 +215,7 @@ func doInstall(cmdline []string) {
 		}
 	}
 	// Seems we are cross compiling, work around forbidden GOBIN
-	goinstall := goToolArch(*arch, "install", buildFlags(env)...)
+	goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...)
 	goinstall.Args = append(goinstall.Args, "-v")
 	goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...)
 	goinstall.Args = append(goinstall.Args, packages...)
@@ -217,7 +229,7 @@ func doInstall(cmdline []string) {
 			}
 			for name := range pkgs {
 				if name == "main" {
-					gobuild := goToolArch(*arch, "build", buildFlags(env)...)
+					gobuild := goToolArch(*arch, *cc, "build", buildFlags(env)...)
 					gobuild.Args = append(gobuild.Args, "-v")
 					gobuild.Args = append(gobuild.Args, []string{"-o", executablePath(cmd.Name())}...)
 					gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name()))
@@ -245,21 +257,11 @@ func buildFlags(env build.Environment) (flags []string) {
 }
 
 func goTool(subcmd string, args ...string) *exec.Cmd {
-	return goToolArch(runtime.GOARCH, subcmd, args...)
+	return goToolArch(runtime.GOARCH, os.Getenv("CC"), subcmd, args...)
 }
 
-func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd {
-	gocmd := filepath.Join(runtime.GOROOT(), "bin", "go")
-	cmd := exec.Command(gocmd, subcmd)
-	cmd.Args = append(cmd.Args, args...)
-
-	if subcmd == "build" || subcmd == "install" || subcmd == "test" {
-		// Go CGO has a Windows linker error prior to 1.8 (https://github.com/golang/go/issues/8756).
-		// Work around issue by allowing multiple definitions for <1.8 builds.
-		if runtime.GOOS == "windows" && runtime.Version() < "go1.8" {
-			cmd.Args = append(cmd.Args, []string{"-ldflags", "-extldflags -Wl,--allow-multiple-definition"}...)
-		}
-	}
+func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd {
+	cmd := build.GoTool(subcmd, args...)
 	cmd.Env = []string{"GOPATH=" + build.GOPATH()}
 	if arch == "" || arch == runtime.GOARCH {
 		cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
@@ -267,6 +269,9 @@ func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd {
 		cmd.Env = append(cmd.Env, "CGO_ENABLED=1")
 		cmd.Env = append(cmd.Env, "GOARCH="+arch)
 	}
+	if cc != "" {
+		cmd.Env = append(cmd.Env, "CC="+cc)
+	}
 	for _, e := range os.Environ() {
 		if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
 			continue
@@ -282,7 +287,6 @@ func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd {
 
 func doTest(cmdline []string) {
 	var (
-		misspell = flag.Bool("misspell", false, "Whether to run the spell checker")
 		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
 	)
 	flag.CommandLine.Parse(cmdline)
@@ -296,10 +300,7 @@ func doTest(cmdline []string) {
 
 	// Run analysis tools before the tests.
 	build.MustRun(goTool("vet", packages...))
-	if *misspell {
-		// TODO(karalabe): Reenable after false detection is fixed: https://github.com/client9/misspell/issues/105
-		// spellcheck(packages)
-	}
+
 	// Run the actual tests.
 	gotest := goTool("test", buildFlags(env)...)
 	// Test a single package at a time. CI builders are slow
@@ -308,35 +309,42 @@ func doTest(cmdline []string) {
 	if *coverage {
 		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
 	}
+
 	gotest.Args = append(gotest.Args, packages...)
 	build.MustRun(gotest)
 }
 
-// spellcheck runs the client9/misspell spellchecker package on all Go, Cgo and
-// test files in the requested packages.
-func spellcheck(packages []string) {
-	// Ensure the spellchecker is available
-	build.MustRun(goTool("get", "github.com/client9/misspell/cmd/misspell"))
+// runs gometalinter on requested packages
+func doLint(cmdline []string) {
+	flag.CommandLine.Parse(cmdline)
 
-	// Windows chokes on long argument lists, check packages individually
-	for _, pkg := range packages {
-		// The spell checker doesn't work on packages, gather all .go files for it
-		out, err := goTool("list", "-f", "{{.Dir}}{{range .GoFiles}}\n{{.}}{{end}}{{range .CgoFiles}}\n{{.}}{{end}}{{range .TestGoFiles}}\n{{.}}{{end}}", pkg).CombinedOutput()
-		if err != nil {
-			log.Fatalf("source file listing failed: %v\n%s", err, string(out))
-		}
-		// Retrieve the folder and assemble the source list
-		lines := strings.Split(string(out), "\n")
-		root := lines[0]
+	packages := []string{"./..."}
+	if len(flag.CommandLine.Args()) > 0 {
+		packages = flag.CommandLine.Args()
+	}
+	// Get metalinter and install all supported linters
+	build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v2"))
+	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), "--install")
 
-		sources := make([]string, 0, len(lines)-1)
-		for _, line := range lines[1:] {
-			if line = strings.TrimSpace(line); line != "" {
-				sources = append(sources, filepath.Join(root, line))
-			}
-		}
-		// Run the spell checker for this particular package
-		build.MustRunCommand(filepath.Join(GOBIN, "misspell"), append([]string{"-error"}, sources...)...)
+	// Run fast linters batched together
+	configs := []string{
+		"--vendor",
+		"--tests",
+		"--disable-all",
+		"--enable=goimports",
+		"--enable=varcheck",
+		"--enable=vet",
+		"--enable=gofmt",
+		"--enable=misspell",
+		"--enable=goconst",
+		"--min-occurrences=6", // for goconst
+	}
+	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
+
+	// Run slow linters one by one
+	for _, linter := range []string{"unconvert", "gosimple"} {
+		configs = []string{"--vendor", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter}
+		build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
 	}
 }
 
@@ -721,9 +729,9 @@ func doAndroidArchive(cmdline []string) {
 		log.Fatal("Please ensure ANDROID_NDK points to your Android NDK")
 	}
 	// Build the Android archive and Maven resources
-	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile"))
+	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
 	build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK")))
-	build.MustRun(gomobileTool("bind", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
+	build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
 
 	if *local {
 		// If we're building locally, copy bundle to build dir and skip Maven
@@ -747,22 +755,27 @@ func doAndroidArchive(cmdline []string) {
 	os.Rename(archive, meta.Package+".aar")
 	if *signer != "" && *deploy != "" {
 		// Import the signing key into the local GPG instance
-		if b64key := os.Getenv(*signer); b64key != "" {
-			key, err := base64.StdEncoding.DecodeString(b64key)
-			if err != nil {
-				log.Fatalf("invalid base64 %s", *signer)
-			}
-			gpg := exec.Command("gpg", "--import")
-			gpg.Stdin = bytes.NewReader(key)
-			build.MustRun(gpg)
+		b64key := os.Getenv(*signer)
+		key, err := base64.StdEncoding.DecodeString(b64key)
+		if err != nil {
+			log.Fatalf("invalid base64 %s", *signer)
+		}
+		gpg := exec.Command("gpg", "--import")
+		gpg.Stdin = bytes.NewReader(key)
+		build.MustRun(gpg)
+
+		keyID, err := build.PGPKeyID(string(key))
+		if err != nil {
+			log.Fatal(err)
 		}
 		// Upload the artifacts to Sonatype and/or Maven Central
 		repo := *deploy + "/service/local/staging/deploy/maven2"
 		if meta.Develop {
 			repo = *deploy + "/content/repositories/snapshots"
 		}
-		build.MustRunCommand("mvn", "gpg:sign-and-deploy-file",
+		build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X",
 			"-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh",
+			"-Dgpg.keyname="+keyID,
 			"-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar")
 	}
 }
@@ -772,9 +785,10 @@ func gomobileTool(subcmd string, args ...string) *exec.Cmd {
 	cmd.Args = append(cmd.Args, args...)
 	cmd.Env = []string{
 		"GOPATH=" + build.GOPATH(),
+		"PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"),
 	}
 	for _, e := range os.Environ() {
-		if strings.HasPrefix(e, "GOPATH=") {
+		if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "PATH=") {
 			continue
 		}
 		cmd.Env = append(cmd.Env, e)
@@ -841,9 +855,9 @@ func doXCodeFramework(cmdline []string) {
 	env := build.Env()
 
 	// Build the iOS XCode framework
-	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile"))
+	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
 	build.MustRun(gomobileTool("init"))
-	bind := gomobileTool("bind", "--target", "ios", "--tags", "ios", "-v", "github.com/ethereum/go-ethereum/mobile")
+	bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "--tags", "ios", "-v", "github.com/ethereum/go-ethereum/mobile")
 
 	if *local {
 		// If we're building locally, use the build folder and stop afterwards

build/deb.control

@@ -2,7 +2,7 @@ Source: {{.Name}}
 Section: science
 Priority: extra
 Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.8
+Build-Depends: debhelper (>= 8.0.0), golang-1.10
 Standards-Version: 3.9.5
 Homepage: https://ethereum.org
 Vcs-Git: git://github.com/ethereum/go-ethereum.git

build/deb.rules

@@ -5,7 +5,7 @@
 #export DH_VERBOSE=1
 
 override_dh_auto_build:
-	build/env.sh /usr/lib/go-1.8/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+	build/env.sh /usr/lib/go-1.10/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

build/goimports.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+find_files() {
+  find . -not \( \
+      \( \
+        -wholename '.github' \
+        -o -wholename './build/_workspace' \
+        -o -wholename './build/bin' \
+        -o -wholename './crypto/bn256' \
+        -o -wholename '*/vendor/*' \
+      \) -prune \
+    \) -name '*.go'
+}
+
+GOFMT="gofmt -s -w";
+GOIMPORTS="goimports -w";
+find_files | xargs $GOFMT;
+find_files | xargs $GOIMPORTS;

build/update-license.go

@@ -55,10 +55,9 @@ var (
 		"crypto/sha3/",
 		"internal/jsre/deps",
 		"log/",
+		"common/bitutil/bitutil",
 		// don't license generated files
-		"contracts/chequebook/contract/",
-		"contracts/ens/contract/",
-		"contracts/release/contract.go",
+		"contracts/chequebook/contract/code.go",
 	}
 
 	// paths with this prefix are licensed as GPL. all other files are LGPL.

cmd/abigen/main.go

@@ -29,7 +29,7 @@ import (
 )
 
 var (
-	abiFlag = flag.String("abi", "", "Path to the Ethereum contract ABI json to bind")
+	abiFlag = flag.String("abi", "", "Path to the Ethereum contract ABI json to bind, - for STDIN")
 	binFlag = flag.String("bin", "", "Path to the Ethereum contract bytecode (generate deploy method)")
 	typFlag = flag.String("type", "", "Struct name for the binding (default = package name)")
 
@@ -75,16 +75,27 @@ func main() {
 		bins  []string
 		types []string
 	)
-	if *solFlag != "" {
+	if *solFlag != "" || *abiFlag == "-" {
 		// Generate the list of types to exclude from binding
 		exclude := make(map[string]bool)
 		for _, kind := range strings.Split(*excFlag, ",") {
 			exclude[strings.ToLower(kind)] = true
 		}
-		contracts, err := compiler.CompileSolidity(*solcFlag, *solFlag)
-		if err != nil {
-			fmt.Printf("Failed to build Solidity contract: %v\n", err)
-			os.Exit(-1)
+
+		var contracts map[string]*compiler.Contract
+		var err error
+		if *solFlag != "" {
+			contracts, err = compiler.CompileSolidity(*solcFlag, *solFlag)
+			if err != nil {
+				fmt.Printf("Failed to build Solidity contract: %v\n", err)
+				os.Exit(-1)
+			}
+		} else {
+			contracts, err = contractsFromStdin()
+			if err != nil {
+				fmt.Printf("Failed to read input ABIs from STDIN: %v\n", err)
+				os.Exit(-1)
+			}
 		}
 		// Gather all non-excluded contract for binding
 		for name, contract := range contracts {
@@ -138,3 +149,12 @@ func main() {
 		os.Exit(-1)
 	}
 }
+
+func contractsFromStdin() (map[string]*compiler.Contract, error) {
+	bytes, err := ioutil.ReadAll(os.Stdin)
+	if err != nil {
+		return nil, err
+	}
+
+	return compiler.ParseCombinedJSON(bytes, "", "", "", "")
+}

cmd/bootnode/main.go

@@ -21,6 +21,7 @@ import (
 	"crypto/ecdsa"
 	"flag"
 	"fmt"
+	"net"
 	"os"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -96,12 +97,37 @@ func main() {
 		}
 	}
 
+	addr, err := net.ResolveUDPAddr("udp", *listenAddr)
+	if err != nil {
+		utils.Fatalf("-ResolveUDPAddr: %v", err)
+	}
+	conn, err := net.ListenUDP("udp", addr)
+	if err != nil {
+		utils.Fatalf("-ListenUDP: %v", err)
+	}
+
+	realaddr := conn.LocalAddr().(*net.UDPAddr)
+	if natm != nil {
+		if !realaddr.IP.IsLoopback() {
+			go nat.Map(natm, nil, "udp", realaddr.Port, realaddr.Port, "ethereum discovery")
+		}
+		// TODO: react to external IP changes over time.
+		if ext, err := natm.ExternalIP(); err == nil {
+			realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port}
+		}
+	}
+
 	if *runv5 {
-		if _, err := discv5.ListenUDP(nodeKey, *listenAddr, natm, "", restrictList); err != nil {
+		if _, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", restrictList); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	} else {
-		if _, err := discover.ListenUDP(nodeKey, *listenAddr, natm, "", restrictList); err != nil {
+		cfg := discover.Config{
+			PrivateKey:   nodeKey,
+			AnnounceAddr: realaddr,
+			NetRestrict:  restrictList,
+		}
+		if _, err := discover.ListenUDP(conn, cfg); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	}

cmd/clef/README.md

@@ -0,0 +1,877 @@
+Clef
+----
+Clef can be used to sign transactions and data and is meant as a replacement for geth's account management.
+This allows DApps not to depend on geth's account management. When a DApp wants to sign data it can send the data to
+the signer, the signer will then provide the user with context and asks the user for permission to sign the data. If
+the users grants the signing request the signer will send the signature back to the DApp.
+  
+This setup allows a DApp to connect to a remote Ethereum node and send transactions that are locally signed. This can
+help in situations when a DApp is connected to a remote node because a local Ethereum node is not available, not
+synchronised with the chain or a particular Ethereum node that has no built-in (or limited) account management.
+  
+Clef can run as a daemon on the same machine, or off a usb-stick like [usb armory](https://inversepath.com/usbarmory),
+or a separate VM in a [QubesOS](https://www.qubes-os.org/) type os setup.
+
+Check out 
+
+* the [tutorial](tutorial.md) for some concrete examples on how the signer works.
+* the [setup docs](docs/setup.md) for some information on how to configure it to work on QubesOS or USBArmory. 
+
+
+## Command line flags
+Clef accepts the following command line options:
+```
+COMMANDS:
+   init    Initialize the signer, generate secret storage
+   attest  Attest that a js-file is to be used
+   addpw   Store a credential for a keystore file
+   help    Shows a list of commands or help for one command
+
+GLOBAL OPTIONS:
+   --loglevel value        log level to emit to the screen (default: 4)
+   --keystore value        Directory for the keystore (default: "$HOME/.ethereum/keystore")
+   --configdir value       Directory for clef configuration (default: "$HOME/.clef")
+   --networkid value       Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby) (default: 1)
+   --lightkdf              Reduce key-derivation RAM & CPU usage at some expense of KDF strength
+   --nousb                 Disables monitoring for and managing USB hardware wallets
+   --rpcaddr value         HTTP-RPC server listening interface (default: "localhost")
+   --rpcport value         HTTP-RPC server listening port (default: 8550)
+   --signersecret value    A file containing the password used to encrypt signer credentials, e.g. keystore credentials and ruleset hash
+   --4bytedb value         File containing 4byte-identifiers (default: "./4byte.json")
+   --4bytedb-custom value  File used for writing new 4byte-identifiers submitted via API (default: "./4byte-custom.json")
+   --auditlog value        File used to emit audit logs. Set to "" to disable (default: "audit.log")
+   --rules value           Enable rule-engine (default: "rules.json")
+   --stdio-ui              Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when the signer is started by an external process.
+   --stdio-ui-test         Mechanism to test interface between signer and UI. Requires 'stdio-ui'.
+   --help, -h              show help
+   --version, -v           print the version
+
+```
+
+
+Example:
+```
+signer -keystore /my/keystore -chainid 4
+```
+
+
+## Security model
+
+The security model of the signer is as follows:
+
+* One critical component (the signer binary / daemon) is responsible for handling cryptographic operations: signing, private keys, encryption/decryption of keystore files.
+* The signer binary has a well-defined 'external' API.
+* The 'external' API is considered UNTRUSTED.
+* The signer binary also communicates with whatever process that invoked the binary, via stdin/stdout.
+  * This channel is considered 'trusted'. Over this channel, approvals and passwords are communicated.
+
+The general flow for signing a transaction using e.g. geth is as follows:
+![image](sign_flow.png)
+
+In this case, `geth` would be started with `--externalsigner=http://localhost:8550` and would relay requests to `eth.sendTransaction`.
+
+## TODOs
+
+Some snags and todos
+
+* [ ] The signer should take a startup param "--no-change", for UIs that do not contain the capability
+   to perform changes to things, only approve/deny. Such a UI should be able to start the signer in
+   a more secure mode by telling it that it only wants approve/deny capabilities.
+
+* [x] It would be nice if the signer could collect new 4byte-id:s/method selectors, and have a
+secondary database for those (`4byte_custom.json`). Users could then (optionally) submit their collections for
+inclusion upstream.
+
+* It should be possible to configure the signer to check if an account is indeed known to it, before
+passing on to the UI. The reason it currently does not, is that it would make it possible to enumerate
+accounts if it immediately returned "unknown account".
+* [x] It should be possible to configure the signer to auto-allow listing (certain) accounts, instead of asking every time.
+* [x] Done Upon startup, the signer should spit out some info to the caller (particularly important when executed in `stdio-ui`-mode),
+invoking methods with the following info:
+  * [x] Version info about the signer
+  * [x] Address of API (http/ipc)
+  * [ ] List of known accounts
+* [ ] Have a default timeout on signing operations, so that if the user has not answered withing e.g. 60 seconds, the request is rejected.
+* [ ] `account_signRawTransaction`
+* [ ] `account_bulkSignTransactions([] transactions)` should
+   * only exist if enabled via config/flag
+   * only allow non-data-sending transactions
+   * all txs must use the same `from`-account
+   * let the user confirm, showing
+      * the total amount
+      * the number of unique recipients
+
+* Geth todos
+    - The signer should pass the `Origin` header as call-info to the UI. As of right now, the way that info about the request is
+put together is a bit of a hack into the http server. This could probably be greatly improved
+    - Relay: Geth should be started in `geth --external_signer localhost:8550`.
+    - Currently, the Geth APIs use `common.Address` in the arguments to transaction submission (e.g `to` field). This
+  type is 20 `bytes`, and is incapable of carrying checksum information. The signer uses `common.MixedcaseAddress`, which
+  retains the original input.
+    - The Geth api should switch to use the same type, and relay `to`-account verbatim to the external api.
+
+* [x] Storage
+    * [x] An encrypted key-value storage should be implemented
+    * See [rules.md](rules.md) for more info about this.
+
+* Another potential thing to introduce is pairing.
+  * To prevent spurious requests which users just accept, implement a way to "pair" the caller with the signer (external API).
+  * Thus geth/mist/cpp would cryptographically handshake and afterwards the caller would be allowed to make signing requests.
+  * This feature would make the addition of rules less dangerous.
+
+* Wallets / accounts. Add API methods for wallets.
+
+## Communication
+
+### External API
+
+The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRPC standard as Geth. The messages are
+expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification).
+
+Some of these call can require user interaction. Clients must be aware that responses
+may be delayed significanlty or may never be received if a users decides to ignore the confirmation request.
+
+The External API is **untrusted** : it does not accept credentials over this api, nor does it expect
+that requests have any authority.
+
+### UI API
+
+The signer has one native console-based UI, for operation without any standalone tools.
+However, there is also an API to communicate with an external UI. To enable that UI,
+the signer needs to be executed with the `--stdio-ui` option, which allocates the
+`stdin`/`stdout` for the UI-api.
+
+An example (insecure) proof-of-concept of has been implemented in `pythonsigner.py`.
+
+The model is as follows:
+
+* The user starts the UI app (`pythonsigner.py`).
+* The UI app starts the `signer` with `--stdio-ui`, and listens to the
+process output for confirmation-requests.
+* The `signer` opens the external http api.
+* When the `signer` receives requests, it sends a `jsonrpc` request via `stdout`.
+* The UI app prompts the user accordingly, and responds to the `signer`
+* The `signer` signs (or not), and responds to the original request.
+
+## External API
+
+See the [external api changelog](extapi_changelog.md) for information about changes to this API.
+
+### Encoding
+- number: positive integers that are hex encoded
+- data: hex encoded data
+- string: ASCII string
+
+All hex encoded values must be prefixed with `0x`.
+
+## Methods
+
+### account_new
+
+#### Create new password protected account
+
+The signer will generate a new private key, encrypts it according to [web3 keystore spec](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) and stores it in the keystore directory.
+The client is responsible for creating a backup of the keystore. If the keystore is lost there is no method of retrieving lost accounts.
+
+#### Arguments
+
+None
+
+#### Result
+  - address [string]: account address that is derived from the generated key
+  - url [string]: location of the keyfile
+  
+#### Sample call
+```json
+{
+  "id": 0,
+  "jsonrpc": "2.0",
+  "method": "account_new",
+  "params": []
+}
+
+{
+  "id": 0,
+  "jsonrpc": "2.0",
+  "result": {
+    "address": "0xbea9183f8f4f03d427f6bcea17388bdff1cab133",
+    "url": "keystore:///my/keystore/UTC--2017-08-24T08-40-15.419655028Z--bea9183f8f4f03d427f6bcea17388bdff1cab133"
+  }
+}
+```
+
+### account_list
+
+#### List available accounts
+   List all accounts that this signer currently manages
+
+#### Arguments
+
+None
+
+#### Result
+  - array with account records:
+     - account.address [string]: account address that is derived from the generated key
+     - account.type [string]: type of the 
+     - account.url [string]: location of the account
+  
+#### Sample call
+```json
+{
+  "id": 1,
+  "jsonrpc": "2.0",
+  "method": "account_list"
+}
+
+{
+  "id": 1,
+  "jsonrpc": "2.0",
+  "result": [
+    {
+      "address": "0xafb2f771f58513609765698f65d3f2f0224a956f",
+      "type": "account",
+      "url": "keystore:///tmp/keystore/UTC--2017-08-24T07-26-47.162109726Z--afb2f771f58513609765698f65d3f2f0224a956f"
+    },
+    {
+      "address": "0xbea9183f8f4f03d427f6bcea17388bdff1cab133",
+      "type": "account",
+      "url": "keystore:///tmp/keystore/UTC--2017-08-24T08-40-15.419655028Z--bea9183f8f4f03d427f6bcea17388bdff1cab133"
+    }
+  ]
+}
+```
+
+### account_signTransaction
+
+#### Sign transactions
+   Signs a transactions and responds with the signed transaction in RLP encoded form.
+
+#### Arguments
+  2. transaction object:
+     - `from` [address]: account to send the transaction from
+     - `to` [address]: receiver account. If omitted or `0x`, will cause contract creation.
+     - `gas` [number]: maximum amount of gas to burn
+     - `gasPrice` [number]: gas price
+     - `value` [number:optional]: amount of Wei to send with the transaction
+     - `data` [data:optional]:  input data
+     - `nonce` [number]: account nonce
+  3. method signature [string:optional]
+     - The method signature, if present, is to aid decoding the calldata. Should consist of `methodname(paramtype,...)`, e.g. `transfer(uint256,address)`. The signer may use this data to parse the supplied calldata, and show the user. The data, however, is considered totally untrusted, and reliability is not expected.
+
+
+#### Result
+  - signed transaction in RLP encoded form [data]
+  
+#### Sample call
+```json
+{
+  "id": 2,
+  "jsonrpc": "2.0",
+  "method": "account_signTransaction",
+  "params": [
+    {
+      "from": "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db",
+      "gas": "0x55555",
+      "gasPrice": "0x1234",
+      "input": "0xabcd",
+      "nonce": "0x0",
+      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+      "value": "0x1234"
+    }
+  ]
+}
+```
+Response
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 67,
+  "error": {
+    "code": -32000,
+    "message": "Request denied"
+  }
+}
+```
+#### Sample call with ABI-data
+
+
+```json
+{
+  "jsonrpc": "2.0",
+  "method": "account_signTransaction",
+  "params": [
+    {
+      "from": "0x694267f14675d7e1b9494fd8d72fefe1755710fa",
+      "gas": "0x333",
+      "gasPrice": "0x1",
+      "nonce": "0x0",
+      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+      "value": "0x0",
+      "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"
+    },
+    "safeSend(address)"
+  ],
+  "id": 67
+}
+```
+Response
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 67,
+  "result": {
+    "raw": "0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
+    "tx": {
+      "nonce": "0x0",
+      "gasPrice": "0x1",
+      "gas": "0x333",
+      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+      "value": "0x0",
+      "input": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
+      "v": "0x26",
+      "r": "0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e",
+      "s": "0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
+      "hash": "0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"
+    }
+  }
+}
+```
+
+Bash example:
+```bash
+#curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
+
+{"jsonrpc":"2.0","id":67,"result":{"raw":"0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","tx":{"nonce":"0x0","gasPrice":"0x1","gas":"0x333","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0","value":"0x0","input":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012","v":"0x26","r":"0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e","s":"0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","hash":"0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"}}}
+```
+
+
+### account_sign
+
+#### Sign data
+   Signs a chunk of data and returns the calculated signature.
+
+#### Arguments
+  - account [address]: account to sign with
+  - data [data]: data to sign
+
+#### Result
+  - calculated signature [data]
+  
+#### Sample call
+```json
+{
+  "id": 3,
+  "jsonrpc": "2.0",
+  "method": "account_sign",
+  "params": [
+    "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db",
+    "0xaabbccdd"
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 3,
+  "jsonrpc": "2.0",
+  "result": "0x5b6693f153b48ec1c706ba4169960386dbaa6903e249cc79a8e6ddc434451d417e1e57327872c7f538beeb323c300afa9999a3d4a5de6caf3be0d5ef832b67ef1c"
+}
+```
+
+### account_ecRecover
+
+#### Recover address
+   Derive the address from the account that was used to sign data from the data and signature.
+   
+#### Arguments
+  - data [data]: data that was signed
+  - signature [data]: the signature to verify
+
+#### Result
+  - derived account [address]
+  
+#### Sample call
+```json
+{
+  "id": 4,
+  "jsonrpc": "2.0",
+  "method": "account_ecRecover",
+  "params": [
+    "0xaabbccdd",
+    "0x5b6693f153b48ec1c706ba4169960386dbaa6903e249cc79a8e6ddc434451d417e1e57327872c7f538beeb323c300afa9999a3d4a5de6caf3be0d5ef832b67ef1c"
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 4,
+  "jsonrpc": "2.0",
+  "result": "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db"
+}
+
+```
+
+### account_import
+
+#### Import account
+   Import a private key into the keystore. The imported key is expected to be encrypted according to the web3 keystore
+   format.
+   
+#### Arguments
+  - account [object]: key in [web3 keystore format](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) (retrieved with account_export) 
+
+#### Result
+  - imported key [object]:
+     - key.address [address]: address of the imported key
+     - key.type [string]: type of the account
+     - key.url [string]: key URL
+  
+#### Sample call
+```json
+{
+  "id": 6,
+  "jsonrpc": "2.0",
+  "method": "account_import",
+  "params": [
+    {
+      "address": "c7412fc59930fd90099c917a50e5f11d0934b2f5",
+      "crypto": {
+        "cipher": "aes-128-ctr",
+        "cipherparams": {
+          "iv": "401c39a7c7af0388491c3d3ecb39f532"
+        },
+        "ciphertext": "eb045260b18dd35cd0e6d99ead52f8fa1e63a6b0af2d52a8de198e59ad783204",
+        "kdf": "scrypt",
+        "kdfparams": {
+          "dklen": 32,
+          "n": 262144,
+          "p": 1,
+          "r": 8,
+          "salt": "9a657e3618527c9b5580ded60c12092e5038922667b7b76b906496f021bb841a"
+        },
+        "mac": "880dc10bc06e9cec78eb9830aeb1e7a4a26b4c2c19615c94acb632992b952806"
+      },
+      "id": "09bccb61-b8d3-4e93-bf4f-205a8194f0b9",
+      "version": 3
+    },
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 6,
+  "jsonrpc": "2.0",
+  "result": {
+    "address": "0xc7412fc59930fd90099c917a50e5f11d0934b2f5",
+    "type": "account",
+    "url": "keystore:///tmp/keystore/UTC--2017-08-24T11-00-42.032024108Z--c7412fc59930fd90099c917a50e5f11d0934b2f5"
+  }
+}
+```
+
+### account_export
+
+#### Export account from keystore
+   Export a private key from the keystore. The exported private key is encrypted with the original passphrase. When the
+   key is imported later this passphrase is required.
+   
+#### Arguments
+  - account [address]: export private key that is associated with this account
+
+#### Result
+  - exported key, see [web3 keystore format](https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition) for
+  more information
+  
+#### Sample call
+```json
+{
+  "id": 5,
+  "jsonrpc": "2.0",
+  "method": "account_export",
+  "params": [
+    "0xc7412fc59930fd90099c917a50e5f11d0934b2f5"
+  ]
+}
+```
+Response
+
+```json
+{
+  "id": 5,
+  "jsonrpc": "2.0",
+  "result": {
+    "address": "c7412fc59930fd90099c917a50e5f11d0934b2f5",
+    "crypto": {
+      "cipher": "aes-128-ctr",
+      "cipherparams": {
+        "iv": "401c39a7c7af0388491c3d3ecb39f532"
+      },
+      "ciphertext": "eb045260b18dd35cd0e6d99ead52f8fa1e63a6b0af2d52a8de198e59ad783204",
+      "kdf": "scrypt",
+      "kdfparams": {
+        "dklen": 32,
+        "n": 262144,
+        "p": 1,
+        "r": 8,
+        "salt": "9a657e3618527c9b5580ded60c12092e5038922667b7b76b906496f021bb841a"
+      },
+      "mac": "880dc10bc06e9cec78eb9830aeb1e7a4a26b4c2c19615c94acb632992b952806"
+    },
+    "id": "09bccb61-b8d3-4e93-bf4f-205a8194f0b9",
+    "version": 3
+  }
+}
+```
+
+
+
+## UI API
+
+These methods needs to be implemented by a UI listener.
+
+By starting the signer with the switch `--stdio-ui-test`, the signer will invoke all known methods, and expect the UI to respond with
+denials. This can be used during development to ensure that the API is (at least somewhat) correctly implemented.
+See `pythonsigner`, which can be invoked via `python3 pythonsigner.py test` to perform the 'denial-handshake-test'.
+
+All methods in this API uses object-based parameters, so that there can be no mixups of parameters: each piece of data is accessed by key.
+
+See the [ui api changelog](intapi_changelog.md) for information about changes to this API.
+
+OBS! A slight deviation from `json` standard is in place: every request and response should be confined to a single line.
+Whereas the `json` specification allows for linebreaks, linebreaks __should not__ be used in this communication channel, to make
+things simpler for both parties.
+
+### ApproveTx
+
+Invoked when there's a transaction for approval.
+
+
+#### Sample call
+
+Here's a method invocation:
+```bash
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
+```
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "0x0x694267f14675d7e1b9494fd8d72fefe1755710fa",
+        "to": "0x0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+        "gas": "0x333",
+        "gasPrice": "0x1",
+        "value": "0x0",
+        "nonce": "0x0",
+        "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
+        "input": null
+      },
+      "call_info": [
+          {
+            "type": "WARNING",
+            "message": "Invalid checksum on to-address"
+          },
+          {
+            "type": "Info",
+            "message": "safeSend(address: 0x0000000000000000000000000000000000000012)"
+          }
+        ],
+      "meta": {
+        "remote": "127.0.0.1:48486",
+        "local": "localhost:8550",
+        "scheme": "HTTP/1.1"
+      }
+    }
+  ]
+}
+
+```
+
+The same method invocation, but with invalid data:
+```bash
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000002000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
+```
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "0x0x694267f14675d7e1b9494fd8d72fefe1755710fa",
+        "to": "0x0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+        "gas": "0x333",
+        "gasPrice": "0x1",
+        "value": "0x0",
+        "nonce": "0x0",
+        "data": "0x4401a6e40000000000000002000000000000000000000000000000000000000000000012",
+        "input": null
+      },
+      "call_info": [
+          {
+            "type": "WARNING",
+            "message": "Invalid checksum on to-address"
+          },
+          {
+            "type": "WARNING",
+            "message": "Transaction data did not match ABI-interface: WARNING: Supplied data is stuffed with extra data. \nWant 0000000000000002000000000000000000000000000000000000000000000012\nHave 0000000000000000000000000000000000000000000000000000000000000012\nfor method safeSend(address)"
+          }
+        ],
+      "meta": {
+        "remote": "127.0.0.1:48492",
+        "local": "localhost:8550",
+        "scheme": "HTTP/1.1"
+      }
+    }
+  ]
+}
+
+
+```
+
+One which has missing `to`, but with no `data`:
+
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 3,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "",
+        "to": null,
+        "gas": "0x0",
+        "gasPrice": "0x0",
+        "value": "0x0",
+        "nonce": "0x0",
+        "data": null,
+        "input": null
+      },
+      "call_info": [
+          {
+            "type": "CRITICAL",
+            "message": "Tx will create contract with empty code!"
+          }
+        ],
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+```
+
+### ApproveExport
+
+Invoked when a request to export an account has been made.
+
+#### Sample call
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 7,
+  "method": "ApproveExport",
+  "params": [
+    {
+      "address": "0x0000000000000000000000000000000000000000",
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+
+```
+
+### ApproveListing
+
+Invoked when a request for account listing has been made.
+
+#### Sample call
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 5,
+  "method": "ApproveListing",
+  "params": [
+    {
+      "accounts": [
+        {
+          "type": "Account",
+          "url": "keystore:///home/bazonk/.ethereum/keystore/UTC--2017-11-20T14-44-54.089682944Z--123409812340981234098123409812deadbeef42",
+          "address": "0x123409812340981234098123409812deadbeef42"
+        },
+        {
+          "type": "Account",
+          "url": "keystore:///home/bazonk/.ethereum/keystore/UTC--2017-11-23T21-59-03.199240693Z--cafebabedeadbeef34098123409812deadbeef42",
+          "address": "0xcafebabedeadbeef34098123409812deadbeef42"
+        }
+      ],
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+
+```
+
+
+### ApproveSignData
+
+#### Sample call
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 4,
+  "method": "ApproveSignData",
+  "params": [
+    {
+      "address": "0x123409812340981234098123409812deadbeef42",
+      "raw_data": "0x01020304",
+      "message": "\u0019Ethereum Signed Message:\n4\u0001\u0002\u0003\u0004",
+      "hash": "0x7e3a4e7a9d1744bc5c675c25e1234ca8ed9162bd17f78b9085e48047c15ac310",
+      "meta": {
+        "remote": "signer binary",
+        "local": "main",
+        "scheme": "in-proc"
+      }
+    }
+  ]
+}
+
+```
+
+### ShowInfo
+
+The UI should show the info to the user. Does not expect response.
+
+#### Sample call
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 9,
+  "method": "ShowInfo",
+  "params": [
+    {
+      "text": "Tests completed"
+    }
+  ]
+}
+
+```
+
+### ShowError
+
+The UI should show the info to the user. Does not expect response.
+
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 2,
+  "method": "ShowError",
+  "params": [
+    {
+      "text": "Testing 'ShowError'"
+    }
+  ]
+}
+
+```
+
+### OnApproved
+
+`OnApprovedTx` is called when a transaction has been approved and signed. The call contains the return value that will be sent to the external caller.  The return value from this method is ignored - the reason for having this callback is to allow the ruleset to keep track of approved transactions.
+
+When implementing rate-limited rules, this callback should be used.
+
+TLDR; Use this method to keep track of signed transactions, instead of using the data in `ApproveTx`.
+
+### OnSignerStartup
+
+This method provide the UI with information about what API version the signer uses (both internal and external) aswell as build-info and external api,
+in k/v-form.
+
+Example call:
+```json
+
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "OnSignerStartup",
+  "params": [
+    {
+      "info": {
+        "extapi_http": "http://localhost:8550",
+        "extapi_ipc": null,
+        "extapi_version": "2.0.0",
+        "intapi_version": "1.2.0"
+      }
+    }
+  ]
+}
+
+```
+
+
+### Rules for UI apis
+
+A UI should conform to the following rules.
+
+* A UI MUST NOT load any external resources that were not embedded/part of the UI package.
+  * For example, not load icons, stylesheets from the internet
+  * Not load files from the filesystem, unless they reside in the same local directory (e.g. config files)
+* A Graphical UI MUST show the blocky-identicon for ethereum addresses.
+* A UI MUST warn display approproate warning if the destination-account is formatted with invalid checksum.
+* A UI MUST NOT open any ports or services
+  * The signer opens the public port
+* A UI SHOULD verify the permissions on the signer binary, and refuse to execute or warn if permissions allow non-user write.
+* A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed
+* A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts
+  * The signer provides accounts
+* A UI SHOULD, to the best extent possible, use static linking / bundling, so that requried libraries are bundled
+along with the UI.
+
+
+### UI Implementations 
+
+There are a couple of implementation for a UI. We'll try to keep this list up to date. 
+
+| Name | Repo | UI type| No external resources| Blocky support| Verifies permissions | Hash information | No secondary storage | Statically linked| Can modify parameters|
+| ---- | ---- | -------| ---- | ---- | ---- |---- | ---- | ---- | ---- |
+| QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: |  :+1: (partially)|
+| GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: |  :x: |
+| Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: |  :x: |

cmd/clef/docs/qubes/qubes-client.py

@@ -0,0 +1,23 @@
+"""
+This implements a dispatcher which listens to localhost:8550, and proxies
+requests via qrexec to the service qubes.EthSign on a target domain
+"""
+
+import http.server
+import socketserver,subprocess
+
+PORT=8550
+TARGET_DOMAIN= 'debian-work'
+
+class Dispatcher(http.server.BaseHTTPRequestHandler):
+    def do_POST(self):
+        post_data = self.rfile.read(int(self.headers['Content-Length']))
+        p = subprocess.Popen(['/usr/bin/qrexec-client-vm',TARGET_DOMAIN,'qubes.Clefsign'],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        output = p.communicate(post_data)[0]
+        self.wfile.write(output)
+
+
+with socketserver.TCPServer(("",PORT), Dispatcher) as httpd:
+    print("Serving at port", PORT)
+    httpd.serve_forever()
+

cmd/clef/docs/qubes/qubes.Clefsign

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+SIGNER_BIN="/home/user/tools/clef/clef"
+SIGNER_CMD="/home/user/tools/gtksigner/gtkui.py -s $SIGNER_BIN"
+
+# Start clef if not already started
+if [ ! -S /home/user/.clef/clef.ipc ]; then
+	$SIGNER_CMD &
+	sleep 1
+fi
+
+# Should be started by now
+if [ -S /home/user/.clef/clef.ipc ]; then
+    # Post incoming request to HTTP channel
+	curl -H "Content-Type: application/json" -X POST -d @- http://localhost:8550 2>/dev/null
+fi

cmd/clef/docs/setup.md

@@ -0,0 +1,198 @@
+# Setting up Clef
+
+This document describes how Clef can be used in a more secure manner than executing it from your everyday laptop, 
+in order to ensure that the keys remain safe in the event that your computer should get compromised. 
+
+## Qubes OS
+
+
+### Background 
+
+The Qubes operating system is based around virtual machines (qubes), where a set of virtual machines are configured, typically for 
+different purposes such as:
+
+- personal
+   - Your personal email, browsing etc
+- work
+  - Work email etc
+- vault
+  - a VM without network access, where gpg-keys and/or keepass credentials are stored. 
+
+A couple of dedicated virtual machines handle externalities:
+
+- sys-net provides networking to all other (network-enabled) machines
+- sys-firewall handles firewall rules
+- sys-usb handles USB devices, and can map usb-devices to certain qubes.
+
+The goal of this document is to describe how we can set up clef to provide secure transaction
+signing from a `vault` vm, to another networked qube which runs Dapps.
+
+### Setup
+
+There are two ways that this can be achieved: integrated via Qubes or integrated via networking. 
+
+
+#### 1. Qubes Integrated
+
+Qubes provdes a facility for inter-qubes communication via `qrexec`. A qube can request to make a cross-qube RPC request 
+to another qube. The OS then asks the user if the call is permitted. 
+
+![Example](qubes/qrexec-example.png)
+
+A policy-file can be created to allow such interaction. On the `target` domain, a service is invoked which can read the
+`stdin` from the `client` qube. 
+
+This is how [Split GPG](https://www.qubes-os.org/doc/split-gpg/) is implemented. We can set up Clef the same way:
+
+##### Server
+
+![Clef via qrexec](qubes/clef_qubes_qrexec.png)
+
+On the `target` qubes, we need to define the rpc service.
+
+[qubes.Clefsign](qubes/qubes.Clefsign):
+
+```bash
+#!/bin/bash
+
+SIGNER_BIN="/home/user/tools/clef/clef"
+SIGNER_CMD="/home/user/tools/gtksigner/gtkui.py -s $SIGNER_BIN"
+
+# Start clef if not already started
+if [ ! -S /home/user/.clef/clef.ipc ]; then
+	$SIGNER_CMD &
+	sleep 1
+fi
+
+# Should be started by now
+if [ -S /home/user/.clef/clef.ipc ]; then
+    # Post incoming request to HTTP channel
+	curl -H "Content-Type: application/json" -X POST -d @- http://localhost:8550 2>/dev/null
+fi
+
+```
+This RPC service is not complete (see notes about HTTP headers below), but works as a proof-of-concept. 
+It will forward the data received on `stdin` (forwarded by the OS) to Clef's HTTP channel.  
+
+It would have been possible to send data directly to the `/home/user/.clef/.clef.ipc` 
+socket via e.g `nc -U /home/user/.clef/clef.ipc`, but the reason for sending the request 
+data over `HTTP` instead of `IPC` is that we want the ability to forward `HTTP` headers.
+
+To enable the service:
+
+``` bash
+sudo cp qubes.Clefsign /etc/qubes-rpc/
+sudo chmod +x /etc/qubes-rpc/ qubes.Clefsign
+```
+
+This setup uses [gtksigner](https://github.com/holiman/gtksigner), which is a very minimal GTK-based UI that works well 
+with minimal requirements. 
+
+##### Client
+
+
+On the `client` qube, we need to create a listener which will receive the request from the Dapp, and proxy it. 
+
+
+[qubes-client.py](qubes/client/qubes-client.py):
+
+```python
+
+"""
+This implements a dispatcher which listens to localhost:8550, and proxies
+requests via qrexec to the service qubes.EthSign on a target domain
+"""
+
+import http.server
+import socketserver,subprocess
+
+PORT=8550
+TARGET_DOMAIN= 'debian-work'
+
+class Dispatcher(http.server.BaseHTTPRequestHandler):
+    def do_POST(self):
+        post_data = self.rfile.read(int(self.headers['Content-Length']))
+        p = subprocess.Popen(['/usr/bin/qrexec-client-vm',TARGET_DOMAIN,'qubes.Clefsign'],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        output = p.communicate(post_data)[0]
+        self.wfile.write(output)
+
+
+with socketserver.TCPServer(("",PORT), Dispatcher) as httpd:
+    print("Serving at port", PORT)
+    httpd.serve_forever()
+
+
+```
+
+#### Testing
+
+To test the flow, if we have set up `debian-work` as the `target`, we can do
+ 
+```bash
+$ cat newaccnt.json 
+{ "id": 0, "jsonrpc": "2.0","method": "account_new","params": []}
+
+$ cat newaccnt.json| qrexec-client-vm debian-work qubes.Clefsign
+```
+
+This should pop up first a dialog to allow the IPC call:
+
+![one](qubes/qubes_newaccount-1.png)
+
+Followed by a GTK-dialog to approve the operation
+
+![two](qubes/qubes_newaccount-2.png)
+
+To test the full flow, we use the client wrapper. Start it on the `client` qube:
+```
+[user@work qubes]$ python3 qubes-client.py 
+```
+
+Make the request over http (`client` qube):
+```
+[user@work clef]$ cat newaccnt.json | curl -X POST -d @- http://localhost:8550
+```
+And it should show the same popups again. 
+
+##### Pros and cons
+
+The benefits of this setup are:
+
+- This is the qubes-os intended model for inter-qube communication,
+- and thus benefits from qubes-os dialogs and policies for user approval
+
+However, it comes with a couple of drawbacks:
+
+- The `qubes-gpg-client` must forward the http request via RPC to the `target` qube. When doing so, the proxy
+  will either drop important headers, or replace them. 
+  - The `Host` header is most likely `localhost` 
+  - The `Origin` header must be forwarded
+  - Information about the remote ip must be added as a `X-Forwarded-For`. However, Clef cannot always trust an `XFF` header, 
+  since malicious clients may lie about `XFF` in order to fool the http server into believing it comes from another address.
+- Even with a policy in place to allow rpc-calls between `caller` and `target`, there will be several popups:
+  - One qubes-specific where the user specifies the `target` vm
+  - One clef-specific to approve the transaction
+  
+
+#### 2. Network integrated
+
+The second way to set up Clef on a qubes system is to allow networking, and have Clef listen to a port which is accessible
+form other qubes. 
+
+![Clef via http](qubes/clef_qubes_http.png)
+
+
+
+
+## USBArmory
+
+The [USB armory](https://inversepath.com/usbarmory) is an open source hardware design with an 800 Mhz ARM processor. It is a pocket-size
+computer. When inserted into a laptop, it identifies itself as a USB network interface, basically adding another network
+to your computer. Over this new network interface, you can SSH into the device. 
+
+Running Clef off a USB armory means that you can use the armory as a very versatile offline computer, which only
+ever connects to a local network between your computer and the device itself.
+
+Needless to say, the while this model should be fairly secure against remote attacks, an attacker with physical access
+to the USB Armory would trivially be able to extract the contents of the device filesystem. 
+

cmd/clef/extapi_changelog.md

@@ -0,0 +1,25 @@
+### Changelog for external API
+
+
+
+#### 2.0.0
+
+* Commit `73abaf04b1372fa4c43201fb1b8019fe6b0a6f8d`, move `from` into `transaction` object in `signTransaction`. This
+makes the `accounts_signTransaction` identical to the old `eth_signTransaction`.
+
+
+#### 1.0.0
+
+Initial release.
+
+### Versioning
+
+The API uses [semantic versioning](https://semver.org/).
+
+TLDR; Given a version number MAJOR.MINOR.PATCH, increment the:
+
+* MAJOR version when you make incompatible API changes,
+* MINOR version when you add functionality in a backwards-compatible manner, and
+* PATCH version when you make backwards-compatible bug fixes.
+
+Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.

cmd/clef/intapi_changelog.md

@@ -0,0 +1,86 @@
+### Changelog for internal API (ui-api)
+
+### 2.0.0
+
+* Modify how `call_info` on a transaction is conveyed. New format:
+
+```
+{
+  "jsonrpc": "2.0",
+  "id": 2,
+  "method": "ApproveTx",
+  "params": [
+    {
+      "transaction": {
+        "from": "0x82A2A876D39022B3019932D30Cd9c97ad5616813",
+        "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
+        "gas": "0x333",
+        "gasPrice": "0x123",
+        "value": "0x10",
+        "nonce": "0x0",
+        "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
+        "input": null
+      },
+      "call_info": [
+        {
+          "type": "WARNING",
+          "message": "Invalid checksum on to-address"
+        },
+        {
+          "type": "WARNING",
+          "message": "Tx contains data, but provided ABI signature could not be matched: Did not match: test (0 matches)"
+        }
+      ],
+      "meta": {
+        "remote": "127.0.0.1:54286",
+        "local": "localhost:8550",
+        "scheme": "HTTP/1.1"
+      }
+    }
+  ]
+}
+```
+
+#### 1.2.0
+
+* Add `OnStartup` method, to provide the UI with information about what API version
+the signer uses (both internal and external) aswell as build-info and external api.
+
+Example call:
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "method": "OnSignerStartup",
+  "params": [
+    {
+      "info": {
+        "extapi_http": "http://localhost:8550",
+        "extapi_ipc": null,
+        "extapi_version": "2.0.0",
+        "intapi_version": "1.2.0"
+      }
+    }
+  ]
+}
+```
+
+#### 1.1.0
+
+* Add `OnApproved` method
+
+#### 1.0.0
+
+Initial release.
+
+### Versioning
+
+The API uses [semantic versioning](https://semver.org/).
+
+TLDR; Given a version number MAJOR.MINOR.PATCH, increment the:
+
+* MAJOR version when you make incompatible API changes,
+* MINOR version when you add functionality in a backwards-compatible manner, and
+* PATCH version when you make backwards-compatible bug fixes.
+
+Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.

cmd/clef/main.go

@@ -0,0 +1,640 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// signer is a utility that can be used so sign transactions and
+// arbitrary data.
+package main
+
+import (
+	"bufio"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/signal"
+	"os/user"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/signer/core"
+	"github.com/ethereum/go-ethereum/signer/rules"
+	"github.com/ethereum/go-ethereum/signer/storage"
+	"gopkg.in/urfave/cli.v1"
+)
+
+// ExternalAPIVersion -- see extapi_changelog.md
+const ExternalAPIVersion = "2.0.0"
+
+// InternalAPIVersion -- see intapi_changelog.md
+const InternalAPIVersion = "2.0.0"
+
+const legalWarning = `
+WARNING! 
+
+Clef is alpha software, and not yet publically released. This software has _not_ been audited, and there
+are no guarantees about the workings of this software. It may contain severe flaws. You should not use this software
+unless you agree to take full responsibility for doing so, and know what you are doing. 
+
+TLDR; THIS IS NOT PRODUCTION-READY SOFTWARE! 
+
+`
+
+var (
+	logLevelFlag = cli.IntFlag{
+		Name:  "loglevel",
+		Value: 4,
+		Usage: "log level to emit to the screen",
+	}
+	keystoreFlag = cli.StringFlag{
+		Name:  "keystore",
+		Value: filepath.Join(node.DefaultDataDir(), "keystore"),
+		Usage: "Directory for the keystore",
+	}
+	configdirFlag = cli.StringFlag{
+		Name:  "configdir",
+		Value: DefaultConfigDir(),
+		Usage: "Directory for Clef configuration",
+	}
+	rpcPortFlag = cli.IntFlag{
+		Name:  "rpcport",
+		Usage: "HTTP-RPC server listening port",
+		Value: node.DefaultHTTPPort + 5,
+	}
+	signerSecretFlag = cli.StringFlag{
+		Name:  "signersecret",
+		Usage: "A file containing the password used to encrypt Clef credentials, e.g. keystore credentials and ruleset hash",
+	}
+	dBFlag = cli.StringFlag{
+		Name:  "4bytedb",
+		Usage: "File containing 4byte-identifiers",
+		Value: "./4byte.json",
+	}
+	customDBFlag = cli.StringFlag{
+		Name:  "4bytedb-custom",
+		Usage: "File used for writing new 4byte-identifiers submitted via API",
+		Value: "./4byte-custom.json",
+	}
+	auditLogFlag = cli.StringFlag{
+		Name:  "auditlog",
+		Usage: "File used to emit audit logs. Set to \"\" to disable",
+		Value: "audit.log",
+	}
+	ruleFlag = cli.StringFlag{
+		Name:  "rules",
+		Usage: "Enable rule-engine",
+		Value: "rules.json",
+	}
+	stdiouiFlag = cli.BoolFlag{
+		Name: "stdio-ui",
+		Usage: "Use STDIN/STDOUT as a channel for an external UI. " +
+			"This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user " +
+			"interface, and can be used when Clef is started by an external process.",
+	}
+	testFlag = cli.BoolFlag{
+		Name:  "stdio-ui-test",
+		Usage: "Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.",
+	}
+	app         = cli.NewApp()
+	initCommand = cli.Command{
+		Action:    utils.MigrateFlags(initializeSecrets),
+		Name:      "init",
+		Usage:     "Initialize the signer, generate secret storage",
+		ArgsUsage: "",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			configdirFlag,
+		},
+		Description: `
+The init command generates a master seed which Clef can use to store credentials and data needed for 
+the rule-engine to work.`,
+	}
+	attestCommand = cli.Command{
+		Action:    utils.MigrateFlags(attestFile),
+		Name:      "attest",
+		Usage:     "Attest that a js-file is to be used",
+		ArgsUsage: "<sha256sum>",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			configdirFlag,
+			signerSecretFlag,
+		},
+		Description: `
+The attest command stores the sha256 of the rule.js-file that you want to use for automatic processing of 
+incoming requests. 
+
+Whenever you make an edit to the rule file, you need to use attestation to tell 
+Clef that the file is 'safe' to execute.`,
+	}
+
+	addCredentialCommand = cli.Command{
+		Action:    utils.MigrateFlags(addCredential),
+		Name:      "addpw",
+		Usage:     "Store a credential for a keystore file",
+		ArgsUsage: "<address> <password>",
+		Flags: []cli.Flag{
+			logLevelFlag,
+			configdirFlag,
+			signerSecretFlag,
+		},
+		Description: `
+The addpw command stores a password for a given address (keyfile). If you invoke it with only one parameter, it will 
+remove any stored credential for that address (keyfile)
+`,
+	}
+)
+
+func init() {
+	app.Name = "Clef"
+	app.Usage = "Manage Ethereum account operations"
+	app.Flags = []cli.Flag{
+		logLevelFlag,
+		keystoreFlag,
+		configdirFlag,
+		utils.NetworkIdFlag,
+		utils.LightKDFFlag,
+		utils.NoUSBFlag,
+		utils.RPCListenAddrFlag,
+		utils.RPCVirtualHostsFlag,
+		utils.IPCDisabledFlag,
+		utils.IPCPathFlag,
+		utils.RPCEnabledFlag,
+		rpcPortFlag,
+		signerSecretFlag,
+		dBFlag,
+		customDBFlag,
+		auditLogFlag,
+		ruleFlag,
+		stdiouiFlag,
+		testFlag,
+	}
+	app.Action = signer
+	app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand}
+
+}
+func main() {
+	if err := app.Run(os.Args); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func initializeSecrets(c *cli.Context) error {
+	if err := initialize(c); err != nil {
+		return err
+	}
+	configDir := c.String(configdirFlag.Name)
+
+	masterSeed := make([]byte, 256)
+	n, err := io.ReadFull(rand.Reader, masterSeed)
+	if err != nil {
+		return err
+	}
+	if n != len(masterSeed) {
+		return fmt.Errorf("failed to read enough random")
+	}
+	err = os.Mkdir(configDir, 0700)
+	if err != nil && !os.IsExist(err) {
+		return err
+	}
+	location := filepath.Join(configDir, "secrets.dat")
+	if _, err := os.Stat(location); err == nil {
+		return fmt.Errorf("file %v already exists, will not overwrite", location)
+	}
+	err = ioutil.WriteFile(location, masterSeed, 0700)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("A master seed has been generated into %s\n", location)
+	fmt.Printf(`
+This is required to be able to store credentials, such as : 
+* Passwords for keystores (used by rule engine)
+* Storage for javascript rules
+* Hash of rule-file
+
+You should treat that file with utmost secrecy, and make a backup of it. 
+NOTE: This file does not contain your accounts. Those need to be backed up separately!
+
+`)
+	return nil
+}
+func attestFile(ctx *cli.Context) error {
+	if len(ctx.Args()) < 1 {
+		utils.Fatalf("This command requires an argument.")
+	}
+	if err := initialize(ctx); err != nil {
+		return err
+	}
+
+	stretchedKey, err := readMasterKey(ctx)
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	configDir := ctx.String(configdirFlag.Name)
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+	confKey := crypto.Keccak256([]byte("config"), stretchedKey)
+
+	// Initialize the encrypted storages
+	configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confKey)
+	val := ctx.Args().First()
+	configStorage.Put("ruleset_sha256", val)
+	log.Info("Ruleset attestation updated", "sha256", val)
+	return nil
+}
+
+func addCredential(ctx *cli.Context) error {
+	if len(ctx.Args()) < 1 {
+		utils.Fatalf("This command requires at leaste one argument.")
+	}
+	if err := initialize(ctx); err != nil {
+		return err
+	}
+
+	stretchedKey, err := readMasterKey(ctx)
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	configDir := ctx.String(configdirFlag.Name)
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
+
+	// Initialize the encrypted storages
+	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
+	key := ctx.Args().First()
+	value := ""
+	if len(ctx.Args()) > 1 {
+		value = ctx.Args().Get(1)
+	}
+	pwStorage.Put(key, value)
+	log.Info("Credential store updated", "key", key)
+	return nil
+}
+
+func initialize(c *cli.Context) error {
+	// Set up the logger to print everything
+	logOutput := os.Stdout
+	if c.Bool(stdiouiFlag.Name) {
+		logOutput = os.Stderr
+		// If using the stdioui, we can't do the 'confirm'-flow
+		fmt.Fprintf(logOutput, legalWarning)
+	} else {
+		if !confirm(legalWarning) {
+			return fmt.Errorf("aborted by user")
+		}
+	}
+
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int(logLevelFlag.Name)), log.StreamHandler(logOutput, log.TerminalFormat(true))))
+	return nil
+}
+
+func signer(c *cli.Context) error {
+	if err := initialize(c); err != nil {
+		return err
+	}
+	var (
+		ui core.SignerUI
+	)
+	if c.Bool(stdiouiFlag.Name) {
+		log.Info("Using stdin/stdout as UI-channel")
+		ui = core.NewStdIOUI()
+	} else {
+		log.Info("Using CLI as UI-channel")
+		ui = core.NewCommandlineUI()
+	}
+	db, err := core.NewAbiDBFromFiles(c.String(dBFlag.Name), c.String(customDBFlag.Name))
+	if err != nil {
+		utils.Fatalf(err.Error())
+	}
+	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", c.String("4bytedb"))
+
+	var (
+		api core.ExternalAPI
+	)
+
+	configDir := c.String(configdirFlag.Name)
+	if stretchedKey, err := readMasterKey(c); err != nil {
+		log.Info("No master seed provided, rules disabled")
+	} else {
+
+		if err != nil {
+			utils.Fatalf(err.Error())
+		}
+		vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
+
+		// Generate domain specific keys
+		pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
+		jskey := crypto.Keccak256([]byte("jsstorage"), stretchedKey)
+		confkey := crypto.Keccak256([]byte("config"), stretchedKey)
+
+		// Initialize the encrypted storages
+		pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
+		jsStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "jsstorage.json"), jskey)
+		configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
+
+		//Do we have a rule-file?
+		ruleJS, err := ioutil.ReadFile(c.String(ruleFlag.Name))
+		if err != nil {
+			log.Info("Could not load rulefile, rules not enabled", "file", "rulefile")
+		} else {
+			hasher := sha256.New()
+			hasher.Write(ruleJS)
+			shasum := hasher.Sum(nil)
+			storedShasum := configStorage.Get("ruleset_sha256")
+			if storedShasum != hex.EncodeToString(shasum) {
+				log.Info("Could not validate ruleset hash, rules not enabled", "got", hex.EncodeToString(shasum), "expected", storedShasum)
+			} else {
+				// Initialize rules
+				ruleEngine, err := rules.NewRuleEvaluator(ui, jsStorage, pwStorage)
+				if err != nil {
+					utils.Fatalf(err.Error())
+				}
+				ruleEngine.Init(string(ruleJS))
+				ui = ruleEngine
+				log.Info("Rule engine configured", "file", c.String(ruleFlag.Name))
+			}
+		}
+	}
+
+	apiImpl := core.NewSignerAPI(
+		c.Int64(utils.NetworkIdFlag.Name),
+		c.String(keystoreFlag.Name),
+		c.Bool(utils.NoUSBFlag.Name),
+		ui, db,
+		c.Bool(utils.LightKDFFlag.Name))
+
+	api = apiImpl
+
+	// Audit logging
+	if logfile := c.String(auditLogFlag.Name); logfile != "" {
+		api, err = core.NewAuditLogger(logfile, api)
+		if err != nil {
+			utils.Fatalf(err.Error())
+		}
+		log.Info("Audit logs configured", "file", logfile)
+	}
+	// register signer API with server
+	var (
+		extapiURL = "n/a"
+		ipcapiURL = "n/a"
+	)
+	rpcAPI := []rpc.API{
+		{
+			Namespace: "account",
+			Public:    true,
+			Service:   api,
+			Version:   "1.0"},
+	}
+	if c.Bool(utils.RPCEnabledFlag.Name) {
+
+		vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name))
+		cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name))
+
+		// start http server
+		httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
+		listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts)
+		if err != nil {
+			utils.Fatalf("Could not start RPC api: %v", err)
+		}
+		extapiURL = fmt.Sprintf("http://%s", httpEndpoint)
+		log.Info("HTTP endpoint opened", "url", extapiURL)
+
+		defer func() {
+			listener.Close()
+			log.Info("HTTP endpoint closed", "url", httpEndpoint)
+		}()
+
+	}
+	if !c.Bool(utils.IPCDisabledFlag.Name) {
+		if c.IsSet(utils.IPCPathFlag.Name) {
+			ipcapiURL = c.String(utils.IPCPathFlag.Name)
+		} else {
+			ipcapiURL = filepath.Join(configDir, "clef.ipc")
+		}
+
+		listener, _, err := rpc.StartIPCEndpoint(ipcapiURL, rpcAPI)
+		if err != nil {
+			utils.Fatalf("Could not start IPC api: %v", err)
+		}
+		log.Info("IPC endpoint opened", "url", ipcapiURL)
+		defer func() {
+			listener.Close()
+			log.Info("IPC endpoint closed", "url", ipcapiURL)
+		}()
+
+	}
+
+	if c.Bool(testFlag.Name) {
+		log.Info("Performing UI test")
+		go testExternalUI(apiImpl)
+	}
+	ui.OnSignerStartup(core.StartupInfo{
+		Info: map[string]interface{}{
+			"extapi_version": ExternalAPIVersion,
+			"intapi_version": InternalAPIVersion,
+			"extapi_http":    extapiURL,
+			"extapi_ipc":     ipcapiURL,
+		},
+	})
+
+	abortChan := make(chan os.Signal)
+	signal.Notify(abortChan, os.Interrupt)
+
+	sig := <-abortChan
+	log.Info("Exiting...", "signal", sig)
+
+	return nil
+}
+
+// splitAndTrim splits input separated by a comma
+// and trims excessive white space from the substrings.
+func splitAndTrim(input string) []string {
+	result := strings.Split(input, ",")
+	for i, r := range result {
+		result[i] = strings.TrimSpace(r)
+	}
+	return result
+}
+
+// DefaultConfigDir is the default config directory to use for the vaults and other
+// persistence requirements.
+func DefaultConfigDir() string {
+	// Try to place the data folder in the user's home dir
+	home := homeDir()
+	if home != "" {
+		if runtime.GOOS == "darwin" {
+			return filepath.Join(home, "Library", "Signer")
+		} else if runtime.GOOS == "windows" {
+			return filepath.Join(home, "AppData", "Roaming", "Signer")
+		} else {
+			return filepath.Join(home, ".clef")
+		}
+	}
+	// As we cannot guess a stable location, return empty and handle later
+	return ""
+}
+
+func homeDir() string {
+	if home := os.Getenv("HOME"); home != "" {
+		return home
+	}
+	if usr, err := user.Current(); err == nil {
+		return usr.HomeDir
+	}
+	return ""
+}
+func readMasterKey(ctx *cli.Context) ([]byte, error) {
+	var (
+		file      string
+		configDir = ctx.String(configdirFlag.Name)
+	)
+	if ctx.IsSet(signerSecretFlag.Name) {
+		file = ctx.String(signerSecretFlag.Name)
+	} else {
+		file = filepath.Join(configDir, "secrets.dat")
+	}
+	if err := checkFile(file); err != nil {
+		return nil, err
+	}
+	masterKey, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	if len(masterKey) < 256 {
+		return nil, fmt.Errorf("master key of insufficient length, expected >255 bytes, got %d", len(masterKey))
+	}
+	// Create vault location
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterKey)[:10]))
+	err = os.Mkdir(vaultLocation, 0700)
+	if err != nil && !os.IsExist(err) {
+		return nil, err
+	}
+	//!TODO, use KDF to stretch the master key
+	//			stretched_key := stretch_key(master_key)
+
+	return masterKey, nil
+}
+
+// checkFile is a convenience function to check if a file
+// * exists
+// * is mode 0600
+func checkFile(filename string) error {
+	info, err := os.Stat(filename)
+	if err != nil {
+		return fmt.Errorf("failed stat on %s: %v", filename, err)
+	}
+	// Check the unix permission bits
+	if info.Mode().Perm()&077 != 0 {
+		return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String())
+	}
+	return nil
+}
+
+// confirm displays a text and asks for user confirmation
+func confirm(text string) bool {
+	fmt.Printf(text)
+	fmt.Printf("\nEnter 'ok' to proceed:\n>")
+
+	text, err := bufio.NewReader(os.Stdin).ReadString('\n')
+	if err != nil {
+		log.Crit("Failed to read user input", "err", err)
+	}
+
+	if text := strings.TrimSpace(text); text == "ok" {
+		return true
+	}
+	return false
+}
+
+func testExternalUI(api *core.SignerAPI) {
+
+	ctx := context.WithValue(context.Background(), "remote", "clef binary")
+	ctx = context.WithValue(ctx, "scheme", "in-proc")
+	ctx = context.WithValue(ctx, "local", "main")
+
+	errs := make([]string, 0)
+
+	api.UI.ShowInfo("Testing 'ShowInfo'")
+	api.UI.ShowError("Testing 'ShowError'")
+
+	checkErr := func(method string, err error) {
+		if err != nil && err != core.ErrRequestDenied {
+			errs = append(errs, fmt.Sprintf("%v: %v", method, err.Error()))
+		}
+	}
+	var err error
+
+	_, err = api.SignTransaction(ctx, core.SendTxArgs{From: common.MixedcaseAddress{}}, nil)
+	checkErr("SignTransaction", err)
+	_, err = api.Sign(ctx, common.MixedcaseAddress{}, common.Hex2Bytes("01020304"))
+	checkErr("Sign", err)
+	_, err = api.List(ctx)
+	checkErr("List", err)
+	_, err = api.New(ctx)
+	checkErr("New", err)
+	_, err = api.Export(ctx, common.Address{})
+	checkErr("Export", err)
+	_, err = api.Import(ctx, json.RawMessage{})
+	checkErr("Import", err)
+
+	api.UI.ShowInfo("Tests completed")
+
+	if len(errs) > 0 {
+		log.Error("Got errors")
+		for _, e := range errs {
+			log.Error(e)
+		}
+	} else {
+		log.Info("No errors")
+	}
+
+}
+
+/**
+//Create Account
+
+curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_new","params":["test"],"id":67}' localhost:8550
+
+// List accounts
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_list","params":[""],"id":67}' http://localhost:8550/
+
+// Make Transaction
+// safeSend(0x12)
+// 4401a6e40000000000000000000000000000000000000000000000000000000000000012
+
+// supplied abi
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x82A2A876D39022B3019932D30Cd9c97ad5616813","gas":"0x333","gasPrice":"0x123","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x10", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"test"],"id":67}' http://localhost:8550/
+
+// Not supplied
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x82A2A876D39022B3019932D30Cd9c97ad5616813","gas":"0x333","gasPrice":"0x123","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x10", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"}],"id":67}' http://localhost:8550/
+
+// Sign data
+
+curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_sign","params":["0x694267f14675d7e1b9494fd8d72fefe1755710fa","bazonk gaz baz"],"id":67}' http://localhost:8550/
+
+
+**/

cmd/clef/pythonsigner.py

@@ -0,0 +1,179 @@
+import os,sys, subprocess
+from tinyrpc.transports import ServerTransport
+from tinyrpc.protocols.jsonrpc import JSONRPCProtocol
+from tinyrpc.dispatch import public,RPCDispatcher
+from tinyrpc.server import RPCServer
+
+""" This is a POC example of how to write a custom UI for Clef. The UI starts the
+clef process with the '--stdio-ui' option, and communicates with clef using standard input / output.
+
+The standard input/output is a relatively secure way to communicate, as it does not require opening any ports
+or IPC files. Needless to say, it does not protect against memory inspection mechanisms where an attacker
+can access process memory."""
+
+try:
+    import urllib.parse as urlparse
+except ImportError:
+    import urllib as urlparse
+
+class StdIOTransport(ServerTransport):
+    """ Uses std input/output for RPC """
+    def receive_message(self):
+        return None, urlparse.unquote(sys.stdin.readline())
+
+    def send_reply(self, context, reply):
+        print(reply)
+
+class PipeTransport(ServerTransport):
+    """ Uses std a pipe for RPC """
+
+    def __init__(self,input, output):
+        self.input = input
+        self.output = output
+
+    def receive_message(self):
+        data = self.input.readline()
+        print(">> {}".format( data))
+        return None, urlparse.unquote(data)
+
+    def send_reply(self, context, reply):
+        print("<< {}".format( reply))
+        self.output.write(reply)
+        self.output.write("\n")
+
+class StdIOHandler():
+
+    def __init__(self):
+        pass
+
+    @public
+    def ApproveTx(self,req):
+        """
+        Example request:
+        {
+            "jsonrpc": "2.0",
+            "method": "ApproveTx",
+            "params": [{
+                "transaction": {
+                    "to": "0xae967917c465db8578ca9024c205720b1a3651A9",
+                    "gas": "0x333",
+                    "gasPrice": "0x123",
+                    "value": "0x10",
+                    "data": "0xd7a5865800000000000000000000000000000000000000000000000000000000000000ff",
+                    "nonce": "0x0"
+                },
+                "from": "0xAe967917c465db8578ca9024c205720b1a3651A9",
+                "call_info": "Warning! Could not validate ABI-data against calldata\nSupplied ABI spec does not contain method signature in data: 0xd7a58658",
+                "meta": {
+                    "remote": "127.0.0.1:34572",
+                    "local": "localhost:8550",
+                    "scheme": "HTTP/1.1"
+                }
+            }],
+            "id": 1
+        }
+
+        :param transaction: transaction info
+        :param call_info: info abou the call, e.g. if ABI info could not be
+        :param meta: metadata about the request, e.g. where the call comes from
+        :return: 
+        """
+        transaction = req.get('transaction')
+        _from       = req.get('from')
+        call_info   = req.get('call_info')
+        meta        = req.get('meta')
+
+        return {
+            "approved" : False,
+            #"transaction" : transaction,
+  #          "from" : _from,
+#            "password" : None,
+        }
+
+    @public
+    def ApproveSignData(self, req):
+        """ Example request
+
+        """
+        return {"approved": False, "password" : None}
+
+    @public
+    def ApproveExport(self, req):
+        """ Example request
+
+        """
+        return {"approved" : False}
+
+    @public
+    def ApproveImport(self, req):
+        """ Example request
+
+        """
+        return { "approved" : False, "old_password": "", "new_password": ""}
+
+    @public
+    def ApproveListing(self, req):
+        """ Example request
+
+        """
+        return {'accounts': []}
+
+    @public
+    def ApproveNewAccount(self, req):
+        """
+        Example request
+
+        :return:
+        """
+        return {"approved": False,
+                #"password": ""
+                }
+
+    @public
+    def ShowError(self,message = {}):
+        """
+        Example request:
+
+        {"jsonrpc":"2.0","method":"ShowInfo","params":{"message":"Testing 'ShowError'"},"id":1}
+
+        :param message: to show
+        :return: nothing
+        """
+        if 'text' in message.keys():
+            sys.stderr.write("Error: {}\n".format( message['text']))
+        return
+
+    @public
+    def ShowInfo(self,message = {}):
+        """
+        Example request
+        {"jsonrpc":"2.0","method":"ShowInfo","params":{"message":"Testing 'ShowInfo'"},"id":0}
+
+        :param message: to display
+        :return:nothing
+        """
+
+        if 'text' in message.keys():
+            sys.stdout.write("Error: {}\n".format( message['text']))
+        return
+
+def main(args):
+
+    cmd = ["./clef", "--stdio-ui"]
+    if len(args) > 0 and args[0] == "test":
+        cmd.extend(["--stdio-ui-test"])
+    print("cmd: {}".format(" ".join(cmd)))
+    dispatcher = RPCDispatcher()
+    dispatcher.register_instance(StdIOHandler(), '')
+    # line buffered
+    p = subprocess.Popen(cmd, bufsize=1, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+
+    rpc_server = RPCServer(
+        PipeTransport(p.stdout, p.stdin),
+        JSONRPCProtocol(),
+        dispatcher
+    )
+    rpc_server.serve_forever()
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

cmd/clef/rules.md

@@ -0,0 +1,236 @@
+# Rules
+
+The `signer` binary contains a ruleset engine, implemented with [OttoVM](https://github.com/robertkrimen/otto)
+
+It enables usecases like the following:
+
+* I want to auto-approve transactions with contract `CasinoDapp`, with up to `0.05 ether` in value to maximum `1 ether` per 24h period
+* I want to auto-approve transaction to contract `EthAlarmClock` with `data`=`0xdeadbeef`, if `value=0`, `gas < 44k` and `gasPrice < 40Gwei`
+
+The two main features that are required for this to work well are;
+
+1. Rule Implementation: how to create, manage and interpret rules in a flexible but secure manner
+2. Credential managements and credentials; how to provide auto-unlock without exposing keys unnecessarily.
+
+The section below deals with both of them
+
+## Rule Implementation
+
+A ruleset file is implemented as a `js` file. Under the hood, the ruleset-engine is a `SignerUI`, implementing the same methods as the `json-rpc` methods
+defined in the UI protocol. Example:
+
+```javascript
+
+function asBig(str){
+    if(str.slice(0,2) == "0x"){ return new BigNumber(str.slice(2),16)}
+    return new BigNumber(str)
+}
+
+// Approve transactions to a certain contract if value is below a certain limit
+function ApproveTx(req){
+
+    var limit = big.Newint("0xb1a2bc2ec50000")
+	var value = asBig(req.transaction.value);
+
+	if(req.transaction.to.toLowerCase()=="0xae967917c465db8578ca9024c205720b1a3651a9")
+	    && value.lt(limit) ){
+	    return "Approve"
+	 }
+    // If we return "Reject", it will be rejected.
+    // By not returning anything, it will be passed to the next UI, for manual processing
+}
+
+//Approve listings if request made from IPC
+function ApproveListing(req){
+    if (req.metadata.scheme == "ipc"){ return "Approve"}
+}
+
+```
+
+Whenever the external API is called (and the ruleset is enabled), the `signer` calls the UI, which is an instance of a ruleset-engine. The ruleset-engine
+invokes the corresponding method. In doing so, there are three possible outcomes:
+
+1. JS returns "Approve"
+  * Auto-approve request
+2. JS returns "Reject"
+  * Auto-reject request
+3. Error occurs, or something else is returned
+  * Pass on to `next` ui: the regular UI channel.
+
+A more advanced example can be found below, "Example 1: ruleset for a rate-limited window", using `storage` to `Put` and `Get` `string`s by key.
+
+* At the time of writing, storage only exists as an ephemeral unencrypted implementation, to be used during testing.
+
+### Things to note
+
+The Otto vm has a few [caveats](https://github.com/robertkrimen/otto):
+
+* "use strict" will parse, but does nothing.
+* The regular expression engine (re2/regexp) is not fully compatible with the ECMA5 specification.
+* Otto targets ES5. ES6 features (eg: Typed Arrays) are not supported.
+
+Additionally, a few more have been added
+
+* The rule execution cannot load external javascript files.
+* The only preloaded libary is [`bignumber.js`](https://github.com/MikeMcl/bignumber.js) version `2.0.3`. This one is fairly old, and is not aligned with the documentation at the github repository.
+* Each invocation is made in a fresh virtual machine. This means that you cannot store data in global variables between invocations. This is a deliberate choice -- if you want to store data, use the disk-backed `storage`, since rules should not rely on ephemeral data.
+* Javascript API parameters are _always_ an object. This is also a design choice, to ensure that parameters are accessed by _key_ and not by order. This is to prevent mistakes due to missing parameters or parameter changes.
+* The JS engine has access to `storage` and `console`.
+
+#### Security considerations
+
+##### Security of ruleset
+
+Some security precautions can be made, such as:
+
+* Never load `ruleset.js` unless the file is `readonly` (`r-??-??-?`). If the user wishes to modify the ruleset, he must make it writeable and then set back to readonly.
+  * This is to prevent attacks where files are dropped on the users disk.
+* Since we're going to have to have some form of secure storage (not defined in this section), we could also store the `sha3` of the `ruleset.js` file in there.
+  * If the user wishes to modify the ruleset, he'd then have to perform e.g. `signer --attest /path/to/ruleset --credential <creds>`
+
+##### Security of implementation
+
+The drawbacks of this very flexible solution is that the `signer` needs to contain a javascript engine. This is pretty simple to implement, since it's already
+implemented for `geth`. There are no known security vulnerabilities in, nor have we had any security-problems with it so far.
+
+The javascript engine would be an added attack surface; but if the validation of `rulesets` is made good (with hash-based attestation), the actual javascript cannot be considered
+an attack surface -- if an attacker can control the ruleset, a much simpler attack would be to implement an "always-approve" rule instead of exploiting the js vm. The only benefit
+to be gained from attacking the actual `signer` process from the `js` side would be if it could somehow extract cryptographic keys from memory.
+
+##### Security in usability
+
+Javascript is flexible, but also easy to get wrong, especially when users assume that `js` can handle large integers natively. Typical errors
+include trying to multiply `gasCost` with `gas` without using `bigint`:s.
+
+It's unclear whether any other DSL could be more secure; since there's always the possibility of erroneously implementing a rule.
+
+
+## Credential management
+
+The ability to auto-approve transaction means that the signer needs to have necessary credentials to decrypt keyfiles. These passwords are hereafter called `ksp` (keystore pass).
+
+### Example implementation
+
+Upon startup of the signer, the signer is given a switch: `--seed <path/to/masterseed>`
+The `seed` contains a blob of bytes, which is the master seed for the `signer`.
+
+The `signer` uses the `seed` to:
+
+* Generate the `path` where the settings are stored.
+  * `./settings/1df094eb-c2b1-4689-90dd-790046d38025/vault.dat`
+  * `./settings/1df094eb-c2b1-4689-90dd-790046d38025/rules.js`
+* Generate the encryption password for `vault.dat`.
+
+The `vault.dat` would be an encrypted container storing the following information:
+
+* `ksp` entries
+* `sha256` hash of `rules.js`
+* Information about pair:ed callers (not yet specified)
+
+### Security considerations
+
+This would leave it up to the user to ensure that the `path/to/masterseed` is handled in a secure way. It's difficult to get around this, although one could
+imagine leveraging OS-level keychains where supported. The setup is however in general similar to how ssh-keys are  stored in `.ssh/`.
+
+
+# Implementation status
+
+This is now implemented (with ephemeral non-encrypted storage for now, so not yet enabled).
+
+## Example 1: ruleset for a rate-limited window
+
+
+```javascript
+
+	function big(str){
+		if(str.slice(0,2) == "0x"){ return new BigNumber(str.slice(2),16)}
+		return new BigNumber(str)
+	}
+
+	// Time window: 1 week
+	var window = 1000* 3600*24*7;
+
+	// Limit : 1 ether
+	var limit = new BigNumber("1e18");
+
+	function isLimitOk(transaction){
+		var value = big(transaction.value)
+		// Start of our window function
+		var windowstart = new Date().getTime() - window;
+
+		var txs = [];
+		var stored = storage.Get('txs');
+
+		if(stored != ""){
+			txs = JSON.parse(stored)
+		}
+		// First, remove all that have passed out of the time-window
+		var newtxs = txs.filter(function(tx){return tx.tstamp > windowstart});
+		console.log(txs, newtxs.length);
+
+		// Secondly, aggregate the current sum
+		sum = new BigNumber(0)
+
+		sum = newtxs.reduce(function(agg, tx){ return big(tx.value).plus(agg)}, sum);
+		console.log("ApproveTx > Sum so far", sum);
+		console.log("ApproveTx > Requested", value.toNumber());
+
+		// Would we exceed weekly limit ?
+		return sum.plus(value).lt(limit)
+
+	}
+	function ApproveTx(r){
+		if (isLimitOk(r.transaction)){
+			return "Approve"
+		}
+		return "Nope"
+	}
+
+	/**
+	* OnApprovedTx(str) is called when a transaction has been approved and signed. The parameter
+ 	* 'response_str' contains the return value that will be sent to the external caller.
+	* The return value from this method is ignore - the reason for having this callback is to allow the
+	* ruleset to keep track of approved transactions.
+	*
+	* When implementing rate-limited rules, this callback should be used.
+	* If a rule responds with neither 'Approve' nor 'Reject' - the tx goes to manual processing. If the user
+	* then accepts the transaction, this method will be called.
+	*
+	* TLDR; Use this method to keep track of signed transactions, instead of using the data in ApproveTx.
+	*/
+ 	function OnApprovedTx(resp){
+		var value = big(resp.tx.value)
+		var txs = []
+		// Load stored transactions
+		var stored = storage.Get('txs');
+		if(stored != ""){
+			txs = JSON.parse(stored)
+		}
+		// Add this to the storage
+		txs.push({tstamp: new Date().getTime(), value: value});
+		storage.Put("txs", JSON.stringify(txs));
+	}
+
+```
+
+## Example 2: allow destination
+
+```javascript
+
+	function ApproveTx(r){
+		if(r.transaction.from.toLowerCase()=="0x0000000000000000000000000000000000001337"){ return "Approve"}
+		if(r.transaction.from.toLowerCase()=="0x000000000000000000000000000000000000dead"){ return "Reject"}
+		// Otherwise goes to manual processing
+	}
+
+```
+
+## Example 3: Allow listing
+
+```javascript
+
+    function ApproveListing(){
+        return "Approve"
+    }
+
+```

cmd/clef/tutorial.md

@@ -0,0 +1,198 @@
+## Initializing the signer
+
+First, initialize the master seed.
+
+```text
+#./signer init
+
+WARNING!
+
+The signer is alpha software, and not yet publically released. This software has _not_ been audited, and there
+are no guarantees about the workings of this software. It may contain severe flaws. You should not use this software
+unless you agree to take full responsibility for doing so, and know what you are doing.
+
+TLDR; THIS IS NOT PRODUCTION-READY SOFTWARE!
+
+
+Enter 'ok' to proceed:
+>ok
+A master seed has been generated into /home/martin/.signer/secrets.dat
+
+This is required to be able to store credentials, such as :
+* Passwords for keystores (used by rule engine)
+* Storage for javascript rules
+* Hash of rule-file
+
+You should treat that file with utmost secrecy, and make a backup of it.
+NOTE: This file does not contain your accounts. Those need to be backed up separately!
+```
+
+(for readability purposes, we'll remove the WARNING printout in the rest of this document)
+
+## Creating rules
+
+Now, you can create a rule-file.
+
+```javascript
+function ApproveListing(){
+    return "Approve"
+}
+```
+Get the `sha256` hash....
+```text
+#sha256sum rules.js
+6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72  rules.js
+```
+...And then `attest` the file:
+```text
+#./signer attest 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
+
+INFO [02-21|12:14:38] Ruleset attestation updated              sha256=6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
+```
+At this point, we then start the signer with the rule-file:
+
+```text
+#./signer --rules rules.json
+
+INFO [02-21|12:15:18] Using CLI as UI-channel
+INFO [02-21|12:15:18] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [02-21|12:15:18] Could not load rulefile, rules not enabled file=rulefile
+DEBUG[02-21|12:15:18] FS scan times                            list=35.335µs set=5.536µs diff=5.073µs
+DEBUG[02-21|12:15:18] Ledger support enabled
+DEBUG[02-21|12:15:18] Trezor support enabled
+INFO [02-21|12:15:18] Audit logs configured                    file=audit.log
+INFO [02-21|12:15:18] HTTP endpoint opened                     url=http://localhost:8550
+------- Signer info -------
+* extapi_http : http://localhost:8550
+* extapi_ipc : <nil>
+* extapi_version : 2.0.0
+* intapi_version : 1.2.0
+
+```
+
+Any list-requests will now be auto-approved by our rule-file.
+
+## Under the hood
+
+While doing the operations above, these files have been created:
+
+```text
+#ls -laR ~/.signer/
+/home/martin/.signer/:
+total 16
+drwx------  3 martin martin 4096 feb 21 12:14 .
+drwxr-xr-x 71 martin martin 4096 feb 21 12:12 ..
+drwx------  2 martin martin 4096 feb 21 12:14 43f73718397aa54d1b22
+-rwx------  1 martin martin  256 feb 21 12:12 secrets.dat
+
+/home/martin/.signer/43f73718397aa54d1b22:
+total 12
+drwx------ 2 martin martin 4096 feb 21 12:14 .
+drwx------ 3 martin martin 4096 feb 21 12:14 ..
+-rw------- 1 martin martin  159 feb 21 12:14 config.json
+
+#cat /home/martin/.signer/43f73718397aa54d1b22/config.json
+{"ruleset_sha256":{"iv":"6v4W4tfJxj3zZFbl","c":"6dt5RTDiTq93yh1qDEjpsat/tsKG7cb+vr3sza26IPL2fvsQ6ZoqFx++CPUa8yy6fD9Bbq41L01ehkKHTG3pOAeqTW6zc/+t0wv3AB6xPmU="}}
+
+```
+
+In `~/.signer`, the `secrets.dat` file was created, containing the `master_seed`.
+The `master_seed` was then used to derive a few other things:
+
+- `vault_location` : in this case `43f73718397aa54d1b22` .
+   - Thus, if you use a different `master_seed`, another `vault_location` will be used that does not conflict with each other.
+   - Example: `signer --signersecret /path/to/afile ...`
+- `config.json` which is the encrypted key/value storage for configuration data, containing the key `ruleset_sha256`.
+
+
+## Adding credentials
+
+In order to make more useful rules; sign transactions, the signer needs access to the passwords needed to unlock keystores.
+
+```text
+#./signer addpw 0x694267f14675d7e1b9494fd8d72fefe1755710fa test
+
+INFO [02-21|13:43:21] Credential store updated                 key=0x694267f14675d7e1b9494fd8d72fefe1755710fa
+```
+## More advanced rules
+
+Now let's update the rules to make use of credentials
+
+```javascript
+function ApproveListing(){
+    return "Approve"
+}
+function ApproveSignData(r){
+    if( r.address.toLowerCase() == "0x694267f14675d7e1b9494fd8d72fefe1755710fa")
+    {
+        if(r.message.indexOf("bazonk") >= 0){
+            return "Approve"
+        }
+        return "Reject"
+    }
+    // Otherwise goes to manual processing
+}
+
+```
+In this example,
+* any requests to sign data with the account `0x694...` will be
+    * auto-approved if the message contains with `bazonk`,
+    * and auto-rejected if it does not.
+    * Any other signing-requests will be passed along for manual approve/reject.
+
+..attest the new file
+```text
+#sha256sum rules.js
+2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f  rules.js
+
+#./signer attest 2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f
+
+INFO [02-21|14:36:30] Ruleset attestation updated              sha256=2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f
+```
+
+And start the signer:
+
+```
+#./signer --rules rules.js
+
+INFO [02-21|14:41:56] Using CLI as UI-channel
+INFO [02-21|14:41:56] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [02-21|14:41:56] Rule engine configured                   file=rules.js
+DEBUG[02-21|14:41:56] FS scan times                            list=34.607µs set=4.509µs diff=4.87µs
+DEBUG[02-21|14:41:56] Ledger support enabled
+DEBUG[02-21|14:41:56] Trezor support enabled
+INFO [02-21|14:41:56] Audit logs configured                    file=audit.log
+INFO [02-21|14:41:56] HTTP endpoint opened                     url=http://localhost:8550
+------- Signer info -------
+* extapi_version : 2.0.0
+* intapi_version : 1.2.0
+* extapi_http : http://localhost:8550
+* extapi_ipc : <nil>
+INFO [02-21|14:41:56] error occurred during execution          error="ReferenceError: 'OnSignerStartup' is not defined"
+```
+And then test signing, once with `bazonk` and once without:
+
+```
+#curl -H "Content-Type: application/json" -X POST --data "{\"jsonrpc\":\"2.0\",\"method\":\"account_sign\",\"params\":[\"0x694267f14675d7e1b9494fd8d72fefe1755710fa\",\"0x$(xxd -pu <<< '  bazonk baz gaz')\"],\"id\":67}" http://localhost:8550/
+{"jsonrpc":"2.0","id":67,"result":"0x93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c"}
+
+#curl -H "Content-Type: application/json" -X POST --data "{\"jsonrpc\":\"2.0\",\"method\":\"account_sign\",\"params\":[\"0x694267f14675d7e1b9494fd8d72fefe1755710fa\",\"0x$(xxd -pu <<< '  bonk baz gaz')\"],\"id\":67}" http://localhost:8550/
+{"jsonrpc":"2.0","id":67,"error":{"code":-32000,"message":"Request denied"}}
+
+```
+
+Meanwhile, in the signer output:
+```text
+INFO [02-21|14:42:41] Op approved
+INFO [02-21|14:42:56] Op rejected
+```
+
+The signer also stores all traffic over the external API in a log file. The last 4 lines shows the two requests and their responses:
+
+```text
+#tail audit.log -n 4
+t=2018-02-21T14:42:41+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49706\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=202062617a6f6e6b2062617a2067617a0a
+t=2018-02-21T14:42:42+0100 lvl=info msg=Sign       api=signer type=response data=93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c error=nil
+t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49708\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=2020626f6e6b2062617a2067617a0a
+t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=response data=                                                                                                                                   error="Request denied"
+```

cmd/ethkey/README.md

@@ -0,0 +1,41 @@
+ethkey
+======
+
+ethkey is a simple command-line tool for working with Ethereum keyfiles.
+
+
+# Usage
+
+### `ethkey generate`
+
+Generate a new keyfile.
+If you want to use an existing private key to use in the keyfile, it can be 
+specified by setting `--privatekey` with the location of the file containing the 
+private key.
+
+
+### `ethkey inspect <keyfile>`
+
+Print various information about the keyfile.
+Private key information can be printed by using the `--private` flag;
+make sure to use this feature with great caution!
+
+
+### `ethkey sign <keyfile> <message/file>`
+
+Sign the message with a keyfile.
+It is possible to refer to a file containing the message.
+
+
+### `ethkey verify <address> <signature> <message/file>`
+
+Verify the signature of the message.
+It is possible to refer to a file containing the message.
+
+
+## Passphrases
+
+For every command that uses a keyfile, you will be prompted to provide the 
+passphrase for decrypting the keyfile.  To avoid this message, it is possible
+to pass the passphrase by using the `--passphrase` flag pointing to a file that
+contains the passphrase.

cmd/ethkey/changepassphrase.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"gopkg.in/urfave/cli.v1"
+)
+
+var newPassphraseFlag = cli.StringFlag{
+	Name:  "newpasswordfile",
+	Usage: "the file that contains the new passphrase for the keyfile",
+}
+
+var commandChangePassphrase = cli.Command{
+	Name:      "changepassphrase",
+	Usage:     "change the passphrase on a keyfile",
+	ArgsUsage: "<keyfile>",
+	Description: `
+Change the passphrase of a keyfile.`,
+	Flags: []cli.Flag{
+		passphraseFlag,
+		newPassphraseFlag,
+	},
+	Action: func(ctx *cli.Context) error {
+		keyfilepath := ctx.Args().First()
+
+		// Read key from file.
+		keyjson, err := ioutil.ReadFile(keyfilepath)
+		if err != nil {
+			utils.Fatalf("Failed to read the keyfile at '%s': %v", keyfilepath, err)
+		}
+
+		// Decrypt key with passphrase.
+		passphrase := getPassphrase(ctx)
+		key, err := keystore.DecryptKey(keyjson, passphrase)
+		if err != nil {
+			utils.Fatalf("Error decrypting key: %v", err)
+		}
+
+		// Get a new passphrase.
+		fmt.Println("Please provide a new passphrase")
+		var newPhrase string
+		if passFile := ctx.String(newPassphraseFlag.Name); passFile != "" {
+			content, err := ioutil.ReadFile(passFile)
+			if err != nil {
+				utils.Fatalf("Failed to read new passphrase file '%s': %v", passFile, err)
+			}
+			newPhrase = strings.TrimRight(string(content), "\r\n")
+		} else {
+			newPhrase = promptPassphrase(true)
+		}
+
+		// Encrypt the key with the new passphrase.
+		newJson, err := keystore.EncryptKey(key, newPhrase, keystore.StandardScryptN, keystore.StandardScryptP)
+		if err != nil {
+			utils.Fatalf("Error encrypting with new passphrase: %v", err)
+		}
+
+		// Then write the new keyfile in place of the old one.
+		if err := ioutil.WriteFile(keyfilepath, newJson, 600); err != nil {
+			utils.Fatalf("Error writing new keyfile to disk: %v", err)
+		}
+
+		// Don't print anything.  Just return successfully,
+		// producing a positive exit code.
+		return nil
+	},
+}

cmd/ethkey/generate.go

@@ -0,0 +1,118 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"crypto/ecdsa"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/pborman/uuid"
+	"gopkg.in/urfave/cli.v1"
+)
+
+type outputGenerate struct {
+	Address      string
+	AddressEIP55 string
+}
+
+var commandGenerate = cli.Command{
+	Name:      "generate",
+	Usage:     "generate new keyfile",
+	ArgsUsage: "[ <keyfile> ]",
+	Description: `
+Generate a new keyfile.
+
+If you want to encrypt an existing private key, it can be specified by setting
+--privatekey with the location of the file containing the private key.
+`,
+	Flags: []cli.Flag{
+		passphraseFlag,
+		jsonFlag,
+		cli.StringFlag{
+			Name:  "privatekey",
+			Usage: "file containing a raw private key to encrypt",
+		},
+	},
+	Action: func(ctx *cli.Context) error {
+		// Check if keyfile path given and make sure it doesn't already exist.
+		keyfilepath := ctx.Args().First()
+		if keyfilepath == "" {
+			keyfilepath = defaultKeyfileName
+		}
+		if _, err := os.Stat(keyfilepath); err == nil {
+			utils.Fatalf("Keyfile already exists at %s.", keyfilepath)
+		} else if !os.IsNotExist(err) {
+			utils.Fatalf("Error checking if keyfile exists: %v", err)
+		}
+
+		var privateKey *ecdsa.PrivateKey
+		var err error
+		if file := ctx.String("privatekey"); file != "" {
+			// Load private key from file.
+			privateKey, err = crypto.LoadECDSA(file)
+			if err != nil {
+				utils.Fatalf("Can't load private key: %v", err)
+			}
+		} else {
+			// If not loaded, generate random.
+			privateKey, err = crypto.GenerateKey()
+			if err != nil {
+				utils.Fatalf("Failed to generate random private key: %v", err)
+			}
+		}
+
+		// Create the keyfile object with a random UUID.
+		id := uuid.NewRandom()
+		key := &keystore.Key{
+			Id:         id,
+			Address:    crypto.PubkeyToAddress(privateKey.PublicKey),
+			PrivateKey: privateKey,
+		}
+
+		// Encrypt key with passphrase.
+		passphrase := promptPassphrase(true)
+		keyjson, err := keystore.EncryptKey(key, passphrase, keystore.StandardScryptN, keystore.StandardScryptP)
+		if err != nil {
+			utils.Fatalf("Error encrypting key: %v", err)
+		}
+
+		// Store the file to disk.
+		if err := os.MkdirAll(filepath.Dir(keyfilepath), 0700); err != nil {
+			utils.Fatalf("Could not create directory %s", filepath.Dir(keyfilepath))
+		}
+		if err := ioutil.WriteFile(keyfilepath, keyjson, 0600); err != nil {
+			utils.Fatalf("Failed to write keyfile to %s: %v", keyfilepath, err)
+		}
+
+		// Output some information.
+		out := outputGenerate{
+			Address: key.Address.Hex(),
+		}
+		if ctx.Bool(jsonFlag.Name) {
+			mustPrintJSON(out)
+		} else {
+			fmt.Println("Address:", out.Address)
+		}
+		return nil
+	},
+}

cmd/ethkey/inspect.go

@@ -0,0 +1,91 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/crypto"
+	"gopkg.in/urfave/cli.v1"
+)
+
+type outputInspect struct {
+	Address    string
+	PublicKey  string
+	PrivateKey string
+}
+
+var commandInspect = cli.Command{
+	Name:      "inspect",
+	Usage:     "inspect a keyfile",
+	ArgsUsage: "<keyfile>",
+	Description: `
+Print various information about the keyfile.
+
+Private key information can be printed by using the --private flag;
+make sure to use this feature with great caution!`,
+	Flags: []cli.Flag{
+		passphraseFlag,
+		jsonFlag,
+		cli.BoolFlag{
+			Name:  "private",
+			Usage: "include the private key in the output",
+		},
+	},
+	Action: func(ctx *cli.Context) error {
+		keyfilepath := ctx.Args().First()
+
+		// Read key from file.
+		keyjson, err := ioutil.ReadFile(keyfilepath)
+		if err != nil {
+			utils.Fatalf("Failed to read the keyfile at '%s': %v", keyfilepath, err)
+		}
+
+		// Decrypt key with passphrase.
+		passphrase := getPassphrase(ctx)
+		key, err := keystore.DecryptKey(keyjson, passphrase)
+		if err != nil {
+			utils.Fatalf("Error decrypting key: %v", err)
+		}
+
+		// Output all relevant information we can retrieve.
+		showPrivate := ctx.Bool("private")
+		out := outputInspect{
+			Address: key.Address.Hex(),
+			PublicKey: hex.EncodeToString(
+				crypto.FromECDSAPub(&key.PrivateKey.PublicKey)),
+		}
+		if showPrivate {
+			out.PrivateKey = hex.EncodeToString(crypto.FromECDSA(key.PrivateKey))
+		}
+
+		if ctx.Bool(jsonFlag.Name) {
+			mustPrintJSON(out)
+		} else {
+			fmt.Println("Address:       ", out.Address)
+			fmt.Println("Public key:    ", out.PublicKey)
+			if showPrivate {
+				fmt.Println("Private key:   ", out.PrivateKey)
+			}
+		}
+		return nil
+	},
+}

cmd/ethkey/main.go

@@ -17,22 +17,48 @@
 package main
 
 import (
+	"fmt"
+	"os"
+
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/swarm/storage"
 	"gopkg.in/urfave/cli.v1"
 )
 
-func cleandb(ctx *cli.Context) {
-	args := ctx.Args()
-	if len(args) != 1 {
-		utils.Fatalf("Need path to chunks database as the first and only argument")
-	}
+const (
+	defaultKeyfileName = "keyfile.json"
+)
 
-	chunkDbPath := args[0]
-	hash := storage.MakeHashFunc("SHA3")
-	dbStore, err := storage.NewDbStore(chunkDbPath, hash, 10000000, 0)
-	if err != nil {
-		utils.Fatalf("Cannot initialise dbstore: %v", err)
+// Git SHA1 commit hash of the release (set via linker flags)
+var gitCommit = ""
+
+var app *cli.App
+
+func init() {
+	app = utils.NewApp(gitCommit, "an Ethereum key manager")
+	app.Commands = []cli.Command{
+		commandGenerate,
+		commandInspect,
+		commandChangePassphrase,
+		commandSignMessage,
+		commandVerifyMessage,
+	}
+}
+
+// Commonly used command line flags.
+var (
+	passphraseFlag = cli.StringFlag{
+		Name:  "passwordfile",
+		Usage: "the file that contains the passphrase for the keyfile",
+	}
+	jsonFlag = cli.BoolFlag{
+		Name:  "json",
+		Usage: "output JSON instead of human-readable format",
+	}
+)
+
+func main() {
+	if err := app.Run(os.Args); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
 	}
-	dbStore.Cleanup()
 }