params: 1.6.6 stable

consensus/clique: choose valid votes

.dockerignore

@@ -0,0 +1,3 @@
+.git
+build/_workspace
+build/_bin

.mailmap

@@ -69,7 +69,7 @@ RJ Catalano <rj@erisindustries.com>
 
 Nchinda Nchinda <nchinda2@gmail.com>
 
-Aron Fischer <homotopycolimit@users.noreply.github.com>
+Aron Fischer <github@aron.guru> <homotopycolimit@users.noreply.github.com>
 
 Vlad Gluhovsky <gluk256@users.noreply.github.com>
 
@@ -90,3 +90,22 @@ Nick Johnson <arachnid@notdot.net>
 Henning Diedrich <hd@eonblast.com>
 Henning Diedrich <hd@eonblast.com> Drake Burroughs <wildfyre@hotmail.com>
 
+Felix Lange <fjl@twurst.com>
+Felix Lange <fjl@twurst.com> <fjl@users.noreply.github.com>
+
+Максим Чусовлянов <mchusovlianov@gmail.com>
+
+Louis Holbrook <dev@holbrook.no>
+Louis Holbrook <dev@holbrook.no> <nolash@users.noreply.github.com>
+
+Thomas Bocek <tom@tomp2p.net>
+
+Victor Tran <vu.tran54@gmail.com>
+
+Justin Drake <drakefjustin@gmail.com>
+
+Frank Wang <eternnoir@gmail.com>
+
+Gary Rong <garyrong0905@gmail.com>
+
+Guillaume Nicolas <guin56@gmail.com>

.travis.yml

@@ -5,23 +5,44 @@ matrix:
   include:
     - os: linux
       dist: trusty
-      go: 1.5.4
-      env:
-        - GO15VENDOREXPERIMENT=1
+      sudo: required
+      go: 1.7.6
+      script:
+        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
+        - sudo modprobe fuse
+        - sudo chmod 666 /dev/fuse
+        - sudo chown root:$USER /etc/fuse.conf
+        - go run build/ci.go install
+        - go run build/ci.go test -coverage
+
+    # These are the latest Go versions.
     - os: linux
       dist: trusty
-      go: 1.6.2
-    - os: linux
-      dist: trusty
-      go: 1.7.4
+      sudo: required
+      go: 1.8.3
+      script:
+        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse
+        - sudo modprobe fuse
+        - sudo chmod 666 /dev/fuse
+        - sudo chown root:$USER /etc/fuse.conf
+        - go run build/ci.go install
+        - go run build/ci.go test -coverage -misspell
+
     - os: osx
-      go: 1.7.4
+      go: 1.8.3
+      sudo: required
+      script:
+        - brew update
+        - brew install caskroom/cask/brew-cask
+        - brew cask install osxfuse
+        - go run build/ci.go install
+        - go run build/ci.go test -coverage -misspell
 
     # This builder does the Ubuntu PPA and Linux Azure uploads
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.7.4
+      go: 1.8.3
       env:
         - ubuntu-ppa
         - azure-linux
@@ -32,6 +53,7 @@ matrix:
             - debhelper
             - dput
             - gcc-multilib
+            - fakeroot
       script:
         # Build for the primary platforms that Trusty can manage
         - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
@@ -53,29 +75,81 @@ matrix:
         - CC=aarch64-linux-gnu-gcc go run build/ci.go install -arch arm64
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
-    # This builder does the OSX Azure, Android Maven and Azure and iOS CocoaPods and Azure uploads
+    # This builder does the Linux Azure MIPS xgo uploads
+    - os: linux
+      dist: trusty
+      sudo: required
+      services:
+        - docker
+      go: 1.8.3
+      env:
+        - azure-linux-mips
+      script:
+        - go run build/ci.go xgo --alltools -- --targets=linux/mips --ldflags '-extldflags "-static"' -v
+        - for bin in build/bin/*-linux-mips; do mv -f "${bin}" "${bin/-linux-mips/}"; done
+        - go run build/ci.go archive -arch mips -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+
+        - go run build/ci.go xgo --alltools -- --targets=linux/mipsle --ldflags '-extldflags "-static"' -v
+        - for bin in build/bin/*-linux-mipsle; do mv -f "${bin}" "${bin/-linux-mipsle/}"; done
+        - go run build/ci.go archive -arch mipsle -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+
+        - go run build/ci.go xgo --alltools -- --targets=linux/mips64 --ldflags '-extldflags "-static"' -v
+        - for bin in build/bin/*-linux-mips64; do mv -f "${bin}" "${bin/-linux-mips64/}"; done
+        - go run build/ci.go archive -arch mips64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+
+        - go run build/ci.go xgo --alltools -- --targets=linux/mips64le --ldflags '-extldflags "-static"' -v
+        - for bin in build/bin/*-linux-mips64le; do mv -f "${bin}" "${bin/-linux-mips64le/}"; done
+        - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+
+    # This builder does the Android Maven and Azure uploads
+    - os: linux
+      dist: precise # Needed for the android tools
+      addons:
+        apt:
+          packages:
+            - oracle-java8-installer
+            - oracle-java8-set-default
+      language: android
+      android:
+        components:
+          - platform-tools
+          - tools
+          - android-15
+          - android-19
+          - android-24
+      env:
+        - azure-android
+        - maven-android
+      before_install:
+        - curl https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz | tar -xz
+        - export PATH=`pwd`/go/bin:$PATH
+        - export GOROOT=`pwd`/go
+        - export GOPATH=$HOME/go
+      script:
+        # Build the Android archive and upload it to Maven Central and Azure
+        - curl https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip -o android-ndk-r14b.zip
+        - unzip -q android-ndk-r14b.zip && rm android-ndk-r14b.zip
+        - mv android-ndk-r14b $HOME
+        - export ANDROID_NDK=$HOME/android-ndk-r14b
+
+        - mkdir -p $GOPATH/src/github.com/ethereum
+        - ln -s `pwd` $GOPATH/src/github.com/ethereum
+        - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
+
+    # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
     - os: osx
-      go: 1.7.4
+      go: 1.8.3
       env:
         - azure-osx
-        - mobile
+        - azure-ios
+        - cocoapods-ios
       script:
         - go run build/ci.go install
         - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -upload gethstore/builds
 
-        # Build the Android archive and upload it to Maven Central and Azure
-        - brew update
-        - brew install android-sdk maven gpg
-        - alias gpg="gpg2"
-
-        - export ANDROID_HOME=/usr/local/opt/android-sdk
-        - echo "y" | android update sdk --no-ui --filter `android list sdk | grep "SDK Platform Android" | grep -E 'API 15|API 19|API 24' | awk '{print $1}' | cut -d '-' -f 1 | tr '\n' ','`
-
-        - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
-
         # Build the iOS framework and upload it to CocoaPods and Azure
-        - gem uninstall cocoapods -a
-        - gem install cocoapods --pre
+        - gem uninstall cocoapods -a -x
+        - gem install cocoapods
 
         - mv ~/.cocoapods/repos/master ~/.cocoapods/repos/master.bak
         - sed -i '.bak' 's/repo.join/!repo.join/g' $(dirname `gem which cocoapods`)/cocoapods/sources_manager.rb
@@ -86,11 +160,21 @@ matrix:
 
         - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
 
+    # This builder does the Azure archive purges to avoid accumulating junk
+    - os: linux
+      dist: trusty
+      sudo: required
+      go: 1.8.3
+      env:
+        - azure-purge
+      script:
+        - go run build/ci.go purge -store gethstore/builds -days 14
+
 install:
   - go get golang.org/x/tools/cmd/cover
 script:
   - go run build/ci.go install
-  - go run build/ci.go test -coverage -vet -misspell
+  - go run build/ci.go test -coverage
 
 notifications:
   webhooks:

AUTHORS

@@ -3,24 +3,30 @@
 Ales Katona <ales@coinbase.com>
 Alex Leverington <alex@ethdev.com>
 Alexandre Van de Sande <alex.vandesande@ethdev.com>
-Aron Fischer <homotopycolimit@users.noreply.github.com>
+Aron Fischer <github@aron.guru>
 Bas van Kervel <bas@ethdev.com>
 Benjamin Brent <benjamin@benjaminbrent.com>
+Brian Schroeder <bts@gmail.com>
 Casey Detrio <cdetrio@gmail.com>
 Christoph Jentzsch <jentzsch.software@gmail.com>
 Daniel A. Nagy <nagy.da@gmail.com>
+Diego Siqueira <DiSiqueira@users.noreply.github.com>
 Elliot Shepherd <elliot@identitii.com>
 Enrique Fynn <enriquefynn@gmail.com>
 Ethan Buchman <ethan@coinculture.info>
 Fabian Vogelsteller <fabian@frozeman.de>
 Fabio Berger <fabioberger1991@gmail.com>
 Felix Lange <fjl@twurst.com>
+Frank Wang <eternnoir@gmail.com>
+Gary Rong <garyrong0905@gmail.com>
 Gregg Dourgarian <greggd@tempworks.com>
+Guillaume Nicolas <guin56@gmail.com>
 Gustav Simonsson <gustav.simonsson@gmail.com>
 Hao Bryan Cheng <haobcheng@gmail.com>
 Henning Diedrich <hd@eonblast.com>
 Isidoro Ghezzi <isidoro.ghezzi@icloud.com>
 Jae Kwon <jkwon.work@gmail.com>
+Jamie Pitts <james.pitts@gmail.com>
 Jason Carver <jacarver@linkedin.com>
 Jeff R. Allen <jra@nella.org>
 Jeffrey Wilcke <jeffrey@ethereum.org>
@@ -28,15 +34,20 @@ Jens Agerberg <github@agerberg.me>
 Jonathan Brown <jbrown@bluedroplet.com>
 Joseph Chow <ethereum@outlook.com>
 Justin Clark-Casey <justincc@justincc.org>
+Justin Drake <drakefjustin@gmail.com>
 Kenji Siu <kenji@isuntv.com>
 Kobi Gurkan <kobigurk@gmail.com>
 Lefteris Karapetsas <lefteris@refu.co>
 Leif Jurvetson <leijurv@gmail.com>
+Lewis Marshall <lewis@lmars.net>
+Louis Holbrook <dev@holbrook.no>
+Luca Zeug <luclu@users.noreply.github.com>
 Maran Hidskes <maran.hidskes@gmail.com>
 Marek Kotewicz <marek.kotewicz@gmail.com>
 Martin Holst Swende <martin@swende.se>
 Matthew Di Ferrante <mattdf@users.noreply.github.com>
 Matthew Wampler-Doty <matthew.wampler.doty@gmail.com>
+Micah Zoltu <micah@zoltu.net>
 Nchinda Nchinda <nchinda2@gmail.com>
 Nick Dodson <silentcicero@outlook.com>
 Nick Johnson <arachnid@notdot.net>
@@ -47,17 +58,28 @@ RJ Catalano <rj@erisindustries.com>
 Ramesh Nair <ram@hiddentao.com>
 Ricardo Catalinas Jiménez <r@untroubled.be>
 Rémy Roy <remyroy@remyroy.com>
+Shintaro Kaneko <kaneshin0120@gmail.com>
 Stein Dekker <dekker.stein@gmail.com>
 Steven Roose <stevenroose@gmail.com>
 Taylor Gerring <taylor.gerring@gmail.com>
 Thomas Bocek <tom@tomp2p.net>
 Tosh Camille <tochecamille@gmail.com>
+Valentin Wüstholz <wuestholz@users.noreply.github.com>
+Victor Farazdagi <simple.square@gmail.com>
+Victor Tran <vu.tran54@gmail.com>
 Viktor Trón <viktor.tron@gmail.com>
 Ville Sundell <github@solarius.fi>
 Vincent G <caktux@gmail.com>
 Vitalik Buterin <v@buterin.com>
+Vivek Anand <vivekanand1101@users.noreply.github.com>
 Vlad Gluhovsky <gluk256@users.noreply.github.com>
 Yohann Léon <sybiload@gmail.com>
 Yoichi Hirai <i@yoichihirai.com>
+Zahoor Mohamed <zahoor@zahoor.in>
 Zsolt Felföldi <zsfelfoldi@gmail.com>
+holisticode <holistic.computing@gmail.com>
+ken10100147 <sunhongping@kanjian.com>
+ligi <ligi@ligi.de>
+xiekeyang <xiekeyang@users.noreply.github.com>
 ΞTHΞЯSPHΞЯΞ <{viktor.tron,nagydani,zsfelfoldi}@gmail.com>
+Максим Чусовлянов <mchusovlianov@gmail.com>

Dockerfile

@@ -1,14 +1,15 @@
-FROM alpine:3.3
+FROM alpine:3.5
 
 ADD . /go-ethereum
 RUN \
-  apk add --update git go make gcc musl-dev         && \
+  apk add --update git go make gcc musl-dev linux-headers && \
   (cd go-ethereum && make geth)                           && \
-  cp go-ethereum/build/bin/geth /geth               && \
-  apk del git go make gcc musl-dev                  && \
+  cp go-ethereum/build/bin/geth /usr/local/bin/           && \
+  apk del git go make gcc musl-dev linux-headers          && \
   rm -rf /go-ethereum && rm -rf /var/cache/apk/*
 
 EXPOSE 8545
 EXPOSE 30303
+EXPOSE 30303/udp
 
-ENTRYPOINT ["/geth"]
+ENTRYPOINT ["geth"]

Makefile

@@ -40,6 +40,15 @@ test: all
 clean:
 	rm -fr build/_workspace/pkg/ $(GOBIN)/*
 
+# The devtools target installs tools required for 'go generate'.
+# You need to put $GOBIN (or $GOPATH/bin) in your PATH to use 'go generate'.
+
+devtools:
+	env GOBIN= go get -u golang.org/x/tools/cmd/stringer
+	env GOBIN= go get -u github.com/jteeuwen/go-bindata/go-bindata
+	env GOBIN= go get -u github.com/fjl/gencodec
+	env GOBIN= go install ./cmd/abigen
+
 # Cross Compilation Targets (xgo)
 
 geth-cross: geth-linux geth-darwin geth-windows geth-android geth-ios
@@ -51,12 +60,12 @@ geth-linux: geth-linux-386 geth-linux-amd64 geth-linux-arm geth-linux-mips64 get
 	@ls -ld $(GOBIN)/geth-linux-*
 
 geth-linux-386:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/386 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/386 -v ./cmd/geth
 	@echo "Linux 386 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep 386
 
 geth-linux-amd64:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/amd64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/amd64 -v ./cmd/geth
 	@echo "Linux amd64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep amd64
 
@@ -65,32 +74,42 @@ geth-linux-arm: geth-linux-arm-5 geth-linux-arm-6 geth-linux-arm-7 geth-linux-ar
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm
 
 geth-linux-arm-5:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-5 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm-5 -v ./cmd/geth
 	@echo "Linux ARMv5 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm-5
 
 geth-linux-arm-6:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-6 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm-6 -v ./cmd/geth
 	@echo "Linux ARMv6 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm-6
 
 geth-linux-arm-7:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm-7 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm-7 -v ./cmd/geth
 	@echo "Linux ARMv7 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm-7
 
 geth-linux-arm64:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/arm64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm64 -v ./cmd/geth
 	@echo "Linux ARM64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep arm64
 
+geth-linux-mips:
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/mips --ldflags '-extldflags "-static"' -v ./cmd/geth
+	@echo "Linux MIPS cross compilation done:"
+	@ls -ld $(GOBIN)/geth-linux-* | grep mips
+
+geth-linux-mipsle:
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/mipsle --ldflags '-extldflags "-static"' -v ./cmd/geth
+	@echo "Linux MIPSle cross compilation done:"
+	@ls -ld $(GOBIN)/geth-linux-* | grep mipsle
+
 geth-linux-mips64:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/mips64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/mips64 --ldflags '-extldflags "-static"' -v ./cmd/geth
 	@echo "Linux MIPS64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep mips64
 
 geth-linux-mips64le:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=linux/mips64le -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/mips64le --ldflags '-extldflags "-static"' -v ./cmd/geth
 	@echo "Linux MIPS64le cross compilation done:"
 	@ls -ld $(GOBIN)/geth-linux-* | grep mips64le
 
@@ -99,12 +118,12 @@ geth-darwin: geth-darwin-386 geth-darwin-amd64
 	@ls -ld $(GOBIN)/geth-darwin-*
 
 geth-darwin-386:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=darwin/386 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=darwin/386 -v ./cmd/geth
 	@echo "Darwin 386 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-darwin-* | grep 386
 
 geth-darwin-amd64:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=darwin/amd64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=darwin/amd64 -v ./cmd/geth
 	@echo "Darwin amd64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-darwin-* | grep amd64
 
@@ -113,11 +132,11 @@ geth-windows: geth-windows-386 geth-windows-amd64
 	@ls -ld $(GOBIN)/geth-windows-*
 
 geth-windows-386:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=windows/386 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=windows/386 -v ./cmd/geth
 	@echo "Windows 386 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-windows-* | grep 386
 
 geth-windows-amd64:
-	build/env.sh go run build/ci.go xgo -- --go=$(GO) --dest=$(GOBIN) --targets=windows/amd64 -v ./cmd/geth
+	build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=windows/amd64 -v ./cmd/geth
 	@echo "Windows amd64 cross compilation done:"
 	@ls -ld $(GOBIN)/geth-windows-* | grep amd64

README.md

@@ -16,7 +16,7 @@ For prerequisites and detailed build instructions please read the
 [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum)
 on the wiki.
 
-Building geth requires both a Go and a C compiler.
+Building geth requires both a Go (version 1.7 or later) and a C compiler.
 You can install them using your favourite package manager.
 Once the dependencies are installed, run
 
@@ -70,7 +70,7 @@ This command will:
    (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API)
    as well as Geth's own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs).
    This too is optional and if you leave it out you can always attach to an already running Geth instance
-   with `geth --attach`.
+   with `geth attach`.
 
 ### Full node on the Ethereum test network
 
@@ -84,21 +84,39 @@ $ geth --testnet --fast --cache=512 console
 ```
 
 The `--fast`, `--cache` flags and `console` subcommand have the exact same meaning as above and they
-are equially useful on the testnet too. Please see above for their explanations if you've skipped to
+are equally useful on the testnet too. Please see above for their explanations if you've skipped to
 here.
 
 Specifying the `--testnet` flag however will reconfigure your Geth instance a bit:
 
  * Instead of using the default data directory (`~/.ethereum` on Linux for example), Geth will nest
-   itself one level deeper into a `testnet` subfolder (`~/.ethereum/testnet` on Linux).
+   itself one level deeper into a `testnet` subfolder (`~/.ethereum/testnet` on Linux). Note, on OSX
+   and Linux this also means that attaching to a running testnet node requires the use of a custom
+   endpoint since `geth attach` will try to attach to a production node endpoint by default. E.g.
+   `geth attach <datadir>/testnet/geth.ipc`. Windows users are not affected by this.
  * Instead of connecting the main Ethereum network, the client will connect to the test network,
    which uses different P2P bootnodes, different network IDs and genesis states.
    
 *Note: Although there are some internal protective measures to prevent transactions from crossing
-over between the main network and test network (different starting nonces), you should make sure to
-always use separate accounts for play-money and real-money. Unless you manually move accounts, Geth
-will by default correctly separate the two networks and will not make any accounts available between
-them.*
+over between the main network and test network, you should make sure to always use separate accounts
+for play-money and real-money. Unless you manually move accounts, Geth will by default correctly
+separate the two networks and will not make any accounts available between them.*
+
+### Configuration
+
+As an alternative to passing the numerous flags to the `geth` binary, you can also pass a configuration file via:
+
+```
+$ geth --config /path/to/your_config.toml
+```
+
+To get an idea how the file should look like you can use the `dumpconfig` subcommand to export your existing configuration:
+
+```
+$ geth --your-favourite-flags dumpconfig
+```
+
+*Note: This works only with geth v1.6.0 and above*
 
 #### Docker quick start
 
@@ -118,7 +136,7 @@ As a developer, sooner rather than later you'll want to start interacting with G
 network via your own programs and not manually through the console. To aid this, Geth has built in
 support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC) and
 [Geth specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)). These can be
-exposed via HTTP, WebSockets and IPC (unix sockets on unix based platroms, and named pipes on Windows).
+exposed via HTTP, WebSockets and IPC (unix sockets on unix based platforms, and named pipes on Windows).
 
 The IPC interface is enabled by default and exposes all the APIs supported by Geth, whereas the HTTP
 and WS interfaces need to manually be enabled and only expose a subset of APIs due to security reasons.
@@ -161,6 +179,12 @@ and agree upon. This consists of a small JSON file (e.g. call it `genesis.json`)
 
 ```json
 {
+  "config": {
+        "chainId": 0,
+        "homesteadBlock": 0,
+        "eip155Block": 0,
+        "eip158Block": 0
+    },
   "alloc"      : {},
   "coinbase"   : "0x0000000000000000000000000000000000000000",
   "difficulty" : "0x20000",

VERSION

@@ -1 +1 @@
-1.5.7
+1.6.6

accounts/abi/abi.go

@@ -17,6 +17,7 @@
 package abi
 
 import (
+	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -129,16 +130,15 @@ func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
 	var size int
 	var offset int
 	if t.Type.IsSlice {
-
 		// get the offset which determines the start of this array ...
-		offset = int(common.BytesToBig(output[index : index+32]).Uint64())
+		offset = int(binary.BigEndian.Uint64(output[index+24 : index+32]))
 		if offset+32 > len(output) {
 			return nil, fmt.Errorf("abi: cannot marshal in to go slice: offset %d would go over slice boundary (len=%d)", len(output), offset+32)
 		}
 
 		slice = output[offset:]
 		// ... starting with the size of the array in elements ...
-		size = int(common.BytesToBig(slice[:32]).Uint64())
+		size = int(binary.BigEndian.Uint64(slice[24:32]))
 		slice = slice[32:]
 		// ... and make sure that we've at the very least the amount of bytes
 		// available in the buffer.
@@ -147,7 +147,7 @@ func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
 		}
 
 		// reslice to match the required size
-		slice = slice[:(size * 32)]
+		slice = slice[:size*32]
 	} else if t.Type.IsArray {
 		//get the number of elements in the array
 		size = t.Type.SliceSize
@@ -165,33 +165,12 @@ func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
 			inter        interface{}             // interface type
 			returnOutput = slice[i*32 : i*32+32] // the return output
 		)
-
 		// set inter to the correct type (cast)
 		switch elem.T {
 		case IntTy, UintTy:
-			bigNum := common.BytesToBig(returnOutput)
-			switch t.Type.Kind {
-			case reflect.Uint8:
-				inter = uint8(bigNum.Uint64())
-			case reflect.Uint16:
-				inter = uint16(bigNum.Uint64())
-			case reflect.Uint32:
-				inter = uint32(bigNum.Uint64())
-			case reflect.Uint64:
-				inter = bigNum.Uint64()
-			case reflect.Int8:
-				inter = int8(bigNum.Int64())
-			case reflect.Int16:
-				inter = int16(bigNum.Int64())
-			case reflect.Int32:
-				inter = int32(bigNum.Int64())
-			case reflect.Int64:
-				inter = bigNum.Int64()
-			default:
-				inter = common.BytesToBig(returnOutput)
-			}
+			inter = readInteger(t.Type.Kind, returnOutput)
 		case BoolTy:
-			inter = common.BytesToBig(returnOutput).Uint64() > 0
+			inter = !allZero(returnOutput)
 		case AddressTy:
 			inter = common.BytesToAddress(returnOutput)
 		case HashTy:
@@ -207,6 +186,38 @@ func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
 	return refSlice.Interface(), nil
 }
 
+func readInteger(kind reflect.Kind, b []byte) interface{} {
+	switch kind {
+	case reflect.Uint8:
+		return uint8(b[len(b)-1])
+	case reflect.Uint16:
+		return binary.BigEndian.Uint16(b[len(b)-2:])
+	case reflect.Uint32:
+		return binary.BigEndian.Uint32(b[len(b)-4:])
+	case reflect.Uint64:
+		return binary.BigEndian.Uint64(b[len(b)-8:])
+	case reflect.Int8:
+		return int8(b[len(b)-1])
+	case reflect.Int16:
+		return int16(binary.BigEndian.Uint16(b[len(b)-2:]))
+	case reflect.Int32:
+		return int32(binary.BigEndian.Uint32(b[len(b)-4:]))
+	case reflect.Int64:
+		return int64(binary.BigEndian.Uint64(b[len(b)-8:]))
+	default:
+		return new(big.Int).SetBytes(b)
+	}
+}
+
+func allZero(b []byte) bool {
+	for _, byte := range b {
+		if byte != 0 {
+			return false
+		}
+	}
+	return true
+}
+
 // toGoType parses the input and casts it to the proper type defined by the ABI
 // argument in T.
 func toGoType(i int, t Argument, output []byte) (interface{}, error) {
@@ -226,12 +237,12 @@ func toGoType(i int, t Argument, output []byte) (interface{}, error) {
 	switch t.Type.T {
 	case StringTy, BytesTy: // variable arrays are written at the end of the return bytes
 		// parse offset from which we should start reading
-		offset := int(common.BytesToBig(output[index : index+32]).Uint64())
+		offset := int(binary.BigEndian.Uint64(output[index+24 : index+32]))
 		if offset+32 > len(output) {
 			return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32)
 		}
 		// parse the size up until we should be reading
-		size := int(common.BytesToBig(output[offset : offset+32]).Uint64())
+		size := int(binary.BigEndian.Uint64(output[offset+24 : offset+32]))
 		if offset+32+size > len(output) {
 			return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32+size)
 		}
@@ -245,32 +256,9 @@ func toGoType(i int, t Argument, output []byte) (interface{}, error) {
 	// convert the bytes to whatever is specified by the ABI.
 	switch t.Type.T {
 	case IntTy, UintTy:
-		bigNum := common.BytesToBig(returnOutput)
-
-		// If the type is a integer convert to the integer type
-		// specified by the ABI.
-		switch t.Type.Kind {
-		case reflect.Uint8:
-			return uint8(bigNum.Uint64()), nil
-		case reflect.Uint16:
-			return uint16(bigNum.Uint64()), nil
-		case reflect.Uint32:
-			return uint32(bigNum.Uint64()), nil
-		case reflect.Uint64:
-			return uint64(bigNum.Uint64()), nil
-		case reflect.Int8:
-			return int8(bigNum.Int64()), nil
-		case reflect.Int16:
-			return int16(bigNum.Int64()), nil
-		case reflect.Int32:
-			return int32(bigNum.Int64()), nil
-		case reflect.Int64:
-			return int64(bigNum.Int64()), nil
-		case reflect.Ptr:
-			return bigNum, nil
-		}
+		return readInteger(t.Type.Kind, returnOutput), nil
 	case BoolTy:
-		return common.BytesToBig(returnOutput).Uint64() > 0, nil
+		return !allZero(returnOutput), nil
 	case AddressTy:
 		return common.BytesToAddress(returnOutput), nil
 	case HashTy:

accounts/abi/bind/auth.go

@@ -22,7 +22,7 @@ import (
 	"io"
 	"io/ioutil"
 
-	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -35,7 +35,7 @@ func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) {
 	if err != nil {
 		return nil, err
 	}
-	key, err := accounts.DecryptKey(json, passphrase)
+	key, err := keystore.DecryptKey(json, passphrase)
 	if err != nil {
 		return nil, err
 	}

accounts/abi/bind/backend.go

@@ -17,13 +17,13 @@
 package bind
 
 import (
+	"context"
 	"errors"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"golang.org/x/net/context"
 )
 
 var (

accounts/abi/bind/backends/simulated.go

@@ -17,6 +17,7 @@
 package backends
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"math/big"
@@ -25,6 +26,8 @@ import (
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -32,12 +35,8 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/params"
-	"golang.org/x/net/context"
 )
 
-// Default chain configuration which sets homestead phase at block 0 (i.e. no frontier)
-var chainConfig = &params.ChainConfig{HomesteadBlock: big.NewInt(0), EIP150Block: new(big.Int), EIP158Block: new(big.Int)}
-
 // This nil assignment ensures compile time that SimulatedBackend implements bind.ContractBackend.
 var _ bind.ContractBackend = (*SimulatedBackend)(nil)
 
@@ -58,11 +57,12 @@ type SimulatedBackend struct {
 
 // NewSimulatedBackend creates a new binding backend using a simulated blockchain
 // for testing purposes.
-func NewSimulatedBackend(accounts ...core.GenesisAccount) *SimulatedBackend {
+func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend {
 	database, _ := ethdb.NewMemDatabase()
-	core.WriteGenesisBlockForTesting(database, accounts...)
-	blockchain, _ := core.NewBlockChain(database, chainConfig, new(core.FakePow), new(event.TypeMux))
-	backend := &SimulatedBackend{database: database, blockchain: blockchain}
+	genesis := core.Genesis{Config: params.AllProtocolChanges, Alloc: alloc}
+	genesis.MustCommit(database)
+	blockchain, _ := core.NewBlockChain(database, genesis.Config, ethash.NewFaker(), new(event.TypeMux), vm.Config{})
+	backend := &SimulatedBackend{database: database, blockchain: blockchain, config: genesis.Config}
 	backend.rollback()
 	return backend
 }
@@ -88,7 +88,7 @@ func (b *SimulatedBackend) Rollback() {
 }
 
 func (b *SimulatedBackend) rollback() {
-	blocks, _ := core.GenerateChain(chainConfig, b.blockchain.CurrentBlock(), b.database, 1, func(int, *core.BlockGen) {})
+	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), b.database, 1, func(int, *core.BlockGen) {})
 	b.pendingBlock = blocks[0]
 	b.pendingState, _ = state.New(b.pendingBlock.Root(), b.database)
 }
@@ -201,10 +201,32 @@ func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error
 func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMsg) (*big.Int, error) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
-	defer b.pendingState.RevertToSnapshot(b.pendingState.Snapshot())
 
+	// Binary search the gas requirement, as it may be higher than the amount used
+	var lo, hi uint64
+	if call.Gas != nil {
+		hi = call.Gas.Uint64()
+	} else {
+		hi = b.pendingBlock.GasLimit().Uint64()
+	}
+	for lo+1 < hi {
+		// Take a guess at the gas, and check transaction validity
+		mid := (hi + lo) / 2
+		call.Gas = new(big.Int).SetUint64(mid)
+
+		snapshot := b.pendingState.Snapshot()
 		_, gas, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
-	return gas, err
+		b.pendingState.RevertToSnapshot(snapshot)
+
+		// If the transaction became invalid or used all the gas (failed), raise the gas limit
+		if err != nil || gas.Cmp(call.Gas) == 0 {
+			lo = mid
+			continue
+		}
+		// Otherwise assume the transaction succeeded, lower the gas limit
+		hi = mid
+	}
+	return new(big.Int).SetUint64(hi), nil
 }
 
 // callContract implemens common code between normal and pending contract calls.
@@ -214,7 +236,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 	if call.GasPrice == nil {
 		call.GasPrice = big.NewInt(1)
 	}
-	if call.Gas == nil || call.Gas.BitLen() == 0 {
+	if call.Gas == nil || call.Gas.Sign() == 0 {
 		call.Gas = big.NewInt(50000000)
 	}
 	if call.Value == nil {
@@ -222,15 +244,15 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 	}
 	// Set infinite balance to the fake caller account.
 	from := statedb.GetOrNewStateObject(call.From)
-	from.SetBalance(common.MaxBig)
+	from.SetBalance(math.MaxBig256)
 	// Execute the call.
 	msg := callmsg{call}
 
-	evmContext := core.NewEVMContext(msg, block.Header(), b.blockchain)
+	evmContext := core.NewEVMContext(msg, block.Header(), b.blockchain, nil)
 	// Create a new environment which holds all relevant information
 	// about the transaction and calling mechanisms.
-	vmenv := vm.NewEVM(evmContext, statedb, chainConfig, vm.Config{})
-	gaspool := new(core.GasPool).AddGas(common.MaxBig)
+	vmenv := vm.NewEVM(evmContext, statedb, b.config, vm.Config{})
+	gaspool := new(core.GasPool).AddGas(math.MaxBig256)
 	ret, gasUsed, _, err := core.NewStateTransition(vmenv, msg, gaspool).TransitionDb()
 	return ret, gasUsed, err
 }
@@ -250,7 +272,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 		panic(fmt.Errorf("invalid transaction nonce: got %d, want %d", tx.Nonce(), nonce))
 	}
 
-	blocks, _ := core.GenerateChain(chainConfig, b.blockchain.CurrentBlock(), b.database, 1, func(number int, block *core.BlockGen) {
+	blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), b.database, 1, func(number int, block *core.BlockGen) {
 		for _, tx := range b.pendingBlock.Transactions() {
 			block.AddTx(tx)
 		}

accounts/abi/bind/base.go

@@ -17,6 +17,7 @@
 package bind
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"math/big"
@@ -26,7 +27,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
-	"golang.org/x/net/context"
 )
 
 // SignerFn is a signer function callback when a contract requires a method to
@@ -36,6 +36,7 @@ type SignerFn func(types.Signer, common.Address, *types.Transaction) (*types.Tra
 // CallOpts is the collection of options to fine tune a contract call request.
 type CallOpts struct {
 	Pending bool           // Whether to operate on the pending state or the last known one
+	From    common.Address // Optional the sender address, otherwise the first account is used
 
 	Context context.Context // Network context to support cancellation and timeouts (nil = no timeout)
 }
@@ -108,7 +109,7 @@ func (c *BoundContract) Call(opts *CallOpts, result interface{}, method string,
 		return err
 	}
 	var (
-		msg    = ethereum.CallMsg{To: &c.address, Data: input}
+		msg    = ethereum.CallMsg{From: opts.From, To: &c.address, Data: input}
 		ctx    = ensureContext(opts.Context)
 		code   []byte
 		output []byte

accounts/abi/bind/bind_test.go

@@ -169,7 +169,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
 
 			// Deploy an interaction tester contract and call a transaction on it
 			_, _, interactor, err := DeployInteractor(auth, sim, "Deploy string")
@@ -210,7 +210,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
 
 			// Deploy a tuple tester contract and execute a structured call on it
 			_, _, getter, err := DeployGetter(auth, sim)
@@ -242,7 +242,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
 
 			// Deploy a tuple tester contract and execute a structured call on it
 			_, _, tupler, err := DeployTupler(auth, sim)
@@ -284,7 +284,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
 
 			// Deploy a slice tester contract and execute a n array call on it
 			_, _, slicer, err := DeploySlicer(auth, sim)
@@ -318,7 +318,7 @@ var bindTests = []struct {
 			// Generate a new random account and a funded simulator
 			key, _ := crypto.GenerateKey()
 			auth := bind.NewKeyedTransactor(key)
-			sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)})
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
 
 			// Deploy a default method invoker contract and execute its default method
 			_, _, defaulter, err := DeployDefaulter(auth, sim)
@@ -351,7 +351,7 @@ var bindTests = []struct {
 		`[{"constant":true,"inputs":[],"name":"String","outputs":[{"name":"","type":"string"}],"type":"function"}]`,
 		`
 			// Create a simulator and wrap a non-deployed contract
-			sim := backends.NewSimulatedBackend()
+			sim := backends.NewSimulatedBackend(nil)
 
 			nonexistent, err := NewNonExistent(common.Address{}, sim)
 			if err != nil {
@@ -365,6 +365,88 @@ var bindTests = []struct {
 			}
 		`,
 	},
+	// Tests that gas estimation works for contracts with	weird gas mechanics too.
+	{
+		`FunkyGasPattern`,
+		`
+			contract FunkyGasPattern {
+				string public field;
+
+				function SetField(string value) {
+					// This check will screw gas estimation! Good, good!
+					if (msg.gas < 100000) {
+						throw;
+					}
+					field = value;
+				}
+			}
+		`,
+		`606060405261021c806100126000396000f3606060405260e060020a600035046323fcf32a81146100265780634f28bf0e1461007b575b005b6040805160206004803580820135601f8101849004840285018401909552848452610024949193602493909291840191908190840183828082843750949650505050505050620186a05a101561014e57610002565b6100db60008054604080516020601f600260001961010060018816150201909516949094049384018190048102820181019092528281529291908301828280156102145780601f106101e957610100808354040283529160200191610214565b60405180806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600302600f01f150905090810190601f16801561013b5780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b505050565b8060006000509080519060200190828054600181600116156101000203166002900490600052602060002090601f016020900481019282601f106101b557805160ff19168380011785555b506101499291505b808211156101e557600081556001016101a1565b82800160010185558215610199579182015b828111156101995782518260005055916020019190600101906101c7565b5090565b820191906000526020600020905b8154815290600101906020018083116101f757829003601f168201915b50505050508156`,
+		`[{"constant":false,"inputs":[{"name":"value","type":"string"}],"name":"SetField","outputs":[],"type":"function"},{"constant":true,"inputs":[],"name":"field","outputs":[{"name":"","type":"string"}],"type":"function"}]`,
+		`
+			// Generate a new random account and a funded simulator
+			key, _ := crypto.GenerateKey()
+			auth := bind.NewKeyedTransactor(key)
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+
+			// Deploy a funky gas pattern contract
+			_, _, limiter, err := DeployFunkyGasPattern(auth, sim)
+			if err != nil {
+				t.Fatalf("Failed to deploy funky contract: %v", err)
+			}
+			sim.Commit()
+
+			// Set the field with automatic estimation and check that it succeeds
+			auth.GasLimit = nil
+			if _, err := limiter.SetField(auth, "automatic"); err != nil {
+				t.Fatalf("Failed to call automatically gased transaction: %v", err)
+			}
+			sim.Commit()
+
+			if field, _ := limiter.Field(nil); field != "automatic" {
+				t.Fatalf("Field mismatch: have %v, want %v", field, "automatic")
+			}
+		`,
+	},
+	// Test that constant functions can be called from an (optional) specified address
+	{
+		`CallFrom`,
+		`
+			contract CallFrom {
+				function callFrom() constant returns(address) {
+					return msg.sender;
+				}
+			}
+		`, `6060604052346000575b6086806100176000396000f300606060405263ffffffff60e060020a60003504166349f8e98281146022575b6000565b34600057602c6055565b6040805173ffffffffffffffffffffffffffffffffffffffff9092168252519081900360200190f35b335b905600a165627a7a72305820aef6b7685c0fa24ba6027e4870404a57df701473fe4107741805c19f5138417c0029`,
+		`[{"constant":true,"inputs":[],"name":"callFrom","outputs":[{"name":"","type":"address"}],"payable":false,"type":"function"}]`,
+		`
+			// Generate a new random account and a funded simulator
+			key, _ := crypto.GenerateKey()
+			auth := bind.NewKeyedTransactor(key)
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+
+			// Deploy a sender tester contract and execute a structured call on it
+			_, _, callfrom, err := DeployCallFrom(auth, sim)
+			if err != nil {
+				t.Fatalf("Failed to deploy sender contract: %v", err)
+			}
+			sim.Commit()
+
+			if res, err := callfrom.CallFrom(nil); err != nil {
+				t.Errorf("Failed to call constant function: %v", err)
+			} else if res != (common.Address{}) {
+				t.Errorf("Invalid address returned, want: %x, got: %x", (common.Address{}), res)
+			}
+
+			for _, addr := range []common.Address{common.Address{}, common.Address{1}, common.Address{2}} {
+				if res, err := callfrom.CallFrom(&bind.CallOpts{From: addr}); err != nil {
+					t.Fatalf("Failed to call constant function: %v", err)
+				} else if res != addr {
+					t.Fatalf("Invalid address returned, want: %x, got: %x", addr, res)
+				}
+			}
+		`,
+	},
 }
 
 // Tests that packages generated by the binder can be successfully compiled and
@@ -376,7 +458,7 @@ func TestBindings(t *testing.T) {
 		t.Skip("go sdk not found for testing")
 	}
 	// Skip the test if the go-ethereum sources are symlinked (https://github.com/golang/go/issues/14845)
-	linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend())\n}")
+	linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil))\n}")
 	linkTestDeps, err := imports.Process("", []byte(linkTestCode), nil)
 	if err != nil {
 		t.Fatalf("failed check for goimports symlink bug: %v", err)

accounts/abi/bind/util.go

@@ -17,31 +17,31 @@
 package bind
 
 import (
+	"context"
 	"fmt"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
-	"golang.org/x/net/context"
+	"github.com/ethereum/go-ethereum/log"
 )
 
 // WaitMined waits for tx to be mined on the blockchain.
 // It stops waiting when the context is canceled.
 func WaitMined(ctx context.Context, b DeployBackend, tx *types.Transaction) (*types.Receipt, error) {
-	queryTicker := time.NewTicker(1 * time.Second)
+	queryTicker := time.NewTicker(time.Second)
 	defer queryTicker.Stop()
-	loghash := tx.Hash().Hex()[:8]
+
+	logger := log.New("hash", tx.Hash())
 	for {
 		receipt, err := b.TransactionReceipt(ctx, tx.Hash())
 		if receipt != nil {
 			return receipt, nil
 		}
 		if err != nil {
-			glog.V(logger.Detail).Infof("tx %x error: %v", loghash, err)
+			logger.Trace("Receipt retrieval failed", "err", err)
 		} else {
-			glog.V(logger.Detail).Infof("tx %x not yet mined...", loghash)
+			logger.Trace("Transaction not yet mined")
 		}
 		// Wait for the next round.
 		select {

accounts/abi/bind/util_test.go

@@ -17,6 +17,7 @@
 package bind_test
 
 import (
+	"context"
 	"math/big"
 	"testing"
 	"time"
@@ -27,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
-	"golang.org/x/net/context"
 )
 
 var testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
@@ -53,9 +53,8 @@ var waitDeployedTests = map[string]struct {
 
 func TestWaitDeployed(t *testing.T) {
 	for name, test := range waitDeployedTests {
-		backend := backends.NewSimulatedBackend(core.GenesisAccount{
-			Address: crypto.PubkeyToAddress(testKey.PublicKey),
-			Balance: big.NewInt(10000000000),
+		backend := backends.NewSimulatedBackend(core.GenesisAlloc{
+			crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)},
 		})
 
 		// Create the transaction.

accounts/abi/numbers.go

@@ -21,6 +21,7 @@ import (
 	"reflect"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/math"
 )
 
 var (
@@ -58,7 +59,7 @@ var (
 
 // U256 converts a big Int into a 256bit EVM number.
 func U256(n *big.Int) []byte {
-	return common.LeftPadBytes(common.U256(n).Bytes(), 32)
+	return math.PaddedBigBytes(math.U256(n), 32)
 }
 
 // packNum packs the given number (using the reflect value) and will cast it to appropriate number representation

accounts/abi/packing.go

@@ -20,6 +20,7 @@ import (
 	"reflect"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/math"
 )
 
 // packBytesSlice packs the given bytes as [L, V] as the canonical representation
@@ -45,9 +46,9 @@ func packElement(t Type, reflectValue reflect.Value) []byte {
 		return common.LeftPadBytes(reflectValue.Bytes(), 32)
 	case BoolTy:
 		if reflectValue.Bool() {
-			return common.LeftPadBytes(common.Big1.Bytes(), 32)
+			return math.PaddedBigBytes(common.Big1, 32)
 		} else {
-			return common.LeftPadBytes(common.Big0.Bytes(), 32)
+			return math.PaddedBigBytes(common.Big0, 32)
 		}
 	case BytesTy:
 		if reflectValue.Kind() == reflect.Array {

accounts/abi/type_test.go

@@ -55,8 +55,8 @@ func TestTypeRegexp(t *testing.T) {
 		{"string[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[]"}},
 		{"string[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[2]"}},
 		{"address", Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}},
-		{"address[]", Type{IsSlice: true, SliceSize: -1,Kind: reflect.Array, Type:address_t, T: AddressTy, Size:20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
-		{"address[2]", Type{IsArray: true, SliceSize: 2,Kind: reflect.Array, Type:address_t, T: AddressTy, Size:20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
+		{"address[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Array, Type: address_t, T: AddressTy, Size: 20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
+		{"address[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Array, Type: address_t, T: AddressTy, Size: 20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
 
 		// TODO when fixed types are implemented properly
 		// {"fixed", Type{}},

accounts/account_manager.go

@@ -1,350 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package accounts implements encrypted storage of secp256k1 private keys.
-//
-// Keys are stored as encrypted JSON files according to the Web3 Secret Storage specification.
-// See https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition for more information.
-package accounts
-
-import (
-	"crypto/ecdsa"
-	crand "crypto/rand"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"runtime"
-	"sync"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
-)
-
-var (
-	ErrLocked  = errors.New("account is locked")
-	ErrNoMatch = errors.New("no key for given address or file")
-	ErrDecrypt = errors.New("could not decrypt key with given passphrase")
-)
-
-// Account represents a stored key.
-// When used as an argument, it selects a unique key file to act on.
-type Account struct {
-	Address common.Address // Ethereum account address derived from the key
-
-	// File contains the key file name.
-	// When Acccount is used as an argument to select a key, File can be left blank to
-	// select just by address or set to the basename or absolute path of a file in the key
-	// directory. Accounts returned by Manager will always contain an absolute path.
-	File string
-}
-
-func (acc *Account) MarshalJSON() ([]byte, error) {
-	return []byte(`"` + acc.Address.Hex() + `"`), nil
-}
-
-func (acc *Account) UnmarshalJSON(raw []byte) error {
-	return json.Unmarshal(raw, &acc.Address)
-}
-
-// Manager manages a key storage directory on disk.
-type Manager struct {
-	cache    *addrCache
-	keyStore keyStore
-	mu       sync.RWMutex
-	unlocked map[common.Address]*unlocked
-}
-
-type unlocked struct {
-	*Key
-	abort chan struct{}
-}
-
-// NewManager creates a manager for the given directory.
-func NewManager(keydir string, scryptN, scryptP int) *Manager {
-	keydir, _ = filepath.Abs(keydir)
-	am := &Manager{keyStore: &keyStorePassphrase{keydir, scryptN, scryptP}}
-	am.init(keydir)
-	return am
-}
-
-// NewPlaintextManager creates a manager for the given directory.
-// Deprecated: Use NewManager.
-func NewPlaintextManager(keydir string) *Manager {
-	keydir, _ = filepath.Abs(keydir)
-	am := &Manager{keyStore: &keyStorePlain{keydir}}
-	am.init(keydir)
-	return am
-}
-
-func (am *Manager) init(keydir string) {
-	am.unlocked = make(map[common.Address]*unlocked)
-	am.cache = newAddrCache(keydir)
-	// TODO: In order for this finalizer to work, there must be no references
-	// to am. addrCache doesn't keep a reference but unlocked keys do,
-	// so the finalizer will not trigger until all timed unlocks have expired.
-	runtime.SetFinalizer(am, func(m *Manager) {
-		m.cache.close()
-	})
-}
-
-// HasAddress reports whether a key with the given address is present.
-func (am *Manager) HasAddress(addr common.Address) bool {
-	return am.cache.hasAddress(addr)
-}
-
-// Accounts returns all key files present in the directory.
-func (am *Manager) Accounts() []Account {
-	return am.cache.accounts()
-}
-
-// DeleteAccount deletes the key matched by account if the passphrase is correct.
-// If a contains no filename, the address must match a unique key.
-func (am *Manager) DeleteAccount(a Account, passphrase string) error {
-	// Decrypting the key isn't really necessary, but we do
-	// it anyway to check the password and zero out the key
-	// immediately afterwards.
-	a, key, err := am.getDecryptedKey(a, passphrase)
-	if key != nil {
-		zeroKey(key.PrivateKey)
-	}
-	if err != nil {
-		return err
-	}
-	// The order is crucial here. The key is dropped from the
-	// cache after the file is gone so that a reload happening in
-	// between won't insert it into the cache again.
-	err = os.Remove(a.File)
-	if err == nil {
-		am.cache.delete(a)
-	}
-	return err
-}
-
-// Sign calculates a ECDSA signature for the given hash. The produced signature
-// is in the [R || S || V] format where V is 0 or 1.
-func (am *Manager) Sign(addr common.Address, hash []byte) ([]byte, error) {
-	am.mu.RLock()
-	defer am.mu.RUnlock()
-
-	unlockedKey, found := am.unlocked[addr]
-	if !found {
-		return nil, ErrLocked
-	}
-	return crypto.Sign(hash, unlockedKey.PrivateKey)
-}
-
-// SignWithPassphrase signs hash if the private key matching the given address
-// can be decrypted with the given passphrase. The produced signature is in the
-// [R || S || V] format where V is 0 or 1.
-func (am *Manager) SignWithPassphrase(a Account, passphrase string, hash []byte) (signature []byte, err error) {
-	_, key, err := am.getDecryptedKey(a, passphrase)
-	if err != nil {
-		return nil, err
-	}
-	defer zeroKey(key.PrivateKey)
-	return crypto.Sign(hash, key.PrivateKey)
-}
-
-// Unlock unlocks the given account indefinitely.
-func (am *Manager) Unlock(a Account, passphrase string) error {
-	return am.TimedUnlock(a, passphrase, 0)
-}
-
-// Lock removes the private key with the given address from memory.
-func (am *Manager) Lock(addr common.Address) error {
-	am.mu.Lock()
-	if unl, found := am.unlocked[addr]; found {
-		am.mu.Unlock()
-		am.expire(addr, unl, time.Duration(0)*time.Nanosecond)
-	} else {
-		am.mu.Unlock()
-	}
-	return nil
-}
-
-// TimedUnlock unlocks the given account with the passphrase. The account
-// stays unlocked for the duration of timeout. A timeout of 0 unlocks the account
-// until the program exits. The account must match a unique key file.
-//
-// If the account address is already unlocked for a duration, TimedUnlock extends or
-// shortens the active unlock timeout. If the address was previously unlocked
-// indefinitely the timeout is not altered.
-func (am *Manager) TimedUnlock(a Account, passphrase string, timeout time.Duration) error {
-	a, key, err := am.getDecryptedKey(a, passphrase)
-	if err != nil {
-		return err
-	}
-
-	am.mu.Lock()
-	defer am.mu.Unlock()
-	u, found := am.unlocked[a.Address]
-	if found {
-		if u.abort == nil {
-			// The address was unlocked indefinitely, so unlocking
-			// it with a timeout would be confusing.
-			zeroKey(key.PrivateKey)
-			return nil
-		} else {
-			// Terminate the expire goroutine and replace it below.
-			close(u.abort)
-		}
-	}
-	if timeout > 0 {
-		u = &unlocked{Key: key, abort: make(chan struct{})}
-		go am.expire(a.Address, u, timeout)
-	} else {
-		u = &unlocked{Key: key}
-	}
-	am.unlocked[a.Address] = u
-	return nil
-}
-
-// Find resolves the given account into a unique entry in the keystore.
-func (am *Manager) Find(a Account) (Account, error) {
-	am.cache.maybeReload()
-	am.cache.mu.Lock()
-	a, err := am.cache.find(a)
-	am.cache.mu.Unlock()
-	return a, err
-}
-
-func (am *Manager) getDecryptedKey(a Account, auth string) (Account, *Key, error) {
-	a, err := am.Find(a)
-	if err != nil {
-		return a, nil, err
-	}
-	key, err := am.keyStore.GetKey(a.Address, a.File, auth)
-	return a, key, err
-}
-
-func (am *Manager) expire(addr common.Address, u *unlocked, timeout time.Duration) {
-	t := time.NewTimer(timeout)
-	defer t.Stop()
-	select {
-	case <-u.abort:
-		// just quit
-	case <-t.C:
-		am.mu.Lock()
-		// only drop if it's still the same key instance that dropLater
-		// was launched with. we can check that using pointer equality
-		// because the map stores a new pointer every time the key is
-		// unlocked.
-		if am.unlocked[addr] == u {
-			zeroKey(u.PrivateKey)
-			delete(am.unlocked, addr)
-		}
-		am.mu.Unlock()
-	}
-}
-
-// NewAccount generates a new key and stores it into the key directory,
-// encrypting it with the passphrase.
-func (am *Manager) NewAccount(passphrase string) (Account, error) {
-	_, account, err := storeNewKey(am.keyStore, crand.Reader, passphrase)
-	if err != nil {
-		return Account{}, err
-	}
-	// Add the account to the cache immediately rather
-	// than waiting for file system notifications to pick it up.
-	am.cache.add(account)
-	return account, nil
-}
-
-// AccountByIndex returns the ith account.
-func (am *Manager) AccountByIndex(i int) (Account, error) {
-	accounts := am.Accounts()
-	if i < 0 || i >= len(accounts) {
-		return Account{}, fmt.Errorf("account index %d out of range [0, %d]", i, len(accounts)-1)
-	}
-	return accounts[i], nil
-}
-
-// Export exports as a JSON key, encrypted with newPassphrase.
-func (am *Manager) Export(a Account, passphrase, newPassphrase string) (keyJSON []byte, err error) {
-	_, key, err := am.getDecryptedKey(a, passphrase)
-	if err != nil {
-		return nil, err
-	}
-	var N, P int
-	if store, ok := am.keyStore.(*keyStorePassphrase); ok {
-		N, P = store.scryptN, store.scryptP
-	} else {
-		N, P = StandardScryptN, StandardScryptP
-	}
-	return EncryptKey(key, newPassphrase, N, P)
-}
-
-// Import stores the given encrypted JSON key into the key directory.
-func (am *Manager) Import(keyJSON []byte, passphrase, newPassphrase string) (Account, error) {
-	key, err := DecryptKey(keyJSON, passphrase)
-	if key != nil && key.PrivateKey != nil {
-		defer zeroKey(key.PrivateKey)
-	}
-	if err != nil {
-		return Account{}, err
-	}
-	return am.importKey(key, newPassphrase)
-}
-
-// ImportECDSA stores the given key into the key directory, encrypting it with the passphrase.
-func (am *Manager) ImportECDSA(priv *ecdsa.PrivateKey, passphrase string) (Account, error) {
-	key := newKeyFromECDSA(priv)
-	if am.cache.hasAddress(key.Address) {
-		return Account{}, fmt.Errorf("account already exists")
-	}
-
-	return am.importKey(key, passphrase)
-}
-
-func (am *Manager) importKey(key *Key, passphrase string) (Account, error) {
-	a := Account{Address: key.Address, File: am.keyStore.JoinPath(keyFileName(key.Address))}
-	if err := am.keyStore.StoreKey(a.File, key, passphrase); err != nil {
-		return Account{}, err
-	}
-	am.cache.add(a)
-	return a, nil
-}
-
-// Update changes the passphrase of an existing account.
-func (am *Manager) Update(a Account, passphrase, newPassphrase string) error {
-	a, key, err := am.getDecryptedKey(a, passphrase)
-	if err != nil {
-		return err
-	}
-	return am.keyStore.StoreKey(a.File, key, newPassphrase)
-}
-
-// ImportPreSaleKey decrypts the given Ethereum presale wallet and stores
-// a key file in the key directory. The key file is encrypted with the same passphrase.
-func (am *Manager) ImportPreSaleKey(keyJSON []byte, passphrase string) (Account, error) {
-	a, _, err := importPreSaleKey(am.keyStore, keyJSON, passphrase)
-	if err != nil {
-		return a, err
-	}
-	am.cache.add(a)
-	return a, nil
-}
-
-// zeroKey zeroes a private key in memory.
-func zeroKey(k *ecdsa.PrivateKey) {
-	b := k.D.Bits()
-	for i := range b {
-		b[i] = 0
-	}
-}

accounts/accounts.go

@@ -0,0 +1,155 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package accounts implements high level Ethereum account management.
+package accounts
+
+import (
+	"math/big"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
+)
+
+// Account represents an Ethereum account located at a specific location defined
+// by the optional URL field.
+type Account struct {
+	Address common.Address `json:"address"` // Ethereum account address derived from the key
+	URL     URL            `json:"url"`     // Optional resource locator within a backend
+}
+
+// Wallet represents a software or hardware wallet that might contain one or more
+// accounts (derived from the same seed).
+type Wallet interface {
+	// URL retrieves the canonical path under which this wallet is reachable. It is
+	// user by upper layers to define a sorting order over all wallets from multiple
+	// backends.
+	URL() URL
+
+	// Status returns a textual status to aid the user in the current state of the
+	// wallet.
+	Status() string
+
+	// Open initializes access to a wallet instance. It is not meant to unlock or
+	// decrypt account keys, rather simply to establish a connection to hardware
+	// wallets and/or to access derivation seeds.
+	//
+	// The passphrase parameter may or may not be used by the implementation of a
+	// particular wallet instance. The reason there is no passwordless open method
+	// is to strive towards a uniform wallet handling, oblivious to the different
+	// backend providers.
+	//
+	// Please note, if you open a wallet, you must close it to release any allocated
+	// resources (especially important when working with hardware wallets).
+	Open(passphrase string) error
+
+	// Close releases any resources held by an open wallet instance.
+	Close() error
+
+	// Accounts retrieves the list of signing accounts the wallet is currently aware
+	// of. For hierarchical deterministic wallets, the list will not be exhaustive,
+	// rather only contain the accounts explicitly pinned during account derivation.
+	Accounts() []Account
+
+	// Contains returns whether an account is part of this particular wallet or not.
+	Contains(account Account) bool
+
+	// Derive attempts to explicitly derive a hierarchical deterministic account at
+	// the specified derivation path. If requested, the derived account will be added
+	// to the wallet's tracked account list.
+	Derive(path DerivationPath, pin bool) (Account, error)
+
+	// SelfDerive sets a base account derivation path from which the wallet attempts
+	// to discover non zero accounts and automatically add them to list of tracked
+	// accounts.
+	//
+	// Note, self derivaton will increment the last component of the specified path
+	// opposed to decending into a child path to allow discovering accounts starting
+	// from non zero components.
+	//
+	// You can disable automatic account discovery by calling SelfDerive with a nil
+	// chain state reader.
+	SelfDerive(base DerivationPath, chain ethereum.ChainStateReader)
+
+	// SignHash requests the wallet to sign the given hash.
+	//
+	// It looks up the account specified either solely via its address contained within,
+	// or optionally with the aid of any location metadata from the embedded URL field.
+	//
+	// If the wallet requires additional authentication to sign the request (e.g.
+	// a password to decrypt the account, or a PIN code o verify the transaction),
+	// an AuthNeededError instance will be returned, containing infos for the user
+	// about which fields or actions are needed. The user may retry by providing
+	// the needed details via SignHashWithPassphrase, or by other means (e.g. unlock
+	// the account in a keystore).
+	SignHash(account Account, hash []byte) ([]byte, error)
+
+	// SignTx requests the wallet to sign the given transaction.
+	//
+	// It looks up the account specified either solely via its address contained within,
+	// or optionally with the aid of any location metadata from the embedded URL field.
+	//
+	// If the wallet requires additional authentication to sign the request (e.g.
+	// a password to decrypt the account, or a PIN code o verify the transaction),
+	// an AuthNeededError instance will be returned, containing infos for the user
+	// about which fields or actions are needed. The user may retry by providing
+	// the needed details via SignTxWithPassphrase, or by other means (e.g. unlock
+	// the account in a keystore).
+	SignTx(account Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error)
+
+	// SignHashWithPassphrase requests the wallet to sign the given hash with the
+	// given passphrase as extra authentication information.
+	//
+	// It looks up the account specified either solely via its address contained within,
+	// or optionally with the aid of any location metadata from the embedded URL field.
+	SignHashWithPassphrase(account Account, passphrase string, hash []byte) ([]byte, error)
+
+	// SignTxWithPassphrase requests the wallet to sign the given transaction, with the
+	// given passphrase as extra authentication information.
+	//
+	// It looks up the account specified either solely via its address contained within,
+	// or optionally with the aid of any location metadata from the embedded URL field.
+	SignTxWithPassphrase(account Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error)
+}
+
+// Backend is a "wallet provider" that may contain a batch of accounts they can
+// sign transactions with and upon request, do so.
+type Backend interface {
+	// Wallets retrieves the list of wallets the backend is currently aware of.
+	//
+	// The returned wallets are not opened by default. For software HD wallets this
+	// means that no base seeds are decrypted, and for hardware wallets that no actual
+	// connection is established.
+	//
+	// The resulting wallet list will be sorted alphabetically based on its internal
+	// URL assigned by the backend. Since wallets (especially hardware) may come and
+	// go, the same wallet might appear at a different positions in the list during
+	// subsequent retrievals.
+	Wallets() []Wallet
+
+	// Subscribe creates an async subscription to receive notifications when the
+	// backend detects the arrival or departure of a wallet.
+	Subscribe(sink chan<- WalletEvent) event.Subscription
+}
+
+// WalletEvent is an event fired by an account backend when a wallet arrival or
+// departure is detected.
+type WalletEvent struct {
+	Wallet Wallet // Wallet instance arrived or departed
+	Arrive bool   // Whether the wallet was added or removed
+}

accounts/accounts_test.go

@@ -1,224 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package accounts
-
-import (
-	"io/ioutil"
-	"os"
-	"runtime"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-)
-
-var testSigData = make([]byte, 32)
-
-func TestManager(t *testing.T) {
-	dir, am := tmpManager(t, true)
-	defer os.RemoveAll(dir)
-
-	a, err := am.NewAccount("foo")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !strings.HasPrefix(a.File, dir) {
-		t.Errorf("account file %s doesn't have dir prefix", a.File)
-	}
-	stat, err := os.Stat(a.File)
-	if err != nil {
-		t.Fatalf("account file %s doesn't exist (%v)", a.File, err)
-	}
-	if runtime.GOOS != "windows" && stat.Mode() != 0600 {
-		t.Fatalf("account file has wrong mode: got %o, want %o", stat.Mode(), 0600)
-	}
-	if !am.HasAddress(a.Address) {
-		t.Errorf("HasAccount(%x) should've returned true", a.Address)
-	}
-	if err := am.Update(a, "foo", "bar"); err != nil {
-		t.Errorf("Update error: %v", err)
-	}
-	if err := am.DeleteAccount(a, "bar"); err != nil {
-		t.Errorf("DeleteAccount error: %v", err)
-	}
-	if common.FileExist(a.File) {
-		t.Errorf("account file %s should be gone after DeleteAccount", a.File)
-	}
-	if am.HasAddress(a.Address) {
-		t.Errorf("HasAccount(%x) should've returned true after DeleteAccount", a.Address)
-	}
-}
-
-func TestSign(t *testing.T) {
-	dir, am := tmpManager(t, true)
-	defer os.RemoveAll(dir)
-
-	pass := "" // not used but required by API
-	a1, err := am.NewAccount(pass)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := am.Unlock(a1, ""); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := am.Sign(a1.Address, testSigData); err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestSignWithPassphrase(t *testing.T) {
-	dir, am := tmpManager(t, true)
-	defer os.RemoveAll(dir)
-
-	pass := "passwd"
-	acc, err := am.NewAccount(pass)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if _, unlocked := am.unlocked[acc.Address]; unlocked {
-		t.Fatal("expected account to be locked")
-	}
-
-	_, err = am.SignWithPassphrase(acc, pass, testSigData)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if _, unlocked := am.unlocked[acc.Address]; unlocked {
-		t.Fatal("expected account to be locked")
-	}
-
-	if _, err = am.SignWithPassphrase(acc, "invalid passwd", testSigData); err == nil {
-		t.Fatal("expected SignHash to fail with invalid password")
-	}
-}
-
-func TestTimedUnlock(t *testing.T) {
-	dir, am := tmpManager(t, true)
-	defer os.RemoveAll(dir)
-
-	pass := "foo"
-	a1, err := am.NewAccount(pass)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Signing without passphrase fails because account is locked
-	_, err = am.Sign(a1.Address, testSigData)
-	if err != ErrLocked {
-		t.Fatal("Signing should've failed with ErrLocked before unlocking, got ", err)
-	}
-
-	// Signing with passphrase works
-	if err = am.TimedUnlock(a1, pass, 100*time.Millisecond); err != nil {
-		t.Fatal(err)
-	}
-
-	// Signing without passphrase works because account is temp unlocked
-	_, err = am.Sign(a1.Address, testSigData)
-	if err != nil {
-		t.Fatal("Signing shouldn't return an error after unlocking, got ", err)
-	}
-
-	// Signing fails again after automatic locking
-	time.Sleep(250 * time.Millisecond)
-	_, err = am.Sign(a1.Address, testSigData)
-	if err != ErrLocked {
-		t.Fatal("Signing should've failed with ErrLocked timeout expired, got ", err)
-	}
-}
-
-func TestOverrideUnlock(t *testing.T) {
-	dir, am := tmpManager(t, false)
-	defer os.RemoveAll(dir)
-
-	pass := "foo"
-	a1, err := am.NewAccount(pass)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Unlock indefinitely.
-	if err = am.TimedUnlock(a1, pass, 5*time.Minute); err != nil {
-		t.Fatal(err)
-	}
-
-	// Signing without passphrase works because account is temp unlocked
-	_, err = am.Sign(a1.Address, testSigData)
-	if err != nil {
-		t.Fatal("Signing shouldn't return an error after unlocking, got ", err)
-	}
-
-	// reset unlock to a shorter period, invalidates the previous unlock
-	if err = am.TimedUnlock(a1, pass, 100*time.Millisecond); err != nil {
-		t.Fatal(err)
-	}
-
-	// Signing without passphrase still works because account is temp unlocked
-	_, err = am.Sign(a1.Address, testSigData)
-	if err != nil {
-		t.Fatal("Signing shouldn't return an error after unlocking, got ", err)
-	}
-
-	// Signing fails again after automatic locking
-	time.Sleep(250 * time.Millisecond)
-	_, err = am.Sign(a1.Address, testSigData)
-	if err != ErrLocked {
-		t.Fatal("Signing should've failed with ErrLocked timeout expired, got ", err)
-	}
-}
-
-// This test should fail under -race if signing races the expiration goroutine.
-func TestSignRace(t *testing.T) {
-	dir, am := tmpManager(t, false)
-	defer os.RemoveAll(dir)
-
-	// Create a test account.
-	a1, err := am.NewAccount("")
-	if err != nil {
-		t.Fatal("could not create the test account", err)
-	}
-
-	if err := am.TimedUnlock(a1, "", 15*time.Millisecond); err != nil {
-		t.Fatal("could not unlock the test account", err)
-	}
-	end := time.Now().Add(500 * time.Millisecond)
-	for time.Now().Before(end) {
-		if _, err := am.Sign(a1.Address, testSigData); err == ErrLocked {
-			return
-		} else if err != nil {
-			t.Errorf("Sign error: %v", err)
-			return
-		}
-		time.Sleep(1 * time.Millisecond)
-	}
-	t.Errorf("Account did not lock within the timeout")
-}
-
-func tmpManager(t *testing.T, encrypted bool) (string, *Manager) {
-	d, err := ioutil.TempDir("", "eth-keystore-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	new := NewPlaintextManager
-	if encrypted {
-		new = func(kd string) *Manager { return NewManager(kd, veryLightScryptN, veryLightScryptP) }
-	}
-	return d, new(d)
-}

accounts/errors.go

@@ -0,0 +1,68 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package accounts
+
+import (
+	"errors"
+	"fmt"
+)
+
+// ErrUnknownAccount is returned for any requested operation for which no backend
+// provides the specified account.
+var ErrUnknownAccount = errors.New("unknown account")
+
+// ErrUnknownWallet is returned for any requested operation for which no backend
+// provides the specified wallet.
+var ErrUnknownWallet = errors.New("unknown wallet")
+
+// ErrNotSupported is returned when an operation is requested from an account
+// backend that it does not support.
+var ErrNotSupported = errors.New("not supported")
+
+// ErrInvalidPassphrase is returned when a decryption operation receives a bad
+// passphrase.
+var ErrInvalidPassphrase = errors.New("invalid passphrase")
+
+// ErrWalletAlreadyOpen is returned if a wallet is attempted to be opened the
+// second time.
+var ErrWalletAlreadyOpen = errors.New("wallet already open")
+
+// ErrWalletClosed is returned if a wallet is attempted to be opened the
+// secodn time.
+var ErrWalletClosed = errors.New("wallet closed")
+
+// AuthNeededError is returned by backends for signing requests where the user
+// is required to provide further authentication before signing can succeed.
+//
+// This usually means either that a password needs to be supplied, or perhaps a
+// one time PIN code displayed by some hardware device.
+type AuthNeededError struct {
+	Needed string // Extra authentication the user needs to provide
+}
+
+// NewAuthNeededError creates a new authentication error with the extra details
+// about the needed fields set.
+func NewAuthNeededError(needed string) error {
+	return &AuthNeededError{
+		Needed: needed,
+	}
+}
+
+// Error implements the standard error interfacel.
+func (err *AuthNeededError) Error() string {
+	return fmt.Sprintf("authentication needed: %s", err.Needed)
+}

accounts/hd.go

@@ -0,0 +1,130 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package accounts
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"math/big"
+	"strings"
+)
+
+// DefaultRootDerivationPath is the root path to which custom derivation endpoints
+// are appended. As such, the first account will be at m/44'/60'/0'/0, the second
+// at m/44'/60'/0'/1, etc.
+var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0}
+
+// DefaultBaseDerivationPath is the base path from which custom derivation endpoints
+// are incremented. As such, the first account will be at m/44'/60'/0'/0, the second
+// at m/44'/60'/0'/1, etc.
+var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}
+
+// DerivationPath represents the computer friendly version of a hierarchical
+// deterministic wallet account derivaion path.
+//
+// The BIP-32 spec https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki
+// defines derivation paths to be of the form:
+//
+//   m / purpose' / coin_type' / account' / change / address_index
+//
+// The BIP-44 spec https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki
+// defines that the `purpose` be 44' (or 0x8000002C) for crypto currencies, and
+// SLIP-44 https://github.com/satoshilabs/slips/blob/master/slip-0044.md assigns
+// the `coin_type` 60' (or 0x8000003C) to Ethereum.
+//
+// The root path for Ethereum is m/44'/60'/0'/0 according to the specification
+// from https://github.com/ethereum/EIPs/issues/84, albeit it's not set in stone
+// yet whether accounts should increment the last component or the children of
+// that. We will go with the simpler approach of incrementing the last component.
+type DerivationPath []uint32
+
+// ParseDerivationPath converts a user specified derivation path string to the
+// internal binary representation.
+//
+// Full derivation paths need to start with the `m/` prefix, relative derivation
+// paths (which will get appended to the default root path) must not have prefixes
+// in front of the first element. Whitespace is ignored.
+func ParseDerivationPath(path string) (DerivationPath, error) {
+	var result DerivationPath
+
+	// Handle absolute or relative paths
+	components := strings.Split(path, "/")
+	switch {
+	case len(components) == 0:
+		return nil, errors.New("empty derivation path")
+
+	case strings.TrimSpace(components[0]) == "":
+		return nil, errors.New("ambiguous path: use 'm/' prefix for absolute paths, or no leading '/' for relative ones")
+
+	case strings.TrimSpace(components[0]) == "m":
+		components = components[1:]
+
+	default:
+		result = append(result, DefaultRootDerivationPath...)
+	}
+	// All remaining components are relative, append one by one
+	if len(components) == 0 {
+		return nil, errors.New("empty derivation path") // Empty relative paths
+	}
+	for _, component := range components {
+		// Ignore any user added whitespace
+		component = strings.TrimSpace(component)
+		var value uint32
+
+		// Handle hardened paths
+		if strings.HasSuffix(component, "'") {
+			value = 0x80000000
+			component = strings.TrimSpace(strings.TrimSuffix(component, "'"))
+		}
+		// Handle the non hardened component
+		bigval, ok := new(big.Int).SetString(component, 0)
+		if !ok {
+			return nil, fmt.Errorf("invalid component: %s", component)
+		}
+		max := math.MaxUint32 - value
+		if bigval.Sign() < 0 || bigval.Cmp(big.NewInt(int64(max))) > 0 {
+			if value == 0 {
+				return nil, fmt.Errorf("component %v out of allowed range [0, %d]", bigval, max)
+			}
+			return nil, fmt.Errorf("component %v out of allowed hardened range [0, %d]", bigval, max)
+		}
+		value += uint32(bigval.Uint64())
+
+		// Append and repeat
+		result = append(result, value)
+	}
+	return result, nil
+}
+
+// String implements the stringer interface, converting a binary derivation path
+// to its canonical representation.
+func (path DerivationPath) String() string {
+	result := "m"
+	for _, component := range path {
+		var hardened bool
+		if component >= 0x80000000 {
+			component -= 0x80000000
+			hardened = true
+		}
+		result = fmt.Sprintf("%s/%d", result, component)
+		if hardened {
+			result += "'"
+		}
+	}
+	return result
+}

accounts/hd_test.go

@@ -0,0 +1,79 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package accounts
+
+import (
+	"reflect"
+	"testing"
+)
+
+// Tests that HD derivation paths can be correctly parsed into our internal binary
+// representation.
+func TestHDPathParsing(t *testing.T) {
+	tests := []struct {
+		input  string
+		output DerivationPath
+	}{
+		// Plain absolute derivation paths
+		{"m/44'/60'/0'/0", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+		{"m/44'/60'/0'/128", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 128}},
+		{"m/44'/60'/0'/0'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+		{"m/44'/60'/0'/128'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 128}},
+		{"m/2147483692/2147483708/2147483648/0", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+		{"m/2147483692/2147483708/2147483648/2147483648", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+
+		// Plain relative derivation paths
+		{"0", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+		{"128", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 128}},
+		{"0'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+		{"128'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 128}},
+		{"2147483648", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+
+		// Hexadecimal absolute derivation paths
+		{"m/0x2C'/0x3c'/0x00'/0x00", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+		{"m/0x2C'/0x3c'/0x00'/0x80", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 128}},
+		{"m/0x2C'/0x3c'/0x00'/0x00'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+		{"m/0x2C'/0x3c'/0x00'/0x80'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 128}},
+		{"m/0x8000002C/0x8000003c/0x80000000/0x00", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+		{"m/0x8000002C/0x8000003c/0x80000000/0x80000000", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+
+		// Hexadecimal relative derivation paths
+		{"0x00", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+		{"0x80", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 128}},
+		{"0x00'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+		{"0x80'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 128}},
+		{"0x80000000", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+
+		// Weird inputs just to ensure they work
+		{"	m  /   44			'\n/\n   60	\n\n\t'   /\n0 ' /\t\t	0", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+
+		// Invaid derivation paths
+		{"", nil},              // Empty relative derivation path
+		{"m", nil},             // Empty absolute derivation path
+		{"m/", nil},            // Missing last derivation component
+		{"/44'/60'/0'/0", nil}, // Absolute path without m prefix, might be user error
+		{"m/2147483648'", nil}, // Overflows 32 bit integer
+		{"m/-1'", nil},         // Cannot contain negative number
+	}
+	for i, tt := range tests {
+		if path, err := ParseDerivationPath(tt.input); !reflect.DeepEqual(path, tt.output) {
+			t.Errorf("test %d: parse mismatch: have %v (%v), want %v", i, path, err, tt.output)
+		} else if path == nil && err == nil {
+			t.Errorf("test %d: nil path and error: %v", i, err)
+		}
+	}
+}

accounts/keystore/account_cache.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"bufio"
@@ -28,9 +28,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 )
 
 // Minimum amount of time between cache reloads. This limit applies if the platform does
@@ -38,23 +38,23 @@ import (
 // exist yet, the code will attempt to create a watcher at most this often.
 const minReloadInterval = 2 * time.Second
 
-type accountsByFile []Account
+type accountsByURL []accounts.Account
 
-func (s accountsByFile) Len() int           { return len(s) }
-func (s accountsByFile) Less(i, j int) bool { return s[i].File < s[j].File }
-func (s accountsByFile) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s accountsByURL) Len() int           { return len(s) }
+func (s accountsByURL) Less(i, j int) bool { return s[i].URL.Cmp(s[j].URL) < 0 }
+func (s accountsByURL) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 
 // AmbiguousAddrError is returned when attempting to unlock
 // an address for which more than one file exists.
 type AmbiguousAddrError struct {
 	Addr    common.Address
-	Matches []Account
+	Matches []accounts.Account
 }
 
 func (err *AmbiguousAddrError) Error() string {
 	files := ""
 	for i, a := range err.Matches {
-		files += a.File
+		files += a.URL.Path
 		if i < len(err.Matches)-1 {
 			files += ", "
 		}
@@ -62,60 +62,63 @@ func (err *AmbiguousAddrError) Error() string {
 	return fmt.Sprintf("multiple keys match address (%s)", files)
 }
 
-// addrCache is a live index of all accounts in the keystore.
-type addrCache struct {
+// accountCache is a live index of all accounts in the keystore.
+type accountCache struct {
 	keydir   string
 	watcher  *watcher
 	mu       sync.Mutex
-	all      accountsByFile
-	byAddr   map[common.Address][]Account
+	all      accountsByURL
+	byAddr   map[common.Address][]accounts.Account
 	throttle *time.Timer
+	notify   chan struct{}
 }
 
-func newAddrCache(keydir string) *addrCache {
-	ac := &addrCache{
+func newAccountCache(keydir string) (*accountCache, chan struct{}) {
+	ac := &accountCache{
 		keydir: keydir,
-		byAddr: make(map[common.Address][]Account),
+		byAddr: make(map[common.Address][]accounts.Account),
+		notify: make(chan struct{}, 1),
 	}
 	ac.watcher = newWatcher(ac)
-	return ac
+	return ac, ac.notify
 }
 
-func (ac *addrCache) accounts() []Account {
+func (ac *accountCache) accounts() []accounts.Account {
 	ac.maybeReload()
 	ac.mu.Lock()
 	defer ac.mu.Unlock()
-	cpy := make([]Account, len(ac.all))
+	cpy := make([]accounts.Account, len(ac.all))
 	copy(cpy, ac.all)
 	return cpy
 }
 
-func (ac *addrCache) hasAddress(addr common.Address) bool {
+func (ac *accountCache) hasAddress(addr common.Address) bool {
 	ac.maybeReload()
 	ac.mu.Lock()
 	defer ac.mu.Unlock()
 	return len(ac.byAddr[addr]) > 0
 }
 
-func (ac *addrCache) add(newAccount Account) {
+func (ac *accountCache) add(newAccount accounts.Account) {
 	ac.mu.Lock()
 	defer ac.mu.Unlock()
 
-	i := sort.Search(len(ac.all), func(i int) bool { return ac.all[i].File >= newAccount.File })
+	i := sort.Search(len(ac.all), func(i int) bool { return ac.all[i].URL.Cmp(newAccount.URL) >= 0 })
 	if i < len(ac.all) && ac.all[i] == newAccount {
 		return
 	}
 	// newAccount is not in the cache.
-	ac.all = append(ac.all, Account{})
+	ac.all = append(ac.all, accounts.Account{})
 	copy(ac.all[i+1:], ac.all[i:])
 	ac.all[i] = newAccount
 	ac.byAddr[newAccount.Address] = append(ac.byAddr[newAccount.Address], newAccount)
 }
 
 // note: removed needs to be unique here (i.e. both File and Address must be set).
-func (ac *addrCache) delete(removed Account) {
+func (ac *accountCache) delete(removed accounts.Account) {
 	ac.mu.Lock()
 	defer ac.mu.Unlock()
+
 	ac.all = removeAccount(ac.all, removed)
 	if ba := removeAccount(ac.byAddr[removed.Address], removed); len(ba) == 0 {
 		delete(ac.byAddr, removed.Address)
@@ -124,7 +127,7 @@ func (ac *addrCache) delete(removed Account) {
 	}
 }
 
-func removeAccount(slice []Account, elem Account) []Account {
+func removeAccount(slice []accounts.Account, elem accounts.Account) []accounts.Account {
 	for i := range slice {
 		if slice[i] == elem {
 			return append(slice[:i], slice[i+1:]...)
@@ -134,43 +137,44 @@ func removeAccount(slice []Account, elem Account) []Account {
 }
 
 // find returns the cached account for address if there is a unique match.
-// The exact matching rules are explained by the documentation of Account.
+// The exact matching rules are explained by the documentation of accounts.Account.
 // Callers must hold ac.mu.
-func (ac *addrCache) find(a Account) (Account, error) {
+func (ac *accountCache) find(a accounts.Account) (accounts.Account, error) {
 	// Limit search to address candidates if possible.
 	matches := ac.all
 	if (a.Address != common.Address{}) {
 		matches = ac.byAddr[a.Address]
 	}
-	if a.File != "" {
+	if a.URL.Path != "" {
 		// If only the basename is specified, complete the path.
-		if !strings.ContainsRune(a.File, filepath.Separator) {
-			a.File = filepath.Join(ac.keydir, a.File)
+		if !strings.ContainsRune(a.URL.Path, filepath.Separator) {
+			a.URL.Path = filepath.Join(ac.keydir, a.URL.Path)
 		}
 		for i := range matches {
-			if matches[i].File == a.File {
+			if matches[i].URL == a.URL {
 				return matches[i], nil
 			}
 		}
 		if (a.Address == common.Address{}) {
-			return Account{}, ErrNoMatch
+			return accounts.Account{}, ErrNoMatch
 		}
 	}
 	switch len(matches) {
 	case 1:
 		return matches[0], nil
 	case 0:
-		return Account{}, ErrNoMatch
+		return accounts.Account{}, ErrNoMatch
 	default:
-		err := &AmbiguousAddrError{Addr: a.Address, Matches: make([]Account, len(matches))}
+		err := &AmbiguousAddrError{Addr: a.Address, Matches: make([]accounts.Account, len(matches))}
 		copy(err.Matches, matches)
-		return Account{}, err
+		return accounts.Account{}, err
 	}
 }
 
-func (ac *addrCache) maybeReload() {
+func (ac *accountCache) maybeReload() {
 	ac.mu.Lock()
 	defer ac.mu.Unlock()
+
 	if ac.watcher.running {
 		return // A watcher is running and will keep the cache up-to-date.
 	}
@@ -188,21 +192,25 @@ func (ac *addrCache) maybeReload() {
 	ac.throttle.Reset(minReloadInterval)
 }
 
-func (ac *addrCache) close() {
+func (ac *accountCache) close() {
 	ac.mu.Lock()
 	ac.watcher.close()
 	if ac.throttle != nil {
 		ac.throttle.Stop()
 	}
+	if ac.notify != nil {
+		close(ac.notify)
+		ac.notify = nil
+	}
 	ac.mu.Unlock()
 }
 
 // reload caches addresses of existing accounts.
 // Callers must hold ac.mu.
-func (ac *addrCache) reload() {
+func (ac *accountCache) reload() {
 	accounts, err := ac.scan()
-	if err != nil && glog.V(logger.Debug) {
-		glog.Errorf("can't load keys: %v", err)
+	if err != nil {
+		log.Debug("Failed to reload keystore contents", "err", err)
 	}
 	ac.all = accounts
 	sort.Sort(ac.all)
@@ -212,10 +220,14 @@ func (ac *addrCache) reload() {
 	for _, a := range accounts {
 		ac.byAddr[a.Address] = append(ac.byAddr[a.Address], a)
 	}
-	glog.V(logger.Debug).Infof("reloaded keys, cache has %d accounts", len(ac.all))
+	select {
+	case ac.notify <- struct{}{}:
+	default:
+	}
+	log.Debug("Reloaded keystore contents", "accounts", len(ac.all))
 }
 
-func (ac *addrCache) scan() ([]Account, error) {
+func (ac *accountCache) scan() ([]accounts.Account, error) {
 	files, err := ioutil.ReadDir(ac.keydir)
 	if err != nil {
 		return nil, err
@@ -223,7 +235,7 @@ func (ac *addrCache) scan() ([]Account, error) {
 
 	var (
 		buf     = new(bufio.Reader)
-		addrs   []Account
+		addrs   []accounts.Account
 		keyJSON struct {
 			Address string `json:"address"`
 		}
@@ -231,12 +243,14 @@ func (ac *addrCache) scan() ([]Account, error) {
 	for _, fi := range files {
 		path := filepath.Join(ac.keydir, fi.Name())
 		if skipKeyFile(fi) {
-			glog.V(logger.Detail).Infof("ignoring file %s", path)
+			log.Trace("Ignoring file on account scan", "path", path)
 			continue
 		}
+		logger := log.New("path", path)
+
 		fd, err := os.Open(path)
 		if err != nil {
-			glog.V(logger.Detail).Infoln(err)
+			logger.Trace("Failed to open keystore file", "err", err)
 			continue
 		}
 		buf.Reset(fd)
@@ -246,11 +260,11 @@ func (ac *addrCache) scan() ([]Account, error) {
 		addr := common.HexToAddress(keyJSON.Address)
 		switch {
 		case err != nil:
-			glog.V(logger.Debug).Infof("can't decode key %s: %v", path, err)
+			logger.Debug("Failed to decode keystore key", "err", err)
 		case (addr == common.Address{}):
-			glog.V(logger.Debug).Infof("can't decode key %s: missing or zero address", path)
+			logger.Debug("Failed to decode keystore key", "err", "missing or zero address")
 		default:
-			addrs = append(addrs, Account{Address: addr, File: path})
+			addrs = append(addrs, accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}})
 		}
 		fd.Close()
 	}

accounts/keystore/account_cache_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"fmt"
@@ -28,23 +28,24 @@ import (
 
 	"github.com/cespare/cp"
 	"github.com/davecgh/go-spew/spew"
+	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 )
 
 var (
 	cachetestDir, _   = filepath.Abs(filepath.Join("testdata", "keystore"))
-	cachetestAccounts = []Account{
+	cachetestAccounts = []accounts.Account{
 		{
 			Address: common.HexToAddress("7ef5a6135f1fd6a02593eedc869c6d41d934aef8"),
-			File:    filepath.Join(cachetestDir, "UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(cachetestDir, "UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8")},
 		},
 		{
 			Address: common.HexToAddress("f466859ead1932d743d622cb74fc058882e8648a"),
-			File:    filepath.Join(cachetestDir, "aaa"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(cachetestDir, "aaa")},
 		},
 		{
 			Address: common.HexToAddress("289d485d9771714cce91d3393d764e1311907acc"),
-			File:    filepath.Join(cachetestDir, "zzz"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(cachetestDir, "zzz")},
 		},
 	}
 )
@@ -52,29 +53,36 @@ var (
 func TestWatchNewFile(t *testing.T) {
 	t.Parallel()
 
-	dir, am := tmpManager(t, false)
+	dir, ks := tmpKeyStore(t, false)
 	defer os.RemoveAll(dir)
 
 	// Ensure the watcher is started before adding any files.
-	am.Accounts()
+	ks.Accounts()
 	time.Sleep(200 * time.Millisecond)
 
 	// Move in the files.
-	wantAccounts := make([]Account, len(cachetestAccounts))
+	wantAccounts := make([]accounts.Account, len(cachetestAccounts))
 	for i := range cachetestAccounts {
-		a := cachetestAccounts[i]
-		a.File = filepath.Join(dir, filepath.Base(a.File))
-		wantAccounts[i] = a
-		if err := cp.CopyFile(a.File, cachetestAccounts[i].File); err != nil {
+		wantAccounts[i] = accounts.Account{
+			Address: cachetestAccounts[i].Address,
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(dir, filepath.Base(cachetestAccounts[i].URL.Path))},
+		}
+		if err := cp.CopyFile(wantAccounts[i].URL.Path, cachetestAccounts[i].URL.Path); err != nil {
 			t.Fatal(err)
 		}
 	}
 
-	// am should see the accounts.
-	var list []Account
+	// ks should see the accounts.
+	var list []accounts.Account
 	for d := 200 * time.Millisecond; d < 5*time.Second; d *= 2 {
-		list = am.Accounts()
+		list = ks.Accounts()
 		if reflect.DeepEqual(list, wantAccounts) {
+			// ks should have also received change notifications
+			select {
+			case <-ks.changes:
+			default:
+				t.Fatalf("wasn't notified of new accounts")
+			}
 			return
 		}
 		time.Sleep(d)
@@ -85,12 +93,12 @@ func TestWatchNewFile(t *testing.T) {
 func TestWatchNoDir(t *testing.T) {
 	t.Parallel()
 
-	// Create am but not the directory that it watches.
+	// Create ks but not the directory that it watches.
 	rand.Seed(time.Now().UnixNano())
 	dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-watch-test-%d-%d", os.Getpid(), rand.Int()))
-	am := NewManager(dir, LightScryptN, LightScryptP)
+	ks := NewKeyStore(dir, LightScryptN, LightScryptP)
 
-	list := am.Accounts()
+	list := ks.Accounts()
 	if len(list) > 0 {
 		t.Error("initial account list not empty:", list)
 	}
@@ -100,16 +108,22 @@ func TestWatchNoDir(t *testing.T) {
 	os.MkdirAll(dir, 0700)
 	defer os.RemoveAll(dir)
 	file := filepath.Join(dir, "aaa")
-	if err := cp.CopyFile(file, cachetestAccounts[0].File); err != nil {
+	if err := cp.CopyFile(file, cachetestAccounts[0].URL.Path); err != nil {
 		t.Fatal(err)
 	}
 
-	// am should see the account.
-	wantAccounts := []Account{cachetestAccounts[0]}
-	wantAccounts[0].File = file
+	// ks should see the account.
+	wantAccounts := []accounts.Account{cachetestAccounts[0]}
+	wantAccounts[0].URL = accounts.URL{Scheme: KeyStoreScheme, Path: file}
 	for d := 200 * time.Millisecond; d < 8*time.Second; d *= 2 {
-		list = am.Accounts()
+		list = ks.Accounts()
 		if reflect.DeepEqual(list, wantAccounts) {
+			// ks should have also received change notifications
+			select {
+			case <-ks.changes:
+			default:
+				t.Fatalf("wasn't notified of new accounts")
+			}
 			return
 		}
 		time.Sleep(d)
@@ -118,7 +132,7 @@ func TestWatchNoDir(t *testing.T) {
 }
 
 func TestCacheInitialReload(t *testing.T) {
-	cache := newAddrCache(cachetestDir)
+	cache, _ := newAccountCache(cachetestDir)
 	accounts := cache.accounts()
 	if !reflect.DeepEqual(accounts, cachetestAccounts) {
 		t.Fatalf("got initial accounts: %swant %s", spew.Sdump(accounts), spew.Sdump(cachetestAccounts))
@@ -126,55 +140,55 @@ func TestCacheInitialReload(t *testing.T) {
 }
 
 func TestCacheAddDeleteOrder(t *testing.T) {
-	cache := newAddrCache("testdata/no-such-dir")
+	cache, _ := newAccountCache("testdata/no-such-dir")
 	cache.watcher.running = true // prevent unexpected reloads
 
-	accounts := []Account{
+	accs := []accounts.Account{
 		{
 			Address: common.HexToAddress("095e7baea6a6c7c4c2dfeb977efac326af552d87"),
-			File:    "-309830980",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "-309830980"},
 		},
 		{
 			Address: common.HexToAddress("2cac1adea150210703ba75ed097ddfe24e14f213"),
-			File:    "ggg",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "ggg"},
 		},
 		{
 			Address: common.HexToAddress("8bda78331c916a08481428e4b07c96d3e916d165"),
-			File:    "zzzzzz-the-very-last-one.keyXXX",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "zzzzzz-the-very-last-one.keyXXX"},
 		},
 		{
 			Address: common.HexToAddress("d49ff4eeb0b2686ed89c0fc0f2b6ea533ddbbd5e"),
-			File:    "SOMETHING.key",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "SOMETHING.key"},
 		},
 		{
 			Address: common.HexToAddress("7ef5a6135f1fd6a02593eedc869c6d41d934aef8"),
-			File:    "UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8"},
 		},
 		{
 			Address: common.HexToAddress("f466859ead1932d743d622cb74fc058882e8648a"),
-			File:    "aaa",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "aaa"},
 		},
 		{
 			Address: common.HexToAddress("289d485d9771714cce91d3393d764e1311907acc"),
-			File:    "zzz",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "zzz"},
 		},
 	}
-	for _, a := range accounts {
+	for _, a := range accs {
 		cache.add(a)
 	}
 	// Add some of them twice to check that they don't get reinserted.
-	cache.add(accounts[0])
-	cache.add(accounts[2])
+	cache.add(accs[0])
+	cache.add(accs[2])
 
 	// Check that the account list is sorted by filename.
-	wantAccounts := make([]Account, len(accounts))
-	copy(wantAccounts, accounts)
-	sort.Sort(accountsByFile(wantAccounts))
+	wantAccounts := make([]accounts.Account, len(accs))
+	copy(wantAccounts, accs)
+	sort.Sort(accountsByURL(wantAccounts))
 	list := cache.accounts()
 	if !reflect.DeepEqual(list, wantAccounts) {
-		t.Fatalf("got accounts: %s\nwant %s", spew.Sdump(accounts), spew.Sdump(wantAccounts))
+		t.Fatalf("got accounts: %s\nwant %s", spew.Sdump(accs), spew.Sdump(wantAccounts))
 	}
-	for _, a := range accounts {
+	for _, a := range accs {
 		if !cache.hasAddress(a.Address) {
 			t.Errorf("expected hasAccount(%x) to return true", a.Address)
 		}
@@ -184,13 +198,13 @@ func TestCacheAddDeleteOrder(t *testing.T) {
 	}
 
 	// Delete a few keys from the cache.
-	for i := 0; i < len(accounts); i += 2 {
+	for i := 0; i < len(accs); i += 2 {
 		cache.delete(wantAccounts[i])
 	}
-	cache.delete(Account{Address: common.HexToAddress("fd9bd350f08ee3c0c19b85a8e16114a11a60aa4e"), File: "something"})
+	cache.delete(accounts.Account{Address: common.HexToAddress("fd9bd350f08ee3c0c19b85a8e16114a11a60aa4e"), URL: accounts.URL{Scheme: KeyStoreScheme, Path: "something"}})
 
 	// Check content again after deletion.
-	wantAccountsAfterDelete := []Account{
+	wantAccountsAfterDelete := []accounts.Account{
 		wantAccounts[1],
 		wantAccounts[3],
 		wantAccounts[5],
@@ -211,63 +225,63 @@ func TestCacheAddDeleteOrder(t *testing.T) {
 
 func TestCacheFind(t *testing.T) {
 	dir := filepath.Join("testdata", "dir")
-	cache := newAddrCache(dir)
+	cache, _ := newAccountCache(dir)
 	cache.watcher.running = true // prevent unexpected reloads
 
-	accounts := []Account{
+	accs := []accounts.Account{
 		{
 			Address: common.HexToAddress("095e7baea6a6c7c4c2dfeb977efac326af552d87"),
-			File:    filepath.Join(dir, "a.key"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(dir, "a.key")},
 		},
 		{
 			Address: common.HexToAddress("2cac1adea150210703ba75ed097ddfe24e14f213"),
-			File:    filepath.Join(dir, "b.key"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(dir, "b.key")},
 		},
 		{
 			Address: common.HexToAddress("d49ff4eeb0b2686ed89c0fc0f2b6ea533ddbbd5e"),
-			File:    filepath.Join(dir, "c.key"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(dir, "c.key")},
 		},
 		{
 			Address: common.HexToAddress("d49ff4eeb0b2686ed89c0fc0f2b6ea533ddbbd5e"),
-			File:    filepath.Join(dir, "c2.key"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(dir, "c2.key")},
 		},
 	}
-	for _, a := range accounts {
+	for _, a := range accs {
 		cache.add(a)
 	}
 
-	nomatchAccount := Account{
+	nomatchAccount := accounts.Account{
 		Address: common.HexToAddress("f466859ead1932d743d622cb74fc058882e8648a"),
-		File:    filepath.Join(dir, "something"),
+		URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(dir, "something")},
 	}
 	tests := []struct {
-		Query      Account
-		WantResult Account
+		Query      accounts.Account
+		WantResult accounts.Account
 		WantError  error
 	}{
 		// by address
-		{Query: Account{Address: accounts[0].Address}, WantResult: accounts[0]},
+		{Query: accounts.Account{Address: accs[0].Address}, WantResult: accs[0]},
 		// by file
-		{Query: Account{File: accounts[0].File}, WantResult: accounts[0]},
+		{Query: accounts.Account{URL: accs[0].URL}, WantResult: accs[0]},
 		// by basename
-		{Query: Account{File: filepath.Base(accounts[0].File)}, WantResult: accounts[0]},
+		{Query: accounts.Account{URL: accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Base(accs[0].URL.Path)}}, WantResult: accs[0]},
 		// by file and address
-		{Query: accounts[0], WantResult: accounts[0]},
+		{Query: accs[0], WantResult: accs[0]},
 		// ambiguous address, tie resolved by file
-		{Query: accounts[2], WantResult: accounts[2]},
+		{Query: accs[2], WantResult: accs[2]},
 		// ambiguous address error
 		{
-			Query: Account{Address: accounts[2].Address},
+			Query: accounts.Account{Address: accs[2].Address},
 			WantError: &AmbiguousAddrError{
-				Addr:    accounts[2].Address,
-				Matches: []Account{accounts[2], accounts[3]},
+				Addr:    accs[2].Address,
+				Matches: []accounts.Account{accs[2], accs[3]},
 			},
 		},
 		// no match error
 		{Query: nomatchAccount, WantError: ErrNoMatch},
-		{Query: Account{File: nomatchAccount.File}, WantError: ErrNoMatch},
-		{Query: Account{File: filepath.Base(nomatchAccount.File)}, WantError: ErrNoMatch},
-		{Query: Account{Address: nomatchAccount.Address}, WantError: ErrNoMatch},
+		{Query: accounts.Account{URL: nomatchAccount.URL}, WantError: ErrNoMatch},
+		{Query: accounts.Account{URL: accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Base(nomatchAccount.URL.Path)}}, WantError: ErrNoMatch},
+		{Query: accounts.Account{Address: nomatchAccount.Address}, WantError: ErrNoMatch},
 	}
 	for i, test := range tests {
 		a, err := cache.find(test.Query)

accounts/keystore/key.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"bytes"
@@ -29,9 +29,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/crypto/secp256k1"
 	"github.com/pborman/uuid"
 )
 
@@ -124,14 +124,13 @@ func (k *Key) UnmarshalJSON(j []byte) (err error) {
 	if err != nil {
 		return err
 	}
-
-	privkey, err := hex.DecodeString(keyJSON.PrivateKey)
+	privkey, err := crypto.HexToECDSA(keyJSON.PrivateKey)
 	if err != nil {
 		return err
 	}
 
 	k.Address = common.BytesToAddress(addr)
-	k.PrivateKey = crypto.ToECDSA(privkey)
+	k.PrivateKey = privkey
 
 	return nil
 }
@@ -156,7 +155,7 @@ func NewKeyForDirectICAP(rand io.Reader) *Key {
 		panic("key generation: could not read from random source: " + err.Error())
 	}
 	reader := bytes.NewReader(randBytes)
-	privateKeyECDSA, err := ecdsa.GenerateKey(secp256k1.S256(), reader)
+	privateKeyECDSA, err := ecdsa.GenerateKey(crypto.S256(), reader)
 	if err != nil {
 		panic("key generation: ecdsa.GenerateKey failed: " + err.Error())
 	}
@@ -168,20 +167,20 @@ func NewKeyForDirectICAP(rand io.Reader) *Key {
 }
 
 func newKey(rand io.Reader) (*Key, error) {
-	privateKeyECDSA, err := ecdsa.GenerateKey(secp256k1.S256(), rand)
+	privateKeyECDSA, err := ecdsa.GenerateKey(crypto.S256(), rand)
 	if err != nil {
 		return nil, err
 	}
 	return newKeyFromECDSA(privateKeyECDSA), nil
 }
 
-func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, Account, error) {
+func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Account, error) {
 	key, err := newKey(rand)
 	if err != nil {
-		return nil, Account{}, err
+		return nil, accounts.Account{}, err
 	}
-	a := Account{Address: key.Address, File: ks.JoinPath(keyFileName(key.Address))}
-	if err := ks.StoreKey(a.File, key, auth); err != nil {
+	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))}}
+	if err := ks.StoreKey(a.URL.Path, key, auth); err != nil {
 		zeroKey(key.PrivateKey)
 		return nil, a, err
 	}

accounts/keystore/keystore.go

@@ -0,0 +1,493 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package keystore implements encrypted storage of secp256k1 private keys.
+//
+// Keys are stored as encrypted JSON files according to the Web3 Secret Storage specification.
+// See https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition for more information.
+package keystore
+
+import (
+	"crypto/ecdsa"
+	crand "crypto/rand"
+	"errors"
+	"fmt"
+	"math/big"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/event"
+)
+
+var (
+	ErrLocked  = accounts.NewAuthNeededError("password or unlock")
+	ErrNoMatch = errors.New("no key for given address or file")
+	ErrDecrypt = errors.New("could not decrypt key with given passphrase")
+)
+
+// KeyStoreType is the reflect type of a keystore backend.
+var KeyStoreType = reflect.TypeOf(&KeyStore{})
+
+// KeyStoreScheme is the protocol scheme prefixing account and wallet URLs.
+var KeyStoreScheme = "keystore"
+
+// Maximum time between wallet refreshes (if filesystem notifications don't work).
+const walletRefreshCycle = 3 * time.Second
+
+// KeyStore manages a key storage directory on disk.
+type KeyStore struct {
+	storage  keyStore                     // Storage backend, might be cleartext or encrypted
+	cache    *accountCache                // In-memory account cache over the filesystem storage
+	changes  chan struct{}                // Channel receiving change notifications from the cache
+	unlocked map[common.Address]*unlocked // Currently unlocked account (decrypted private keys)
+
+	wallets     []accounts.Wallet       // Wallet wrappers around the individual key files
+	updateFeed  event.Feed              // Event feed to notify wallet additions/removals
+	updateScope event.SubscriptionScope // Subscription scope tracking current live listeners
+	updating    bool                    // Whether the event notification loop is running
+
+	mu sync.RWMutex
+}
+
+type unlocked struct {
+	*Key
+	abort chan struct{}
+}
+
+// NewKeyStore creates a keystore for the given directory.
+func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore {
+	keydir, _ = filepath.Abs(keydir)
+	ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP}}
+	ks.init(keydir)
+	return ks
+}
+
+// NewPlaintextKeyStore creates a keystore for the given directory.
+// Deprecated: Use NewKeyStore.
+func NewPlaintextKeyStore(keydir string) *KeyStore {
+	keydir, _ = filepath.Abs(keydir)
+	ks := &KeyStore{storage: &keyStorePlain{keydir}}
+	ks.init(keydir)
+	return ks
+}
+
+func (ks *KeyStore) init(keydir string) {
+	// Lock the mutex since the account cache might call back with events
+	ks.mu.Lock()
+	defer ks.mu.Unlock()
+
+	// Initialize the set of unlocked keys and the account cache
+	ks.unlocked = make(map[common.Address]*unlocked)
+	ks.cache, ks.changes = newAccountCache(keydir)
+
+	// TODO: In order for this finalizer to work, there must be no references
+	// to ks. addressCache doesn't keep a reference but unlocked keys do,
+	// so the finalizer will not trigger until all timed unlocks have expired.
+	runtime.SetFinalizer(ks, func(m *KeyStore) {
+		m.cache.close()
+	})
+	// Create the initial list of wallets from the cache
+	accs := ks.cache.accounts()
+	ks.wallets = make([]accounts.Wallet, len(accs))
+	for i := 0; i < len(accs); i++ {
+		ks.wallets[i] = &keystoreWallet{account: accs[i], keystore: ks}
+	}
+}
+
+// Wallets implements accounts.Backend, returning all single-key wallets from the
+// keystore directory.
+func (ks *KeyStore) Wallets() []accounts.Wallet {
+	// Make sure the list of wallets is in sync with the account cache
+	ks.refreshWallets()
+
+	ks.mu.RLock()
+	defer ks.mu.RUnlock()
+
+	cpy := make([]accounts.Wallet, len(ks.wallets))
+	copy(cpy, ks.wallets)
+	return cpy
+}
+
+// refreshWallets retrieves the current account list and based on that does any
+// necessary wallet refreshes.
+func (ks *KeyStore) refreshWallets() {
+	// Retrieve the current list of accounts
+	ks.mu.Lock()
+	accs := ks.cache.accounts()
+
+	// Transform the current list of wallets into the new one
+	wallets := make([]accounts.Wallet, 0, len(accs))
+	events := []accounts.WalletEvent{}
+
+	for _, account := range accs {
+		// Drop wallets while they were in front of the next account
+		for len(ks.wallets) > 0 && ks.wallets[0].URL().Cmp(account.URL) < 0 {
+			events = append(events, accounts.WalletEvent{Wallet: ks.wallets[0], Arrive: false})
+			ks.wallets = ks.wallets[1:]
+		}
+		// If there are no more wallets or the account is before the next, wrap new wallet
+		if len(ks.wallets) == 0 || ks.wallets[0].URL().Cmp(account.URL) > 0 {
+			wallet := &keystoreWallet{account: account, keystore: ks}
+
+			events = append(events, accounts.WalletEvent{Wallet: wallet, Arrive: true})
+			wallets = append(wallets, wallet)
+			continue
+		}
+		// If the account is the same as the first wallet, keep it
+		if ks.wallets[0].Accounts()[0] == account {
+			wallets = append(wallets, ks.wallets[0])
+			ks.wallets = ks.wallets[1:]
+			continue
+		}
+	}
+	// Drop any leftover wallets and set the new batch
+	for _, wallet := range ks.wallets {
+		events = append(events, accounts.WalletEvent{Wallet: wallet, Arrive: false})
+	}
+	ks.wallets = wallets
+	ks.mu.Unlock()
+
+	// Fire all wallet events and return
+	for _, event := range events {
+		ks.updateFeed.Send(event)
+	}
+}
+
+// Subscribe implements accounts.Backend, creating an async subscription to
+// receive notifications on the addition or removal of keystore wallets.
+func (ks *KeyStore) Subscribe(sink chan<- accounts.WalletEvent) event.Subscription {
+	// We need the mutex to reliably start/stop the update loop
+	ks.mu.Lock()
+	defer ks.mu.Unlock()
+
+	// Subscribe the caller and track the subscriber count
+	sub := ks.updateScope.Track(ks.updateFeed.Subscribe(sink))
+
+	// Subscribers require an active notification loop, start it
+	if !ks.updating {
+		ks.updating = true
+		go ks.updater()
+	}
+	return sub
+}
+
+// updater is responsible for maintaining an up-to-date list of wallets stored in
+// the keystore, and for firing wallet addition/removal events. It listens for
+// account change events from the underlying account cache, and also periodically
+// forces a manual refresh (only triggers for systems where the filesystem notifier
+// is not running).
+func (ks *KeyStore) updater() {
+	for {
+		// Wait for an account update or a refresh timeout
+		select {
+		case <-ks.changes:
+		case <-time.After(walletRefreshCycle):
+		}
+		// Run the wallet refresher
+		ks.refreshWallets()
+
+		// If all our subscribers left, stop the updater
+		ks.mu.Lock()
+		if ks.updateScope.Count() == 0 {
+			ks.updating = false
+			ks.mu.Unlock()
+			return
+		}
+		ks.mu.Unlock()
+	}
+}
+
+// HasAddress reports whether a key with the given address is present.
+func (ks *KeyStore) HasAddress(addr common.Address) bool {
+	return ks.cache.hasAddress(addr)
+}
+
+// Accounts returns all key files present in the directory.
+func (ks *KeyStore) Accounts() []accounts.Account {
+	return ks.cache.accounts()
+}
+
+// Delete deletes the key matched by account if the passphrase is correct.
+// If the account contains no filename, the address must match a unique key.
+func (ks *KeyStore) Delete(a accounts.Account, passphrase string) error {
+	// Decrypting the key isn't really necessary, but we do
+	// it anyway to check the password and zero out the key
+	// immediately afterwards.
+	a, key, err := ks.getDecryptedKey(a, passphrase)
+	if key != nil {
+		zeroKey(key.PrivateKey)
+	}
+	if err != nil {
+		return err
+	}
+	// The order is crucial here. The key is dropped from the
+	// cache after the file is gone so that a reload happening in
+	// between won't insert it into the cache again.
+	err = os.Remove(a.URL.Path)
+	if err == nil {
+		ks.cache.delete(a)
+		ks.refreshWallets()
+	}
+	return err
+}
+
+// SignHash calculates a ECDSA signature for the given hash. The produced
+// signature is in the [R || S || V] format where V is 0 or 1.
+func (ks *KeyStore) SignHash(a accounts.Account, hash []byte) ([]byte, error) {
+	// Look up the key to sign with and abort if it cannot be found
+	ks.mu.RLock()
+	defer ks.mu.RUnlock()
+
+	unlockedKey, found := ks.unlocked[a.Address]
+	if !found {
+		return nil, ErrLocked
+	}
+	// Sign the hash using plain ECDSA operations
+	return crypto.Sign(hash, unlockedKey.PrivateKey)
+}
+
+// SignTx signs the given transaction with the requested account.
+func (ks *KeyStore) SignTx(a accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	// Look up the key to sign with and abort if it cannot be found
+	ks.mu.RLock()
+	defer ks.mu.RUnlock()
+
+	unlockedKey, found := ks.unlocked[a.Address]
+	if !found {
+		return nil, ErrLocked
+	}
+	// Depending on the presence of the chain ID, sign with EIP155 or homestead
+	if chainID != nil {
+		return types.SignTx(tx, types.NewEIP155Signer(chainID), unlockedKey.PrivateKey)
+	}
+	return types.SignTx(tx, types.HomesteadSigner{}, unlockedKey.PrivateKey)
+}
+
+// SignHashWithPassphrase signs hash if the private key matching the given address
+// can be decrypted with the given passphrase. The produced signature is in the
+// [R || S || V] format where V is 0 or 1.
+func (ks *KeyStore) SignHashWithPassphrase(a accounts.Account, passphrase string, hash []byte) (signature []byte, err error) {
+	_, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return nil, err
+	}
+	defer zeroKey(key.PrivateKey)
+	return crypto.Sign(hash, key.PrivateKey)
+}
+
+// SignTxWithPassphrase signs the transaction if the private key matching the
+// given address can be decrypted with the given passphrase.
+func (ks *KeyStore) SignTxWithPassphrase(a accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	_, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return nil, err
+	}
+	defer zeroKey(key.PrivateKey)
+
+	// Depending on the presence of the chain ID, sign with EIP155 or homestead
+	if chainID != nil {
+		return types.SignTx(tx, types.NewEIP155Signer(chainID), key.PrivateKey)
+	}
+	return types.SignTx(tx, types.HomesteadSigner{}, key.PrivateKey)
+}
+
+// Unlock unlocks the given account indefinitely.
+func (ks *KeyStore) Unlock(a accounts.Account, passphrase string) error {
+	return ks.TimedUnlock(a, passphrase, 0)
+}
+
+// Lock removes the private key with the given address from memory.
+func (ks *KeyStore) Lock(addr common.Address) error {
+	ks.mu.Lock()
+	if unl, found := ks.unlocked[addr]; found {
+		ks.mu.Unlock()
+		ks.expire(addr, unl, time.Duration(0)*time.Nanosecond)
+	} else {
+		ks.mu.Unlock()
+	}
+	return nil
+}
+
+// TimedUnlock unlocks the given account with the passphrase. The account
+// stays unlocked for the duration of timeout. A timeout of 0 unlocks the account
+// until the program exits. The account must match a unique key file.
+//
+// If the account address is already unlocked for a duration, TimedUnlock extends or
+// shortens the active unlock timeout. If the address was previously unlocked
+// indefinitely the timeout is not altered.
+func (ks *KeyStore) TimedUnlock(a accounts.Account, passphrase string, timeout time.Duration) error {
+	a, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return err
+	}
+
+	ks.mu.Lock()
+	defer ks.mu.Unlock()
+	u, found := ks.unlocked[a.Address]
+	if found {
+		if u.abort == nil {
+			// The address was unlocked indefinitely, so unlocking
+			// it with a timeout would be confusing.
+			zeroKey(key.PrivateKey)
+			return nil
+		}
+		// Terminate the expire goroutine and replace it below.
+		close(u.abort)
+	}
+	if timeout > 0 {
+		u = &unlocked{Key: key, abort: make(chan struct{})}
+		go ks.expire(a.Address, u, timeout)
+	} else {
+		u = &unlocked{Key: key}
+	}
+	ks.unlocked[a.Address] = u
+	return nil
+}
+
+// Find resolves the given account into a unique entry in the keystore.
+func (ks *KeyStore) Find(a accounts.Account) (accounts.Account, error) {
+	ks.cache.maybeReload()
+	ks.cache.mu.Lock()
+	a, err := ks.cache.find(a)
+	ks.cache.mu.Unlock()
+	return a, err
+}
+
+func (ks *KeyStore) getDecryptedKey(a accounts.Account, auth string) (accounts.Account, *Key, error) {
+	a, err := ks.Find(a)
+	if err != nil {
+		return a, nil, err
+	}
+	key, err := ks.storage.GetKey(a.Address, a.URL.Path, auth)
+	return a, key, err
+}
+
+func (ks *KeyStore) expire(addr common.Address, u *unlocked, timeout time.Duration) {
+	t := time.NewTimer(timeout)
+	defer t.Stop()
+	select {
+	case <-u.abort:
+		// just quit
+	case <-t.C:
+		ks.mu.Lock()
+		// only drop if it's still the same key instance that dropLater
+		// was launched with. we can check that using pointer equality
+		// because the map stores a new pointer every time the key is
+		// unlocked.
+		if ks.unlocked[addr] == u {
+			zeroKey(u.PrivateKey)
+			delete(ks.unlocked, addr)
+		}
+		ks.mu.Unlock()
+	}
+}
+
+// NewAccount generates a new key and stores it into the key directory,
+// encrypting it with the passphrase.
+func (ks *KeyStore) NewAccount(passphrase string) (accounts.Account, error) {
+	_, account, err := storeNewKey(ks.storage, crand.Reader, passphrase)
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	// Add the account to the cache immediately rather
+	// than waiting for file system notifications to pick it up.
+	ks.cache.add(account)
+	ks.refreshWallets()
+	return account, nil
+}
+
+// Export exports as a JSON key, encrypted with newPassphrase.
+func (ks *KeyStore) Export(a accounts.Account, passphrase, newPassphrase string) (keyJSON []byte, err error) {
+	_, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return nil, err
+	}
+	var N, P int
+	if store, ok := ks.storage.(*keyStorePassphrase); ok {
+		N, P = store.scryptN, store.scryptP
+	} else {
+		N, P = StandardScryptN, StandardScryptP
+	}
+	return EncryptKey(key, newPassphrase, N, P)
+}
+
+// Import stores the given encrypted JSON key into the key directory.
+func (ks *KeyStore) Import(keyJSON []byte, passphrase, newPassphrase string) (accounts.Account, error) {
+	key, err := DecryptKey(keyJSON, passphrase)
+	if key != nil && key.PrivateKey != nil {
+		defer zeroKey(key.PrivateKey)
+	}
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	return ks.importKey(key, newPassphrase)
+}
+
+// ImportECDSA stores the given key into the key directory, encrypting it with the passphrase.
+func (ks *KeyStore) ImportECDSA(priv *ecdsa.PrivateKey, passphrase string) (accounts.Account, error) {
+	key := newKeyFromECDSA(priv)
+	if ks.cache.hasAddress(key.Address) {
+		return accounts.Account{}, fmt.Errorf("account already exists")
+	}
+	return ks.importKey(key, passphrase)
+}
+
+func (ks *KeyStore) importKey(key *Key, passphrase string) (accounts.Account, error) {
+	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.storage.JoinPath(keyFileName(key.Address))}}
+	if err := ks.storage.StoreKey(a.URL.Path, key, passphrase); err != nil {
+		return accounts.Account{}, err
+	}
+	ks.cache.add(a)
+	ks.refreshWallets()
+	return a, nil
+}
+
+// Update changes the passphrase of an existing account.
+func (ks *KeyStore) Update(a accounts.Account, passphrase, newPassphrase string) error {
+	a, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return err
+	}
+	return ks.storage.StoreKey(a.URL.Path, key, newPassphrase)
+}
+
+// ImportPreSaleKey decrypts the given Ethereum presale wallet and stores
+// a key file in the key directory. The key file is encrypted with the same passphrase.
+func (ks *KeyStore) ImportPreSaleKey(keyJSON []byte, passphrase string) (accounts.Account, error) {
+	a, _, err := importPreSaleKey(ks.storage, keyJSON, passphrase)
+	if err != nil {
+		return a, err
+	}
+	ks.cache.add(a)
+	ks.refreshWallets()
+	return a, nil
+}
+
+// zeroKey zeroes a private key in memory.
+func zeroKey(k *ecdsa.PrivateKey) {
+	b := k.D.Bits()
+	for i := range b {
+		b[i] = 0
+	}
+}

accounts/keystore/keystore_passphrase.go

@@ -23,7 +23,7 @@ The crypto is documented at https://github.com/ethereum/wiki/wiki/Web3-Secret-St
 
 */
 
-package accounts
+package keystore
 
 import (
 	"bytes"
@@ -36,6 +36,7 @@ import (
 	"path/filepath"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/crypto/randentropy"
 	"github.com/pborman/uuid"
@@ -115,8 +116,7 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
 		return nil, err
 	}
 	encryptKey := derivedKey[:16]
-	keyBytes0 := crypto.FromECDSA(key.PrivateKey)
-	keyBytes := common.LeftPadBytes(keyBytes0, 32)
+	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
 
 	iv := randentropy.GetEntropyCSPRNG(aes.BlockSize) // 16
 	cipherText, err := aesCTRXOR(encryptKey, keyBytes, iv)
@@ -182,7 +182,8 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) {
 	if err != nil {
 		return nil, err
 	}
-	key := crypto.ToECDSA(keyBytes)
+	key := crypto.ToECDSAUnsafe(keyBytes)
+
 	return &Key{
 		Id:         uuid.UUID(keyId),
 		Address:    crypto.PubkeyToAddress(key.PublicKey),

accounts/keystore/keystore_passphrase_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"io/ioutil"
@@ -46,7 +46,7 @@ func TestKeyEncryptDecrypt(t *testing.T) {
 		// Decrypt with the correct password
 		key, err := DecryptKey(keyjson, password)
 		if err != nil {
-			t.Errorf("test %d: json key failed to decrypt: %v", i, err)
+			t.Fatalf("test %d: json key failed to decrypt: %v", i, err)
 		}
 		if key.Address != address {
 			t.Errorf("test %d: key address mismatch: have %x, want %x", i, key.Address, address)

accounts/keystore/keystore_plain.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"encoding/json"

accounts/keystore/keystore_plain_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"crypto/rand"
@@ -30,7 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
-func tmpKeyStore(t *testing.T, encrypted bool) (dir string, ks keyStore) {
+func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) {
 	d, err := ioutil.TempDir("", "geth-keystore-test")
 	if err != nil {
 		t.Fatal(err)
@@ -44,7 +44,7 @@ func tmpKeyStore(t *testing.T, encrypted bool) (dir string, ks keyStore) {
 }
 
 func TestKeyStorePlain(t *testing.T) {
-	dir, ks := tmpKeyStore(t, false)
+	dir, ks := tmpKeyStoreIface(t, false)
 	defer os.RemoveAll(dir)
 
 	pass := "" // not used but required by API
@@ -52,7 +52,7 @@ func TestKeyStorePlain(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	k2, err := ks.GetKey(k1.Address, account.File, pass)
+	k2, err := ks.GetKey(k1.Address, account.URL.Path, pass)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -65,7 +65,7 @@ func TestKeyStorePlain(t *testing.T) {
 }
 
 func TestKeyStorePassphrase(t *testing.T) {
-	dir, ks := tmpKeyStore(t, true)
+	dir, ks := tmpKeyStoreIface(t, true)
 	defer os.RemoveAll(dir)
 
 	pass := "foo"
@@ -73,7 +73,7 @@ func TestKeyStorePassphrase(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	k2, err := ks.GetKey(k1.Address, account.File, pass)
+	k2, err := ks.GetKey(k1.Address, account.URL.Path, pass)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -86,7 +86,7 @@ func TestKeyStorePassphrase(t *testing.T) {
 }
 
 func TestKeyStorePassphraseDecryptionFail(t *testing.T) {
-	dir, ks := tmpKeyStore(t, true)
+	dir, ks := tmpKeyStoreIface(t, true)
 	defer os.RemoveAll(dir)
 
 	pass := "foo"
@@ -94,13 +94,13 @@ func TestKeyStorePassphraseDecryptionFail(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if _, err = ks.GetKey(k1.Address, account.File, "bar"); err != ErrDecrypt {
+	if _, err = ks.GetKey(k1.Address, account.URL.Path, "bar"); err != ErrDecrypt {
 		t.Fatalf("wrong error for invalid passphrase\ngot %q\nwant %q", err, ErrDecrypt)
 	}
 }
 
 func TestImportPreSaleKey(t *testing.T) {
-	dir, ks := tmpKeyStore(t, true)
+	dir, ks := tmpKeyStoreIface(t, true)
 	defer os.RemoveAll(dir)
 
 	// file content of a presale key file generated with:
@@ -115,8 +115,8 @@ func TestImportPreSaleKey(t *testing.T) {
 	if account.Address != common.HexToAddress("d4584b5f6229b7be90727b0fc8c6b91bb427821f") {
 		t.Errorf("imported account has wrong address %x", account.Address)
 	}
-	if !strings.HasPrefix(account.File, dir) {
-		t.Errorf("imported account file not in keystore directory: %q", account.File)
+	if !strings.HasPrefix(account.URL.Path, dir) {
+		t.Errorf("imported account file not in keystore directory: %q", account.URL)
 	}
 }
 
@@ -142,19 +142,19 @@ func TestV3_PBKDF2_1(t *testing.T) {
 
 func TestV3_PBKDF2_2(t *testing.T) {
 	t.Parallel()
-	tests := loadKeyStoreTestV3("../tests/files/KeyStoreTests/basic_tests.json", t)
+	tests := loadKeyStoreTestV3("../../tests/files/KeyStoreTests/basic_tests.json", t)
 	testDecryptV3(tests["test1"], t)
 }
 
 func TestV3_PBKDF2_3(t *testing.T) {
 	t.Parallel()
-	tests := loadKeyStoreTestV3("../tests/files/KeyStoreTests/basic_tests.json", t)
+	tests := loadKeyStoreTestV3("../../tests/files/KeyStoreTests/basic_tests.json", t)
 	testDecryptV3(tests["python_generated_test_with_odd_iv"], t)
 }
 
 func TestV3_PBKDF2_4(t *testing.T) {
 	t.Parallel()
-	tests := loadKeyStoreTestV3("../tests/files/KeyStoreTests/basic_tests.json", t)
+	tests := loadKeyStoreTestV3("../../tests/files/KeyStoreTests/basic_tests.json", t)
 	testDecryptV3(tests["evilnonce"], t)
 }
 
@@ -166,7 +166,7 @@ func TestV3_Scrypt_1(t *testing.T) {
 
 func TestV3_Scrypt_2(t *testing.T) {
 	t.Parallel()
-	tests := loadKeyStoreTestV3("../tests/files/KeyStoreTests/basic_tests.json", t)
+	tests := loadKeyStoreTestV3("../../tests/files/KeyStoreTests/basic_tests.json", t)
 	testDecryptV3(tests["test2"], t)
 }
 

accounts/keystore/keystore_test.go

@@ -0,0 +1,365 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package keystore
+
+import (
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"runtime"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/event"
+)
+
+var testSigData = make([]byte, 32)
+
+func TestKeyStore(t *testing.T) {
+	dir, ks := tmpKeyStore(t, true)
+	defer os.RemoveAll(dir)
+
+	a, err := ks.NewAccount("foo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(a.URL.Path, dir) {
+		t.Errorf("account file %s doesn't have dir prefix", a.URL)
+	}
+	stat, err := os.Stat(a.URL.Path)
+	if err != nil {
+		t.Fatalf("account file %s doesn't exist (%v)", a.URL, err)
+	}
+	if runtime.GOOS != "windows" && stat.Mode() != 0600 {
+		t.Fatalf("account file has wrong mode: got %o, want %o", stat.Mode(), 0600)
+	}
+	if !ks.HasAddress(a.Address) {
+		t.Errorf("HasAccount(%x) should've returned true", a.Address)
+	}
+	if err := ks.Update(a, "foo", "bar"); err != nil {
+		t.Errorf("Update error: %v", err)
+	}
+	if err := ks.Delete(a, "bar"); err != nil {
+		t.Errorf("Delete error: %v", err)
+	}
+	if common.FileExist(a.URL.Path) {
+		t.Errorf("account file %s should be gone after Delete", a.URL)
+	}
+	if ks.HasAddress(a.Address) {
+		t.Errorf("HasAccount(%x) should've returned true after Delete", a.Address)
+	}
+}
+
+func TestSign(t *testing.T) {
+	dir, ks := tmpKeyStore(t, true)
+	defer os.RemoveAll(dir)
+
+	pass := "" // not used but required by API
+	a1, err := ks.NewAccount(pass)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := ks.Unlock(a1, ""); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ks.SignHash(accounts.Account{Address: a1.Address}, testSigData); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSignWithPassphrase(t *testing.T) {
+	dir, ks := tmpKeyStore(t, true)
+	defer os.RemoveAll(dir)
+
+	pass := "passwd"
+	acc, err := ks.NewAccount(pass)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, unlocked := ks.unlocked[acc.Address]; unlocked {
+		t.Fatal("expected account to be locked")
+	}
+
+	_, err = ks.SignHashWithPassphrase(acc, pass, testSigData)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, unlocked := ks.unlocked[acc.Address]; unlocked {
+		t.Fatal("expected account to be locked")
+	}
+
+	if _, err = ks.SignHashWithPassphrase(acc, "invalid passwd", testSigData); err == nil {
+		t.Fatal("expected SignHashWithPassphrase to fail with invalid password")
+	}
+}
+
+func TestTimedUnlock(t *testing.T) {
+	dir, ks := tmpKeyStore(t, true)
+	defer os.RemoveAll(dir)
+
+	pass := "foo"
+	a1, err := ks.NewAccount(pass)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Signing without passphrase fails because account is locked
+	_, err = ks.SignHash(accounts.Account{Address: a1.Address}, testSigData)
+	if err != ErrLocked {
+		t.Fatal("Signing should've failed with ErrLocked before unlocking, got ", err)
+	}
+
+	// Signing with passphrase works
+	if err = ks.TimedUnlock(a1, pass, 100*time.Millisecond); err != nil {
+		t.Fatal(err)
+	}
+
+	// Signing without passphrase works because account is temp unlocked
+	_, err = ks.SignHash(accounts.Account{Address: a1.Address}, testSigData)
+	if err != nil {
+		t.Fatal("Signing shouldn't return an error after unlocking, got ", err)
+	}
+
+	// Signing fails again after automatic locking
+	time.Sleep(250 * time.Millisecond)
+	_, err = ks.SignHash(accounts.Account{Address: a1.Address}, testSigData)
+	if err != ErrLocked {
+		t.Fatal("Signing should've failed with ErrLocked timeout expired, got ", err)
+	}
+}
+
+func TestOverrideUnlock(t *testing.T) {
+	dir, ks := tmpKeyStore(t, false)
+	defer os.RemoveAll(dir)
+
+	pass := "foo"
+	a1, err := ks.NewAccount(pass)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Unlock indefinitely.
+	if err = ks.TimedUnlock(a1, pass, 5*time.Minute); err != nil {
+		t.Fatal(err)
+	}
+
+	// Signing without passphrase works because account is temp unlocked
+	_, err = ks.SignHash(accounts.Account{Address: a1.Address}, testSigData)
+	if err != nil {
+		t.Fatal("Signing shouldn't return an error after unlocking, got ", err)
+	}
+
+	// reset unlock to a shorter period, invalidates the previous unlock
+	if err = ks.TimedUnlock(a1, pass, 100*time.Millisecond); err != nil {
+		t.Fatal(err)
+	}
+
+	// Signing without passphrase still works because account is temp unlocked
+	_, err = ks.SignHash(accounts.Account{Address: a1.Address}, testSigData)
+	if err != nil {
+		t.Fatal("Signing shouldn't return an error after unlocking, got ", err)
+	}
+
+	// Signing fails again after automatic locking
+	time.Sleep(250 * time.Millisecond)
+	_, err = ks.SignHash(accounts.Account{Address: a1.Address}, testSigData)
+	if err != ErrLocked {
+		t.Fatal("Signing should've failed with ErrLocked timeout expired, got ", err)
+	}
+}
+
+// This test should fail under -race if signing races the expiration goroutine.
+func TestSignRace(t *testing.T) {
+	dir, ks := tmpKeyStore(t, false)
+	defer os.RemoveAll(dir)
+
+	// Create a test account.
+	a1, err := ks.NewAccount("")
+	if err != nil {
+		t.Fatal("could not create the test account", err)
+	}
+
+	if err := ks.TimedUnlock(a1, "", 15*time.Millisecond); err != nil {
+		t.Fatal("could not unlock the test account", err)
+	}
+	end := time.Now().Add(500 * time.Millisecond)
+	for time.Now().Before(end) {
+		if _, err := ks.SignHash(accounts.Account{Address: a1.Address}, testSigData); err == ErrLocked {
+			return
+		} else if err != nil {
+			t.Errorf("Sign error: %v", err)
+			return
+		}
+		time.Sleep(1 * time.Millisecond)
+	}
+	t.Errorf("Account did not lock within the timeout")
+}
+
+// Tests that the wallet notifier loop starts and stops correctly based on the
+// addition and removal of wallet event subscriptions.
+func TestWalletNotifierLifecycle(t *testing.T) {
+	// Create a temporary kesytore to test with
+	dir, ks := tmpKeyStore(t, false)
+	defer os.RemoveAll(dir)
+
+	// Ensure that the notification updater is not running yet
+	time.Sleep(250 * time.Millisecond)
+	ks.mu.RLock()
+	updating := ks.updating
+	ks.mu.RUnlock()
+
+	if updating {
+		t.Errorf("wallet notifier running without subscribers")
+	}
+	// Subscribe to the wallet feed and ensure the updater boots up
+	updates := make(chan accounts.WalletEvent)
+
+	subs := make([]event.Subscription, 2)
+	for i := 0; i < len(subs); i++ {
+		// Create a new subscription
+		subs[i] = ks.Subscribe(updates)
+
+		// Ensure the notifier comes online
+		time.Sleep(250 * time.Millisecond)
+		ks.mu.RLock()
+		updating = ks.updating
+		ks.mu.RUnlock()
+
+		if !updating {
+			t.Errorf("sub %d: wallet notifier not running after subscription", i)
+		}
+	}
+	// Unsubscribe and ensure the updater terminates eventually
+	for i := 0; i < len(subs); i++ {
+		// Close an existing subscription
+		subs[i].Unsubscribe()
+
+		// Ensure the notifier shuts down at and only at the last close
+		for k := 0; k < int(walletRefreshCycle/(250*time.Millisecond))+2; k++ {
+			ks.mu.RLock()
+			updating = ks.updating
+			ks.mu.RUnlock()
+
+			if i < len(subs)-1 && !updating {
+				t.Fatalf("sub %d: event notifier stopped prematurely", i)
+			}
+			if i == len(subs)-1 && !updating {
+				return
+			}
+			time.Sleep(250 * time.Millisecond)
+		}
+	}
+	t.Errorf("wallet notifier didn't terminate after unsubscribe")
+}
+
+// Tests that wallet notifications and correctly fired when accounts are added
+// or deleted from the keystore.
+func TestWalletNotifications(t *testing.T) {
+	// Create a temporary kesytore to test with
+	dir, ks := tmpKeyStore(t, false)
+	defer os.RemoveAll(dir)
+
+	// Subscribe to the wallet feed
+	updates := make(chan accounts.WalletEvent, 1)
+	sub := ks.Subscribe(updates)
+	defer sub.Unsubscribe()
+
+	// Randomly add and remove account and make sure events and wallets are in sync
+	live := make(map[common.Address]accounts.Account)
+	for i := 0; i < 1024; i++ {
+		// Execute a creation or deletion and ensure event arrival
+		if create := len(live) == 0 || rand.Int()%4 > 0; create {
+			// Add a new account and ensure wallet notifications arrives
+			account, err := ks.NewAccount("")
+			if err != nil {
+				t.Fatalf("failed to create test account: %v", err)
+			}
+			select {
+			case event := <-updates:
+				if !event.Arrive {
+					t.Errorf("departure event on account creation")
+				}
+				if event.Wallet.Accounts()[0] != account {
+					t.Errorf("account mismatch on created wallet: have %v, want %v", event.Wallet.Accounts()[0], account)
+				}
+			default:
+				t.Errorf("wallet arrival event not fired on account creation")
+			}
+			live[account.Address] = account
+		} else {
+			// Select a random account to delete (crude, but works)
+			var account accounts.Account
+			for _, a := range live {
+				account = a
+				break
+			}
+			// Remove an account and ensure wallet notifiaction arrives
+			if err := ks.Delete(account, ""); err != nil {
+				t.Fatalf("failed to delete test account: %v", err)
+			}
+			select {
+			case event := <-updates:
+				if event.Arrive {
+					t.Errorf("arrival event on account deletion")
+				}
+				if event.Wallet.Accounts()[0] != account {
+					t.Errorf("account mismatch on deleted wallet: have %v, want %v", event.Wallet.Accounts()[0], account)
+				}
+			default:
+				t.Errorf("wallet departure event not fired on account creation")
+			}
+			delete(live, account.Address)
+		}
+		// Retrieve the list of wallets and ensure it matches with our required live set
+		liveList := make([]accounts.Account, 0, len(live))
+		for _, account := range live {
+			liveList = append(liveList, account)
+		}
+		sort.Sort(accountsByURL(liveList))
+
+		wallets := ks.Wallets()
+		if len(liveList) != len(wallets) {
+			t.Errorf("wallet list doesn't match required accounts: have %v, want %v", wallets, liveList)
+		} else {
+			for j, wallet := range wallets {
+				if accs := wallet.Accounts(); len(accs) != 1 {
+					t.Errorf("wallet %d: contains invalid number of accounts: have %d, want 1", j, len(accs))
+				} else if accs[0] != liveList[j] {
+					t.Errorf("wallet %d: account mismatch: have %v, want %v", j, accs[0], liveList[j])
+				}
+			}
+		}
+	}
+}
+
+func tmpKeyStore(t *testing.T, encrypted bool) (string, *KeyStore) {
+	d, err := ioutil.TempDir("", "eth-keystore-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	new := NewPlaintextKeyStore
+	if encrypted {
+		new = func(kd string) *KeyStore { return NewKeyStore(kd, veryLightScryptN, veryLightScryptP) }
+	}
+	return d, new(d)
+}

accounts/keystore/keystore_wallet.go

@@ -0,0 +1,139 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package keystore
+
+import (
+	"math/big"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+// keystoreWallet implements the accounts.Wallet interface for the original
+// keystore.
+type keystoreWallet struct {
+	account  accounts.Account // Single account contained in this wallet
+	keystore *KeyStore        // Keystore where the account originates from
+}
+
+// URL implements accounts.Wallet, returning the URL of the account within.
+func (w *keystoreWallet) URL() accounts.URL {
+	return w.account.URL
+}
+
+// Status implements accounts.Wallet, always returning "open", since there is no
+// concept of open/close for plain keystore accounts.
+func (w *keystoreWallet) Status() string {
+	w.keystore.mu.RLock()
+	defer w.keystore.mu.RUnlock()
+
+	if _, ok := w.keystore.unlocked[w.account.Address]; ok {
+		return "Unlocked"
+	}
+	return "Locked"
+}
+
+// Open implements accounts.Wallet, but is a noop for plain wallets since there
+// is no connection or decryption step necessary to access the list of accounts.
+func (w *keystoreWallet) Open(passphrase string) error { return nil }
+
+// Close implements accounts.Wallet, but is a noop for plain wallets since is no
+// meaningful open operation.
+func (w *keystoreWallet) Close() error { return nil }
+
+// Accounts implements accounts.Wallet, returning an account list consisting of
+// a single account that the plain kestore wallet contains.
+func (w *keystoreWallet) Accounts() []accounts.Account {
+	return []accounts.Account{w.account}
+}
+
+// Contains implements accounts.Wallet, returning whether a particular account is
+// or is not wrapped by this wallet instance.
+func (w *keystoreWallet) Contains(account accounts.Account) bool {
+	return account.Address == w.account.Address && (account.URL == (accounts.URL{}) || account.URL == w.account.URL)
+}
+
+// Derive implements accounts.Wallet, but is a noop for plain wallets since there
+// is no notion of hierarchical account derivation for plain keystore accounts.
+func (w *keystoreWallet) Derive(path accounts.DerivationPath, pin bool) (accounts.Account, error) {
+	return accounts.Account{}, accounts.ErrNotSupported
+}
+
+// SelfDerive implements accounts.Wallet, but is a noop for plain wallets since
+// there is no notion of hierarchical account derivation for plain keystore accounts.
+func (w *keystoreWallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {}
+
+// SignHash implements accounts.Wallet, attempting to sign the given hash with
+// the given account. If the wallet does not wrap this particular account, an
+// error is returned to avoid account leakage (even though in theory we may be
+// able to sign via our shared keystore backend).
+func (w *keystoreWallet) SignHash(account accounts.Account, hash []byte) ([]byte, error) {
+	// Make sure the requested account is contained within
+	if account.Address != w.account.Address {
+		return nil, accounts.ErrUnknownAccount
+	}
+	if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignHash(account, hash)
+}
+
+// SignTx implements accounts.Wallet, attempting to sign the given transaction
+// with the given account. If the wallet does not wrap this particular account,
+// an error is returned to avoid account leakage (even though in theory we may
+// be able to sign via our shared keystore backend).
+func (w *keystoreWallet) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	// Make sure the requested account is contained within
+	if account.Address != w.account.Address {
+		return nil, accounts.ErrUnknownAccount
+	}
+	if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignTx(account, tx, chainID)
+}
+
+// SignHashWithPassphrase implements accounts.Wallet, attempting to sign the
+// given hash with the given account using passphrase as extra authentication.
+func (w *keystoreWallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
+	// Make sure the requested account is contained within
+	if account.Address != w.account.Address {
+		return nil, accounts.ErrUnknownAccount
+	}
+	if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignHashWithPassphrase(account, passphrase, hash)
+}
+
+// SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given
+// transaction with the given account using passphrase as extra authentication.
+func (w *keystoreWallet) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	// Make sure the requested account is contained within
+	if account.Address != w.account.Address {
+		return nil, accounts.ErrUnknownAccount
+	}
+	if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignTxWithPassphrase(account, passphrase, tx, chainID)
+}

accounts/keystore/presale.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"crypto/aes"
@@ -25,20 +25,21 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/pborman/uuid"
 	"golang.org/x/crypto/pbkdf2"
 )
 
 // creates a Key and stores that in the given KeyStore by decrypting a presale key JSON
-func importPreSaleKey(keyStore keyStore, keyJSON []byte, password string) (Account, *Key, error) {
+func importPreSaleKey(keyStore keyStore, keyJSON []byte, password string) (accounts.Account, *Key, error) {
 	key, err := decryptPreSaleKey(keyJSON, password)
 	if err != nil {
-		return Account{}, nil, err
+		return accounts.Account{}, nil, err
 	}
 	key.Id = uuid.NewRandom()
-	a := Account{Address: key.Address, File: keyStore.JoinPath(keyFileName(key.Address))}
-	err = keyStore.StoreKey(a.File, key, password)
+	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: keyStore.JoinPath(keyFileName(key.Address))}}
+	err = keyStore.StoreKey(a.URL.Path, key, password)
 	return a, key, err
 }
 
@@ -73,7 +74,8 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error
 		return nil, err
 	}
 	ethPriv := crypto.Keccak256(plainText)
-	ecKey := crypto.ToECDSA(ethPriv)
+	ecKey := crypto.ToECDSAUnsafe(ethPriv)
+
 	key = &Key{
 		Id:         nil,
 		Address:    crypto.PubkeyToAddress(ecKey.PublicKey),

accounts/keystore/watch.go

@@ -16,25 +16,24 @@
 
 // +build darwin,!ios freebsd linux,!arm64 netbsd solaris
 
-package accounts
+package keystore
 
 import (
 	"time"
 
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/rjeczalik/notify"
 )
 
 type watcher struct {
-	ac       *addrCache
+	ac       *accountCache
 	starting bool
 	running  bool
 	ev       chan notify.EventInfo
 	quit     chan struct{}
 }
 
-func newWatcher(ac *addrCache) *watcher {
+func newWatcher(ac *accountCache) *watcher {
 	return &watcher{
 		ac:   ac,
 		ev:   make(chan notify.EventInfo, 10),
@@ -64,15 +63,16 @@ func (w *watcher) loop() {
 		w.starting = false
 		w.ac.mu.Unlock()
 	}()
+	logger := log.New("path", w.ac.keydir)
 
-	err := notify.Watch(w.ac.keydir, w.ev, notify.All)
-	if err != nil {
-		glog.V(logger.Detail).Infof("can't watch %s: %v", w.ac.keydir, err)
+	if err := notify.Watch(w.ac.keydir, w.ev, notify.All); err != nil {
+		logger.Trace("Failed to watch keystore folder", "err", err)
 		return
 	}
 	defer notify.Stop(w.ev)
-	glog.V(logger.Detail).Infof("now watching %s", w.ac.keydir)
-	defer glog.V(logger.Detail).Infof("no longer watching %s", w.ac.keydir)
+
+	logger.Trace("Started watching keystore folder")
+	defer logger.Trace("Stopped watching keystore folder")
 
 	w.ac.mu.Lock()
 	w.running = true

accounts/keystore/watch_fallback.go

@@ -19,10 +19,10 @@
 // This is the fallback implementation of directory watching.
 // It is used on unsupported platforms.
 
-package accounts
+package keystore
 
 type watcher struct{ running bool }
 
-func newWatcher(*addrCache) *watcher { return new(watcher) }
+func newWatcher(*accountCache) *watcher { return new(watcher) }
 func (*watcher) start()                 {}
 func (*watcher) close()                 {}

accounts/manager.go

@@ -0,0 +1,198 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package accounts
+
+import (
+	"reflect"
+	"sort"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/event"
+)
+
+// Manager is an overarching account manager that can communicate with various
+// backends for signing transactions.
+type Manager struct {
+	backends map[reflect.Type][]Backend // Index of backends currently registered
+	updaters []event.Subscription       // Wallet update subscriptions for all backends
+	updates  chan WalletEvent           // Subscription sink for backend wallet changes
+	wallets  []Wallet                   // Cache of all wallets from all registered backends
+
+	feed event.Feed // Wallet feed notifying of arrivals/departures
+
+	quit chan chan error
+	lock sync.RWMutex
+}
+
+// NewManager creates a generic account manager to sign transaction via various
+// supported backends.
+func NewManager(backends ...Backend) *Manager {
+	// Subscribe to wallet notifications from all backends
+	updates := make(chan WalletEvent, 4*len(backends))
+
+	subs := make([]event.Subscription, len(backends))
+	for i, backend := range backends {
+		subs[i] = backend.Subscribe(updates)
+	}
+	// Retrieve the initial list of wallets from the backends and sort by URL
+	var wallets []Wallet
+	for _, backend := range backends {
+		wallets = merge(wallets, backend.Wallets()...)
+	}
+	// Assemble the account manager and return
+	am := &Manager{
+		backends: make(map[reflect.Type][]Backend),
+		updaters: subs,
+		updates:  updates,
+		wallets:  wallets,
+		quit:     make(chan chan error),
+	}
+	for _, backend := range backends {
+		kind := reflect.TypeOf(backend)
+		am.backends[kind] = append(am.backends[kind], backend)
+	}
+	go am.update()
+
+	return am
+}
+
+// Close terminates the account manager's internal notification processes.
+func (am *Manager) Close() error {
+	errc := make(chan error)
+	am.quit <- errc
+	return <-errc
+}
+
+// update is the wallet event loop listening for notifications from the backends
+// and updating the cache of wallets.
+func (am *Manager) update() {
+	// Close all subscriptions when the manager terminates
+	defer func() {
+		am.lock.Lock()
+		for _, sub := range am.updaters {
+			sub.Unsubscribe()
+		}
+		am.updaters = nil
+		am.lock.Unlock()
+	}()
+
+	// Loop until termination
+	for {
+		select {
+		case event := <-am.updates:
+			// Wallet event arrived, update local cache
+			am.lock.Lock()
+			if event.Arrive {
+				am.wallets = merge(am.wallets, event.Wallet)
+			} else {
+				am.wallets = drop(am.wallets, event.Wallet)
+			}
+			am.lock.Unlock()
+
+			// Notify any listeners of the event
+			am.feed.Send(event)
+
+		case errc := <-am.quit:
+			// Manager terminating, return
+			errc <- nil
+			return
+		}
+	}
+}
+
+// Backends retrieves the backend(s) with the given type from the account manager.
+func (am *Manager) Backends(kind reflect.Type) []Backend {
+	return am.backends[kind]
+}
+
+// Wallets returns all signer accounts registered under this account manager.
+func (am *Manager) Wallets() []Wallet {
+	am.lock.RLock()
+	defer am.lock.RUnlock()
+
+	cpy := make([]Wallet, len(am.wallets))
+	copy(cpy, am.wallets)
+	return cpy
+}
+
+// Wallet retrieves the wallet associated with a particular URL.
+func (am *Manager) Wallet(url string) (Wallet, error) {
+	am.lock.RLock()
+	defer am.lock.RUnlock()
+
+	parsed, err := parseURL(url)
+	if err != nil {
+		return nil, err
+	}
+	for _, wallet := range am.Wallets() {
+		if wallet.URL() == parsed {
+			return wallet, nil
+		}
+	}
+	return nil, ErrUnknownWallet
+}
+
+// Find attempts to locate the wallet corresponding to a specific account. Since
+// accounts can be dynamically added to and removed from wallets, this method has
+// a linear runtime in the number of wallets.
+func (am *Manager) Find(account Account) (Wallet, error) {
+	am.lock.RLock()
+	defer am.lock.RUnlock()
+
+	for _, wallet := range am.wallets {
+		if wallet.Contains(account) {
+			return wallet, nil
+		}
+	}
+	return nil, ErrUnknownAccount
+}
+
+// Subscribe creates an async subscription to receive notifications when the
+// manager detects the arrival or departure of a wallet from any of its backends.
+func (am *Manager) Subscribe(sink chan<- WalletEvent) event.Subscription {
+	return am.feed.Subscribe(sink)
+}
+
+// merge is a sorted analogue of append for wallets, where the ordering of the
+// origin list is preserved by inserting new wallets at the correct position.
+//
+// The original slice is assumed to be already sorted by URL.
+func merge(slice []Wallet, wallets ...Wallet) []Wallet {
+	for _, wallet := range wallets {
+		n := sort.Search(len(slice), func(i int) bool { return slice[i].URL().Cmp(wallet.URL()) >= 0 })
+		if n == len(slice) {
+			slice = append(slice, wallet)
+			continue
+		}
+		slice = append(slice[:n], append([]Wallet{wallet}, slice[n:]...)...)
+	}
+	return slice
+}
+
+// drop is the couterpart of merge, which looks up wallets from within the sorted
+// cache and removes the ones specified.
+func drop(slice []Wallet, wallets ...Wallet) []Wallet {
+	for _, wallet := range wallets {
+		n := sort.Search(len(slice), func(i int) bool { return slice[i].URL().Cmp(wallet.URL()) >= 0 })
+		if n == len(slice) {
+			// Wallet not found, may happen during startup
+			continue
+		}
+		slice = append(slice[:n], slice[n+1:]...)
+	}
+	return slice
+}

accounts/url.go

@@ -0,0 +1,88 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package accounts
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+)
+
+// URL represents the canonical identification URL of a wallet or account.
+//
+// It is a simplified version of url.URL, with the important limitations (which
+// are considered features here) that it contains value-copyable components only,
+// as well as that it doesn't do any URL encoding/decoding of special characters.
+//
+// The former is important to allow an account to be copied without leaving live
+// references to the original version, whereas the latter is important to ensure
+// one single canonical form opposed to many allowed ones by the RFC 3986 spec.
+//
+// As such, these URLs should not be used outside of the scope of an Ethereum
+// wallet or account.
+type URL struct {
+	Scheme string // Protocol scheme to identify a capable account backend
+	Path   string // Path for the backend to identify a unique entity
+}
+
+// parseURL converts a user supplied URL into the accounts specific structure.
+func parseURL(url string) (URL, error) {
+	parts := strings.Split(url, "://")
+	if len(parts) != 2 || parts[0] == "" {
+		return URL{}, errors.New("protocol scheme missing")
+	}
+	return URL{
+		Scheme: parts[0],
+		Path:   parts[1],
+	}, nil
+}
+
+// String implements the stringer interface.
+func (u URL) String() string {
+	if u.Scheme != "" {
+		return fmt.Sprintf("%s://%s", u.Scheme, u.Path)
+	}
+	return u.Path
+}
+
+// TerminalString implements the log.TerminalStringer interface.
+func (u URL) TerminalString() string {
+	url := u.String()
+	if len(url) > 32 {
+		return url[:31] + "…"
+	}
+	return url
+}
+
+// MarshalJSON implements the json.Marshaller interface.
+func (u URL) MarshalJSON() ([]byte, error) {
+	return json.Marshal(u.String())
+}
+
+// Cmp compares x and y and returns:
+//
+//   -1 if x <  y
+//    0 if x == y
+//   +1 if x >  y
+//
+func (u URL) Cmp(url URL) int {
+	if u.Scheme == url.Scheme {
+		return strings.Compare(u.Path, url.Path)
+	}
+	return strings.Compare(u.Scheme, url.Scheme)
+}

accounts/usbwallet/ledger_hub.go

@@ -0,0 +1,217 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// This file contains the implementation for interacting with the Ledger hardware
+// wallets. The wire protocol spec can be found in the Ledger Blue GitHub repo:
+// https://raw.githubusercontent.com/LedgerHQ/blue-app-eth/master/doc/ethapp.asc
+
+package usbwallet
+
+import (
+	"errors"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/karalabe/hid"
+)
+
+// LedgerScheme is the protocol scheme prefixing account and wallet URLs.
+var LedgerScheme = "ledger"
+
+// ledgerDeviceIDs are the known device IDs that Ledger wallets use.
+var ledgerDeviceIDs = []deviceID{
+	{Vendor: 0x2c97, Product: 0x0000}, // Ledger Blue
+	{Vendor: 0x2c97, Product: 0x0001}, // Ledger Nano S
+}
+
+// Maximum time between wallet refreshes (if USB hotplug notifications don't work).
+const ledgerRefreshCycle = time.Second
+
+// Minimum time between wallet refreshes to avoid USB trashing.
+const ledgerRefreshThrottling = 500 * time.Millisecond
+
+// LedgerHub is a accounts.Backend that can find and handle Ledger hardware wallets.
+type LedgerHub struct {
+	refreshed   time.Time               // Time instance when the list of wallets was last refreshed
+	wallets     []accounts.Wallet       // List of Ledger devices currently tracking
+	updateFeed  event.Feed              // Event feed to notify wallet additions/removals
+	updateScope event.SubscriptionScope // Subscription scope tracking current live listeners
+	updating    bool                    // Whether the event notification loop is running
+
+	quit chan chan error
+
+	stateLock sync.RWMutex // Protects the internals of the hub from racey access
+
+	// TODO(karalabe): remove if hotplug lands on Windows
+	commsPend int        // Number of operations blocking enumeration
+	commsLock sync.Mutex // Lock protecting the pending counter and enumeration
+}
+
+// NewLedgerHub creates a new hardware wallet manager for Ledger devices.
+func NewLedgerHub() (*LedgerHub, error) {
+	if !hid.Supported() {
+		return nil, errors.New("unsupported platform")
+	}
+	hub := &LedgerHub{
+		quit: make(chan chan error),
+	}
+	hub.refreshWallets()
+	return hub, nil
+}
+
+// Wallets implements accounts.Backend, returning all the currently tracked USB
+// devices that appear to be Ledger hardware wallets.
+func (hub *LedgerHub) Wallets() []accounts.Wallet {
+	// Make sure the list of wallets is up to date
+	hub.refreshWallets()
+
+	hub.stateLock.RLock()
+	defer hub.stateLock.RUnlock()
+
+	cpy := make([]accounts.Wallet, len(hub.wallets))
+	copy(cpy, hub.wallets)
+	return cpy
+}
+
+// refreshWallets scans the USB devices attached to the machine and updates the
+// list of wallets based on the found devices.
+func (hub *LedgerHub) refreshWallets() {
+	// Don't scan the USB like crazy it the user fetches wallets in a loop
+	hub.stateLock.RLock()
+	elapsed := time.Since(hub.refreshed)
+	hub.stateLock.RUnlock()
+
+	if elapsed < ledgerRefreshThrottling {
+		return
+	}
+	// Retrieve the current list of Ledger devices
+	var ledgers []hid.DeviceInfo
+
+	if runtime.GOOS == "linux" {
+		// hidapi on Linux opens the device during enumeration to retrieve some infos,
+		// breaking the Ledger protocol if that is waiting for user confirmation. This
+		// is a bug acknowledged at Ledger, but it won't be fixed on old devices so we
+		// need to prevent concurrent comms ourselves. The more elegant solution would
+		// be to ditch enumeration in favor of hutplug events, but that don't work yet
+		// on Windows so if we need to hack it anyway, this is more elegant for now.
+		hub.commsLock.Lock()
+		if hub.commsPend > 0 { // A confirmation is pending, don't refresh
+			hub.commsLock.Unlock()
+			return
+		}
+	}
+	for _, info := range hid.Enumerate(0, 0) { // Can't enumerate directly, one valid ID is the 0 wildcard
+		for _, id := range ledgerDeviceIDs {
+			if info.VendorID == id.Vendor && info.ProductID == id.Product {
+				ledgers = append(ledgers, info)
+				break
+			}
+		}
+	}
+	if runtime.GOOS == "linux" {
+		// See rationale before the enumeration why this is needed and only on Linux.
+		hub.commsLock.Unlock()
+	}
+	// Transform the current list of wallets into the new one
+	hub.stateLock.Lock()
+
+	wallets := make([]accounts.Wallet, 0, len(ledgers))
+	events := []accounts.WalletEvent{}
+
+	for _, ledger := range ledgers {
+		url := accounts.URL{Scheme: LedgerScheme, Path: ledger.Path}
+
+		// Drop wallets in front of the next device or those that failed for some reason
+		for len(hub.wallets) > 0 && (hub.wallets[0].URL().Cmp(url) < 0 || hub.wallets[0].(*ledgerWallet).failed()) {
+			events = append(events, accounts.WalletEvent{Wallet: hub.wallets[0], Arrive: false})
+			hub.wallets = hub.wallets[1:]
+		}
+		// If there are no more wallets or the device is before the next, wrap new wallet
+		if len(hub.wallets) == 0 || hub.wallets[0].URL().Cmp(url) > 0 {
+			wallet := &ledgerWallet{hub: hub, url: &url, info: ledger, log: log.New("url", url)}
+
+			events = append(events, accounts.WalletEvent{Wallet: wallet, Arrive: true})
+			wallets = append(wallets, wallet)
+			continue
+		}
+		// If the device is the same as the first wallet, keep it
+		if hub.wallets[0].URL().Cmp(url) == 0 {
+			wallets = append(wallets, hub.wallets[0])
+			hub.wallets = hub.wallets[1:]
+			continue
+		}
+	}
+	// Drop any leftover wallets and set the new batch
+	for _, wallet := range hub.wallets {
+		events = append(events, accounts.WalletEvent{Wallet: wallet, Arrive: false})
+	}
+	hub.refreshed = time.Now()
+	hub.wallets = wallets
+	hub.stateLock.Unlock()
+
+	// Fire all wallet events and return
+	for _, event := range events {
+		hub.updateFeed.Send(event)
+	}
+}
+
+// Subscribe implements accounts.Backend, creating an async subscription to
+// receive notifications on the addition or removal of Ledger wallets.
+func (hub *LedgerHub) Subscribe(sink chan<- accounts.WalletEvent) event.Subscription {
+	// We need the mutex to reliably start/stop the update loop
+	hub.stateLock.Lock()
+	defer hub.stateLock.Unlock()
+
+	// Subscribe the caller and track the subscriber count
+	sub := hub.updateScope.Track(hub.updateFeed.Subscribe(sink))
+
+	// Subscribers require an active notification loop, start it
+	if !hub.updating {
+		hub.updating = true
+		go hub.updater()
+	}
+	return sub
+}
+
+// updater is responsible for maintaining an up-to-date list of wallets stored in
+// the keystore, and for firing wallet addition/removal events. It listens for
+// account change events from the underlying account cache, and also periodically
+// forces a manual refresh (only triggers for systems where the filesystem notifier
+// is not running).
+func (hub *LedgerHub) updater() {
+	for {
+		// Wait for a USB hotplug event (not supported yet) or a refresh timeout
+		select {
+		//case <-hub.changes: // reenable on hutplug implementation
+		case <-time.After(ledgerRefreshCycle):
+		}
+		// Run the wallet refresher
+		hub.refreshWallets()
+
+		// If all our subscribers left, stop the updater
+		hub.stateLock.Lock()
+		if hub.updateScope.Count() == 0 {
+			hub.updating = false
+			hub.stateLock.Unlock()
+			return
+		}
+		hub.stateLock.Unlock()
+	}
+}

accounts/usbwallet/ledger_wallet.go

@@ -0,0 +1,903 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// This file contains the implementation for interacting with the Ledger hardware
+// wallets. The wire protocol spec can be found in the Ledger Blue GitHub repo:
+// https://raw.githubusercontent.com/LedgerHQ/blue-app-eth/master/doc/ethapp.asc
+
+package usbwallet
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"math/big"
+	"sync"
+	"time"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/karalabe/hid"
+)
+
+// Maximum time between wallet health checks to detect USB unplugs.
+const ledgerHeartbeatCycle = time.Second
+
+// Minimum time to wait between self derivation attempts, even it the user is
+// requesting accounts like crazy.
+const ledgerSelfDeriveThrottling = time.Second
+
+// ledgerOpcode is an enumeration encoding the supported Ledger opcodes.
+type ledgerOpcode byte
+
+// ledgerParam1 is an enumeration encoding the supported Ledger parameters for
+// specific opcodes. The same parameter values may be reused between opcodes.
+type ledgerParam1 byte
+
+// ledgerParam2 is an enumeration encoding the supported Ledger parameters for
+// specific opcodes. The same parameter values may be reused between opcodes.
+type ledgerParam2 byte
+
+const (
+	ledgerOpRetrieveAddress  ledgerOpcode = 0x02 // Returns the public key and Ethereum address for a given BIP 32 path
+	ledgerOpSignTransaction  ledgerOpcode = 0x04 // Signs an Ethereum transaction after having the user validate the parameters
+	ledgerOpGetConfiguration ledgerOpcode = 0x06 // Returns specific wallet application configuration
+
+	ledgerP1DirectlyFetchAddress    ledgerParam1 = 0x00 // Return address directly from the wallet
+	ledgerP1ConfirmFetchAddress     ledgerParam1 = 0x01 // Require a user confirmation before returning the address
+	ledgerP1InitTransactionData     ledgerParam1 = 0x00 // First transaction data block for signing
+	ledgerP1ContTransactionData     ledgerParam1 = 0x80 // Subsequent transaction data block for signing
+	ledgerP2DiscardAddressChainCode ledgerParam2 = 0x00 // Do not return the chain code along with the address
+	ledgerP2ReturnAddressChainCode  ledgerParam2 = 0x01 // Require a user confirmation before returning the address
+)
+
+// errReplyInvalidHeader is the error message returned by a Ledger data exchange
+// if the device replies with a mismatching header. This usually means the device
+// is in browser mode.
+var errReplyInvalidHeader = errors.New("invalid reply header")
+
+// errInvalidVersionReply is the error message returned by a Ledger version retrieval
+// when a response does arrive, but it does not contain the expected data.
+var errInvalidVersionReply = errors.New("invalid version reply")
+
+// ledgerWallet represents a live USB Ledger hardware wallet.
+type ledgerWallet struct {
+	hub *LedgerHub    // USB hub the device originates from (TODO(karalabe): remove if hotplug lands on Windows)
+	url *accounts.URL // Textual URL uniquely identifying this wallet
+
+	info    hid.DeviceInfo // Known USB device infos about the wallet
+	device  *hid.Device    // USB device advertising itself as a Ledger wallet
+	failure error          // Any failure that would make the device unusable
+
+	version  [3]byte                                    // Current version of the Ledger Ethereum app (zero if app is offline)
+	browser  bool                                       // Flag whether the Ledger is in browser mode (reply channel mismatch)
+	accounts []accounts.Account                         // List of derive accounts pinned on the Ledger
+	paths    map[common.Address]accounts.DerivationPath // Known derivation paths for signing operations
+
+	deriveNextPath accounts.DerivationPath   // Next derivation path for account auto-discovery
+	deriveNextAddr common.Address            // Next derived account address for auto-discovery
+	deriveChain    ethereum.ChainStateReader // Blockchain state reader to discover used account with
+	deriveReq      chan chan struct{}        // Channel to request a self-derivation on
+	deriveQuit     chan chan error           // Channel to terminate the self-deriver with
+
+	healthQuit chan chan error
+
+	// Locking a hardware wallet is a bit special. Since hardware devices are lower
+	// performing, any communication with them might take a non negligible amount of
+	// time. Worse still, waiting for user confirmation can take arbitrarily long,
+	// but exclusive communication must be upheld during. Locking the entire wallet
+	// in the mean time however would stall any parts of the system that don't want
+	// to communicate, just read some state (e.g. list the accounts).
+	//
+	// As such, a hardware wallet needs two locks to function correctly. A state
+	// lock can be used to protect the wallet's software-side internal state, which
+	// must not be held exlusively during hardware communication. A communication
+	// lock can be used to achieve exclusive access to the device itself, this one
+	// however should allow "skipping" waiting for operations that might want to
+	// use the device, but can live without too (e.g. account self-derivation).
+	//
+	// Since we have two locks, it's important to know how to properly use them:
+	//   - Communication requires the `device` to not change, so obtaining the
+	//     commsLock should be done after having a stateLock.
+	//   - Communication must not disable read access to the wallet state, so it
+	//     must only ever hold a *read* lock to stateLock.
+	commsLock chan struct{} // Mutex (buf=1) for the USB comms without keeping the state locked
+	stateLock sync.RWMutex  // Protects read and write access to the wallet struct fields
+
+	log log.Logger // Contextual logger to tag the ledger with its id
+}
+
+// URL implements accounts.Wallet, returning the URL of the Ledger device.
+func (w *ledgerWallet) URL() accounts.URL {
+	return *w.url // Immutable, no need for a lock
+}
+
+// Status implements accounts.Wallet, always whether the Ledger is opened, closed
+// or whether the Ethereum app was not started on it.
+func (w *ledgerWallet) Status() string {
+	w.stateLock.RLock() // No device communication, state lock is enough
+	defer w.stateLock.RUnlock()
+
+	if w.failure != nil {
+		return fmt.Sprintf("Failed: %v", w.failure)
+	}
+	if w.device == nil {
+		return "Closed"
+	}
+	if w.browser {
+		return "Ethereum app in browser mode"
+	}
+	if w.offline() {
+		return "Ethereum app offline"
+	}
+	return fmt.Sprintf("Ethereum app v%d.%d.%d online", w.version[0], w.version[1], w.version[2])
+}
+
+// offline returns whether the wallet and the Ethereum app is offline or not.
+//
+// The method assumes that the state lock is held!
+func (w *ledgerWallet) offline() bool {
+	return w.version == [3]byte{0, 0, 0}
+}
+
+// failed returns if the USB device wrapped by the wallet failed for some reason.
+// This is used by the device scanner to report failed wallets as departed.
+//
+// The method assumes that the state lock is *not* held!
+func (w *ledgerWallet) failed() bool {
+	w.stateLock.RLock() // No device communication, state lock is enough
+	defer w.stateLock.RUnlock()
+
+	return w.failure != nil
+}
+
+// Open implements accounts.Wallet, attempting to open a USB connection to the
+// Ledger hardware wallet. The Ledger does not require a user passphrase, so that
+// parameter is silently discarded.
+func (w *ledgerWallet) Open(passphrase string) error {
+	w.stateLock.Lock() // State lock is enough since there's no connection yet at this point
+	defer w.stateLock.Unlock()
+
+	// If the wallet was already opened, don't try to open again
+	if w.device != nil {
+		return accounts.ErrWalletAlreadyOpen
+	}
+	// Otherwise iterate over all USB devices and find this again (no way to directly do this)
+	device, err := w.info.Open()
+	if err != nil {
+		return err
+	}
+	// Wallet seems to be successfully opened, guess if the Ethereum app is running
+	w.device = device
+	w.commsLock = make(chan struct{}, 1)
+	w.commsLock <- struct{}{} // Enable lock
+
+	w.paths = make(map[common.Address]accounts.DerivationPath)
+
+	w.deriveReq = make(chan chan struct{})
+	w.deriveQuit = make(chan chan error)
+	w.healthQuit = make(chan chan error)
+
+	defer func() {
+		go w.heartbeat()
+		go w.selfDerive()
+	}()
+
+	if _, err = w.ledgerDerive(accounts.DefaultBaseDerivationPath); err != nil {
+		// Ethereum app is not running or in browser mode, nothing more to do, return
+		if err == errReplyInvalidHeader {
+			w.browser = true
+		}
+		return nil
+	}
+	// Try to resolve the Ethereum app's version, will fail prior to v1.0.2
+	if w.version, err = w.ledgerVersion(); err != nil {
+		w.version = [3]byte{1, 0, 0} // Assume worst case, can't verify if v1.0.0 or v1.0.1
+	}
+	return nil
+}
+
+// heartbeat is a health check loop for the Ledger wallets to periodically verify
+// whether they are still present or if they malfunctioned. It is needed because:
+//  - libusb on Windows doesn't support hotplug, so we can't detect USB unplugs
+//  - communication timeout on the Ledger requires a device power cycle to fix
+func (w *ledgerWallet) heartbeat() {
+	w.log.Debug("Ledger health-check started")
+	defer w.log.Debug("Ledger health-check stopped")
+
+	// Execute heartbeat checks until termination or error
+	var (
+		errc chan error
+		err  error
+	)
+	for errc == nil && err == nil {
+		// Wait until termination is requested or the heartbeat cycle arrives
+		select {
+		case errc = <-w.healthQuit:
+			// Termination requested
+			continue
+		case <-time.After(ledgerHeartbeatCycle):
+			// Heartbeat time
+		}
+		// Execute a tiny data exchange to see responsiveness
+		w.stateLock.RLock()
+		if w.device == nil {
+			// Terminated while waiting for the lock
+			w.stateLock.RUnlock()
+			continue
+		}
+		<-w.commsLock // Don't lock state while resolving version
+		_, err = w.ledgerVersion()
+		w.commsLock <- struct{}{}
+		w.stateLock.RUnlock()
+
+		if err != nil && err != errInvalidVersionReply {
+			w.stateLock.Lock() // Lock state to tear the wallet down
+			w.failure = err
+			w.close()
+			w.stateLock.Unlock()
+		}
+		// Ignore non hardware related errors
+		err = nil
+	}
+	// In case of error, wait for termination
+	if err != nil {
+		w.log.Debug("Ledger health-check failed", "err", err)
+		errc = <-w.healthQuit
+	}
+	errc <- err
+}
+
+// Close implements accounts.Wallet, closing the USB connection to the Ledger.
+func (w *ledgerWallet) Close() error {
+	// Ensure the wallet was opened
+	w.stateLock.RLock()
+	hQuit, dQuit := w.healthQuit, w.deriveQuit
+	w.stateLock.RUnlock()
+
+	// Terminate the health checks
+	var herr error
+	if hQuit != nil {
+		errc := make(chan error)
+		hQuit <- errc
+		herr = <-errc // Save for later, we *must* close the USB
+	}
+	// Terminate the self-derivations
+	var derr error
+	if dQuit != nil {
+		errc := make(chan error)
+		dQuit <- errc
+		derr = <-errc // Save for later, we *must* close the USB
+	}
+	// Terminate the device connection
+	w.stateLock.Lock()
+	defer w.stateLock.Unlock()
+
+	w.healthQuit = nil
+	w.deriveQuit = nil
+	w.deriveReq = nil
+
+	if err := w.close(); err != nil {
+		return err
+	}
+	if herr != nil {
+		return herr
+	}
+	return derr
+}
+
+// close is the internal wallet closer that terminates the USB connection and
+// resets all the fields to their defaults.
+//
+// Note, close assumes the state lock is held!
+func (w *ledgerWallet) close() error {
+	// Allow duplicate closes, especially for health-check failures
+	if w.device == nil {
+		return nil
+	}
+	// Close the device, clear everything, then return
+	w.device.Close()
+	w.device = nil
+
+	w.browser, w.version = false, [3]byte{}
+	w.accounts, w.paths = nil, nil
+
+	return nil
+}
+
+// Accounts implements accounts.Wallet, returning the list of accounts pinned to
+// the Ledger hardware wallet. If self-derivation was enabled, the account list
+// is periodically expanded based on current chain state.
+func (w *ledgerWallet) Accounts() []accounts.Account {
+	// Attempt self-derivation if it's running
+	reqc := make(chan struct{}, 1)
+	select {
+	case w.deriveReq <- reqc:
+		// Self-derivation request accepted, wait for it
+		<-reqc
+	default:
+		// Self-derivation offline, throttled or busy, skip
+	}
+	// Return whatever account list we ended up with
+	w.stateLock.RLock()
+	defer w.stateLock.RUnlock()
+
+	cpy := make([]accounts.Account, len(w.accounts))
+	copy(cpy, w.accounts)
+	return cpy
+}
+
+// selfDerive is an account derivation loop that upon request attempts to find
+// new non-zero accounts.
+func (w *ledgerWallet) selfDerive() {
+	w.log.Debug("Ledger self-derivation started")
+	defer w.log.Debug("Ledger self-derivation stopped")
+
+	// Execute self-derivations until termination or error
+	var (
+		reqc chan struct{}
+		errc chan error
+		err  error
+	)
+	for errc == nil && err == nil {
+		// Wait until either derivation or termination is requested
+		select {
+		case errc = <-w.deriveQuit:
+			// Termination requested
+			continue
+		case reqc = <-w.deriveReq:
+			// Account discovery requested
+		}
+		// Derivation needs a chain and device access, skip if either unavailable
+		w.stateLock.RLock()
+		if w.device == nil || w.deriveChain == nil || w.offline() {
+			w.stateLock.RUnlock()
+			reqc <- struct{}{}
+			continue
+		}
+		select {
+		case <-w.commsLock:
+		default:
+			w.stateLock.RUnlock()
+			reqc <- struct{}{}
+			continue
+		}
+		// Device lock obtained, derive the next batch of accounts
+		var (
+			accs  []accounts.Account
+			paths []accounts.DerivationPath
+
+			nextAddr = w.deriveNextAddr
+			nextPath = w.deriveNextPath
+
+			context = context.Background()
+		)
+		for empty := false; !empty; {
+			// Retrieve the next derived Ethereum account
+			if nextAddr == (common.Address{}) {
+				if nextAddr, err = w.ledgerDerive(nextPath); err != nil {
+					w.log.Warn("Ledger account derivation failed", "err", err)
+					break
+				}
+			}
+			// Check the account's status against the current chain state
+			var (
+				balance *big.Int
+				nonce   uint64
+			)
+			balance, err = w.deriveChain.BalanceAt(context, nextAddr, nil)
+			if err != nil {
+				w.log.Warn("Ledger balance retrieval failed", "err", err)
+				break
+			}
+			nonce, err = w.deriveChain.NonceAt(context, nextAddr, nil)
+			if err != nil {
+				w.log.Warn("Ledger nonce retrieval failed", "err", err)
+				break
+			}
+			// If the next account is empty, stop self-derivation, but add it nonetheless
+			if balance.Sign() == 0 && nonce == 0 {
+				empty = true
+			}
+			// We've just self-derived a new account, start tracking it locally
+			path := make(accounts.DerivationPath, len(nextPath))
+			copy(path[:], nextPath[:])
+			paths = append(paths, path)
+
+			account := accounts.Account{
+				Address: nextAddr,
+				URL:     accounts.URL{Scheme: w.url.Scheme, Path: fmt.Sprintf("%s/%s", w.url.Path, path)},
+			}
+			accs = append(accs, account)
+
+			// Display a log message to the user for new (or previously empty accounts)
+			if _, known := w.paths[nextAddr]; !known || (!empty && nextAddr == w.deriveNextAddr) {
+				w.log.Info("Ledger discovered new account", "address", nextAddr, "path", path, "balance", balance, "nonce", nonce)
+			}
+			// Fetch the next potential account
+			if !empty {
+				nextAddr = common.Address{}
+				nextPath[len(nextPath)-1]++
+			}
+		}
+		// Self derivation complete, release device lock
+		w.commsLock <- struct{}{}
+		w.stateLock.RUnlock()
+
+		// Insert any accounts successfully derived
+		w.stateLock.Lock()
+		for i := 0; i < len(accs); i++ {
+			if _, ok := w.paths[accs[i].Address]; !ok {
+				w.accounts = append(w.accounts, accs[i])
+				w.paths[accs[i].Address] = paths[i]
+			}
+		}
+		// Shift the self-derivation forward
+		// TODO(karalabe): don't overwrite changes from wallet.SelfDerive
+		w.deriveNextAddr = nextAddr
+		w.deriveNextPath = nextPath
+		w.stateLock.Unlock()
+
+		// Notify the user of termination and loop after a bit of time (to avoid trashing)
+		reqc <- struct{}{}
+		if err == nil {
+			select {
+			case errc = <-w.deriveQuit:
+				// Termination requested, abort
+			case <-time.After(ledgerSelfDeriveThrottling):
+				// Waited enough, willing to self-derive again
+			}
+		}
+	}
+	// In case of error, wait for termination
+	if err != nil {
+		w.log.Debug("Ledger self-derivation failed", "err", err)
+		errc = <-w.deriveQuit
+	}
+	errc <- err
+}
+
+// Contains implements accounts.Wallet, returning whether a particular account is
+// or is not pinned into this Ledger instance. Although we could attempt to resolve
+// unpinned accounts, that would be an non-negligible hardware operation.
+func (w *ledgerWallet) Contains(account accounts.Account) bool {
+	w.stateLock.RLock()
+	defer w.stateLock.RUnlock()
+
+	_, exists := w.paths[account.Address]
+	return exists
+}
+
+// Derive implements accounts.Wallet, deriving a new account at the specific
+// derivation path. If pin is set to true, the account will be added to the list
+// of tracked accounts.
+func (w *ledgerWallet) Derive(path accounts.DerivationPath, pin bool) (accounts.Account, error) {
+	// Try to derive the actual account and update its URL if successful
+	w.stateLock.RLock() // Avoid device disappearing during derivation
+
+	if w.device == nil || w.offline() {
+		w.stateLock.RUnlock()
+		return accounts.Account{}, accounts.ErrWalletClosed
+	}
+	<-w.commsLock // Avoid concurrent hardware access
+	address, err := w.ledgerDerive(path)
+	w.commsLock <- struct{}{}
+
+	w.stateLock.RUnlock()
+
+	// If an error occurred or no pinning was requested, return
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	account := accounts.Account{
+		Address: address,
+		URL:     accounts.URL{Scheme: w.url.Scheme, Path: fmt.Sprintf("%s/%s", w.url.Path, path)},
+	}
+	if !pin {
+		return account, nil
+	}
+	// Pinning needs to modify the state
+	w.stateLock.Lock()
+	defer w.stateLock.Unlock()
+
+	if _, ok := w.paths[address]; !ok {
+		w.accounts = append(w.accounts, account)
+		w.paths[address] = path
+	}
+	return account, nil
+}
+
+// SelfDerive implements accounts.Wallet, trying to discover accounts that the
+// user used previously (based on the chain state), but ones that he/she did not
+// explicitly pin to the wallet manually. To avoid chain head monitoring, self
+// derivation only runs during account listing (and even then throttled).
+func (w *ledgerWallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {
+	w.stateLock.Lock()
+	defer w.stateLock.Unlock()
+
+	w.deriveNextPath = make(accounts.DerivationPath, len(base))
+	copy(w.deriveNextPath[:], base[:])
+
+	w.deriveNextAddr = common.Address{}
+	w.deriveChain = chain
+}
+
+// SignHash implements accounts.Wallet, however signing arbitrary data is not
+// supported for Ledger wallets, so this method will always return an error.
+func (w *ledgerWallet) SignHash(acc accounts.Account, hash []byte) ([]byte, error) {
+	return nil, accounts.ErrNotSupported
+}
+
+// SignTx implements accounts.Wallet. It sends the transaction over to the Ledger
+// wallet to request a confirmation from the user. It returns either the signed
+// transaction or a failure if the user denied the transaction.
+//
+// Note, if the version of the Ethereum application running on the Ledger wallet is
+// too old to sign EIP-155 transactions, but such is requested nonetheless, an error
+// will be returned opposed to silently signing in Homestead mode.
+func (w *ledgerWallet) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	w.stateLock.RLock() // Comms have own mutex, this is for the state fields
+	defer w.stateLock.RUnlock()
+
+	// If the wallet is closed, or the Ethereum app doesn't run, abort
+	if w.device == nil || w.offline() {
+		return nil, accounts.ErrWalletClosed
+	}
+	// Make sure the requested account is contained within
+	path, ok := w.paths[account.Address]
+	if !ok {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Ensure the wallet is capable of signing the given transaction
+	if chainID != nil && w.version[0] <= 1 && w.version[1] <= 0 && w.version[2] <= 2 {
+		return nil, fmt.Errorf("Ledger v%d.%d.%d doesn't support signing this transaction, please update to v1.0.3 at least", w.version[0], w.version[1], w.version[2])
+	}
+	// All infos gathered and metadata checks out, request signing
+	<-w.commsLock
+	defer func() { w.commsLock <- struct{}{} }()
+
+	// Ensure the device isn't screwed with while user confirmation is pending
+	// TODO(karalabe): remove if hotplug lands on Windows
+	w.hub.commsLock.Lock()
+	w.hub.commsPend++
+	w.hub.commsLock.Unlock()
+
+	defer func() {
+		w.hub.commsLock.Lock()
+		w.hub.commsPend--
+		w.hub.commsLock.Unlock()
+	}()
+	return w.ledgerSign(path, account.Address, tx, chainID)
+}
+
+// SignHashWithPassphrase implements accounts.Wallet, however signing arbitrary
+// data is not supported for Ledger wallets, so this method will always return
+// an error.
+func (w *ledgerWallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
+	return nil, accounts.ErrNotSupported
+}
+
+// SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given
+// transaction with the given account using passphrase as extra authentication.
+// Since the Ledger does not support extra passphrases, it is silently ignored.
+func (w *ledgerWallet) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	return w.SignTx(account, tx, chainID)
+}
+
+// ledgerVersion retrieves the current version of the Ethereum wallet app running
+// on the Ledger wallet.
+//
+// The version retrieval protocol is defined as follows:
+//
+//   CLA | INS | P1 | P2 | Lc | Le
+//   ----+-----+----+----+----+---
+//    E0 | 06  | 00 | 00 | 00 | 04
+//
+// With no input data, and the output data being:
+//
+//   Description                                        | Length
+//   ---------------------------------------------------+--------
+//   Flags 01: arbitrary data signature enabled by user | 1 byte
+//   Application major version                          | 1 byte
+//   Application minor version                          | 1 byte
+//   Application patch version                          | 1 byte
+func (w *ledgerWallet) ledgerVersion() ([3]byte, error) {
+	// Send the request and wait for the response
+	reply, err := w.ledgerExchange(ledgerOpGetConfiguration, 0, 0, nil)
+	if err != nil {
+		return [3]byte{}, err
+	}
+	if len(reply) != 4 {
+		return [3]byte{}, errInvalidVersionReply
+	}
+	// Cache the version for future reference
+	var version [3]byte
+	copy(version[:], reply[1:])
+	return version, nil
+}
+
+// ledgerDerive retrieves the currently active Ethereum address from a Ledger
+// wallet at the specified derivation path.
+//
+// The address derivation protocol is defined as follows:
+//
+//   CLA | INS | P1 | P2 | Lc  | Le
+//   ----+-----+----+----+-----+---
+//    E0 | 02  | 00 return address
+//               01 display address and confirm before returning
+//                  | 00: do not return the chain code
+//                  | 01: return the chain code
+//                       | var | 00
+//
+// Where the input data is:
+//
+//   Description                                      | Length
+//   -------------------------------------------------+--------
+//   Number of BIP 32 derivations to perform (max 10) | 1 byte
+//   First derivation index (big endian)              | 4 bytes
+//   ...                                              | 4 bytes
+//   Last derivation index (big endian)               | 4 bytes
+//
+// And the output data is:
+//
+//   Description             | Length
+//   ------------------------+-------------------
+//   Public Key length       | 1 byte
+//   Uncompressed Public Key | arbitrary
+//   Ethereum address length | 1 byte
+//   Ethereum address        | 40 bytes hex ascii
+//   Chain code if requested | 32 bytes
+func (w *ledgerWallet) ledgerDerive(derivationPath []uint32) (common.Address, error) {
+	// Flatten the derivation path into the Ledger request
+	path := make([]byte, 1+4*len(derivationPath))
+	path[0] = byte(len(derivationPath))
+	for i, component := range derivationPath {
+		binary.BigEndian.PutUint32(path[1+4*i:], component)
+	}
+	// Send the request and wait for the response
+	reply, err := w.ledgerExchange(ledgerOpRetrieveAddress, ledgerP1DirectlyFetchAddress, ledgerP2DiscardAddressChainCode, path)
+	if err != nil {
+		return common.Address{}, err
+	}
+	// Discard the public key, we don't need that for now
+	if len(reply) < 1 || len(reply) < 1+int(reply[0]) {
+		return common.Address{}, errors.New("reply lacks public key entry")
+	}
+	reply = reply[1+int(reply[0]):]
+
+	// Extract the Ethereum hex address string
+	if len(reply) < 1 || len(reply) < 1+int(reply[0]) {
+		return common.Address{}, errors.New("reply lacks address entry")
+	}
+	hexstr := reply[1 : 1+int(reply[0])]
+
+	// Decode the hex sting into an Ethereum address and return
+	var address common.Address
+	hex.Decode(address[:], hexstr)
+	return address, nil
+}
+
+// ledgerSign sends the transaction to the Ledger wallet, and waits for the user
+// to confirm or deny the transaction.
+//
+// The transaction signing protocol is defined as follows:
+//
+//   CLA | INS | P1 | P2 | Lc  | Le
+//   ----+-----+----+----+-----+---
+//    E0 | 04  | 00: first transaction data block
+//               80: subsequent transaction data block
+//                  | 00 | variable | variable
+//
+// Where the input for the first transaction block (first 255 bytes) is:
+//
+//   Description                                      | Length
+//   -------------------------------------------------+----------
+//   Number of BIP 32 derivations to perform (max 10) | 1 byte
+//   First derivation index (big endian)              | 4 bytes
+//   ...                                              | 4 bytes
+//   Last derivation index (big endian)               | 4 bytes
+//   RLP transaction chunk                            | arbitrary
+//
+// And the input for subsequent transaction blocks (first 255 bytes) are:
+//
+//   Description           | Length
+//   ----------------------+----------
+//   RLP transaction chunk | arbitrary
+//
+// And the output data is:
+//
+//   Description | Length
+//   ------------+---------
+//   signature V | 1 byte
+//   signature R | 32 bytes
+//   signature S | 32 bytes
+func (w *ledgerWallet) ledgerSign(derivationPath []uint32, address common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	// Flatten the derivation path into the Ledger request
+	path := make([]byte, 1+4*len(derivationPath))
+	path[0] = byte(len(derivationPath))
+	for i, component := range derivationPath {
+		binary.BigEndian.PutUint32(path[1+4*i:], component)
+	}
+	// Create the transaction RLP based on whether legacy or EIP155 signing was requeste
+	var (
+		txrlp []byte
+		err   error
+	)
+	if chainID == nil {
+		if txrlp, err = rlp.EncodeToBytes([]interface{}{tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data()}); err != nil {
+			return nil, err
+		}
+	} else {
+		if txrlp, err = rlp.EncodeToBytes([]interface{}{tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data(), chainID, big.NewInt(0), big.NewInt(0)}); err != nil {
+			return nil, err
+		}
+	}
+	payload := append(path, txrlp...)
+
+	// Send the request and wait for the response
+	var (
+		op    = ledgerP1InitTransactionData
+		reply []byte
+	)
+	for len(payload) > 0 {
+		// Calculate the size of the next data chunk
+		chunk := 255
+		if chunk > len(payload) {
+			chunk = len(payload)
+		}
+		// Send the chunk over, ensuring it's processed correctly
+		reply, err = w.ledgerExchange(ledgerOpSignTransaction, op, 0, payload[:chunk])
+		if err != nil {
+			return nil, err
+		}
+		// Shift the payload and ensure subsequent chunks are marked as such
+		payload = payload[chunk:]
+		op = ledgerP1ContTransactionData
+	}
+	// Extract the Ethereum signature and do a sanity validation
+	if len(reply) != 65 {
+		return nil, errors.New("reply lacks signature")
+	}
+	signature := append(reply[1:], reply[0])
+
+	// Create the correct signer and signature transform based on the chain ID
+	var signer types.Signer
+	if chainID == nil {
+		signer = new(types.HomesteadSigner)
+	} else {
+		signer = types.NewEIP155Signer(chainID)
+		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+	}
+	// Inject the final signature into the transaction and sanity check the sender
+	signed, err := tx.WithSignature(signer, signature)
+	if err != nil {
+		return nil, err
+	}
+	sender, err := types.Sender(signer, signed)
+	if err != nil {
+		return nil, err
+	}
+	if sender != address {
+		return nil, fmt.Errorf("signer mismatch: expected %s, got %s", address.Hex(), sender.Hex())
+	}
+	return signed, nil
+}
+
+// ledgerExchange performs a data exchange with the Ledger wallet, sending it a
+// message and retrieving the response.
+//
+// The common transport header is defined as follows:
+//
+//  Description                           | Length
+//  --------------------------------------+----------
+//  Communication channel ID (big endian) | 2 bytes
+//  Command tag                           | 1 byte
+//  Packet sequence index (big endian)    | 2 bytes
+//  Payload                               | arbitrary
+//
+// The Communication channel ID allows commands multiplexing over the same
+// physical link. It is not used for the time being, and should be set to 0101
+// to avoid compatibility issues with implementations ignoring a leading 00 byte.
+//
+// The Command tag describes the message content. Use TAG_APDU (0x05) for standard
+// APDU payloads, or TAG_PING (0x02) for a simple link test.
+//
+// The Packet sequence index describes the current sequence for fragmented payloads.
+// The first fragment index is 0x00.
+//
+// APDU Command payloads are encoded as follows:
+//
+//  Description              | Length
+//  -----------------------------------
+//  APDU length (big endian) | 2 bytes
+//  APDU CLA                 | 1 byte
+//  APDU INS                 | 1 byte
+//  APDU P1                  | 1 byte
+//  APDU P2                  | 1 byte
+//  APDU length              | 1 byte
+//  Optional APDU data       | arbitrary
+func (w *ledgerWallet) ledgerExchange(opcode ledgerOpcode, p1 ledgerParam1, p2 ledgerParam2, data []byte) ([]byte, error) {
+	// Construct the message payload, possibly split into multiple chunks
+	apdu := make([]byte, 2, 7+len(data))
+
+	binary.BigEndian.PutUint16(apdu, uint16(5+len(data)))
+	apdu = append(apdu, []byte{0xe0, byte(opcode), byte(p1), byte(p2), byte(len(data))}...)
+	apdu = append(apdu, data...)
+
+	// Stream all the chunks to the device
+	header := []byte{0x01, 0x01, 0x05, 0x00, 0x00} // Channel ID and command tag appended
+	chunk := make([]byte, 64)
+	space := len(chunk) - len(header)
+
+	for i := 0; len(apdu) > 0; i++ {
+		// Construct the new message to stream
+		chunk = append(chunk[:0], header...)
+		binary.BigEndian.PutUint16(chunk[3:], uint16(i))
+
+		if len(apdu) > space {
+			chunk = append(chunk, apdu[:space]...)
+			apdu = apdu[space:]
+		} else {
+			chunk = append(chunk, apdu...)
+			apdu = nil
+		}
+		// Send over to the device
+		w.log.Trace("Data chunk sent to the Ledger", "chunk", hexutil.Bytes(chunk))
+		if _, err := w.device.Write(chunk); err != nil {
+			return nil, err
+		}
+	}
+	// Stream the reply back from the wallet in 64 byte chunks
+	var reply []byte
+	chunk = chunk[:64] // Yeah, we surely have enough space
+	for {
+		// Read the next chunk from the Ledger wallet
+		if _, err := io.ReadFull(w.device, chunk); err != nil {
+			return nil, err
+		}
+		w.log.Trace("Data chunk received from the Ledger", "chunk", hexutil.Bytes(chunk))
+
+		// Make sure the transport header matches
+		if chunk[0] != 0x01 || chunk[1] != 0x01 || chunk[2] != 0x05 {
+			return nil, errReplyInvalidHeader
+		}
+		// If it's the first chunk, retrieve the total message length
+		var payload []byte
+
+		if chunk[3] == 0x00 && chunk[4] == 0x00 {
+			reply = make([]byte, 0, int(binary.BigEndian.Uint16(chunk[5:7])))
+			payload = chunk[7:]
+		} else {
+			payload = chunk[5:]
+		}
+		// Append to the reply and stop when filled up
+		if left := cap(reply) - len(reply); left > len(payload) {
+			reply = append(reply, payload...)
+		} else {
+			reply = append(reply, payload[:left]...)
+			break
+		}
+	}
+	return reply[:len(reply)-2], nil
+}

accounts/usbwallet/usbwallet.go

@@ -1,4 +1,4 @@
-// Copyright 2016 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -14,13 +14,12 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// Contains the wrappers from the math/big package that require Go 1.7 and above.
+// Package usbwallet implements support for USB hardware wallets.
+package usbwallet
 
-// +build go1.7
-
-package geth
-
-// GetString returns the value of x as a formatted string in some number base.
-func (bi *BigInt) GetString(base int) string {
-	return bi.bigint.Text(base)
+// deviceID is a combined vendor/product identifier to uniquely identify a USB
+// hardware device.
+type deviceID struct {
+	Vendor  uint16 // The Vendor identifer
+	Product uint16 // The Product identifier
 }

appveyor.yml

@@ -22,8 +22,8 @@ environment:
 
 install:
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.4.windows-%GETH_ARCH%.zip
-  - 7z x go1.7.4.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.3.windows-%GETH_ARCH%.zip
+  - 7z x go1.8.3.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 
@@ -36,4 +36,4 @@ after_build:
 
 test_script:
   - set CGO_ENABLED=1
-  - go run build\ci.go test -vet -coverage
+  - go run build\ci.go test -coverage

build/_vendor/src/golang.org/x/net/LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2009 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

build/ci-notes.md

@@ -1,18 +1,3 @@
-# Vendored Dependencies
-
-Dependencies are almost all vendored in at the standard Go `/vendor` path. This allows
-people to build go-ethereum using the standard toolchain without any particular package
-manager. It also plays nicely with `go get`, not requiring external code to be relied on.
-
-The one single dependent package missing from `vendor` is `golang.org/x/net/context`. As
-this is a package exposed via public library APIs, it must not be vendored as dependent
-code woulnd't be able to instantiate.
-
-To allow reproducible builds of go-ethereum nonetheless that don't need network access
-during build time to fetch `golang.org/x/net/context`, a version was copied into our repo
-at the very specific `/build/_vendor` path, which is added automatically by all CI build
-scripts and the makefile too.
-
 # Debian Packaging
 
 Tagged releases and develop branch commits are available as installable Debian packages
@@ -20,9 +5,9 @@ for Ubuntu. Packages are built for the all Ubuntu versions which are supported b
 Canonical:
 
 - Trusty Tahr (14.04 LTS)
-- Wily Werewolf (15.10)
 - Xenial Xerus (16.04 LTS)
 - Yakkety Yak (16.10)
+- Zesty Zapus (17.04)
 
 Packages of develop branch commits have suffix -unstable and cannot be installed alongside
 the stable version. Switching between release streams requires user intervention.
@@ -36,18 +21,18 @@ variable which Travis CI makes available to certain builds.
 We want to build go-ethereum with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
 our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on
-golang-1.7, which is co-installable alongside the regular golang package. PPA dependencies
+golang-1.8, which is co-installable alongside the regular golang package. PPA dependencies
 can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies
 
 ## Building Packages Locally (for testing)
 
 You need to run Ubuntu to do test packaging.
 
-Add the gophers PPA and install Go 1.7 and Debian packaging tools:
+Add the gophers PPA and install Go 1.8 and Debian packaging tools:
 
     $ sudo apt-add-repository ppa:gophers/ubuntu/archive
     $ sudo apt-get update
-    $ sudo apt-get install build-essential golang-1.7 devscripts debhelper
+    $ sudo apt-get install build-essential golang-1.8 devscripts debhelper
 
 Create the source packages:
 
@@ -55,10 +40,10 @@ Create the source packages:
 
 Then go into the source package directory for your running distribution and build the package:
 
-    $ cd dist/ethereum-unstable-1.5.0+xenial
+    $ cd dist/ethereum-unstable-1.6.0+xenial
     $ dpkg-buildpackage
 
 Built packages are placed in the dist/ directory.
 
     $ cd ..
-    $ dpkg-deb -c geth-unstable_1.5.0+xenial_amd64.deb
+    $ dpkg-deb -c geth-unstable_1.6.0+xenial_amd64.deb

build/ci.go

@@ -23,15 +23,16 @@ Usage: go run ci.go <command> <command flags/arguments>
 
 Available commands are:
 
-   install    [-arch architecture] [ packages... ]                                           -- builds packages and executables
-   test       [ -coverage ] [ -vet ] [ -misspell ] [ packages... ]                           -- runs the tests
-   archive    [-arch architecture] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
+   install    [ -arch architecture ] [ packages... ]                                           -- builds packages and executables
+   test       [ -coverage ] [ -misspell ] [ packages... ]                                      -- runs the tests
+   archive    [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
    importkeys                                                                                  -- imports signing keys from env
    debsrc     [ -signer key-id ] [ -upload dest ]                                              -- creates a debian source package
    nsis                                                                                        -- creates a Windows NSIS installer
    aar        [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an Android archive
    xcode      [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ]                      -- creates an iOS XCode framework
-   xgo        [ options ]                                                                    -- cross builds according to options
+   xgo        [ -alltools ] [ options ]                                                        -- cross builds according to options
+   purge      [ -store blobstore ] [ -days threshold ]                                         -- purges old archives from the blobstore
 
 For all commands, -n prevents execution of external programs (dry run mode).
 
@@ -70,40 +71,55 @@ var (
 	allToolsArchiveFiles = []string{
 		"COPYING",
 		executablePath("abigen"),
+		executablePath("bootnode"),
 		executablePath("evm"),
 		executablePath("geth"),
-		executablePath("swarm"),
+		executablePath("puppeth"),
 		executablePath("rlpdump"),
+		executablePath("swarm"),
+		executablePath("wnode"),
 	}
 
 	// A debian package is created for all executables listed here.
 	debExecutables = []debExecutable{
 		{
-			Name:        "geth",
-			Description: "Ethereum CLI client.",
+			Name:        "abigen",
+			Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.",
 		},
 		{
-			Name:        "rlpdump",
-			Description: "Developer utility tool that prints RLP structures.",
+			Name:        "bootnode",
+			Description: "Ethereum bootnode.",
 		},
 		{
 			Name:        "evm",
 			Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.",
 		},
+		{
+			Name:        "geth",
+			Description: "Ethereum CLI client.",
+		},
+		{
+			Name:        "puppeth",
+			Description: "Ethereum private network manager.",
+		},
+		{
+			Name:        "rlpdump",
+			Description: "Developer utility tool that prints RLP structures.",
+		},
 		{
 			Name:        "swarm",
 			Description: "Ethereum Swarm daemon and tools",
 		},
 		{
-			Name:        "abigen",
-			Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.",
+			Name:        "wnode",
+			Description: "Ethereum Whisper diagnostic tool",
 		},
 	}
 
 	// Distros for which packages are created.
 	// Note: vivid is unsupported because there is no golang-1.6 package for it.
 	// Note: wily is unsupported because it was officially deprecated on lanchpad.
-	debDistros = []string{"trusty", "xenial", "yakkety"}
+	debDistros = []string{"trusty", "xenial", "yakkety", "zesty"}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -141,6 +157,8 @@ func main() {
 		doXCodeFramework(os.Args[2:])
 	case "xgo":
 		doXgo(os.Args[2:])
+	case "purge":
+		doPurge(os.Args[2:])
 	default:
 		log.Fatal("unknown command ", os.Args[1])
 	}
@@ -157,9 +175,9 @@ func doInstall(cmdline []string) {
 
 	// Check Go version. People regularly open issues about compilation
 	// failure with outdated Go. This should save them the trouble.
-	if runtime.Version() < "go1.4" && !strings.HasPrefix(runtime.Version(), "devel") {
+	if runtime.Version() < "go1.7" && !strings.HasPrefix(runtime.Version(), "devel") {
 		log.Println("You have Go version", runtime.Version())
-		log.Println("go-ethereum requires at least Go version 1.4 and cannot")
+		log.Println("go-ethereum requires at least Go version 1.7 and cannot")
 		log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
 		os.Exit(1)
 	}
@@ -168,6 +186,8 @@ func doInstall(cmdline []string) {
 	if flag.NArg() > 0 {
 		packages = flag.Args()
 	}
+	packages = build.ExpandPackagesNoVendor(packages)
+
 	if *arch == "" || *arch == runtime.GOARCH {
 		goinstall := goTool("install", buildFlags(env)...)
 		goinstall.Args = append(goinstall.Args, "-v")
@@ -210,20 +230,16 @@ func doInstall(cmdline []string) {
 }
 
 func buildFlags(env build.Environment) (flags []string) {
-	if os.Getenv("GO_OPENCL") != "" {
-		flags = append(flags, "-tags", "opencl")
+	var ld []string
+	if env.Commit != "" {
+		ld = append(ld, "-X", "main.gitCommit="+env.Commit)
+	}
+	if runtime.GOOS == "darwin" {
+		ld = append(ld, "-s")
 	}
 
-	// Since Go 1.5, the separator char for link time assignments
-	// is '=' and using ' ' prints a warning. However, Go < 1.5 does
-	// not support using '='.
-	sep := " "
-	if runtime.Version() > "go1.5" || strings.Contains(runtime.Version(), "devel") {
-		sep = "="
-	}
-	// Set gitCommit constant via link-time assignment.
-	if env.Commit != "" {
-		flags = append(flags, "-ldflags", "-X main.gitCommit"+sep+env.Commit)
+	if len(ld) > 0 {
+		flags = append(flags, "-ldflags", strings.Join(ld, " "))
 	}
 	return flags
 }
@@ -236,10 +252,15 @@ func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd {
 	gocmd := filepath.Join(runtime.GOROOT(), "bin", "go")
 	cmd := exec.Command(gocmd, subcmd)
 	cmd.Args = append(cmd.Args, args...)
-	cmd.Env = []string{
-		"GO15VENDOREXPERIMENT=1",
-		"GOPATH=" + build.GOPATH(),
+
+	if subcmd == "build" || subcmd == "install" || subcmd == "test" {
+		// Go CGO has a Windows linker error prior to 1.8 (https://github.com/golang/go/issues/8756).
+		// Work around issue by allowing multiple definitions for <1.8 builds.
+		if runtime.GOOS == "windows" && runtime.Version() < "go1.8" {
+			cmd.Args = append(cmd.Args, []string{"-ldflags", "-extldflags -Wl,--allow-multiple-definition"}...)
 		}
+	}
+	cmd.Env = []string{"GOPATH=" + build.GOPATH()}
 	if arch == "" || arch == runtime.GOARCH {
 		cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
 	} else {
@@ -261,38 +282,26 @@ func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd {
 
 func doTest(cmdline []string) {
 	var (
-		vet      = flag.Bool("vet", false, "Whether to run go vet")
 		misspell = flag.Bool("misspell", false, "Whether to run the spell checker")
 		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
 	)
 	flag.CommandLine.Parse(cmdline)
+	env := build.Env()
 
 	packages := []string{"./..."}
 	if len(flag.CommandLine.Args()) > 0 {
 		packages = flag.CommandLine.Args()
 	}
-	if len(packages) == 1 && packages[0] == "./..." {
-		// Resolve ./... manually since go vet will fail on vendored stuff
-		out, err := goTool("list", "./...").CombinedOutput()
-		if err != nil {
-			log.Fatalf("package listing failed: %v\n%s", err, string(out))
-		}
-		packages = []string{}
-		for _, line := range strings.Split(string(out), "\n") {
-			if !strings.Contains(line, "vendor") {
-				packages = append(packages, strings.TrimSpace(line))
-			}
-		}
-	}
+	packages = build.ExpandPackagesNoVendor(packages)
+
 	// Run analysis tools before the tests.
-	if *vet {
 	build.MustRun(goTool("vet", packages...))
-	}
 	if *misspell {
-		spellcheck(packages)
+		// TODO(karalabe): Reenable after false detection is fixed: https://github.com/client9/misspell/issues/105
+		// spellcheck(packages)
 	}
 	// Run the actual tests.
-	gotest := goTool("test")
+	gotest := goTool("test", buildFlags(env)...)
 	// Test a single package at a time. CI builders are slow
 	// and some tests run into timeouts under load.
 	gotest.Args = append(gotest.Args, "-p", "1")
@@ -309,7 +318,7 @@ func spellcheck(packages []string) {
 	// Ensure the spellchecker is available
 	build.MustRun(goTool("get", "github.com/client9/misspell/cmd/misspell"))
 
-	// Windows chokes on long argument lists, check packages individualy
+	// Windows chokes on long argument lists, check packages individually
 	for _, pkg := range packages {
 		// The spell checker doesn't work on packages, gather all .go files for it
 		out, err := goTool("list", "-f", "{{.Dir}}{{range .GoFiles}}\n{{.}}{{end}}{{range .CgoFiles}}\n{{.}}{{end}}{{range .TestGoFiles}}\n{{.}}{{end}}", pkg).CombinedOutput()
@@ -432,6 +441,10 @@ func maybeSkipArchive(env build.Environment) {
 		log.Printf("skipping because this is a PR build")
 		os.Exit(0)
 	}
+	if env.IsCronJob {
+		log.Printf("skipping because this is a cron job")
+		os.Exit(0)
+	}
 	if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") {
 		log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag)
 		os.Exit(0)
@@ -700,9 +713,16 @@ func doAndroidArchive(cmdline []string) {
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
 
+	// Sanity check that the SDK and NDK are installed and set
+	if os.Getenv("ANDROID_HOME") == "" {
+		log.Fatal("Please ensure ANDROID_HOME points to your Android SDK")
+	}
+	if os.Getenv("ANDROID_NDK") == "" {
+		log.Fatal("Please ensure ANDROID_NDK points to your Android NDK")
+	}
 	// Build the Android archive and Maven resources
 	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile"))
-	build.MustRun(gomobileTool("init"))
+	build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK")))
 	build.MustRun(gomobileTool("bind", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
 
 	if *local {
@@ -902,6 +922,9 @@ func newPodMetadata(env build.Environment, archive string) podMetadata {
 // Cross compilation
 
 func doXgo(cmdline []string) {
+	var (
+		alltools = flag.Bool("alltools", false, `Flag whether we're building all known tools, or only on in particular`)
+	)
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
 
@@ -909,8 +932,27 @@ func doXgo(cmdline []string) {
 	gogetxgo := goTool("get", "github.com/karalabe/xgo")
 	build.MustRun(gogetxgo)
 
-	// Execute the actual cross compilation
-	xgo := xgoTool(append(buildFlags(env), flag.Args()...))
+	// If all tools building is requested, build everything the builder wants
+	args := append(buildFlags(env), flag.Args()...)
+
+	if *alltools {
+		args = append(args, []string{"--dest", GOBIN}...)
+		for _, res := range allToolsArchiveFiles {
+			if strings.HasPrefix(res, GOBIN) {
+				// Binary tool found, cross build it explicitly
+				args = append(args, "./"+filepath.Join("cmd", filepath.Base(res)))
+				xgo := xgoTool(args)
+				build.MustRun(xgo)
+				args = args[:len(args)-1]
+			}
+		}
+		return
+	}
+	// Otherwise xxecute the explicit cross compilation
+	path := args[len(args)-1]
+	args = append(args[:len(args)-1], []string{"--dest", GOBIN, path}...)
+
+	xgo := xgoTool(args)
 	build.MustRun(xgo)
 }
 
@@ -928,3 +970,62 @@ func xgoTool(args []string) *exec.Cmd {
 	}
 	return cmd
 }
+
+// Binary distribution cleanups
+
+func doPurge(cmdline []string) {
+	var (
+		store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`)
+		limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`)
+	)
+	flag.CommandLine.Parse(cmdline)
+
+	if env := build.Env(); !env.IsCronJob {
+		log.Printf("skipping because not a cron job")
+		os.Exit(0)
+	}
+	// Create the azure authentication and list the current archives
+	auth := build.AzureBlobstoreConfig{
+		Account:   strings.Split(*store, "/")[0],
+		Token:     os.Getenv("AZURE_BLOBSTORE_TOKEN"),
+		Container: strings.SplitN(*store, "/", 2)[1],
+	}
+	blobs, err := build.AzureBlobstoreList(auth)
+	if err != nil {
+		log.Fatal(err)
+	}
+	// Iterate over the blobs, collect and sort all unstable builds
+	for i := 0; i < len(blobs); i++ {
+		if !strings.Contains(blobs[i].Name, "unstable") {
+			blobs = append(blobs[:i], blobs[i+1:]...)
+			i--
+		}
+	}
+	for i := 0; i < len(blobs); i++ {
+		for j := i + 1; j < len(blobs); j++ {
+			iTime, err := time.Parse(time.RFC1123, blobs[i].Properties.LastModified)
+			if err != nil {
+				log.Fatal(err)
+			}
+			jTime, err := time.Parse(time.RFC1123, blobs[j].Properties.LastModified)
+			if err != nil {
+				log.Fatal(err)
+			}
+			if iTime.After(jTime) {
+				blobs[i], blobs[j] = blobs[j], blobs[i]
+			}
+		}
+	}
+	// Filter out all archives more recent that the given threshold
+	for i, blob := range blobs {
+		timestamp, _ := time.Parse(time.RFC1123, blob.Properties.LastModified)
+		if time.Since(timestamp) < time.Duration(*limit)*24*time.Hour {
+			blobs = blobs[:i]
+			break
+		}
+	}
+	// Delete all marked as such and return
+	if err := build.AzureBlobstoreDelete(auth, blobs); err != nil {
+		log.Fatal(err)
+	}
+}

build/deb.control

@@ -2,7 +2,7 @@ Source: {{.Name}}
 Section: science
 Priority: extra
 Maintainer: {{.Author}}
-Build-Depends: debhelper (>= 8.0.0), golang-1.7
+Build-Depends: debhelper (>= 8.0.0), golang-1.8
 Standards-Version: 3.9.5
 Homepage: https://ethereum.org
 Vcs-Git: git://github.com/ethereum/go-ethereum.git

build/deb.rules

@@ -5,7 +5,7 @@
 #export DH_VERBOSE=1
 
 override_dh_auto_build:
-	build/env.sh /usr/lib/go-1.7/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+	build/env.sh /usr/lib/go-1.8/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 
 override_dh_auto_test:
 

build/env.sh

@@ -20,8 +20,7 @@ fi
 
 # Set up the environment to use the workspace.
 GOPATH="$workspace"
-GO15VENDOREXPERIMENT=1
-export GOPATH GO15VENDOREXPERIMENT
+export GOPATH
 
 # Run the command inside the workspace.
 cd "$ethdir/go-ethereum"

build/update-license.go

@@ -47,13 +47,18 @@ var (
 		// boring stuff
 		"vendor/", "tests/files/", "build/",
 		// don't relicense vendored sources
-		"crypto/sha3/", "crypto/ecies/", "logger/glog/",
+		"cmd/internal/browser",
+		"consensus/ethash/xor.go",
+		"crypto/bn256/",
+		"crypto/ecies/",
 		"crypto/secp256k1/curve.go",
+		"crypto/sha3/",
+		"internal/jsre/deps",
+		"log/",
 		// don't license generated files
 		"contracts/chequebook/contract/",
 		"contracts/ens/contract/",
 		"contracts/release/contract.go",
-		"p2p/discv5/nodeevent_string.go",
 	}
 
 	// paths with this prefix are licensed as GPL. all other files are LGPL.
@@ -284,6 +289,9 @@ func getInfo(files <-chan string, out chan<- *info, wg *sync.WaitGroup) {
 		if !stat.Mode().IsRegular() {
 			continue
 		}
+		if isGenerated(file) {
+			continue
+		}
 		info, err := fileInfo(file)
 		if err != nil {
 			fmt.Printf("ERROR %s: %v\n", file, err)
@@ -294,6 +302,23 @@ func getInfo(files <-chan string, out chan<- *info, wg *sync.WaitGroup) {
 	wg.Done()
 }
 
+func isGenerated(file string) bool {
+	fd, err := os.Open(file)
+	if err != nil {
+		return false
+	}
+	defer fd.Close()
+	buf := make([]byte, 2048)
+	n, _ := fd.Read(buf)
+	buf = buf[:n]
+	for _, l := range bytes.Split(buf, []byte("\n")) {
+		if bytes.HasPrefix(l, []byte("// Code generated")) {
+			return true
+		}
+	}
+	return false
+}
+
 // fileInfo finds the lowest year in which the given file was committed.
 func fileInfo(file string) (*info, error) {
 	info := &info{file: file, Year: int64(time.Now().Year())}

cmd/abigen/main.go

@@ -94,7 +94,9 @@ func main() {
 			abi, _ := json.Marshal(contract.Info.AbiDefinition) // Flatten the compiler parse
 			abis = append(abis, string(abi))
 			bins = append(bins, contract.Code)
-			types = append(types, name)
+
+			nameParts := strings.Split(name, ":")
+			types = append(types, nameParts[len(nameParts)-1])
 		}
 	} else {
 		// Otherwise load up the ABI, optional bytecode and type name from the parameters

cmd/bootnode/main.go

@@ -25,7 +25,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
 	"github.com/ethereum/go-ethereum/p2p/nat"
@@ -42,15 +42,19 @@ func main() {
 		natdesc     = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:<IP>)")
 		netrestrict = flag.String("netrestrict", "", "restrict network communication to the given IP networks (CIDR masks)")
 		runv5       = flag.Bool("v5", false, "run a v5 topic discovery bootnode")
+		verbosity   = flag.Int("verbosity", int(log.LvlInfo), "log verbosity (0-9)")
+		vmodule     = flag.String("vmodule", "", "log verbosity pattern")
 
 		nodeKey *ecdsa.PrivateKey
 		err     error
 	)
-	flag.Var(glog.GetVerbosity(), "verbosity", "log verbosity (0-9)")
-	flag.Var(glog.GetVModule(), "vmodule", "log verbosity pattern")
-	glog.SetToStderr(true)
 	flag.Parse()
 
+	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
+	glogger.Verbosity(log.Lvl(*verbosity))
+	glogger.Vmodule(*vmodule)
+	log.Root().SetHandler(glogger)
+
 	natm, err := nat.Parse(*natdesc)
 	if err != nil {
 		utils.Fatalf("-nat: %v", err)
@@ -64,6 +68,7 @@ func main() {
 		if err = crypto.SaveECDSA(*genKey, nodeKey); err != nil {
 			utils.Fatalf("%v", err)
 		}
+		return
 	case *nodeKeyFile == "" && *nodeKeyHex == "":
 		utils.Fatalf("Use -nodekey or -nodekeyhex to specify a private key")
 	case *nodeKeyFile != "" && *nodeKeyHex != "":

cmd/disasm/main.go

@@ -1,60 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// disasm is a pretty-printer for EVM bytecode.
-package main
-
-import (
-	"encoding/hex"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/core/vm"
-)
-
-func main() {
-	code, err := ioutil.ReadAll(os.Stdin)
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	code, err = hex.DecodeString(strings.TrimSpace(string(code[:])))
-	if err != nil {
-		fmt.Printf("Error: %v\n", err)
-		return
-	}
-	fmt.Printf("%x\n", code)
-
-	for pc := uint64(0); pc < uint64(len(code)); pc++ {
-		op := vm.OpCode(code[pc])
-
-		switch op {
-		case vm.PUSH1, vm.PUSH2, vm.PUSH3, vm.PUSH4, vm.PUSH5, vm.PUSH6, vm.PUSH7, vm.PUSH8, vm.PUSH9, vm.PUSH10, vm.PUSH11, vm.PUSH12, vm.PUSH13, vm.PUSH14, vm.PUSH15, vm.PUSH16, vm.PUSH17, vm.PUSH18, vm.PUSH19, vm.PUSH20, vm.PUSH21, vm.PUSH22, vm.PUSH23, vm.PUSH24, vm.PUSH25, vm.PUSH26, vm.PUSH27, vm.PUSH28, vm.PUSH29, vm.PUSH30, vm.PUSH31, vm.PUSH32:
-			a := uint64(op) - uint64(vm.PUSH1) + 1
-			u := pc + 1 + a
-			if uint64(len(code)) <= pc || uint64(len(code)) < u {
-				fmt.Printf("Error: incomplete push instruction at %v\n", pc)
-				return
-			}
-			fmt.Printf("%-5d  %v  => %x\n", pc, op, code[pc+1:u])
-			pc += a
-		default:
-			fmt.Printf("%-5d  %v\n", pc, op)
-		}
-	}
-}

cmd/ethtest/main.go

@@ -1,222 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// ethtest executes Ethereum JSON tests.
-package main
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/logger/glog"
-	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/tests"
-	"gopkg.in/urfave/cli.v1"
-)
-
-var (
-	continueOnError = false
-	testExtension   = ".json"
-	defaultTest     = "all"
-	defaultDir      = "."
-	allTests        = []string{"BlockTests", "StateTests", "TransactionTests", "VMTests", "RLPTests"}
-	testDirMapping  = map[string]string{"BlockTests": "BlockchainTests"}
-	skipTests       = []string{}
-
-	TestFlag = cli.StringFlag{
-		Name:  "test",
-		Usage: "Test type (string): VMTests, TransactionTests, StateTests, BlockTests",
-		Value: defaultTest,
-	}
-	FileFlag = cli.StringFlag{
-		Name:   "file",
-		Usage:  "Test file or directory. Directories are searched for .json files 1 level deep",
-		Value:  defaultDir,
-		EnvVar: "ETHEREUM_TEST_PATH",
-	}
-	ContinueOnErrorFlag = cli.BoolFlag{
-		Name:  "continue",
-		Usage: "Continue running tests on error (true) or [default] exit immediately (false)",
-	}
-	ReadStdInFlag = cli.BoolFlag{
-		Name:  "stdin",
-		Usage: "Accept input from stdin instead of reading from file",
-	}
-	SkipTestsFlag = cli.StringFlag{
-		Name:  "skip",
-		Usage: "Tests names to skip",
-	}
-	TraceFlag = cli.BoolFlag{
-		Name:  "trace",
-		Usage: "Enable VM tracing",
-	}
-)
-
-func runTestWithReader(test string, r io.Reader) error {
-	glog.Infoln("runTest", test)
-	var err error
-	switch strings.ToLower(test) {
-	case "bk", "block", "blocktest", "blockchaintest", "blocktests", "blockchaintests":
-		err = tests.RunBlockTestWithReader(params.MainNetHomesteadBlock, params.MainNetDAOForkBlock, params.MainNetHomesteadGasRepriceBlock, r, skipTests)
-	case "st", "state", "statetest", "statetests":
-		rs := &params.ChainConfig{HomesteadBlock: params.MainNetHomesteadBlock, DAOForkBlock: params.MainNetDAOForkBlock, DAOForkSupport: true, EIP150Block: params.MainNetHomesteadGasRepriceBlock}
-		err = tests.RunStateTestWithReader(rs, r, skipTests)
-	case "tx", "transactiontest", "transactiontests":
-		rs := &params.ChainConfig{HomesteadBlock: params.MainNetHomesteadBlock, DAOForkBlock: params.MainNetDAOForkBlock, DAOForkSupport: true, EIP150Block: params.MainNetHomesteadGasRepriceBlock}
-		err = tests.RunTransactionTestsWithReader(rs, r, skipTests)
-	case "vm", "vmtest", "vmtests":
-		err = tests.RunVmTestWithReader(r, skipTests)
-	case "rlp", "rlptest", "rlptests":
-		err = tests.RunRLPTestWithReader(r, skipTests)
-	default:
-		err = fmt.Errorf("Invalid test type specified: %v", test)
-	}
-	return err
-}
-
-func getFiles(path string) ([]string, error) {
-	glog.Infoln("getFiles", path)
-	var files []string
-	f, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	fi, err := f.Stat()
-	if err != nil {
-		return nil, err
-	}
-
-	switch mode := fi.Mode(); {
-	case mode.IsDir():
-		fi, _ := ioutil.ReadDir(path)
-		files = make([]string, len(fi))
-		for i, v := range fi {
-			// only go 1 depth and leave directory entires blank
-			if !v.IsDir() && v.Name()[len(v.Name())-len(testExtension):len(v.Name())] == testExtension {
-				files[i] = filepath.Join(path, v.Name())
-				glog.Infoln("Found file", files[i])
-			}
-		}
-	case mode.IsRegular():
-		files = make([]string, 1)
-		files[0] = path
-	}
-
-	return files, nil
-}
-
-func runSuite(test, file string) {
-	var tests []string
-
-	if test == defaultTest {
-		tests = allTests
-	} else {
-		tests = []string{test}
-	}
-
-	for _, curTest := range tests {
-		glog.Infoln("runSuite", curTest, file)
-		var err error
-		var files []string
-		if test == defaultTest {
-			// check if we have an explicit directory mapping for the test
-			if _, ok := testDirMapping[curTest]; ok {
-				files, err = getFiles(filepath.Join(file, testDirMapping[curTest]))
-			} else {
-				// otherwise assume test name
-				files, err = getFiles(filepath.Join(file, curTest))
-			}
-		} else {
-			files, err = getFiles(file)
-		}
-		if err != nil {
-			glog.Fatalln(err)
-		}
-
-		if len(files) == 0 {
-			glog.Warningln("No files matched path")
-		}
-		for _, curFile := range files {
-			// Skip blank entries
-			if len(curFile) == 0 {
-				continue
-			}
-
-			r, err := os.Open(curFile)
-			if err != nil {
-				glog.Fatalln(err)
-			}
-			defer r.Close()
-
-			err = runTestWithReader(curTest, r)
-			if err != nil {
-				if continueOnError {
-					glog.Errorln(err)
-				} else {
-					glog.Fatalln(err)
-				}
-			}
-		}
-	}
-}
-
-func setupApp(c *cli.Context) error {
-	flagTest := c.GlobalString(TestFlag.Name)
-	flagFile := c.GlobalString(FileFlag.Name)
-	continueOnError = c.GlobalBool(ContinueOnErrorFlag.Name)
-	useStdIn := c.GlobalBool(ReadStdInFlag.Name)
-	skipTests = strings.Split(c.GlobalString(SkipTestsFlag.Name), " ")
-
-	if !useStdIn {
-		runSuite(flagTest, flagFile)
-	} else {
-		if err := runTestWithReader(flagTest, os.Stdin); err != nil {
-			glog.Fatalln(err)
-		}
-	}
-	return nil
-}
-
-func main() {
-	glog.SetToStderr(true)
-
-	app := cli.NewApp()
-	app.Name = "ethtest"
-	app.Usage = "go-ethereum test interface"
-	app.Action = setupApp
-	app.Version = "0.2.0"
-	app.Author = "go-ethereum team"
-
-	app.Flags = []cli.Flag{
-		TestFlag,
-		FileFlag,
-		ContinueOnErrorFlag,
-		ReadStdInFlag,
-		SkipTestsFlag,
-		TraceFlag,
-	}
-
-	if err := app.Run(os.Args); err != nil {
-		glog.Fatalln(err)
-	}
-
-}

cmd/evm/compiler.go

@@ -0,0 +1,55 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+
+	"github.com/ethereum/go-ethereum/cmd/evm/internal/compiler"
+
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var compileCommand = cli.Command{
+	Action:    compileCmd,
+	Name:      "compile",
+	Usage:     "compiles easm source to evm binary",
+	ArgsUsage: "<file>",
+}
+
+func compileCmd(ctx *cli.Context) error {
+	debug := ctx.GlobalBool(DebugFlag.Name)
+
+	if len(ctx.Args().First()) == 0 {
+		return errors.New("filename required")
+	}
+
+	fn := ctx.Args().First()
+	src, err := ioutil.ReadFile(fn)
+	if err != nil {
+		return err
+	}
+
+	bin, err := compiler.Compile(fn, src, debug)
+	if err != nil {
+		return err
+	}
+	fmt.Println(bin)
+	return nil
+}

cmd/evm/disasm.go

@@ -0,0 +1,53 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/core/asm"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var disasmCommand = cli.Command{
+	Action:    disasmCmd,
+	Name:      "disasm",
+	Usage:     "disassembles evm binary",
+	ArgsUsage: "<file>",
+}
+
+func disasmCmd(ctx *cli.Context) error {
+	if len(ctx.Args().First()) == 0 {
+		return errors.New("filename required")
+	}
+
+	fn := ctx.Args().First()
+	in, err := ioutil.ReadFile(fn)
+	if err != nil {
+		return err
+	}
+
+	code := strings.TrimSpace(string(in[:]))
+	fmt.Printf("%v\n", code)
+	if err = asm.PrintDisassembled(code); err != nil {
+		return err
+	}
+	return nil
+}

cmd/evm/internal/compiler/compiler.go

@@ -0,0 +1,39 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package compiler
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/core/asm"
+)
+
+func Compile(fn string, src []byte, debug bool) (string, error) {
+	compiler := asm.NewCompiler(debug)
+	compiler.Feed(asm.Lex(fn, src, debug))
+
+	bin, compileErrors := compiler.Compile()
+	if len(compileErrors) > 0 {
+		// report errors
+		for _, err := range compileErrors {
+			fmt.Printf("%s:%v\n", fn, err)
+		}
+		return "", errors.New("compiling failed")
+	}
+	return bin, nil
+}

cmd/evm/json_logger.go

@@ -0,0 +1,67 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"io"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/core/vm"
+)
+
+type JSONLogger struct {
+	encoder *json.Encoder
+	cfg     *vm.LogConfig
+}
+
+func NewJSONLogger(cfg *vm.LogConfig, writer io.Writer) *JSONLogger {
+	return &JSONLogger{json.NewEncoder(writer), cfg}
+}
+
+// CaptureState outputs state information on the logger.
+func (l *JSONLogger) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error {
+	log := vm.StructLog{
+		Pc:         pc,
+		Op:         op,
+		Gas:        gas + cost,
+		GasCost:    cost,
+		MemorySize: memory.Len(),
+		Storage:    nil,
+		Depth:      depth,
+		Err:        err,
+	}
+	if !l.cfg.DisableMemory {
+		log.Memory = memory.Data()
+	}
+	if !l.cfg.DisableStack {
+		log.Stack = stack.Data()
+	}
+	return l.encoder.Encode(log)
+}
+
+// CaptureEnd is triggered at end of execution.
+func (l *JSONLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration) error {
+	type endLog struct {
+		Output  string              `json:"output"`
+		GasUsed math.HexOrDecimal64 `json:"gasUsed"`
+		Time    time.Duration       `json:"time"`
+	}
+	return l.encoder.Encode(endLog{common.Bytes2Hex(output), math.HexOrDecimal64(gasUsed), t})
+}

cmd/evm/main.go

@@ -19,19 +19,10 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
+	"math/big"
 	"os"
-	goruntime "runtime"
-	"time"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/core/vm/runtime"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/logger/glog"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -44,6 +35,18 @@ var (
 		Name:  "debug",
 		Usage: "output full trace logs",
 	}
+	MemProfileFlag = cli.StringFlag{
+		Name:  "memprofile",
+		Usage: "creates a memory profile at the given path",
+	}
+	CPUProfileFlag = cli.StringFlag{
+		Name:  "cpuprofile",
+		Usage: "creates a CPU profile at the given path",
+	}
+	StatDumpFlag = cli.BoolFlag{
+		Name:  "statdump",
+		Usage: "displays stack and heap memory information",
+	}
 	CodeFlag = cli.StringFlag{
 		Name:  "code",
 		Usage: "EVM code",
@@ -52,20 +55,20 @@ var (
 		Name:  "codefile",
 		Usage: "file containing EVM code",
 	}
-	GasFlag = cli.StringFlag{
+	GasFlag = cli.Uint64Flag{
 		Name:  "gas",
 		Usage: "gas limit for the evm",
-		Value: "10000000000",
+		Value: 10000000000,
 	}
-	PriceFlag = cli.StringFlag{
+	PriceFlag = utils.BigFlag{
 		Name:  "price",
 		Usage: "price set for the evm",
-		Value: "0",
+		Value: new(big.Int),
 	}
-	ValueFlag = cli.StringFlag{
+	ValueFlag = utils.BigFlag{
 		Name:  "value",
 		Usage: "value set for the evm",
-		Value: "0",
+		Value: new(big.Int),
 	}
 	DumpFlag = cli.BoolFlag{
 		Name:  "dump",
@@ -75,10 +78,6 @@ var (
 		Name:  "input",
 		Usage: "input for the EVM",
 	}
-	SysStatFlag = cli.BoolFlag{
-		Name:  "sysstat",
-		Usage: "display system stats",
-	}
 	VerbosityFlag = cli.IntFlag{
 		Name:  "verbosity",
 		Usage: "sets the verbosity level",
@@ -91,6 +90,26 @@ var (
 		Name:  "nogasmetering",
 		Usage: "disable gas metering",
 	}
+	GenesisFlag = cli.StringFlag{
+		Name:  "prestate",
+		Usage: "JSON file with prestate (genesis) config",
+	}
+	MachineFlag = cli.BoolFlag{
+		Name:  "json",
+		Usage: "output trace logs in machine readable format (json)",
+	}
+	SenderFlag = cli.StringFlag{
+		Name:  "sender",
+		Usage: "The transaction origin",
+	}
+	DisableMemoryFlag = cli.BoolFlag{
+		Name:  "nomemory",
+		Usage: "disable memory output",
+	}
+	DisableStackFlag = cli.BoolFlag{
+		Name:  "nostack",
+		Usage: "disable stack output",
+	}
 )
 
 func init() {
@@ -98,7 +117,6 @@ func init() {
 		CreateFlag,
 		DebugFlag,
 		VerbosityFlag,
-		SysStatFlag,
 		CodeFlag,
 		CodeFileFlag,
 		GasFlag,
@@ -107,106 +125,20 @@ func init() {
 		DumpFlag,
 		InputFlag,
 		DisableGasMeteringFlag,
+		MemProfileFlag,
+		CPUProfileFlag,
+		StatDumpFlag,
+		GenesisFlag,
+		MachineFlag,
+		SenderFlag,
+		DisableMemoryFlag,
+		DisableStackFlag,
 	}
-	app.Action = run
-}
-
-func run(ctx *cli.Context) error {
-	glog.SetToStderr(true)
-	glog.SetV(ctx.GlobalInt(VerbosityFlag.Name))
-
-	db, _ := ethdb.NewMemDatabase()
-	statedb, _ := state.New(common.Hash{}, db)
-	sender := statedb.CreateAccount(common.StringToAddress("sender"))
-
-	logger := vm.NewStructLogger(nil)
-
-	tstart := time.Now()
-
-	var (
-		code []byte
-		ret  []byte
-		err  error
-	)
-
-	if ctx.GlobalString(CodeFlag.Name) != "" {
-		code = common.Hex2Bytes(ctx.GlobalString(CodeFlag.Name))
-	} else {
-		var hexcode []byte
-		if ctx.GlobalString(CodeFileFlag.Name) != "" {
-			var err error
-			hexcode, err = ioutil.ReadFile(ctx.GlobalString(CodeFileFlag.Name))
-			if err != nil {
-				fmt.Printf("Could not load code from file: %v\n", err)
-				os.Exit(1)
+	app.Commands = []cli.Command{
+		compileCommand,
+		disasmCommand,
+		runCommand,
 	}
-		} else {
-			var err error
-			hexcode, err = ioutil.ReadAll(os.Stdin)
-			if err != nil {
-				fmt.Printf("Could not load code from stdin: %v\n", err)
-				os.Exit(1)
-			}
-		}
-		code = common.Hex2Bytes(string(hexcode[:]))
-	}
-
-	if ctx.GlobalBool(CreateFlag.Name) {
-		input := append(code, common.Hex2Bytes(ctx.GlobalString(InputFlag.Name))...)
-		ret, _, err = runtime.Create(input, &runtime.Config{
-			Origin:   sender.Address(),
-			State:    statedb,
-			GasLimit: common.Big(ctx.GlobalString(GasFlag.Name)),
-			GasPrice: common.Big(ctx.GlobalString(PriceFlag.Name)),
-			Value:    common.Big(ctx.GlobalString(ValueFlag.Name)),
-			EVMConfig: vm.Config{
-				Tracer:             logger,
-				DisableGasMetering: ctx.GlobalBool(DisableGasMeteringFlag.Name),
-			},
-		})
-	} else {
-		receiver := statedb.CreateAccount(common.StringToAddress("receiver"))
-		receiver.SetCode(crypto.Keccak256Hash(code), code)
-
-		ret, err = runtime.Call(receiver.Address(), common.Hex2Bytes(ctx.GlobalString(InputFlag.Name)), &runtime.Config{
-			Origin:   sender.Address(),
-			State:    statedb,
-			GasLimit: common.Big(ctx.GlobalString(GasFlag.Name)),
-			GasPrice: common.Big(ctx.GlobalString(PriceFlag.Name)),
-			Value:    common.Big(ctx.GlobalString(ValueFlag.Name)),
-			EVMConfig: vm.Config{
-				Tracer:             logger,
-				DisableGasMetering: ctx.GlobalBool(DisableGasMeteringFlag.Name),
-			},
-		})
-	}
-	vmdone := time.Since(tstart)
-
-	if ctx.GlobalBool(DumpFlag.Name) {
-		statedb.Commit(true)
-		fmt.Println(string(statedb.Dump()))
-	}
-	vm.StdErrFormat(logger.StructLogs())
-
-	if ctx.GlobalBool(SysStatFlag.Name) {
-		var mem goruntime.MemStats
-		goruntime.ReadMemStats(&mem)
-		fmt.Printf("vm took %v\n", vmdone)
-		fmt.Printf(`alloc:      %d
-tot alloc:  %d
-no. malloc: %d
-heap alloc: %d
-heap objs:  %d
-num gc:     %d
-`, mem.Alloc, mem.TotalAlloc, mem.Mallocs, mem.HeapAlloc, mem.HeapObjects, mem.NumGC)
-	}
-
-	fmt.Printf("OUT: 0x%x", ret)
-	if err != nil {
-		fmt.Printf(" error: %v", err)
-	}
-	fmt.Println()
-	return nil
 }
 
 func main() {

cmd/evm/runner.go

@@ -0,0 +1,239 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"runtime/pprof"
+	"time"
+
+	goruntime "runtime"
+
+	"github.com/ethereum/go-ethereum/cmd/evm/internal/compiler"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/core/vm/runtime"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var runCommand = cli.Command{
+	Action:      runCmd,
+	Name:        "run",
+	Usage:       "run arbitrary evm binary",
+	ArgsUsage:   "<code>",
+	Description: `The run command runs arbitrary EVM code.`,
+}
+
+// readGenesis will read the given JSON format genesis file and return
+// the initialized Genesis structure
+func readGenesis(genesisPath string) *core.Genesis {
+	// Make sure we have a valid genesis JSON
+	//genesisPath := ctx.Args().First()
+	if len(genesisPath) == 0 {
+		utils.Fatalf("Must supply path to genesis JSON file")
+	}
+	file, err := os.Open(genesisPath)
+	if err != nil {
+		utils.Fatalf("Failed to read genesis file: %v", err)
+	}
+	defer file.Close()
+
+	genesis := new(core.Genesis)
+	if err := json.NewDecoder(file).Decode(genesis); err != nil {
+		utils.Fatalf("invalid genesis file: %v", err)
+	}
+	return genesis
+}
+
+func runCmd(ctx *cli.Context) error {
+	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
+	glogger.Verbosity(log.Lvl(ctx.GlobalInt(VerbosityFlag.Name)))
+	log.Root().SetHandler(glogger)
+	logconfig := &vm.LogConfig{
+		DisableMemory: ctx.GlobalBool(DisableMemoryFlag.Name),
+		DisableStack:  ctx.GlobalBool(DisableStackFlag.Name),
+	}
+
+	var (
+		tracer      vm.Tracer
+		debugLogger *vm.StructLogger
+		statedb     *state.StateDB
+		chainConfig *params.ChainConfig
+		sender      = common.StringToAddress("sender")
+	)
+	if ctx.GlobalBool(MachineFlag.Name) {
+		tracer = NewJSONLogger(logconfig, os.Stdout)
+	} else if ctx.GlobalBool(DebugFlag.Name) {
+		debugLogger = vm.NewStructLogger(logconfig)
+		tracer = debugLogger
+	} else {
+		debugLogger = vm.NewStructLogger(logconfig)
+	}
+	if ctx.GlobalString(GenesisFlag.Name) != "" {
+		gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
+		_, statedb = gen.ToBlock()
+		chainConfig = gen.Config
+	} else {
+		var db, _ = ethdb.NewMemDatabase()
+		statedb, _ = state.New(common.Hash{}, db)
+	}
+	if ctx.GlobalString(SenderFlag.Name) != "" {
+		sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name))
+	}
+
+	statedb.CreateAccount(sender)
+
+	var (
+		code []byte
+		ret  []byte
+		err  error
+	)
+	if fn := ctx.Args().First(); len(fn) > 0 {
+		src, err := ioutil.ReadFile(fn)
+		if err != nil {
+			return err
+		}
+
+		bin, err := compiler.Compile(fn, src, false)
+		if err != nil {
+			return err
+		}
+		code = common.Hex2Bytes(bin)
+	} else if ctx.GlobalString(CodeFlag.Name) != "" {
+		code = common.Hex2Bytes(ctx.GlobalString(CodeFlag.Name))
+	} else {
+		var hexcode []byte
+		if ctx.GlobalString(CodeFileFlag.Name) != "" {
+			var err error
+			hexcode, err = ioutil.ReadFile(ctx.GlobalString(CodeFileFlag.Name))
+			if err != nil {
+				fmt.Printf("Could not load code from file: %v\n", err)
+				os.Exit(1)
+			}
+		} else {
+			var err error
+			hexcode, err = ioutil.ReadAll(os.Stdin)
+			if err != nil {
+				fmt.Printf("Could not load code from stdin: %v\n", err)
+				os.Exit(1)
+			}
+		}
+		code = common.Hex2Bytes(string(bytes.TrimRight(hexcode, "\n")))
+	}
+	initialGas := ctx.GlobalUint64(GasFlag.Name)
+	runtimeConfig := runtime.Config{
+		Origin:   sender,
+		State:    statedb,
+		GasLimit: initialGas,
+		GasPrice: utils.GlobalBig(ctx, PriceFlag.Name),
+		Value:    utils.GlobalBig(ctx, ValueFlag.Name),
+		EVMConfig: vm.Config{
+			Tracer:             tracer,
+			Debug:              ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name),
+			DisableGasMetering: ctx.GlobalBool(DisableGasMeteringFlag.Name),
+		},
+	}
+
+	if cpuProfilePath := ctx.GlobalString(CPUProfileFlag.Name); cpuProfilePath != "" {
+		f, err := os.Create(cpuProfilePath)
+		if err != nil {
+			fmt.Println("could not create CPU profile: ", err)
+			os.Exit(1)
+		}
+		if err := pprof.StartCPUProfile(f); err != nil {
+			fmt.Println("could not start CPU profile: ", err)
+			os.Exit(1)
+		}
+		defer pprof.StopCPUProfile()
+	}
+
+	if chainConfig != nil {
+		runtimeConfig.ChainConfig = chainConfig
+	}
+	tstart := time.Now()
+	var leftOverGas uint64
+	if ctx.GlobalBool(CreateFlag.Name) {
+		input := append(code, common.Hex2Bytes(ctx.GlobalString(InputFlag.Name))...)
+		ret, _, leftOverGas, err = runtime.Create(input, &runtimeConfig)
+	} else {
+		receiver := common.StringToAddress("receiver")
+		statedb.SetCode(receiver, code)
+
+		ret, leftOverGas, err = runtime.Call(receiver, common.Hex2Bytes(ctx.GlobalString(InputFlag.Name)), &runtimeConfig)
+	}
+	execTime := time.Since(tstart)
+
+	if ctx.GlobalBool(DumpFlag.Name) {
+		statedb.Commit(true)
+		fmt.Println(string(statedb.Dump()))
+	}
+
+	if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" {
+		f, err := os.Create(memProfilePath)
+		if err != nil {
+			fmt.Println("could not create memory profile: ", err)
+			os.Exit(1)
+		}
+		if err := pprof.WriteHeapProfile(f); err != nil {
+			fmt.Println("could not write memory profile: ", err)
+			os.Exit(1)
+		}
+		f.Close()
+	}
+
+	if ctx.GlobalBool(DebugFlag.Name) {
+		if debugLogger != nil {
+			fmt.Fprintln(os.Stderr, "#### TRACE ####")
+			vm.WriteTrace(os.Stderr, debugLogger.StructLogs())
+		}
+		fmt.Fprintln(os.Stderr, "#### LOGS ####")
+		vm.WriteLogs(os.Stderr, statedb.Logs())
+	}
+
+	if ctx.GlobalBool(StatDumpFlag.Name) {
+		var mem goruntime.MemStats
+		goruntime.ReadMemStats(&mem)
+		fmt.Fprintf(os.Stderr, `evm execution time: %v
+heap objects:       %d
+allocations:        %d
+total allocations:  %d
+GC calls:           %d
+Gas used:           %d
+
+`, execTime, mem.HeapObjects, mem.Alloc, mem.TotalAlloc, mem.NumGC, initialGas-leftOverGas)
+	}
+	if tracer != nil {
+		tracer.CaptureEnd(ret, initialGas-leftOverGas, execTime)
+	} else {
+		fmt.Printf("0x%x\n", ret)
+	}
+
+	if err != nil {
+		fmt.Printf(" error: %v\n", err)
+	}
+	return nil
+}

cmd/faucet/faucet.go

@@ -0,0 +1,543 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// faucet is a Ether faucet backed by a light client.
+package main
+
+//go:generate go-bindata -nometadata -o website.go faucet.html
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"html/template"
+	"io/ioutil"
+	"math"
+	"math/big"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/eth/downloader"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/ethstats"
+	"github.com/ethereum/go-ethereum/les"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/nat"
+	"github.com/ethereum/go-ethereum/params"
+	"golang.org/x/net/websocket"
+)
+
+var (
+	genesisFlag = flag.String("genesis", "", "Genesis json file to seed the chain with")
+	apiPortFlag = flag.Int("apiport", 8080, "Listener port for the HTTP API connection")
+	ethPortFlag = flag.Int("ethport", 30303, "Listener port for the devp2p connection")
+	bootFlag    = flag.String("bootnodes", "", "Comma separated bootnode enode URLs to seed with")
+	netFlag     = flag.Uint64("network", 0, "Network ID to use for the Ethereum protocol")
+	statsFlag   = flag.String("ethstats", "", "Ethstats network monitoring auth string")
+
+	netnameFlag = flag.String("faucet.name", "", "Network name to assign to the faucet")
+	payoutFlag  = flag.Int("faucet.amount", 1, "Number of Ethers to pay out per user request")
+	minutesFlag = flag.Int("faucet.minutes", 1440, "Number of minutes to wait between funding rounds")
+	tiersFlag   = flag.Int("faucet.tiers", 3, "Number of funding tiers to enable (x3 time, x2.5 funds)")
+
+	accJSONFlag = flag.String("account.json", "", "Key json file to fund user requests with")
+	accPassFlag = flag.String("account.pass", "", "Decryption password to access faucet funds")
+
+	githubUser  = flag.String("github.user", "", "GitHub user to authenticate with for Gist access")
+	githubToken = flag.String("github.token", "", "GitHub personal token to access Gists with")
+
+	captchaToken  = flag.String("captcha.token", "", "Recaptcha site key to authenticate client side")
+	captchaSecret = flag.String("captcha.secret", "", "Recaptcha secret key to authenticate server side")
+
+	logFlag = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet")
+)
+
+var (
+	ether = new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)
+)
+
+func main() {
+	// Parse the flags and set up the logger to print everything requested
+	flag.Parse()
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*logFlag), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
+
+	// Construct the payout tiers
+	amounts := make([]string, *tiersFlag)
+	periods := make([]string, *tiersFlag)
+	for i := 0; i < *tiersFlag; i++ {
+		// Calculate the amount for the next tier and format it
+		amount := float64(*payoutFlag) * math.Pow(2.5, float64(i))
+		amounts[i] = fmt.Sprintf("%s Ethers", strconv.FormatFloat(amount, 'f', -1, 64))
+		if amount == 1 {
+			amounts[i] = strings.TrimSuffix(amounts[i], "s")
+		}
+		// Calculate the period for the next tier and format it
+		period := *minutesFlag * int(math.Pow(3, float64(i)))
+		periods[i] = fmt.Sprintf("%d mins", period)
+		if period%60 == 0 {
+			period /= 60
+			periods[i] = fmt.Sprintf("%d hours", period)
+
+			if period%24 == 0 {
+				period /= 24
+				periods[i] = fmt.Sprintf("%d days", period)
+			}
+		}
+		if period == 1 {
+			periods[i] = strings.TrimSuffix(periods[i], "s")
+		}
+	}
+	// Load up and render the faucet website
+	tmpl, err := Asset("faucet.html")
+	if err != nil {
+		log.Crit("Failed to load the faucet template", "err", err)
+	}
+	website := new(bytes.Buffer)
+	err = template.Must(template.New("").Parse(string(tmpl))).Execute(website, map[string]interface{}{
+		"Network":   *netnameFlag,
+		"Amounts":   amounts,
+		"Periods":   periods,
+		"Recaptcha": *captchaToken,
+	})
+	if err != nil {
+		log.Crit("Failed to render the faucet template", "err", err)
+	}
+	// Load and parse the genesis block requested by the user
+	blob, err := ioutil.ReadFile(*genesisFlag)
+	if err != nil {
+		log.Crit("Failed to read genesis block contents", "genesis", *genesisFlag, "err", err)
+	}
+	genesis := new(core.Genesis)
+	if err = json.Unmarshal(blob, genesis); err != nil {
+		log.Crit("Failed to parse genesis block json", "err", err)
+	}
+	// Convert the bootnodes to internal enode representations
+	var enodes []*discv5.Node
+	for _, boot := range strings.Split(*bootFlag, ",") {
+		if url, err := discv5.ParseNode(boot); err == nil {
+			enodes = append(enodes, url)
+		} else {
+			log.Error("Failed to parse bootnode URL", "url", boot, "err", err)
+		}
+	}
+	// Load up the account key and decrypt its password
+	if blob, err = ioutil.ReadFile(*accPassFlag); err != nil {
+		log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err)
+	}
+	pass := string(blob)
+
+	ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP)
+	if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil {
+		log.Crit("Failed to read account key contents", "file", *accJSONFlag, "err", err)
+	}
+	acc, err := ks.Import(blob, pass, pass)
+	if err != nil {
+		log.Crit("Failed to import faucet signer account", "err", err)
+	}
+	ks.Unlock(acc, pass)
+
+	// Assemble and start the faucet light service
+	faucet, err := newFaucet(genesis, *ethPortFlag, enodes, *netFlag, *statsFlag, ks, website.Bytes())
+	if err != nil {
+		log.Crit("Failed to start faucet", "err", err)
+	}
+	defer faucet.close()
+
+	if err := faucet.listenAndServe(*apiPortFlag); err != nil {
+		log.Crit("Failed to launch faucet API", "err", err)
+	}
+}
+
+// request represents an accepted funding request.
+type request struct {
+	Username string             `json:"username"` // GitHub user for displaying an avatar
+	Account  common.Address     `json:"account"`  // Ethereum address being funded
+	Time     time.Time          `json:"time"`     // Timestamp when te request was accepted
+	Tx       *types.Transaction `json:"tx"`       // Transaction funding the account
+}
+
+// faucet represents a crypto faucet backed by an Ethereum light client.
+type faucet struct {
+	config *params.ChainConfig // Chain configurations for signing
+	stack  *node.Node          // Ethereum protocol stack
+	client *ethclient.Client   // Client connection to the Ethereum chain
+	index  []byte              // Index page to serve up on the web
+
+	keystore *keystore.KeyStore // Keystore containing the single signer
+	account  accounts.Account   // Account funding user faucet requests
+	nonce    uint64             // Current pending nonce of the faucet
+	price    *big.Int           // Current gas price to issue funds with
+
+	conns    []*websocket.Conn    // Currently live websocket connections
+	timeouts map[string]time.Time // History of users and their funding timeouts
+	reqs     []*request           // Currently pending funding requests
+	update   chan struct{}        // Channel to signal request updates
+
+	lock sync.RWMutex // Lock protecting the faucet's internals
+}
+
+func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network uint64, stats string, ks *keystore.KeyStore, index []byte) (*faucet, error) {
+	// Assemble the raw devp2p protocol stack
+	stack, err := node.New(&node.Config{
+		Name:    "geth",
+		Version: params.Version,
+		DataDir: filepath.Join(os.Getenv("HOME"), ".faucet"),
+		P2P: p2p.Config{
+			NAT:              nat.Any(),
+			NoDiscovery:      true,
+			DiscoveryV5:      true,
+			ListenAddr:       fmt.Sprintf(":%d", port),
+			DiscoveryV5Addr:  fmt.Sprintf(":%d", port+1),
+			MaxPeers:         25,
+			BootstrapNodesV5: enodes,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	// Assemble the Ethereum light client protocol
+	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+		cfg := eth.DefaultConfig
+		cfg.SyncMode = downloader.LightSync
+		cfg.NetworkId = network
+		cfg.Genesis = genesis
+		return les.New(ctx, &cfg)
+	}); err != nil {
+		return nil, err
+	}
+	// Assemble the ethstats monitoring and reporting service'
+	if stats != "" {
+		if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+			var serv *les.LightEthereum
+			ctx.Service(&serv)
+			return ethstats.New(stats, nil, serv)
+		}); err != nil {
+			return nil, err
+		}
+	}
+	// Boot up the client and ensure it connects to bootnodes
+	if err := stack.Start(); err != nil {
+		return nil, err
+	}
+	for _, boot := range enodes {
+		old, _ := discover.ParseNode(boot.String())
+		stack.Server().AddPeer(old)
+	}
+	// Attach to the client and retrieve and interesting metadatas
+	api, err := stack.Attach()
+	if err != nil {
+		stack.Stop()
+		return nil, err
+	}
+	client := ethclient.NewClient(api)
+
+	return &faucet{
+		config:   genesis.Config,
+		stack:    stack,
+		client:   client,
+		index:    index,
+		keystore: ks,
+		account:  ks.Accounts()[0],
+		timeouts: make(map[string]time.Time),
+		update:   make(chan struct{}, 1),
+	}, nil
+}
+
+// close terminates the Ethereum connection and tears down the faucet.
+func (f *faucet) close() error {
+	return f.stack.Stop()
+}
+
+// listenAndServe registers the HTTP handlers for the faucet and boots it up
+// for service user funding requests.
+func (f *faucet) listenAndServe(port int) error {
+	go f.loop()
+
+	http.HandleFunc("/", f.webHandler)
+	http.Handle("/api", websocket.Handler(f.apiHandler))
+
+	return http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
+}
+
+// webHandler handles all non-api requests, simply flattening and returning the
+// faucet website.
+func (f *faucet) webHandler(w http.ResponseWriter, r *http.Request) {
+	w.Write(f.index)
+}
+
+// apiHandler handles requests for Ether grants and transaction statuses.
+func (f *faucet) apiHandler(conn *websocket.Conn) {
+	// Start tracking the connection and drop at the end
+	f.lock.Lock()
+	f.conns = append(f.conns, conn)
+	f.lock.Unlock()
+
+	defer func() {
+		f.lock.Lock()
+		for i, c := range f.conns {
+			if c == conn {
+				f.conns = append(f.conns[:i], f.conns[i+1:]...)
+				break
+			}
+		}
+		f.lock.Unlock()
+	}()
+	// Send a few initial stats to the client
+	balance, _ := f.client.BalanceAt(context.Background(), f.account.Address, nil)
+	nonce, _ := f.client.NonceAt(context.Background(), f.account.Address, nil)
+
+	websocket.JSON.Send(conn, map[string]interface{}{
+		"funds":    balance.Div(balance, ether),
+		"funded":   nonce,
+		"peers":    f.stack.Server().PeerCount(),
+		"requests": f.reqs,
+	})
+	// Send the initial block to the client
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	header, err := f.client.HeaderByNumber(ctx, nil)
+	cancel()
+
+	if err != nil {
+		log.Error("Failed to retrieve latest header", "err", err)
+	} else {
+		websocket.JSON.Send(conn, header)
+	}
+	// Keep reading requests from the websocket until the connection breaks
+	for {
+		// Fetch the next funding request and validate against github
+		var msg struct {
+			URL     string `json:"url"`
+			Tier    uint   `json:"tier"`
+			Captcha string `json:"captcha"`
+		}
+		if err := websocket.JSON.Receive(conn, &msg); err != nil {
+			return
+		}
+		if !strings.HasPrefix(msg.URL, "https://gist.github.com/") {
+			websocket.JSON.Send(conn, map[string]string{"error": "URL doesn't link to GitHub Gists"})
+			continue
+		}
+		if msg.Tier >= uint(*tiersFlag) {
+			websocket.JSON.Send(conn, map[string]string{"error": "Invalid funding tier requested"})
+			continue
+		}
+		log.Info("Faucet funds requested", "gist", msg.URL, "tier", msg.Tier)
+
+		// If captcha verifications are enabled, make sure we're not dealing with a robot
+		if *captchaToken != "" {
+			form := url.Values{}
+			form.Add("secret", *captchaSecret)
+			form.Add("response", msg.Captcha)
+
+			res, err := http.PostForm("https://www.google.com/recaptcha/api/siteverify", form)
+			if err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				continue
+			}
+			var result struct {
+				Success bool            `json:"success"`
+				Errors  json.RawMessage `json:"error-codes"`
+			}
+			err = json.NewDecoder(res.Body).Decode(&result)
+			res.Body.Close()
+			if err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				continue
+			}
+			if !result.Success {
+				log.Warn("Captcha verification failed", "err", string(result.Errors))
+				websocket.JSON.Send(conn, map[string]string{"error": "Beep-bop, you're a robot!"})
+				continue
+			}
+		}
+		// Retrieve the gist from the GitHub Gist APIs
+		parts := strings.Split(msg.URL, "/")
+		req, _ := http.NewRequest("GET", "https://api.github.com/gists/"+parts[len(parts)-1], nil)
+		if *githubUser != "" {
+			req.SetBasicAuth(*githubUser, *githubToken)
+		}
+		res, err := http.DefaultClient.Do(req)
+		if err != nil {
+			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+			continue
+		}
+		var gist struct {
+			Owner struct {
+				Login string `json:"login"`
+			} `json:"owner"`
+			Files map[string]struct {
+				Content string `json:"content"`
+			} `json:"files"`
+		}
+		err = json.NewDecoder(res.Body).Decode(&gist)
+		res.Body.Close()
+		if err != nil {
+			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+			continue
+		}
+		if gist.Owner.Login == "" {
+			websocket.JSON.Send(conn, map[string]string{"error": "Anonymous Gists not allowed"})
+			continue
+		}
+		// Iterate over all the files and look for Ethereum addresses
+		var address common.Address
+		for _, file := range gist.Files {
+			if len(file.Content) == 2+common.AddressLength*2 {
+				address = common.HexToAddress(file.Content)
+			}
+		}
+		if address == (common.Address{}) {
+			websocket.JSON.Send(conn, map[string]string{"error": "No Ethereum address found to fund"})
+			continue
+		}
+		// Validate the user's existence since the API is unhelpful here
+		if res, err = http.Head("https://github.com/" + gist.Owner.Login); err != nil {
+			websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+			continue
+		}
+		res.Body.Close()
+
+		if res.StatusCode != 200 {
+			websocket.JSON.Send(conn, map[string]string{"error": "Invalid user... boom!"})
+			continue
+		}
+		// Ensure the user didn't request funds too recently
+		f.lock.Lock()
+		var (
+			fund    bool
+			timeout time.Time
+		)
+		if timeout = f.timeouts[gist.Owner.Login]; time.Now().After(timeout) {
+			// User wasn't funded recently, create the funding transaction
+			amount := new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether)
+			amount = new(big.Int).Mul(amount, new(big.Int).Exp(big.NewInt(5), big.NewInt(int64(msg.Tier)), nil))
+			amount = new(big.Int).Div(amount, new(big.Int).Exp(big.NewInt(2), big.NewInt(int64(msg.Tier)), nil))
+
+			tx := types.NewTransaction(f.nonce+uint64(len(f.reqs)), address, amount, big.NewInt(21000), f.price, nil)
+			signed, err := f.keystore.SignTx(f.account, tx, f.config.ChainId)
+			if err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				f.lock.Unlock()
+				continue
+			}
+			// Submit the transaction and mark as funded if successful
+			if err := f.client.SendTransaction(context.Background(), signed); err != nil {
+				websocket.JSON.Send(conn, map[string]string{"error": err.Error()})
+				f.lock.Unlock()
+				continue
+			}
+			f.reqs = append(f.reqs, &request{
+				Username: gist.Owner.Login,
+				Account:  address,
+				Time:     time.Now(),
+				Tx:       signed,
+			})
+			f.timeouts[gist.Owner.Login] = time.Now().Add(time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute)
+			fund = true
+		}
+		f.lock.Unlock()
+
+		// Send an error if too frequent funding, othewise a success
+		if !fund {
+			websocket.JSON.Send(conn, map[string]string{"error": fmt.Sprintf("%s left until next allowance", common.PrettyDuration(timeout.Sub(time.Now())))})
+			continue
+		}
+		websocket.JSON.Send(conn, map[string]string{"success": fmt.Sprintf("Funding request accepted for %s into %s", gist.Owner.Login, address.Hex())})
+		select {
+		case f.update <- struct{}{}:
+		default:
+		}
+	}
+}
+
+// loop keeps waiting for interesting events and pushes them out to connected
+// websockets.
+func (f *faucet) loop() {
+	// Wait for chain events and push them to clients
+	heads := make(chan *types.Header, 16)
+	sub, err := f.client.SubscribeNewHead(context.Background(), heads)
+	if err != nil {
+		log.Crit("Failed to subscribe to head events", "err", err)
+	}
+	defer sub.Unsubscribe()
+
+	for {
+		select {
+		case head := <-heads:
+			// New chain head arrived, query the current stats and stream to clients
+			balance, _ := f.client.BalanceAt(context.Background(), f.account.Address, nil)
+			balance = new(big.Int).Div(balance, ether)
+
+			price, _ := f.client.SuggestGasPrice(context.Background())
+			nonce, _ := f.client.NonceAt(context.Background(), f.account.Address, nil)
+
+			f.lock.Lock()
+			f.price, f.nonce = price, nonce
+			for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
+				f.reqs = f.reqs[1:]
+			}
+			f.lock.Unlock()
+
+			f.lock.RLock()
+			for _, conn := range f.conns {
+				if err := websocket.JSON.Send(conn, map[string]interface{}{
+					"funds":    balance,
+					"funded":   f.nonce,
+					"peers":    f.stack.Server().PeerCount(),
+					"requests": f.reqs,
+				}); err != nil {
+					log.Warn("Failed to send stats to client", "err", err)
+					conn.Close()
+					continue
+				}
+				if err := websocket.JSON.Send(conn, head); err != nil {
+					log.Warn("Failed to send header to client", "err", err)
+					conn.Close()
+				}
+			}
+			f.lock.RUnlock()
+
+		case <-f.update:
+			// Pending requests updated, stream to clients
+			f.lock.RLock()
+			for _, conn := range f.conns {
+				if err := websocket.JSON.Send(conn, map[string]interface{}{"requests": f.reqs}); err != nil {
+					log.Warn("Failed to send requests to client", "err", err)
+					conn.Close()
+				}
+			}
+			f.lock.RUnlock()
+		}
+	}
+}

cmd/faucet/faucet.html

@@ -0,0 +1,150 @@
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<meta charset="utf-8">
+		<meta http-equiv="X-UA-Compatible" content="IE=edge">
+		<meta name="viewport" content="width=device-width, initial-scale=1">
+
+		<title>{{.Network}}: GitHub Faucet</title>
+
+		<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet" />
+		<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet" />
+
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.1.1/jquery.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery-noty/2.4.1/packaged/jquery.noty.packaged.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js"></script>
+		<script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.18.0/moment.min.js"></script>
+
+		<style>
+			.vertical-center {
+				min-height: 100%;
+				min-height: 100vh;
+				display: flex;
+				align-items: center;
+			}
+			.progress {
+				position: relative;
+			}
+			.progress span {
+				position: absolute;
+				display: block;
+				width: 100%;
+				color: white;
+			 }
+			 pre {
+				 padding: 6px;
+				 margin: 0;
+			 }
+		</style>
+	</head>
+
+	<body>
+		<div class="vertical-center">
+			<div class="container">
+				<div class="row" style="margin-bottom: 16px;">
+					<div class="col-lg-12">
+						<h1 style="text-align: center;"><i class="fa fa-bath" aria-hidden="true"></i> {{.Network}} GitHub Authenticated Faucet <i class="fa fa-github-alt" aria-hidden="true"></i></h1>
+					</div>
+				</div>
+				<div class="row">
+					<div class="col-lg-8 col-lg-offset-2">
+						<div class="input-group">
+							<input id="gist" type="text" class="form-control" placeholder="GitHub Gist URL containing your Ethereum address...">
+							<span class="input-group-btn">
+								<button class="btn btn-default dropdown-toggle" type="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Give me Ether	<i class="fa fa-caret-down" aria-hidden="true"></i></button>
+				        <ul class="dropdown-menu dropdown-menu-right">{{range $idx, $amount := .Amounts}}
+				          <li><a style="text-align: center;" onclick="tier={{$idx}}; {{if $.Recaptcha}}grecaptcha.execute(){{else}}submit({{$idx}}){{end}}">{{$amount}} / {{index $.Periods $idx}}</a></li>{{end}}
+				        </ul>
+							</span>
+						</div>{{if .Recaptcha}}
+						<div class="g-recaptcha" data-sitekey="{{.Recaptcha}}" data-callback="submit" data-size="invisible"></div>{{end}}
+					</div>
+				</div>
+				<div class="row" style="margin-top: 32px;">
+						<div class="col-lg-6 col-lg-offset-3">
+						<div class="panel panel-small panel-default">
+							<div class="panel-body" style="padding: 0; overflow: auto; max-height: 300px;">
+								<table id="requests" class="table table-condensed" style="margin: 0;"></table>
+							</div>
+							<div class="panel-footer">
+								<table style="width: 100%"><tr>
+									<td style="text-align: center;"><i class="fa fa-rss" aria-hidden="true"></i> <span id="peers"></span> peers</td>
+									<td style="text-align: center;"><i class="fa fa-database" aria-hidden="true"></i> <span id="block"></span> blocks</td>
+									<td style="text-align: center;"><i class="fa fa-heartbeat" aria-hidden="true"></i> <span id="funds"></span> Ethers</td>
+									<td style="text-align: center;"><i class="fa fa-university" aria-hidden="true"></i> <span id="funded"></span> funded</td>
+								</tr></table>
+							</div>
+						</div>
+					</div>
+				</div>
+				<div class="row" style="margin-top: 32px;">
+					<div class="col-lg-12">
+						<h3>How does this work?</h3>
+						<p>This Ether faucet is running on the {{.Network}} network. To prevent malicious actors from exhausting all available funds or accumulating enough Ether to mount long running spam attacks, requests are tied to GitHub accounts. Anyone having a GitHub account may request funds within the permitted limits.</p>
+						<p>To request funds, simply create a <a href="https://gist.github.com/" target="_about:blank">GitHub Gist</a> with your Ethereum address pasted into the contents (the file name doesn't matter), copy paste the gists URL into the above input box and fire away! You can track the current pending requests below the input field to see how much you have to wait until your turn comes.</p>
+						{{if .Recaptcha}}<em>The faucet is running invisible reCaptcha protection against bots.</em>{{end}}
+					</div>
+				</div>
+			</div>
+		</div>
+
+		<script>
+			// Global variables to hold the current status of the faucet
+			var attempt = 0;
+			var server;
+			var tier = 0;
+
+			// Define the function that submits a gist url to the server
+			var submit = function({{if .Recaptcha}}captcha{{end}}) {
+				server.send(JSON.stringify({url: $("#gist")[0].value, tier: tier{{if .Recaptcha}}, captcha: captcha{{end}}}));{{if .Recaptcha}}
+				grecaptcha.reset();{{end}}
+			};
+			// Define a method to reconnect upon server loss
+			var reconnect = function() {
+				if (attempt % 2 == 0) {
+					server = new WebSocket("wss://" + location.host + "/api");
+				} else {
+					server = new WebSocket("ws://" + location.host + "/api");
+				}
+				attempt++;
+
+				server.onmessage = function(event) {
+					var msg = JSON.parse(event.data);
+					if (msg === null) {
+						return;
+					}
+
+					if (msg.funds !== undefined) {
+						$("#funds").text(msg.funds);
+					}
+					if (msg.funded !== undefined) {
+						$("#funded").text(msg.funded);
+					}
+					if (msg.peers !== undefined) {
+						$("#peers").text(msg.peers);
+					}
+					if (msg.number !== undefined) {
+						$("#block").text(parseInt(msg.number, 16));
+					}
+					if (msg.error !== undefined) {
+						noty({layout: 'topCenter', text: msg.error, type: 'error'});
+					}
+					if (msg.success !== undefined) {
+						noty({layout: 'topCenter', text: msg.success, type: 'success'});
+					}
+					if (msg.requests !== undefined && msg.requests !== null) {
+						var content = "";
+						for (var i=0; i<msg.requests.length; i++) {
+							content += "<tr><td><div style=\"background: url('https://github.com/" + msg.requests[i].username + ".png?size=64'); background-size: cover; width:32px; height: 32px; border-radius: 4px;\"></div></td><td><pre>" + msg.requests[i].account + "</pre></td><td style=\"width: 100%; text-align: center; vertical-align: middle;\">" + moment.duration(moment(msg.requests[i].time).unix()-moment().unix(), 'seconds').humanize(true) + "</td></tr>";
+						}
+						$("#requests").html("<tbody>" + content + "</tbody>");
+					}
+				}
+				server.onclose = function() { setTimeout(reconnect, 3000); };
+			}
+			// Establish a websocket connection to the API server
+			reconnect();
+		</script>{{if .Recaptcha}}
+		<script src="https://www.google.com/recaptcha/api.js" async defer></script>{{end}}
+	</body>
+</html>

cmd/geth/accountcmd.go

@@ -21,11 +21,11 @@ import (
 	"io/ioutil"
 
 	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -40,32 +40,39 @@ var (
 
 will prompt for your password and imports your ether presale account.
 It can be used non-interactively with the --password option taking a
-passwordfile as argument containing the wallet password in plaintext.
-
-`,
+passwordfile as argument containing the wallet password in plaintext.`,
 		Subcommands: []cli.Command{
 			{
-				Action:    importWallet,
+
 				Name:      "import",
 				Usage:     "Import Ethereum presale wallet",
 				ArgsUsage: "<keyFile>",
+				Action:    utils.MigrateFlags(importWallet),
+				Category:  "ACCOUNT COMMANDS",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+					utils.PasswordFileFlag,
+					utils.LightKDFFlag,
+				},
 				Description: `
-TODO: Please write this
-`,
+	geth wallet [options] /path/to/my/presale.wallet
+
+will prompt for your password and imports your ether presale account.
+It can be used non-interactively with the --password option taking a
+passwordfile as argument containing the wallet password in plaintext.`,
 			},
 		},
 	}
+
 	accountCommand = cli.Command{
-		Action:    accountList,
 		Name:     "account",
 		Usage:    "Manage accounts",
-		ArgsUsage: "",
 		Category: "ACCOUNT COMMANDS",
 		Description: `
-Manage accounts lets you create new accounts, list all existing accounts,
-import a private key into a new account.
 
-'            help' shows a list of subcommands or help for one subcommand.
+Manage accounts, list all existing accounts, import a private key into a new
+account, create a new account or update an existing account.
 
 It supports interactive mode, when you are prompted for password as well as
 non-interactive mode where passwords are supplied via a given password file.
@@ -80,36 +87,34 @@ Note that exporting your key in unencrypted format is NOT supported.
 Keys are stored under <DATADIR>/keystore.
 It is safe to transfer the entire directory or the individual keys therein
 between ethereum nodes by simply copying.
-Make sure you backup your keys regularly.
 
-In order to use your account to send transactions, you need to unlock them using
-the '--unlock' option. The argument is a space separated list of addresses or
-indexes. If used non-interactively with a passwordfile, the file should contain
-the respective passwords one per line. If you unlock n accounts and the password
-file contains less than n entries, then the last password is meant to apply to
-all remaining accounts.
-
-And finally. DO NOT FORGET YOUR PASSWORD.
-`,
+Make sure you backup your keys regularly.`,
 		Subcommands: []cli.Command{
 			{
-				Action:    accountList,
 				Name:   "list",
-				Usage:     "Print account addresses",
-				ArgsUsage: " ",
+				Usage:  "Print summary of existing accounts",
+				Action: utils.MigrateFlags(accountList),
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+				},
 				Description: `
-TODO: Please write this
-`,
+Print a short summary of all accounts`,
 			},
 			{
-				Action:    accountCreate,
 				Name:   "new",
 				Usage:  "Create a new account",
-				ArgsUsage: " ",
+				Action: utils.MigrateFlags(accountCreate),
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+					utils.PasswordFileFlag,
+					utils.LightKDFFlag,
+				},
 				Description: `
     geth account new
 
-Creates a new account. Prints the address.
+Creates a new account and prints the address.
 
 The account is saved in encrypted format, you are prompted for a passphrase.
 
@@ -117,17 +122,20 @@ You must remember this passphrase to unlock your account in the future.
 
 For non-interactive use the passphrase can be specified with the --password flag:
 
-    geth --password <passwordfile> account new
-
 Note, this is meant to be used for testing only, it is a bad idea to save your
 password to file or expose in any other way.
 `,
 			},
 			{
-				Action:    accountUpdate,
 				Name:      "update",
 				Usage:     "Update an existing account",
+				Action:    utils.MigrateFlags(accountUpdate),
 				ArgsUsage: "<address>",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+					utils.LightKDFFlag,
+				},
 				Description: `
     geth account update <address>
 
@@ -141,16 +149,22 @@ format to the newest format or change the password for an account.
 
 For non-interactive use the passphrase can be specified with the --password flag:
 
-    geth --password <passwordfile> account update <address>
+    geth account update [options] <address>
 
 Since only one password can be given, only format update can be performed,
 changing your password is only possible interactively.
 `,
 			},
 			{
-				Action:    accountImport,
 				Name:   "import",
 				Usage:  "Import a private key into a new account",
+				Action: utils.MigrateFlags(accountImport),
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.KeyStoreDirFlag,
+					utils.PasswordFileFlag,
+					utils.LightKDFFlag,
+				},
 				ArgsUsage: "<keyFile>",
 				Description: `
     geth account import <keyfile>
@@ -166,7 +180,7 @@ You must remember this passphrase to unlock your account in the future.
 
 For non-interactive use the passphrase can be specified with the -password flag:
 
-    geth --password <passwordfile> account import <keyfile>
+    geth account import [options] <keyfile>
 
 Note:
 As you can directly copy your encrypted accounts to another ethereum instance,
@@ -179,42 +193,47 @@ nodes.
 )
 
 func accountList(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-	for i, acct := range stack.AccountManager().Accounts() {
-		fmt.Printf("Account #%d: {%x} %s\n", i, acct.Address, acct.File)
+	stack, _ := makeConfigNode(ctx)
+	var index int
+	for _, wallet := range stack.AccountManager().Wallets() {
+		for _, account := range wallet.Accounts() {
+			fmt.Printf("Account #%d: {%x} %s\n", index, account.Address, &account.URL)
+			index++
+		}
 	}
 	return nil
 }
 
 // tries unlocking the specified account a few times.
-func unlockAccount(ctx *cli.Context, accman *accounts.Manager, address string, i int, passwords []string) (accounts.Account, string) {
-	account, err := utils.MakeAddress(accman, address)
+func unlockAccount(ctx *cli.Context, ks *keystore.KeyStore, address string, i int, passwords []string) (accounts.Account, string) {
+	account, err := utils.MakeAddress(ks, address)
 	if err != nil {
 		utils.Fatalf("Could not list accounts: %v", err)
 	}
 	for trials := 0; trials < 3; trials++ {
 		prompt := fmt.Sprintf("Unlocking account %s | Attempt %d/%d", address, trials+1, 3)
 		password := getPassPhrase(prompt, false, i, passwords)
-		err = accman.Unlock(account, password)
+		err = ks.Unlock(account, password)
 		if err == nil {
-			glog.V(logger.Info).Infof("Unlocked account %x", account.Address)
+			log.Info("Unlocked account", "address", account.Address.Hex())
 			return account, password
 		}
-		if err, ok := err.(*accounts.AmbiguousAddrError); ok {
-			glog.V(logger.Info).Infof("Unlocked account %x", account.Address)
-			return ambiguousAddrRecovery(accman, err, password), password
+		if err, ok := err.(*keystore.AmbiguousAddrError); ok {
+			log.Info("Unlocked account", "address", account.Address.Hex())
+			return ambiguousAddrRecovery(ks, err, password), password
 		}
-		if err != accounts.ErrDecrypt {
+		if err != keystore.ErrDecrypt {
 			// No need to prompt again if the error is not decryption-related.
 			break
 		}
 	}
 	// All trials expended to unlock account, bail out
 	utils.Fatalf("Failed to unlock account %s (%v)", address, err)
+
 	return accounts.Account{}, ""
 }
 
-// getPassPhrase retrieves the passwor associated with an account, either fetched
+// getPassPhrase retrieves the password associated with an account, either fetched
 // from a list of preloaded passphrases, or requested interactively from the user.
 func getPassPhrase(prompt string, confirmation bool, i int, passwords []string) string {
 	// If a list of passwords was supplied, retrieve from them
@@ -244,15 +263,15 @@ func getPassPhrase(prompt string, confirmation bool, i int, passwords []string)
 	return password
 }
 
-func ambiguousAddrRecovery(am *accounts.Manager, err *accounts.AmbiguousAddrError, auth string) accounts.Account {
+func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrError, auth string) accounts.Account {
 	fmt.Printf("Multiple key files exist for address %x:\n", err.Addr)
 	for _, a := range err.Matches {
-		fmt.Println("  ", a.File)
+		fmt.Println("  ", a.URL)
 	}
 	fmt.Println("Testing your passphrase against all of them...")
 	var match *accounts.Account
 	for _, a := range err.Matches {
-		if err := am.Unlock(a, auth); err == nil {
+		if err := ks.Unlock(a, auth); err == nil {
 			match = &a
 			break
 		}
@@ -260,11 +279,11 @@ func ambiguousAddrRecovery(am *accounts.Manager, err *accounts.AmbiguousAddrErro
 	if match == nil {
 		utils.Fatalf("None of the listed files could be unlocked.")
 	}
-	fmt.Printf("Your passphrase unlocked %s\n", match.File)
+	fmt.Printf("Your passphrase unlocked %s\n", match.URL)
 	fmt.Println("In order to avoid this warning, you need to remove the following duplicate key files:")
 	for _, a := range err.Matches {
 		if a != *match {
-			fmt.Println("  ", a.File)
+			fmt.Println("  ", a.URL)
 		}
 	}
 	return *match
@@ -272,10 +291,11 @@ func ambiguousAddrRecovery(am *accounts.Manager, err *accounts.AmbiguousAddrErro
 
 // accountCreate creates a new account into the keystore defined by the CLI flags.
 func accountCreate(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	password := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx))
 
-	account, err := stack.AccountManager().NewAccount(password)
+	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+	account, err := ks.NewAccount(password)
 	if err != nil {
 		utils.Fatalf("Failed to create account: %v", err)
 	}
@@ -289,12 +309,16 @@ func accountUpdate(ctx *cli.Context) error {
 	if len(ctx.Args()) == 0 {
 		utils.Fatalf("No accounts specified to update")
 	}
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-	account, oldPassword := unlockAccount(ctx, stack.AccountManager(), ctx.Args().First(), 0, nil)
+	stack, _ := makeConfigNode(ctx)
+	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+
+	for _, addr := range ctx.Args() {
+		account, oldPassword := unlockAccount(ctx, ks, addr, 0, nil)
 		newPassword := getPassPhrase("Please give a new password. Do not forget this password.", true, 0, nil)
-	if err := stack.AccountManager().Update(account, oldPassword, newPassword); err != nil {
+		if err := ks.Update(account, oldPassword, newPassword); err != nil {
 			utils.Fatalf("Could not update the account: %v", err)
 		}
+	}
 	return nil
 }
 
@@ -308,9 +332,11 @@ func importWallet(ctx *cli.Context) error {
 		utils.Fatalf("Could not read wallet file: %v", err)
 	}
 
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	passphrase := getPassPhrase("", false, 0, utils.MakePasswordList(ctx))
-	acct, err := stack.AccountManager().ImportPreSaleKey(keyJson, passphrase)
+
+	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+	acct, err := ks.ImportPreSaleKey(keyJson, passphrase)
 	if err != nil {
 		utils.Fatalf("%v", err)
 	}
@@ -327,9 +353,11 @@ func accountImport(ctx *cli.Context) error {
 	if err != nil {
 		utils.Fatalf("Failed to load the private key: %v", err)
 	}
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
+	stack, _ := makeConfigNode(ctx)
 	passphrase := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx))
-	acct, err := stack.AccountManager().ImportECDSA(key, passphrase)
+
+	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+	acct, err := ks.ImportECDSA(key, passphrase)
 	if err != nil {
 		utils.Fatalf("Could not create the account: %v", err)
 	}

cmd/geth/accountcmd_test.go

@@ -35,7 +35,7 @@ import (
 func tmpDatadirWithKeystore(t *testing.T) string {
 	datadir := tmpdir(t)
 	keystore := filepath.Join(datadir, "keystore")
-	source := filepath.Join("..", "..", "accounts", "testdata", "keystore")
+	source := filepath.Join("..", "..", "accounts", "keystore", "testdata", "keystore")
 	if err := cp.CopyAll(keystore, source); err != nil {
 		t.Fatal(err)
 	}
@@ -43,45 +43,45 @@ func tmpDatadirWithKeystore(t *testing.T) string {
 }
 
 func TestAccountListEmpty(t *testing.T) {
-	geth := runGeth(t, "account")
-	geth.expectExit()
+	geth := runGeth(t, "account", "list")
+	geth.ExpectExit()
 }
 
 func TestAccountList(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
-	geth := runGeth(t, "--datadir", datadir, "account")
-	defer geth.expectExit()
+	geth := runGeth(t, "account", "list", "--datadir", datadir)
+	defer geth.ExpectExit()
 	if runtime.GOOS == "windows" {
-		geth.expect(`
-Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} {{.Datadir}}\keystore\UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8
-Account #1: {f466859ead1932d743d622cb74fc058882e8648a} {{.Datadir}}\keystore\aaa
-Account #2: {289d485d9771714cce91d3393d764e1311907acc} {{.Datadir}}\keystore\zzz
+		geth.Expect(`
+Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} keystore://{{.Datadir}}\keystore\UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8
+Account #1: {f466859ead1932d743d622cb74fc058882e8648a} keystore://{{.Datadir}}\keystore\aaa
+Account #2: {289d485d9771714cce91d3393d764e1311907acc} keystore://{{.Datadir}}\keystore\zzz
 `)
 	} else {
-		geth.expect(`
-Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} {{.Datadir}}/keystore/UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8
-Account #1: {f466859ead1932d743d622cb74fc058882e8648a} {{.Datadir}}/keystore/aaa
-Account #2: {289d485d9771714cce91d3393d764e1311907acc} {{.Datadir}}/keystore/zzz
+		geth.Expect(`
+Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} keystore://{{.Datadir}}/keystore/UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8
+Account #1: {f466859ead1932d743d622cb74fc058882e8648a} keystore://{{.Datadir}}/keystore/aaa
+Account #2: {289d485d9771714cce91d3393d764e1311907acc} keystore://{{.Datadir}}/keystore/zzz
 `)
 	}
 }
 
 func TestAccountNew(t *testing.T) {
-	geth := runGeth(t, "--lightkdf", "account", "new")
-	defer geth.expectExit()
-	geth.expect(`
+	geth := runGeth(t, "account", "new", "--lightkdf")
+	defer geth.ExpectExit()
+	geth.Expect(`
 Your new account is locked with a password. Please give a password. Do not forget this password.
 !! Unsupported terminal, password will be echoed.
 Passphrase: {{.InputLine "foobar"}}
 Repeat passphrase: {{.InputLine "foobar"}}
 `)
-	geth.expectRegexp(`Address: \{[0-9a-f]{40}\}\n`)
+	geth.ExpectRegexp(`Address: \{[0-9a-f]{40}\}\n`)
 }
 
 func TestAccountNewBadRepeat(t *testing.T) {
-	geth := runGeth(t, "--lightkdf", "account", "new")
-	defer geth.expectExit()
-	geth.expect(`
+	geth := runGeth(t, "account", "new", "--lightkdf")
+	defer geth.ExpectExit()
+	geth.Expect(`
 Your new account is locked with a password. Please give a password. Do not forget this password.
 !! Unsupported terminal, password will be echoed.
 Passphrase: {{.InputLine "something"}}
@@ -92,11 +92,11 @@ Fatal: Passphrases do not match
 
 func TestAccountUpdate(t *testing.T) {
 	datadir := tmpDatadirWithKeystore(t)
-	geth := runGeth(t,
+	geth := runGeth(t, "account", "update",
 		"--datadir", datadir, "--lightkdf",
-		"account", "update", "f466859ead1932d743d622cb74fc058882e8648a")
-	defer geth.expectExit()
-	geth.expect(`
+		"f466859ead1932d743d622cb74fc058882e8648a")
+	defer geth.ExpectExit()
+	geth.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
 Passphrase: {{.InputLine "foobar"}}
@@ -107,9 +107,9 @@ Repeat passphrase: {{.InputLine "foobar2"}}
 }
 
 func TestWalletImport(t *testing.T) {
-	geth := runGeth(t, "--lightkdf", "wallet", "import", "testdata/guswallet.json")
-	defer geth.expectExit()
-	geth.expect(`
+	geth := runGeth(t, "wallet", "import", "--lightkdf", "testdata/guswallet.json")
+	defer geth.ExpectExit()
+	geth.Expect(`
 !! Unsupported terminal, password will be echoed.
 Passphrase: {{.InputLine "foo"}}
 Address: {d4584b5f6229b7be90727b0fc8c6b91bb427821f}
@@ -122,9 +122,9 @@ Address: {d4584b5f6229b7be90727b0fc8c6b91bb427821f}
 }
 
 func TestWalletImportBadPassword(t *testing.T) {
-	geth := runGeth(t, "--lightkdf", "wallet", "import", "testdata/guswallet.json")
-	defer geth.expectExit()
-	geth.expect(`
+	geth := runGeth(t, "wallet", "import", "--lightkdf", "testdata/guswallet.json")
+	defer geth.ExpectExit()
+	geth.Expect(`
 !! Unsupported terminal, password will be echoed.
 Passphrase: {{.InputLine "wrong"}}
 Fatal: could not decrypt key with given passphrase
@@ -137,18 +137,19 @@ func TestUnlockFlag(t *testing.T) {
 		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
 		"js", "testdata/empty.js")
-	geth.expect(`
+	geth.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
 Passphrase: {{.InputLine "foobar"}}
 `)
-	geth.expectExit()
+	geth.ExpectExit()
 
 	wantMessages := []string{
-		"Unlocked account f466859ead1932d743d622cb74fc058882e8648a",
+		"Unlocked account",
+		"=0xf466859ead1932d743d622cb74fc058882e8648a",
 	}
 	for _, m := range wantMessages {
-		if !strings.Contains(geth.stderrText(), m) {
+		if !strings.Contains(geth.StderrText(), m) {
 			t.Errorf("stderr text does not contain %q", m)
 		}
 	}
@@ -159,8 +160,8 @@ func TestUnlockFlagWrongPassword(t *testing.T) {
 	geth := runGeth(t,
 		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
-	defer geth.expectExit()
-	geth.expect(`
+	defer geth.ExpectExit()
+	geth.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
 Passphrase: {{.InputLine "wrong1"}}
@@ -179,21 +180,22 @@ func TestUnlockFlagMultiIndex(t *testing.T) {
 		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
 		"--unlock", "0,2",
 		"js", "testdata/empty.js")
-	geth.expect(`
+	geth.Expect(`
 Unlocking account 0 | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
 Passphrase: {{.InputLine "foobar"}}
 Unlocking account 2 | Attempt 1/3
 Passphrase: {{.InputLine "foobar"}}
 `)
-	geth.expectExit()
+	geth.ExpectExit()
 
 	wantMessages := []string{
-		"Unlocked account 7ef5a6135f1fd6a02593eedc869c6d41d934aef8",
-		"Unlocked account 289d485d9771714cce91d3393d764e1311907acc",
+		"Unlocked account",
+		"=0x7ef5a6135f1fd6a02593eedc869c6d41d934aef8",
+		"=0x289d485d9771714cce91d3393d764e1311907acc",
 	}
 	for _, m := range wantMessages {
-		if !strings.Contains(geth.stderrText(), m) {
+		if !strings.Contains(geth.StderrText(), m) {
 			t.Errorf("stderr text does not contain %q", m)
 		}
 	}
@@ -205,14 +207,15 @@ func TestUnlockFlagPasswordFile(t *testing.T) {
 		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
 		"--password", "testdata/passwords.txt", "--unlock", "0,2",
 		"js", "testdata/empty.js")
-	geth.expectExit()
+	geth.ExpectExit()
 
 	wantMessages := []string{
-		"Unlocked account 7ef5a6135f1fd6a02593eedc869c6d41d934aef8",
-		"Unlocked account 289d485d9771714cce91d3393d764e1311907acc",
+		"Unlocked account",
+		"=0x7ef5a6135f1fd6a02593eedc869c6d41d934aef8",
+		"=0x289d485d9771714cce91d3393d764e1311907acc",
 	}
 	for _, m := range wantMessages {
-		if !strings.Contains(geth.stderrText(), m) {
+		if !strings.Contains(geth.StderrText(), m) {
 			t.Errorf("stderr text does not contain %q", m)
 		}
 	}
@@ -223,70 +226,71 @@ func TestUnlockFlagPasswordFileWrongPassword(t *testing.T) {
 	geth := runGeth(t,
 		"--datadir", datadir, "--nat", "none", "--nodiscover", "--dev",
 		"--password", "testdata/wrong-passwords.txt", "--unlock", "0,2")
-	defer geth.expectExit()
-	geth.expect(`
+	defer geth.ExpectExit()
+	geth.Expect(`
 Fatal: Failed to unlock account 0 (could not decrypt key with given passphrase)
 `)
 }
 
 func TestUnlockFlagAmbiguous(t *testing.T) {
-	store := filepath.Join("..", "..", "accounts", "testdata", "dupes")
+	store := filepath.Join("..", "..", "accounts", "keystore", "testdata", "dupes")
 	geth := runGeth(t,
 		"--keystore", store, "--nat", "none", "--nodiscover", "--dev",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
 		"js", "testdata/empty.js")
-	defer geth.expectExit()
+	defer geth.ExpectExit()
 
 	// Helper for the expect template, returns absolute keystore path.
-	geth.setTemplateFunc("keypath", func(file string) string {
+	geth.SetTemplateFunc("keypath", func(file string) string {
 		abs, _ := filepath.Abs(filepath.Join(store, file))
 		return abs
 	})
-	geth.expect(`
+	geth.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
 Passphrase: {{.InputLine "foobar"}}
 Multiple key files exist for address f466859ead1932d743d622cb74fc058882e8648a:
-   {{keypath "1"}}
-   {{keypath "2"}}
+   keystore://{{keypath "1"}}
+   keystore://{{keypath "2"}}
 Testing your passphrase against all of them...
-Your passphrase unlocked {{keypath "1"}}
+Your passphrase unlocked keystore://{{keypath "1"}}
 In order to avoid this warning, you need to remove the following duplicate key files:
-   {{keypath "2"}}
+   keystore://{{keypath "2"}}
 `)
-	geth.expectExit()
+	geth.ExpectExit()
 
 	wantMessages := []string{
-		"Unlocked account f466859ead1932d743d622cb74fc058882e8648a",
+		"Unlocked account",
+		"=0xf466859ead1932d743d622cb74fc058882e8648a",
 	}
 	for _, m := range wantMessages {
-		if !strings.Contains(geth.stderrText(), m) {
+		if !strings.Contains(geth.StderrText(), m) {
 			t.Errorf("stderr text does not contain %q", m)
 		}
 	}
 }
 
 func TestUnlockFlagAmbiguousWrongPassword(t *testing.T) {
-	store := filepath.Join("..", "..", "accounts", "testdata", "dupes")
+	store := filepath.Join("..", "..", "accounts", "keystore", "testdata", "dupes")
 	geth := runGeth(t,
 		"--keystore", store, "--nat", "none", "--nodiscover", "--dev",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
-	defer geth.expectExit()
+	defer geth.ExpectExit()
 
 	// Helper for the expect template, returns absolute keystore path.
-	geth.setTemplateFunc("keypath", func(file string) string {
+	geth.SetTemplateFunc("keypath", func(file string) string {
 		abs, _ := filepath.Abs(filepath.Join(store, file))
 		return abs
 	})
-	geth.expect(`
+	geth.Expect(`
 Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
 Passphrase: {{.InputLine "wrong"}}
 Multiple key files exist for address f466859ead1932d743d622cb74fc058882e8648a:
-   {{keypath "1"}}
-   {{keypath "2"}}
+   keystore://{{keypath "1"}}
+   keystore://{{keypath "2"}}
 Testing your passphrase against all of them...
 Fatal: None of the listed files could be unlocked.
 `)
-	geth.expectExit()
+	geth.ExpectExit()
 }

cmd/geth/bugcmd.go

@@ -0,0 +1,109 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/url"
+	"os/exec"
+	"runtime"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/cmd/internal/browser"
+	"github.com/ethereum/go-ethereum/params"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var bugCommand = cli.Command{
+	Action:    utils.MigrateFlags(reportBug),
+	Name:      "bug",
+	Usage:     "opens a window to report a bug on the geth repo",
+	ArgsUsage: " ",
+	Category:  "MISCELLANEOUS COMMANDS",
+}
+
+const issueUrl = "https://github.com/ethereum/go-ethereum/issues/new"
+
+// reportBug reports a bug by opening a new URL to the go-ethereum GH issue
+// tracker and setting default values as the issue body.
+func reportBug(ctx *cli.Context) error {
+	// execute template and write contents to buff
+	var buff bytes.Buffer
+
+	fmt.Fprintln(&buff, header)
+	fmt.Fprintln(&buff, "Version:", params.Version)
+	fmt.Fprintln(&buff, "Go Version:", runtime.Version())
+	fmt.Fprintln(&buff, "OS:", runtime.GOOS)
+	printOSDetails(&buff)
+
+	// open a new GH issue
+	if !browser.Open(issueUrl + "?body=" + url.QueryEscape(buff.String())) {
+		fmt.Printf("Please file a new issue at %s using this template:\n%s", issueUrl, buff.String())
+	}
+	return nil
+}
+
+// copied from the Go source. Copyright 2017 The Go Authors
+func printOSDetails(w io.Writer) {
+	switch runtime.GOOS {
+	case "darwin":
+		printCmdOut(w, "uname -v: ", "uname", "-v")
+		printCmdOut(w, "", "sw_vers")
+	case "linux":
+		printCmdOut(w, "uname -sr: ", "uname", "-sr")
+		printCmdOut(w, "", "lsb_release", "-a")
+	case "openbsd", "netbsd", "freebsd", "dragonfly":
+		printCmdOut(w, "uname -v: ", "uname", "-v")
+	case "solaris":
+		out, err := ioutil.ReadFile("/etc/release")
+		if err == nil {
+			fmt.Fprintf(w, "/etc/release: %s\n", out)
+		} else {
+			fmt.Printf("failed to read /etc/release: %v\n", err)
+		}
+	}
+}
+
+// printCmdOut prints the output of running the given command.
+// It ignores failures; 'go bug' is best effort.
+//
+// copied from the Go source. Copyright 2017 The Go Authors
+func printCmdOut(w io.Writer, prefix, path string, args ...string) {
+	cmd := exec.Command(path, args...)
+	out, err := cmd.Output()
+	if err != nil {
+		fmt.Printf("%s %s: %v\n", path, strings.Join(args, " "), err)
+		return
+	}
+	fmt.Fprintf(w, "%s%s\n", prefix, bytes.TrimSpace(out))
+}
+
+const header = `Please answer these questions before submitting your issue. Thanks!
+
+#### What did you do?
+ 
+#### What did you expect to see?
+ 
+#### What did you see instead?
+ 
+#### System details
+`

cmd/geth/chaincmd.go

@@ -17,9 +17,9 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
-	"path/filepath"
 	"runtime"
 	"strconv"
 	"sync/atomic"
@@ -32,8 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/syndtr/goleveldb/leveldb/util"
 	"gopkg.in/urfave/cli.v1"
@@ -41,99 +40,123 @@ import (
 
 var (
 	initCommand = cli.Command{
-		Action:    initGenesis,
+		Action:    utils.MigrateFlags(initGenesis),
 		Name:      "init",
 		Usage:     "Bootstrap and initialize a new genesis block",
 		ArgsUsage: "<genesisPath>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.LightModeFlag,
+		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 The init command initializes a new genesis block and definition for the network.
 This is a destructive action and changes the network in which you will be
 participating.
-`,
+
+It expects the genesis file as argument.`,
 	}
 	importCommand = cli.Command{
-		Action:    importChain,
+		Action:    utils.MigrateFlags(importChain),
 		Name:      "import",
 		Usage:     "Import a blockchain file",
-		ArgsUsage: "<filename>",
+		ArgsUsage: "<filename> (<filename 2> ... <filename N>) ",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.CacheFlag,
+			utils.LightModeFlag,
+		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
-TODO: Please write this
-`,
+The import command imports blocks from an RLP-encoded form. The form can be one file
+with several RLP-encoded blocks, or several files can be used.
+
+If only one file is used, import error will result in failure. If several files are used, 
+processing will proceed even if an individual RLP-file import failure occurs.`,
 	}
 	exportCommand = cli.Command{
-		Action:    exportChain,
+		Action:    utils.MigrateFlags(exportChain),
 		Name:      "export",
 		Usage:     "Export blockchain into file",
 		ArgsUsage: "<filename> [<blockNumFirst> <blockNumLast>]",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.CacheFlag,
+			utils.LightModeFlag,
+		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
 last block to write. In this mode, the file will be appended
-if already existing.
-`,
-	}
-	upgradedbCommand = cli.Command{
-		Action:    upgradeDB,
-		Name:      "upgradedb",
-		Usage:     "Upgrade chainblock database",
-		ArgsUsage: " ",
-		Category:  "BLOCKCHAIN COMMANDS",
-		Description: `
-TODO: Please write this
-`,
+if already existing.`,
 	}
 	removedbCommand = cli.Command{
-		Action:    removeDB,
+		Action:    utils.MigrateFlags(removeDB),
 		Name:      "removedb",
 		Usage:     "Remove blockchain and state databases",
 		ArgsUsage: " ",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.LightModeFlag,
+		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
-TODO: Please write this
-`,
+Remove blockchain and state databases`,
 	}
 	dumpCommand = cli.Command{
-		Action:    dump,
+		Action:    utils.MigrateFlags(dump),
 		Name:      "dump",
 		Usage:     "Dump a specific block from storage",
 		ArgsUsage: "[<blockHash> | <blockNum>]...",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.CacheFlag,
+			utils.LightModeFlag,
+		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 The arguments are interpreted as block numbers or hashes.
-Use "ethereum dump 0" to dump the genesis block.
-`,
+Use "ethereum dump 0" to dump the genesis block.`,
 	}
 )
 
 // initGenesis will initialise the given JSON format genesis file and writes it as
 // the zero'd block (i.e. genesis) or will fail hard if it can't succeed.
 func initGenesis(ctx *cli.Context) error {
+	// Make sure we have a valid genesis JSON
 	genesisPath := ctx.Args().First()
 	if len(genesisPath) == 0 {
-		utils.Fatalf("must supply path to genesis JSON file")
+		utils.Fatalf("Must supply path to genesis JSON file")
 	}
+	file, err := os.Open(genesisPath)
+	if err != nil {
+		utils.Fatalf("Failed to read genesis file: %v", err)
+	}
+	defer file.Close()
 
+	genesis := new(core.Genesis)
+	if err := json.NewDecoder(file).Decode(genesis); err != nil {
+		utils.Fatalf("invalid genesis file: %v", err)
+	}
+	// Open an initialise both full and light databases
 	stack := makeFullNode(ctx)
-	chaindb := utils.MakeChainDatabase(ctx, stack)
-
-	genesisFile, err := os.Open(genesisPath)
+	for _, name := range []string{"chaindata", "lightchaindata"} {
+		chaindb, err := stack.OpenDatabase(name, 0, 0)
 		if err != nil {
-		utils.Fatalf("failed to read genesis file: %v", err)
+			utils.Fatalf("Failed to open database: %v", err)
 		}
-
-	block, err := core.WriteGenesisBlock(chaindb, genesisFile)
+		_, hash, err := core.SetupGenesisBlock(chaindb, genesis)
 		if err != nil {
-		utils.Fatalf("failed to write genesis block: %v", err)
+			utils.Fatalf("Failed to write genesis block: %v", err)
+		}
+		log.Info("Successfully wrote genesis state", "database", name, "hash", hash)
 	}
-	glog.V(logger.Info).Infof("successfully wrote genesis block and/or chain rule set: %x", block.Hash())
 	return nil
 }
 
 func importChain(ctx *cli.Context) error {
-	if len(ctx.Args()) != 1 {
+	if len(ctx.Args()) < 1 {
 		utils.Fatalf("This command requires an argument.")
 	}
 	stack := makeFullNode(ctx)
@@ -157,9 +180,19 @@ func importChain(ctx *cli.Context) error {
 	}()
 	// Import the chain
 	start := time.Now()
+
+	if len(ctx.Args()) == 1 {
 		if err := utils.ImportChain(chain, ctx.Args().First()); err != nil {
 			utils.Fatalf("Import error: %v", err)
 		}
+	} else {
+		for _, arg := range ctx.Args() {
+			if err := utils.ImportChain(chain, arg); err != nil {
+				log.Error("Import error", "file", arg, "err", err)
+			}
+		}
+	}
+
 	fmt.Printf("Import done in %v.\n\n", time.Since(start))
 
 	// Output pre-compaction stats mostly to see the import trashing
@@ -182,6 +215,10 @@ func importChain(ctx *cli.Context) error {
 	fmt.Printf("Allocations:   %.3f million\n", float64(mem.Mallocs)/1000000)
 	fmt.Printf("GC pause:      %v\n\n", time.Duration(mem.PauseTotalNs))
 
+	if ctx.GlobalIsSet(utils.NoCompactionFlag.Name) {
+		return nil
+	}
+
 	// Compact the entire database to more accurately measure disk io and print the stats
 	start = time.Now()
 	fmt.Println("Compacting entire database...")
@@ -232,72 +269,34 @@ func exportChain(ctx *cli.Context) error {
 }
 
 func removeDB(ctx *cli.Context) error {
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-	dbdir := stack.ResolvePath(utils.ChainDbName(ctx))
-	if !common.FileExist(dbdir) {
-		fmt.Println(dbdir, "does not exist")
-		return nil
-	}
+	stack, _ := makeConfigNode(ctx)
 
+	for _, name := range []string{"chaindata", "lightchaindata"} {
+		// Ensure the database exists in the first place
+		logger := log.New("database", name)
+
+		dbdir := stack.ResolvePath(name)
+		if !common.FileExist(dbdir) {
+			logger.Info("Database doesn't exist, skipping", "path", dbdir)
+			continue
+		}
+		// Confirm removal and execute
 		fmt.Println(dbdir)
 		confirm, err := console.Stdin.PromptConfirm("Remove this database?")
 		switch {
 		case err != nil:
 			utils.Fatalf("%v", err)
 		case !confirm:
-		fmt.Println("Operation aborted")
+			logger.Warn("Database deletion aborted")
 		default:
-		fmt.Println("Removing...")
 			start := time.Now()
 			os.RemoveAll(dbdir)
-		fmt.Printf("Removed in %v\n", time.Since(start))
+			logger.Info("Database successfully deleted", "elapsed", common.PrettyDuration(time.Since(start)))
+		}
 	}
 	return nil
 }
 
-func upgradeDB(ctx *cli.Context) error {
-	glog.Infoln("Upgrading blockchain database")
-
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-	chain, chainDb := utils.MakeChain(ctx, stack)
-	bcVersion := core.GetBlockChainVersion(chainDb)
-	if bcVersion == 0 {
-		bcVersion = core.BlockChainVersion
-	}
-
-	// Export the current chain.
-	filename := fmt.Sprintf("blockchain_%d_%s.chain", bcVersion, time.Now().Format("20060102_150405"))
-	exportFile := filepath.Join(ctx.GlobalString(utils.DataDirFlag.Name), filename)
-	if err := utils.ExportChain(chain, exportFile); err != nil {
-		utils.Fatalf("Unable to export chain for reimport %s", err)
-	}
-	chainDb.Close()
-	if dir := dbDirectory(chainDb); dir != "" {
-		os.RemoveAll(dir)
-	}
-
-	// Import the chain file.
-	chain, chainDb = utils.MakeChain(ctx, stack)
-	core.WriteBlockChainVersion(chainDb, core.BlockChainVersion)
-	err := utils.ImportChain(chain, exportFile)
-	chainDb.Close()
-	if err != nil {
-		utils.Fatalf("Import error %v (a backup is made in %s, use the import command to import it)", err, exportFile)
-	} else {
-		os.Remove(exportFile)
-		glog.Infoln("Import finished")
-	}
-	return nil
-}
-
-func dbDirectory(db ethdb.Database) string {
-	ldb, ok := db.(*ethdb.LDBDatabase)
-	if !ok {
-		return ""
-	}
-	return ldb.Path()
-}
-
 func dump(ctx *cli.Context) error {
 	stack := makeFullNode(ctx)
 	chain, chainDb := utils.MakeChain(ctx, stack)
@@ -329,9 +328,3 @@ func hashish(x string) bool {
 	_, err := strconv.Atoi(x)
 	return err != nil
 }
-
-func closeAll(dbs ...ethdb.Database) {
-	for _, db := range dbs {
-		db.Close()
-	}
-}

cmd/geth/config.go

@@ -0,0 +1,187 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"unicode"
+
+	cli "gopkg.in/urfave/cli.v1"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/contracts/release"
+	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/naoina/toml"
+)
+
+var (
+	dumpConfigCommand = cli.Command{
+		Action:      utils.MigrateFlags(dumpConfig),
+		Name:        "dumpconfig",
+		Usage:       "Show configuration values",
+		ArgsUsage:   "",
+		Flags:       append(nodeFlags, rpcFlags...),
+		Category:    "MISCELLANEOUS COMMANDS",
+		Description: `The dumpconfig command shows configuration values.`,
+	}
+
+	configFileFlag = cli.StringFlag{
+		Name:  "config",
+		Usage: "TOML configuration file",
+	}
+)
+
+// These settings ensure that TOML keys use the same names as Go struct fields.
+var tomlSettings = toml.Config{
+	NormFieldName: func(rt reflect.Type, key string) string {
+		return key
+	},
+	FieldToKey: func(rt reflect.Type, field string) string {
+		return field
+	},
+	MissingField: func(rt reflect.Type, field string) error {
+		link := ""
+		if unicode.IsUpper(rune(rt.Name()[0])) && rt.PkgPath() != "main" {
+			link = fmt.Sprintf(", see https://godoc.org/%s#%s for available fields", rt.PkgPath(), rt.Name())
+		}
+		return fmt.Errorf("field '%s' is not defined in %s%s", field, rt.String(), link)
+	},
+}
+
+type ethstatsConfig struct {
+	URL string `toml:",omitempty"`
+}
+
+type gethConfig struct {
+	Eth      eth.Config
+	Node     node.Config
+	Ethstats ethstatsConfig
+}
+
+func loadConfig(file string, cfg *gethConfig) error {
+	f, err := os.Open(file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	err = tomlSettings.NewDecoder(bufio.NewReader(f)).Decode(cfg)
+	// Add file name to errors that have a line number.
+	if _, ok := err.(*toml.LineError); ok {
+		err = errors.New(file + ", " + err.Error())
+	}
+	return err
+}
+
+func defaultNodeConfig() node.Config {
+	cfg := node.DefaultConfig
+	cfg.Name = clientIdentifier
+	cfg.Version = params.VersionWithCommit(gitCommit)
+	cfg.HTTPModules = append(cfg.HTTPModules, "eth")
+	cfg.WSModules = append(cfg.WSModules, "eth")
+	cfg.IPCPath = "geth.ipc"
+	return cfg
+}
+
+func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
+	// Load defaults.
+	cfg := gethConfig{
+		Eth:  eth.DefaultConfig,
+		Node: defaultNodeConfig(),
+	}
+
+	// Load config file.
+	if file := ctx.GlobalString(configFileFlag.Name); file != "" {
+		if err := loadConfig(file, &cfg); err != nil {
+			utils.Fatalf("%v", err)
+		}
+	}
+
+	// Apply flags.
+	utils.SetNodeConfig(ctx, &cfg.Node)
+	stack, err := node.New(&cfg.Node)
+	if err != nil {
+		utils.Fatalf("Failed to create the protocol stack: %v", err)
+	}
+	utils.SetEthConfig(ctx, stack, &cfg.Eth)
+	if ctx.GlobalIsSet(utils.EthStatsURLFlag.Name) {
+		cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name)
+	}
+
+	return stack, cfg
+}
+
+func makeFullNode(ctx *cli.Context) *node.Node {
+	stack, cfg := makeConfigNode(ctx)
+
+	utils.RegisterEthService(stack, &cfg.Eth)
+
+	// Whisper must be explicitly enabled, but is auto-enabled in --dev mode.
+	shhEnabled := ctx.GlobalBool(utils.WhisperEnabledFlag.Name)
+	shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name)
+	if shhEnabled || shhAutoEnabled {
+		utils.RegisterShhService(stack)
+	}
+
+	// Add the Ethereum Stats daemon if requested.
+	if cfg.Ethstats.URL != "" {
+		utils.RegisterEthStatsService(stack, cfg.Ethstats.URL)
+	}
+
+	// Add the release oracle service so it boots along with node.
+	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
+		config := release.Config{
+			Oracle: relOracle,
+			Major:  uint32(params.VersionMajor),
+			Minor:  uint32(params.VersionMinor),
+			Patch:  uint32(params.VersionPatch),
+		}
+		commit, _ := hex.DecodeString(gitCommit)
+		copy(config.Commit[:], commit)
+		return release.NewReleaseService(ctx, config)
+	}); err != nil {
+		utils.Fatalf("Failed to register the Geth release oracle service: %v", err)
+	}
+	return stack
+}
+
+// dumpConfig is the dumpconfig command.
+func dumpConfig(ctx *cli.Context) error {
+	_, cfg := makeConfigNode(ctx)
+	comment := ""
+
+	if cfg.Eth.Genesis != nil {
+		cfg.Eth.Genesis = nil
+		comment += "# Note: this config doesn't contain the genesis block.\n\n"
+	}
+
+	out, err := tomlSettings.Marshal(&cfg)
+	if err != nil {
+		return err
+	}
+	io.WriteString(os.Stdout, comment)
+	os.Stdout.Write(out)
+	return nil
+}

cmd/geth/consolecmd.go

@@ -29,41 +29,44 @@ import (
 )
 
 var (
+	consoleFlags = []cli.Flag{utils.JSpathFlag, utils.ExecFlag, utils.PreloadJSFlag}
+
 	consoleCommand = cli.Command{
-		Action:    localConsole,
+		Action:   utils.MigrateFlags(localConsole),
 		Name:     "console",
 		Usage:    "Start an interactive JavaScript environment",
-		ArgsUsage: "", // TODO: Write this!
+		Flags:    append(append(nodeFlags, rpcFlags...), consoleFlags...),
 		Category: "CONSOLE COMMANDS",
 		Description: `
 The Geth console is an interactive shell for the JavaScript runtime environment
 which exposes a node admin interface as well as the Ðapp JavaScript API.
-See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console
-`,
+See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console.`,
 	}
+
 	attachCommand = cli.Command{
-		Action:    remoteConsole,
+		Action:    utils.MigrateFlags(remoteConsole),
 		Name:      "attach",
 		Usage:     "Start an interactive JavaScript environment (connect to node)",
-		ArgsUsage: "", // TODO: Write this!
+		ArgsUsage: "[endpoint]",
+		Flags:     append(consoleFlags, utils.DataDirFlag),
 		Category:  "CONSOLE COMMANDS",
 		Description: `
 The Geth console is an interactive shell for the JavaScript runtime environment
 which exposes a node admin interface as well as the Ðapp JavaScript API.
 See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console.
-This command allows to open a console on a running geth node.
-`,
+This command allows to open a console on a running geth node.`,
 	}
+
 	javascriptCommand = cli.Command{
-		Action:    ephemeralConsole,
+		Action:    utils.MigrateFlags(ephemeralConsole),
 		Name:      "js",
 		Usage:     "Execute the specified JavaScript files",
-		ArgsUsage: "", // TODO: Write this!
+		ArgsUsage: "<jsfile> [jsfile...]",
+		Flags:     append(nodeFlags, consoleFlags...),
 		Category:  "CONSOLE COMMANDS",
 		Description: `
 The JavaScript VM exposes a node admin interface as well as the Ðapp
-JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console
-`,
+JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console`,
 	}
 )
 
@@ -81,11 +84,12 @@ func localConsole(ctx *cli.Context) error {
 		utils.Fatalf("Failed to attach to the inproc geth: %v", err)
 	}
 	config := console.Config{
-		DataDir: node.DataDir(),
+		DataDir: utils.MakeDataDir(ctx),
 		DocRoot: ctx.GlobalString(utils.JSpathFlag.Name),
 		Client:  client,
 		Preload: utils.MakeConsolePreloads(ctx),
 	}
+
 	console, err := console.New(config)
 	if err != nil {
 		utils.Fatalf("Failed to start the JavaScript console: %v", err)
@@ -118,17 +122,18 @@ func remoteConsole(ctx *cli.Context) error {
 		Client:  client,
 		Preload: utils.MakeConsolePreloads(ctx),
 	}
+
 	console, err := console.New(config)
 	if err != nil {
 		utils.Fatalf("Failed to start the JavaScript console: %v", err)
 	}
 	defer console.Stop(false)
 
-	// If only a short execution was requested, evaluate and return
 	if script := ctx.GlobalString(utils.ExecFlag.Name); script != "" {
 		console.Evaluate(script)
 		return nil
 	}
+
 	// Otherwise print the welcome screen and enter interactive mode
 	console.Welcome()
 	console.Interactive()
@@ -151,7 +156,7 @@ func dialRPC(endpoint string) (*rpc.Client, error) {
 }
 
 // ephemeralConsole starts a new geth node, attaches an ephemeral JavaScript
-// console to it, and each of the files specified as arguments and tears the
+// console to it, executes each of the files specified as arguments and tears
 // everything down.
 func ephemeralConsole(ctx *cli.Context) error {
 	// Create and start the node based on the CLI flags
@@ -165,11 +170,12 @@ func ephemeralConsole(ctx *cli.Context) error {
 		utils.Fatalf("Failed to attach to the inproc geth: %v", err)
 	}
 	config := console.Config{
-		DataDir: node.DataDir(),
+		DataDir: utils.MakeDataDir(ctx),
 		DocRoot: ctx.GlobalString(utils.JSpathFlag.Name),
 		Client:  client,
 		Preload: utils.MakeConsolePreloads(ctx),
 	}
+
 	console, err := console.New(config)
 	if err != nil {
 		utils.Fatalf("Failed to start the JavaScript console: %v", err)

cmd/geth/consolecmd_test.go

@@ -22,14 +22,17 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
-	"sort"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rpc"
+)
+
+const (
+	ipcAPIs  = "admin:1.0 debug:1.0 eth:1.0 miner:1.0 net:1.0 personal:1.0 rpc:1.0 shh:1.0 txpool:1.0 web3:1.0"
+	httpAPIs = "eth:1.0 net:1.0 rpc:1.0 web3:1.0"
 )
 
 // Tests that a node embedded within a console can be started up properly and
@@ -44,29 +47,26 @@ func TestConsoleWelcome(t *testing.T) {
 		"console")
 
 	// Gather all the infos the welcome message needs to contain
-	geth.setTemplateFunc("goos", func() string { return runtime.GOOS })
-	geth.setTemplateFunc("gover", runtime.Version)
-	geth.setTemplateFunc("gethver", func() string { return params.Version })
-	geth.setTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
-	geth.setTemplateFunc("apis", func() []string {
-		apis := append(strings.Split(rpc.DefaultIPCApis, ","), rpc.MetadataApi)
-		sort.Strings(apis)
-		return apis
-	})
+	geth.SetTemplateFunc("goos", func() string { return runtime.GOOS })
+	geth.SetTemplateFunc("goarch", func() string { return runtime.GOARCH })
+	geth.SetTemplateFunc("gover", runtime.Version)
+	geth.SetTemplateFunc("gethver", func() string { return params.Version })
+	geth.SetTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
+	geth.SetTemplateFunc("apis", func() string { return ipcAPIs })
 
 	// Verify the actual welcome message to the required template
-	geth.expect(`
+	geth.Expect(`
 Welcome to the Geth JavaScript console!
 
-instance: Geth/v{{gethver}}/{{goos}}/{{gover}}
+instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}}
 coinbase: {{.Etherbase}}
 at block: 0 ({{niltime}})
  datadir: {{.Datadir}}
- modules:{{range apis}} {{.}}:1.0{{end}}
+ modules: {{apis}}
 
 > {{.InputLine "exit"}}
 `)
-	geth.expectExit()
+	geth.ExpectExit()
 }
 
 // Tests that a console can be attached to a running node via various means.
@@ -88,10 +88,10 @@ func TestIPCAttachWelcome(t *testing.T) {
 		"--etherbase", coinbase, "--shh", "--ipcpath", ipc)
 
 	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
-	testAttachWelcome(t, geth, "ipc:"+ipc)
+	testAttachWelcome(t, geth, "ipc:"+ipc, ipcAPIs)
 
-	geth.interrupt()
-	geth.expectExit()
+	geth.Interrupt()
+	geth.ExpectExit()
 }
 
 func TestHTTPAttachWelcome(t *testing.T) {
@@ -102,10 +102,10 @@ func TestHTTPAttachWelcome(t *testing.T) {
 		"--etherbase", coinbase, "--rpc", "--rpcport", port)
 
 	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
-	testAttachWelcome(t, geth, "http://localhost:"+port)
+	testAttachWelcome(t, geth, "http://localhost:"+port, httpAPIs)
 
-	geth.interrupt()
-	geth.expectExit()
+	geth.Interrupt()
+	geth.ExpectExit()
 }
 
 func TestWSAttachWelcome(t *testing.T) {
@@ -117,50 +117,42 @@ func TestWSAttachWelcome(t *testing.T) {
 		"--etherbase", coinbase, "--ws", "--wsport", port)
 
 	time.Sleep(2 * time.Second) // Simple way to wait for the RPC endpoint to open
-	testAttachWelcome(t, geth, "ws://localhost:"+port)
+	testAttachWelcome(t, geth, "ws://localhost:"+port, httpAPIs)
 
-	geth.interrupt()
-	geth.expectExit()
+	geth.Interrupt()
+	geth.ExpectExit()
 }
 
-func testAttachWelcome(t *testing.T, geth *testgeth, endpoint string) {
+func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) {
 	// Attach to a running geth note and terminate immediately
 	attach := runGeth(t, "attach", endpoint)
-	defer attach.expectExit()
-	attach.stdin.Close()
+	defer attach.ExpectExit()
+	attach.CloseStdin()
 
 	// Gather all the infos the welcome message needs to contain
-	attach.setTemplateFunc("goos", func() string { return runtime.GOOS })
-	attach.setTemplateFunc("gover", runtime.Version)
-	attach.setTemplateFunc("gethver", func() string { return params.Version })
-	attach.setTemplateFunc("etherbase", func() string { return geth.Etherbase })
-	attach.setTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
-	attach.setTemplateFunc("ipc", func() bool { return strings.HasPrefix(endpoint, "ipc") })
-	attach.setTemplateFunc("datadir", func() string { return geth.Datadir })
-	attach.setTemplateFunc("apis", func() []string {
-		var apis []string
-		if strings.HasPrefix(endpoint, "ipc") {
-			apis = append(strings.Split(rpc.DefaultIPCApis, ","), rpc.MetadataApi)
-		} else {
-			apis = append(strings.Split(rpc.DefaultHTTPApis, ","), rpc.MetadataApi)
-		}
-		sort.Strings(apis)
-		return apis
-	})
+	attach.SetTemplateFunc("goos", func() string { return runtime.GOOS })
+	attach.SetTemplateFunc("goarch", func() string { return runtime.GOARCH })
+	attach.SetTemplateFunc("gover", runtime.Version)
+	attach.SetTemplateFunc("gethver", func() string { return params.Version })
+	attach.SetTemplateFunc("etherbase", func() string { return geth.Etherbase })
+	attach.SetTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
+	attach.SetTemplateFunc("ipc", func() bool { return strings.HasPrefix(endpoint, "ipc") })
+	attach.SetTemplateFunc("datadir", func() string { return geth.Datadir })
+	attach.SetTemplateFunc("apis", func() string { return apis })
 
 	// Verify the actual welcome message to the required template
-	attach.expect(`
+	attach.Expect(`
 Welcome to the Geth JavaScript console!
 
-instance: Geth/v{{gethver}}/{{goos}}/{{gover}}
+instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}}
 coinbase: {{etherbase}}
 at block: 0 ({{niltime}}){{if ipc}}
  datadir: {{datadir}}{{end}}
- modules:{{range apis}} {{.}}:1.0{{end}}
+ modules: {{apis}}
 
 > {{.InputLine "exit" }}
 `)
-	attach.expectExit()
+	attach.ExpectExit()
 }
 
 // trulyRandInt generates a crypto random integer used by the console tests to

cmd/geth/dao_test.go

@@ -84,27 +84,24 @@ var daoGenesisForkBlock = big.NewInt(314)
 // set in the database after various initialization procedures and invocations.
 func TestDAOForkBlockNewChain(t *testing.T) {
 	for i, arg := range []struct {
-		testnet     bool
 		genesis     string
 		expectBlock *big.Int
 		expectVote  bool
 	}{
 		// Test DAO Default Mainnet
-		{false, "", params.MainNetDAOForkBlock, true},
-		// test DAO Default Testnet
-		{true, "", params.TestNetDAOForkBlock, true},
+		{"", params.MainNetDAOForkBlock, true},
 		// test DAO Init Old Privnet
-		{false, daoOldGenesis, nil, false},
+		{daoOldGenesis, nil, false},
 		// test DAO Default No Fork Privnet
-		{false, daoNoForkGenesis, daoGenesisForkBlock, false},
+		{daoNoForkGenesis, daoGenesisForkBlock, false},
 		// test DAO Default Pro Fork Privnet
-		{false, daoProForkGenesis, daoGenesisForkBlock, true},
+		{daoProForkGenesis, daoGenesisForkBlock, true},
 	} {
-		testDAOForkBlockNewChain(t, i, arg.testnet, arg.genesis, arg.expectBlock, arg.expectVote)
+		testDAOForkBlockNewChain(t, i, arg.genesis, arg.expectBlock, arg.expectVote)
 	}
 }
 
-func testDAOForkBlockNewChain(t *testing.T, test int, testnet bool, genesis string, expectBlock *big.Int, expectVote bool) {
+func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBlock *big.Int, expectVote bool) {
 	// Create a temporary data directory to use and inspect later
 	datadir := tmpdir(t)
 	defer os.RemoveAll(datadir)
@@ -115,21 +112,15 @@ func testDAOForkBlockNewChain(t *testing.T, test int, testnet bool, genesis stri
 		if err := ioutil.WriteFile(json, []byte(genesis), 0600); err != nil {
 			t.Fatalf("test %d: failed to write genesis file: %v", test, err)
 		}
-		runGeth(t, "--datadir", datadir, "init", json).cmd.Wait()
+		runGeth(t, "--datadir", datadir, "init", json).WaitExit()
 	} else {
 		// Force chain initialization
 		args := []string{"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--datadir", datadir}
-		if testnet {
-			args = append(args, "--testnet")
-		}
 		geth := runGeth(t, append(args, []string{"--exec", "2+2", "console"}...)...)
-		geth.cmd.Wait()
+		geth.WaitExit()
 	}
 	// Retrieve the DAO config flag from the database
 	path := filepath.Join(datadir, "geth", "chaindata")
-	if testnet && genesis == "" {
-		path = filepath.Join(datadir, "testnet", "geth", "chaindata")
-	}
 	db, err := ethdb.NewLDBDatabase(path, 0, 0)
 	if err != nil {
 		t.Fatalf("test %d: failed to open test database: %v", test, err)
@@ -137,9 +128,6 @@ func testDAOForkBlockNewChain(t *testing.T, test int, testnet bool, genesis stri
 	defer db.Close()
 
 	genesisHash := common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3")
-	if testnet {
-		genesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d")
-	}
 	if genesis != "" {
 		genesisHash = daoGenesisHash
 	}

cmd/geth/genesis_test.go

@@ -97,14 +97,14 @@ func TestCustomGenesis(t *testing.T) {
 		if err := ioutil.WriteFile(json, []byte(tt.genesis), 0600); err != nil {
 			t.Fatalf("test %d: failed to write genesis file: %v", i, err)
 		}
-		runGeth(t, "--datadir", datadir, "init", json).cmd.Wait()
+		runGeth(t, "--datadir", datadir, "init", json).WaitExit()
 
 		// Query the custom genesis block
 		geth := runGeth(t,
 			"--datadir", datadir, "--maxpeers", "0", "--port", "0",
 			"--nodiscover", "--nat", "none", "--ipcdisable",
 			"--exec", tt.query, "console")
-		geth.expectRegexp(tt.result)
-		geth.expectExit()
+		geth.ExpectRegexp(tt.result)
+		geth.ExpectExit()
 	}
 }

cmd/geth/main.go

@@ -18,25 +18,23 @@
 package main
 
 import (
-	"encoding/hex"
 	"fmt"
 	"os"
 	"runtime"
 	"strings"
 	"time"
 
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
-	"github.com/ethereum/go-ethereum/contracts/release"
 	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/internal/debug"
-	"github.com/ethereum/go-ethereum/logger"
-	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rlp"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -51,19 +49,94 @@ var (
 	relOracle = common.HexToAddress("0xfa7b9770ca4cb04296cac84f37736d4041251cdf")
 	// The app that holds all commands and flags.
 	app = utils.NewApp(gitCommit, "the go-ethereum command line interface")
+	// flags that configure the node
+	nodeFlags = []cli.Flag{
+		utils.IdentityFlag,
+		utils.UnlockedAccountFlag,
+		utils.PasswordFileFlag,
+		utils.BootnodesFlag,
+		utils.BootnodesV4Flag,
+		utils.BootnodesV5Flag,
+		utils.DataDirFlag,
+		utils.KeyStoreDirFlag,
+		utils.NoUSBFlag,
+		utils.EthashCacheDirFlag,
+		utils.EthashCachesInMemoryFlag,
+		utils.EthashCachesOnDiskFlag,
+		utils.EthashDatasetDirFlag,
+		utils.EthashDatasetsInMemoryFlag,
+		utils.EthashDatasetsOnDiskFlag,
+		utils.TxPoolPriceLimitFlag,
+		utils.TxPoolPriceBumpFlag,
+		utils.TxPoolAccountSlotsFlag,
+		utils.TxPoolGlobalSlotsFlag,
+		utils.TxPoolAccountQueueFlag,
+		utils.TxPoolGlobalQueueFlag,
+		utils.TxPoolLifetimeFlag,
+		utils.FastSyncFlag,
+		utils.LightModeFlag,
+		utils.SyncModeFlag,
+		utils.LightServFlag,
+		utils.LightPeersFlag,
+		utils.LightKDFFlag,
+		utils.CacheFlag,
+		utils.TrieCacheGenFlag,
+		utils.ListenPortFlag,
+		utils.MaxPeersFlag,
+		utils.MaxPendingPeersFlag,
+		utils.EtherbaseFlag,
+		utils.GasPriceFlag,
+		utils.MinerThreadsFlag,
+		utils.MiningEnabledFlag,
+		utils.TargetGasLimitFlag,
+		utils.NATFlag,
+		utils.NoDiscoverFlag,
+		utils.DiscoveryV5Flag,
+		utils.NetrestrictFlag,
+		utils.NodeKeyFileFlag,
+		utils.NodeKeyHexFlag,
+		utils.WhisperEnabledFlag,
+		utils.DevModeFlag,
+		utils.TestnetFlag,
+		utils.RinkebyFlag,
+		utils.VMEnableDebugFlag,
+		utils.NetworkIdFlag,
+		utils.RPCCORSDomainFlag,
+		utils.EthStatsURLFlag,
+		utils.MetricsEnabledFlag,
+		utils.FakePoWFlag,
+		utils.NoCompactionFlag,
+		utils.GpoBlocksFlag,
+		utils.GpoPercentileFlag,
+		utils.ExtraDataFlag,
+		configFileFlag,
+	}
+
+	rpcFlags = []cli.Flag{
+		utils.RPCEnabledFlag,
+		utils.RPCListenAddrFlag,
+		utils.RPCPortFlag,
+		utils.RPCApiFlag,
+		utils.WSEnabledFlag,
+		utils.WSListenAddrFlag,
+		utils.WSPortFlag,
+		utils.WSApiFlag,
+		utils.WSAllowedOriginsFlag,
+		utils.IPCDisabledFlag,
+		utils.IPCPathFlag,
+	}
 )
 
 func init() {
 	// Initialize the CLI app and start Geth
 	app.Action = geth
 	app.HideVersion = true // we have a command to print the version
-	app.Copyright = "Copyright 2013-2016 The go-ethereum Authors"
+	app.Copyright = "Copyright 2013-2017 The go-ethereum Authors"
 	app.Commands = []cli.Command{
 		// See chaincmd.go:
 		initCommand,
 		importCommand,
 		exportCommand,
-		upgradedbCommand,
 		removedbCommand,
 		dumpCommand,
 		// See monitorcmd.go:
@@ -78,74 +151,15 @@ func init() {
 		// See misccmd.go:
 		makedagCommand,
 		versionCommand,
+		bugCommand,
 		licenseCommand,
+		// See config.go
+		dumpConfigCommand,
 	}
 
-	app.Flags = []cli.Flag{
-		utils.IdentityFlag,
-		utils.UnlockedAccountFlag,
-		utils.PasswordFileFlag,
-		utils.BootnodesFlag,
-		utils.DataDirFlag,
-		utils.KeyStoreDirFlag,
-		utils.FastSyncFlag,
-		utils.LightModeFlag,
-		utils.LightServFlag,
-		utils.LightPeersFlag,
-		utils.LightKDFFlag,
-		utils.CacheFlag,
-		utils.TrieCacheGenFlag,
-		utils.JSpathFlag,
-		utils.ListenPortFlag,
-		utils.MaxPeersFlag,
-		utils.MaxPendingPeersFlag,
-		utils.EtherbaseFlag,
-		utils.GasPriceFlag,
-		utils.MinerThreadsFlag,
-		utils.MiningEnabledFlag,
-		utils.AutoDAGFlag,
-		utils.TargetGasLimitFlag,
-		utils.NATFlag,
-		utils.NatspecEnabledFlag,
-		utils.NoDiscoverFlag,
-		utils.DiscoveryV5Flag,
-		utils.NetrestrictFlag,
-		utils.NodeKeyFileFlag,
-		utils.NodeKeyHexFlag,
-		utils.RPCEnabledFlag,
-		utils.RPCListenAddrFlag,
-		utils.RPCPortFlag,
-		utils.RPCApiFlag,
-		utils.WSEnabledFlag,
-		utils.WSListenAddrFlag,
-		utils.WSPortFlag,
-		utils.WSApiFlag,
-		utils.WSAllowedOriginsFlag,
-		utils.IPCDisabledFlag,
-		utils.IPCApiFlag,
-		utils.IPCPathFlag,
-		utils.ExecFlag,
-		utils.PreloadJSFlag,
-		utils.WhisperEnabledFlag,
-		utils.DevModeFlag,
-		utils.TestNetFlag,
-		utils.VMForceJitFlag,
-		utils.VMJitCacheFlag,
-		utils.VMEnableJitFlag,
-		utils.NetworkIdFlag,
-		utils.RPCCORSDomainFlag,
-		utils.EthStatsURLFlag,
-		utils.MetricsEnabledFlag,
-		utils.FakePoWFlag,
-		utils.SolcPathFlag,
-		utils.GpoMinGasPriceFlag,
-		utils.GpoMaxGasPriceFlag,
-		utils.GpoFullBlockRatioFlag,
-		utils.GpobaseStepDownFlag,
-		utils.GpobaseStepUpFlag,
-		utils.GpobaseCorrectionFactorFlag,
-		utils.ExtraDataFlag,
-	}
+	app.Flags = append(app.Flags, nodeFlags...)
+	app.Flags = append(app.Flags, rpcFlags...)
+	app.Flags = append(app.Flags, consoleFlags...)
 	app.Flags = append(app.Flags, debug.Flags...)
 
 	app.Before = func(ctx *cli.Context) error {
@@ -156,12 +170,6 @@ func init() {
 		// Start system runtime metrics collection
 		go metrics.CollectProcessMetrics(3 * time.Second)
 
-		// This should be the only place where reporting is enabled
-		// because it is not intended to run while testing.
-		// In addition to this check, bad block reports are sent only
-		// for chains with the main network genesis block and network id 1.
-		eth.EnableBadBlockReporting = true
-
 		utils.SetupNetwork(ctx)
 		return nil
 	}
@@ -190,53 +198,6 @@ func geth(ctx *cli.Context) error {
 	return nil
 }
 
-func makeFullNode(ctx *cli.Context) *node.Node {
-	// Create the default extradata and construct the base node
-	var clientInfo = struct {
-		Version   uint
-		Name      string
-		GoVersion string
-		Os        string
-	}{uint(params.VersionMajor<<16 | params.VersionMinor<<8 | params.VersionPatch), clientIdentifier, runtime.Version(), runtime.GOOS}
-	extra, err := rlp.EncodeToBytes(clientInfo)
-	if err != nil {
-		glog.V(logger.Warn).Infoln("error setting canonical miner information:", err)
-	}
-	if uint64(len(extra)) > params.MaximumExtraDataSize.Uint64() {
-		glog.V(logger.Warn).Infoln("error setting canonical miner information: extra exceeds", params.MaximumExtraDataSize)
-		glog.V(logger.Debug).Infof("extra: %x\n", extra)
-		extra = nil
-	}
-	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-	utils.RegisterEthService(ctx, stack, extra)
-
-	// Whisper must be explicitly enabled, but is auto-enabled in --dev mode.
-	shhEnabled := ctx.GlobalBool(utils.WhisperEnabledFlag.Name)
-	shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name)
-	if shhEnabled || shhAutoEnabled {
-		utils.RegisterShhService(stack)
-	}
-	// Add the Ethereum Stats daemon if requested
-	if url := ctx.GlobalString(utils.EthStatsURLFlag.Name); url != "" {
-		utils.RegisterEthStatsService(stack, url)
-	}
-	// Add the release oracle service so it boots along with node.
-	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) {
-		config := release.Config{
-			Oracle: relOracle,
-			Major:  uint32(params.VersionMajor),
-			Minor:  uint32(params.VersionMinor),
-			Patch:  uint32(params.VersionPatch),
-		}
-		commit, _ := hex.DecodeString(gitCommit)
-		copy(config.Commit[:], commit)
-		return release.NewReleaseService(ctx, config)
-	}); err != nil {
-		utils.Fatalf("Failed to register the Geth release oracle service: %v", err)
-	}
-	return stack
-}
-
 // startNode boots up the system node and all registered protocols, after which
 // it unlocks any requested accounts, and starts the RPC/IPC interfaces and the
 // miner.
@@ -245,21 +206,69 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 	utils.StartNode(stack)
 
 	// Unlock any account specifically requested
-	accman := stack.AccountManager()
+	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+
 	passwords := utils.MakePasswordList(ctx)
-	accounts := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",")
-	for i, account := range accounts {
+	unlocks := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",")
+	for i, account := range unlocks {
 		if trimmed := strings.TrimSpace(account); trimmed != "" {
-			unlockAccount(ctx, accman, trimmed, i, passwords)
+			unlockAccount(ctx, ks, trimmed, i, passwords)
 		}
 	}
+	// Register wallet event handlers to open and auto-derive wallets
+	events := make(chan accounts.WalletEvent, 16)
+	stack.AccountManager().Subscribe(events)
+
+	go func() {
+		// Create an chain state reader for self-derivation
+		rpcClient, err := stack.Attach()
+		if err != nil {
+			utils.Fatalf("Failed to attach to self: %v", err)
+		}
+		stateReader := ethclient.NewClient(rpcClient)
+
+		// Open and self derive any wallets already attached
+		for _, wallet := range stack.AccountManager().Wallets() {
+			if err := wallet.Open(""); err != nil {
+				log.Warn("Failed to open wallet", "url", wallet.URL(), "err", err)
+			} else {
+				wallet.SelfDerive(accounts.DefaultBaseDerivationPath, stateReader)
+			}
+		}
+		// Listen for wallet event till termination
+		for event := range events {
+			if event.Arrive {
+				if err := event.Wallet.Open(""); err != nil {
+					log.Warn("New wallet appeared, failed to open", "url", event.Wallet.URL(), "err", err)
+				} else {
+					log.Info("New wallet appeared", "url", event.Wallet.URL(), "status", event.Wallet.Status())
+					event.Wallet.SelfDerive(accounts.DefaultBaseDerivationPath, stateReader)
+				}
+			} else {
+				log.Info("Old wallet dropped", "url", event.Wallet.URL())
+				event.Wallet.Close()
+			}
+		}
+	}()
 	// Start auxiliary services if enabled
 	if ctx.GlobalBool(utils.MiningEnabledFlag.Name) {
+		// Mining only makes sense if a full Ethereum node is running
 		var ethereum *eth.Ethereum
 		if err := stack.Service(&ethereum); err != nil {
 			utils.Fatalf("ethereum service not running: %v", err)
 		}
-		if err := ethereum.StartMining(ctx.GlobalInt(utils.MinerThreadsFlag.Name)); err != nil {
+		// Use a reduced number of threads if requested
+		if threads := ctx.GlobalInt(utils.MinerThreadsFlag.Name); threads > 0 {
+			type threaded interface {
+				SetThreads(threads int)
+			}
+			if th, ok := ethereum.Engine().(threaded); ok {
+				th.SetThreads(threads)
+			}
+		}
+		// Set the gas price to the limits from the CLI and start mining
+		ethereum.TxPool().SetGasPrice(utils.GlobalBig(ctx, utils.GasPriceFlag.Name))
+		if err := ethereum.StartMining(true); err != nil {
 			utils.Fatalf("Failed to start mining: %v", err)
 		}
 	}

cmd/geth/misccmd.go

@@ -25,8 +25,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/ethereum/ethash"
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/params"
 	"gopkg.in/urfave/cli.v1"
@@ -34,7 +34,7 @@ import (
 
 var (
 	makedagCommand = cli.Command{
-		Action:    makedag,
+		Action:    utils.MigrateFlags(makedag),
 		Name:      "makedag",
 		Usage:     "Generate ethash DAG (for testing)",
 		ArgsUsage: "<blockNum> <outputDir>",
@@ -47,7 +47,7 @@ Regular users do not need to execute it.
 `,
 	}
 	versionCommand = cli.Command{
-		Action:    version,
+		Action:    utils.MigrateFlags(version),
 		Name:      "version",
 		Usage:     "Print version numbers",
 		ArgsUsage: " ",
@@ -57,7 +57,7 @@ The output of this command is supposed to be machine-readable.
 `,
 	}
 	licenseCommand = cli.Command{
-		Action:    license,
+		Action:    utils.MigrateFlags(license),
 		Name:      "license",
 		Usage:     "Display license information",
 		ArgsUsage: " ",
@@ -87,7 +87,7 @@ func makedag(ctx *cli.Context) error {
 				utils.Fatalf("Can't find dir")
 			}
 			fmt.Println("making DAG, this could take awhile...")
-			ethash.MakeDAG(blockNum, dir)
+			ethash.MakeDataset(blockNum, dir)
 		}
 	default:
 		wrongArgs()
@@ -101,10 +101,11 @@ func version(ctx *cli.Context) error {
 	if gitCommit != "" {
 		fmt.Println("Git Commit:", gitCommit)
 	}
+	fmt.Println("Architecture:", runtime.GOARCH)
 	fmt.Println("Protocol Versions:", eth.ProtocolVersions)
-	fmt.Println("Network Id:", ctx.GlobalInt(utils.NetworkIdFlag.Name))
+	fmt.Println("Network Id:", eth.DefaultConfig.NetworkId)
 	fmt.Println("Go Version:", runtime.Version())
-	fmt.Println("OS:", runtime.GOOS)
+	fmt.Println("Operating System:", runtime.GOOS)
 	fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH"))
 	fmt.Printf("GOROOT=%s\n", runtime.GOROOT())
 	return nil

cmd/geth/monitorcmd.go

@@ -49,7 +49,7 @@ var (
 		Usage: "Refresh interval in seconds",
 	}
 	monitorCommand = cli.Command{
-		Action:    monitor,
+		Action:    utils.MigrateFlags(monitor), // keep track of migration progress
 		Name:      "monitor",
 		Usage:     "Monitor and visualize node metrics",
 		ArgsUsage: " ",

cmd/geth/run_test.go

@@ -17,18 +17,13 @@
 package main
 
 import (
-	"bufio"
-	"bytes"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
-	"os/exec"
-	"regexp"
-	"sync"
 	"testing"
-	"text/template"
-	"time"
+
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/ethereum/go-ethereum/internal/cmdtest"
 )
 
 func tmpdir(t *testing.T) string {
@@ -40,36 +35,37 @@ func tmpdir(t *testing.T) string {
 }
 
 type testgeth struct {
-	// For total convenience, all testing methods are available.
-	*testing.T
+	*cmdtest.TestCmd
+
 	// template variables for expect
 	Datadir   string
-	Executable string
 	Etherbase string
-	Func       template.FuncMap
-
-	removeDatadir bool
-	cmd           *exec.Cmd
-	stdout        *bufio.Reader
-	stdin         io.WriteCloser
-	stderr        *testlogger
 }
 
 func init() {
-	// Run the app if we're the child process for runGeth.
-	if os.Getenv("GETH_TEST_CHILD") != "" {
+	// Run the app if we've been exec'd as "geth-test" in runGeth.
+	reexec.Register("geth-test", func() {
 		if err := app.Run(os.Args); err != nil {
 			fmt.Fprintln(os.Stderr, err)
 			os.Exit(1)
 		}
 		os.Exit(0)
+	})
+}
+
+func TestMain(m *testing.M) {
+	// check if we have been reexec'd
+	if reexec.Init() {
+		return
 	}
+	os.Exit(m.Run())
 }
 
 // spawns geth with the given command line args. If the args don't set --datadir, the
 // child g gets a temporary data directory.
 func runGeth(t *testing.T, args ...string) *testgeth {
-	tt := &testgeth{T: t, Executable: os.Args[0]}
+	tt := &testgeth{}
+	tt.TestCmd = cmdtest.NewTestCmd(t, tt)
 	for i, arg := range args {
 		switch {
 		case arg == "-datadir" || arg == "--datadir":
@@ -84,215 +80,19 @@ func runGeth(t *testing.T, args ...string) *testgeth {
 	}
 	if tt.Datadir == "" {
 		tt.Datadir = tmpdir(t)
-		tt.removeDatadir = true
+		tt.Cleanup = func() { os.RemoveAll(tt.Datadir) }
 		args = append([]string{"-datadir", tt.Datadir}, args...)
 		// Remove the temporary datadir if something fails below.
 		defer func() {
 			if t.Failed() {
-				os.RemoveAll(tt.Datadir)
+				tt.Cleanup()
 			}
 		}()
 	}
 
-	// Boot "geth". This actually runs the test binary but the init function
-	// will prevent any tests from running.
-	tt.stderr = &testlogger{t: t}
-	tt.cmd = exec.Command(os.Args[0], args...)
-	tt.cmd.Env = append(os.Environ(), "GETH_TEST_CHILD=1")
-	tt.cmd.Stderr = tt.stderr
-	stdout, err := tt.cmd.StdoutPipe()
-	if err != nil {
-		t.Fatal(err)
-	}
-	tt.stdout = bufio.NewReader(stdout)
-	if tt.stdin, err = tt.cmd.StdinPipe(); err != nil {
-		t.Fatal(err)
-	}
-	if err := tt.cmd.Start(); err != nil {
-		t.Fatal(err)
-	}
+	// Boot "geth". This actually runs the test binary but the TestMain
+	// function will prevent any tests from running.
+	tt.Run("geth-test", args...)
+
 	return tt
 }
-
-// InputLine writes the given text to the childs stdin.
-// This method can also be called from an expect template, e.g.:
-//
-//     geth.expect(`Passphrase: {{.InputLine "password"}}`)
-func (tt *testgeth) InputLine(s string) string {
-	io.WriteString(tt.stdin, s+"\n")
-	return ""
-}
-
-func (tt *testgeth) setTemplateFunc(name string, fn interface{}) {
-	if tt.Func == nil {
-		tt.Func = make(map[string]interface{})
-	}
-	tt.Func[name] = fn
-}
-
-// expect runs its argument as a template, then expects the
-// child process to output the result of the template within 5s.
-//
-// If the template starts with a newline, the newline is removed
-// before matching.
-func (tt *testgeth) expect(tplsource string) {
-	// Generate the expected output by running the template.
-	tpl := template.Must(template.New("").Funcs(tt.Func).Parse(tplsource))
-	wantbuf := new(bytes.Buffer)
-	if err := tpl.Execute(wantbuf, tt); err != nil {
-		panic(err)
-	}
-	// Trim exactly one newline at the beginning. This makes tests look
-	// much nicer because all expect strings are at column 0.
-	want := bytes.TrimPrefix(wantbuf.Bytes(), []byte("\n"))
-	if err := tt.matchExactOutput(want); err != nil {
-		tt.Fatal(err)
-	}
-	tt.Logf("Matched stdout text:\n%s", want)
-}
-
-func (tt *testgeth) matchExactOutput(want []byte) error {
-	buf := make([]byte, len(want))
-	n := 0
-	tt.withKillTimeout(func() { n, _ = io.ReadFull(tt.stdout, buf) })
-	buf = buf[:n]
-	if n < len(want) || !bytes.Equal(buf, want) {
-		// Grab any additional buffered output in case of mismatch
-		// because it might help with debugging.
-		buf = append(buf, make([]byte, tt.stdout.Buffered())...)
-		tt.stdout.Read(buf[n:])
-		// Find the mismatch position.
-		for i := 0; i < n; i++ {
-			if want[i] != buf[i] {
-				return fmt.Errorf("Output mismatch at ◊:\n---------------- (stdout text)\n%s◊%s\n---------------- (expected text)\n%s",
-					buf[:i], buf[i:n], want)
-			}
-		}
-		if n < len(want) {
-			return fmt.Errorf("Not enough output, got until ◊:\n---------------- (stdout text)\n%s\n---------------- (expected text)\n%s◊%s",
-				buf, want[:n], want[n:])
-		}
-	}
-	return nil
-}
-
-// expectRegexp expects the child process to output text matching the
-// given regular expression within 5s.
-//
-// Note that an arbitrary amount of output may be consumed by the
-// regular expression. This usually means that expect cannot be used
-// after expectRegexp.
-func (tt *testgeth) expectRegexp(resource string) (*regexp.Regexp, []string) {
-	var (
-		re      = regexp.MustCompile(resource)
-		rtee    = &runeTee{in: tt.stdout}
-		matches []int
-	)
-	tt.withKillTimeout(func() { matches = re.FindReaderSubmatchIndex(rtee) })
-	output := rtee.buf.Bytes()
-	if matches == nil {
-		tt.Fatalf("Output did not match:\n---------------- (stdout text)\n%s\n---------------- (regular expression)\n%s",
-			output, resource)
-		return re, nil
-	}
-	tt.Logf("Matched stdout text:\n%s", output)
-	var submatch []string
-	for i := 0; i < len(matches); i += 2 {
-		submatch = append(submatch, string(output[i:i+1]))
-	}
-	return re, submatch
-}
-
-// expectExit expects the child process to exit within 5s without
-// printing any additional text on stdout.
-func (tt *testgeth) expectExit() {
-	var output []byte
-	tt.withKillTimeout(func() {
-		output, _ = ioutil.ReadAll(tt.stdout)
-	})
-	tt.cmd.Wait()
-	if tt.removeDatadir {
-		os.RemoveAll(tt.Datadir)
-	}
-	if len(output) > 0 {
-		tt.Errorf("Unmatched stdout text:\n%s", output)
-	}
-}
-
-func (tt *testgeth) interrupt() {
-	tt.cmd.Process.Signal(os.Interrupt)
-}
-
-// stderrText returns any stderr output written so far.
-// The returned text holds all log lines after expectExit has
-// returned.
-func (tt *testgeth) stderrText() string {
-	tt.stderr.mu.Lock()
-	defer tt.stderr.mu.Unlock()
-	return tt.stderr.buf.String()
-}
-
-func (tt *testgeth) withKillTimeout(fn func()) {
-	timeout := time.AfterFunc(5*time.Second, func() {
-		tt.Log("killing the child process (timeout)")
-		tt.cmd.Process.Kill()
-		if tt.removeDatadir {
-			os.RemoveAll(tt.Datadir)
-		}
-	})
-	defer timeout.Stop()
-	fn()
-}
-
-// testlogger logs all written lines via t.Log and also
-// collects them for later inspection.
-type testlogger struct {
-	t   *testing.T
-	mu  sync.Mutex
-	buf bytes.Buffer
-}
-
-func (tl *testlogger) Write(b []byte) (n int, err error) {
-	lines := bytes.Split(b, []byte("\n"))
-	for _, line := range lines {
-		if len(line) > 0 {
-			tl.t.Logf("(stderr) %s", line)
-		}
-	}
-	tl.mu.Lock()
-	tl.buf.Write(b)
-	tl.mu.Unlock()
-	return len(b), err
-}
-
-// runeTee collects text read through it into buf.
-type runeTee struct {
-	in interface {
-		io.Reader
-		io.ByteReader
-		io.RuneReader
-	}
-	buf bytes.Buffer
-}
-
-func (rtee *runeTee) Read(b []byte) (n int, err error) {
-	n, err = rtee.in.Read(b)
-	rtee.buf.Write(b[:n])
-	return n, err
-}
-
-func (rtee *runeTee) ReadRune() (r rune, size int, err error) {
-	r, size, err = rtee.in.ReadRune()
-	if err == nil {
-		rtee.buf.WriteRune(r)
-	}
-	return r, size, err
-}
-
-func (rtee *runeTee) ReadByte() (b byte, err error) {
-	b, err = rtee.in.ReadByte()
-	if err == nil {
-		rtee.buf.WriteByte(b)
-	}
-	return b, err
-}

cmd/geth/usage.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"io"
+	"sort"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/internal/debug"
@@ -30,7 +31,7 @@ import (
 var AppHelpTemplate = `NAME:
    {{.App.Name}} - {{.App.Usage}}
 
-   Copyright 2013-2016 The go-ethereum Authors
+   Copyright 2013-2017 The go-ethereum Authors
 
 USAGE:
    {{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}}
@@ -64,19 +65,45 @@ var AppHelpFlagGroups = []flagGroup{
 	{
 		Name: "ETHEREUM",
 		Flags: []cli.Flag{
+			configFileFlag,
 			utils.DataDirFlag,
 			utils.KeyStoreDirFlag,
+			utils.NoUSBFlag,
 			utils.NetworkIdFlag,
-			utils.TestNetFlag,
+			utils.TestnetFlag,
+			utils.RinkebyFlag,
 			utils.DevModeFlag,
+			utils.SyncModeFlag,
+			utils.EthStatsURLFlag,
 			utils.IdentityFlag,
-			utils.FastSyncFlag,
-			utils.LightModeFlag,
 			utils.LightServFlag,
 			utils.LightPeersFlag,
 			utils.LightKDFFlag,
 		},
 	},
+	{
+		Name: "ETHASH",
+		Flags: []cli.Flag{
+			utils.EthashCacheDirFlag,
+			utils.EthashCachesInMemoryFlag,
+			utils.EthashCachesOnDiskFlag,
+			utils.EthashDatasetDirFlag,
+			utils.EthashDatasetsInMemoryFlag,
+			utils.EthashDatasetsOnDiskFlag,
+		},
+	},
+	{
+		Name: "TRANSACTION POOL",
+		Flags: []cli.Flag{
+			utils.TxPoolPriceLimitFlag,
+			utils.TxPoolPriceBumpFlag,
+			utils.TxPoolAccountSlotsFlag,
+			utils.TxPoolGlobalSlotsFlag,
+			utils.TxPoolAccountQueueFlag,
+			utils.TxPoolGlobalQueueFlag,
+			utils.TxPoolLifetimeFlag,
+		},
+	},
 	{
 		Name: "PERFORMANCE TUNING",
 		Flags: []cli.Flag{
@@ -104,7 +131,6 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.WSApiFlag,
 			utils.WSAllowedOriginsFlag,
 			utils.IPCDisabledFlag,
-			utils.IPCApiFlag,
 			utils.IPCPathFlag,
 			utils.RPCCORSDomainFlag,
 			utils.JSpathFlag,
@@ -116,12 +142,15 @@ var AppHelpFlagGroups = []flagGroup{
 		Name: "NETWORKING",
 		Flags: []cli.Flag{
 			utils.BootnodesFlag,
+			utils.BootnodesV4Flag,
+			utils.BootnodesV5Flag,
 			utils.ListenPortFlag,
 			utils.MaxPeersFlag,
 			utils.MaxPendingPeersFlag,
 			utils.NATFlag,
 			utils.NoDiscoverFlag,
 			utils.DiscoveryV5Flag,
+			utils.NetrestrictFlag,
 			utils.NodeKeyFileFlag,
 			utils.NodeKeyHexFlag,
 		},
@@ -131,7 +160,6 @@ var AppHelpFlagGroups = []flagGroup{
 		Flags: []cli.Flag{
 			utils.MiningEnabledFlag,
 			utils.MinerThreadsFlag,
-			utils.AutoDAGFlag,
 			utils.EtherbaseFlag,
 			utils.TargetGasLimitFlag,
 			utils.GasPriceFlag,
@@ -141,45 +169,72 @@ var AppHelpFlagGroups = []flagGroup{
 	{
 		Name: "GAS PRICE ORACLE",
 		Flags: []cli.Flag{
-			utils.GpoMinGasPriceFlag,
-			utils.GpoMaxGasPriceFlag,
-			utils.GpoFullBlockRatioFlag,
-			utils.GpobaseStepDownFlag,
-			utils.GpobaseStepUpFlag,
-			utils.GpobaseCorrectionFactorFlag,
+			utils.GpoBlocksFlag,
+			utils.GpoPercentileFlag,
 		},
 	},
 	{
 		Name: "VIRTUAL MACHINE",
 		Flags: []cli.Flag{
-			utils.VMEnableJitFlag,
-			utils.VMForceJitFlag,
-			utils.VMJitCacheFlag,
+			utils.VMEnableDebugFlag,
 		},
 	},
 	{
 		Name: "LOGGING AND DEBUGGING",
 		Flags: append([]cli.Flag{
-			utils.EthStatsURLFlag,
 			utils.MetricsEnabledFlag,
 			utils.FakePoWFlag,
+			utils.NoCompactionFlag,
 		}, debug.Flags...),
 	},
+	{
+		Name: "DEPRECATED",
+		Flags: []cli.Flag{
+			utils.FastSyncFlag,
+			utils.LightModeFlag,
+		},
+	},
 	{
 		Name: "EXPERIMENTAL",
 		Flags: []cli.Flag{
 			utils.WhisperEnabledFlag,
-			utils.NatspecEnabledFlag,
-		},
-	},
-	{
-		Name: "MISCELLANEOUS",
-		Flags: []cli.Flag{
-			utils.SolcPathFlag,
 		},
 	},
 }
 
+// byCategory sorts an array of flagGroup by Name in the order
+// defined in AppHelpFlagGroups.
+type byCategory []flagGroup
+
+func (a byCategory) Len() int      { return len(a) }
+func (a byCategory) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a byCategory) Less(i, j int) bool {
+	iCat, jCat := a[i].Name, a[j].Name
+	iIdx, jIdx := len(AppHelpFlagGroups), len(AppHelpFlagGroups) // ensure non categorized flags come last
+
+	for i, group := range AppHelpFlagGroups {
+		if iCat == group.Name {
+			iIdx = i
+		}
+		if jCat == group.Name {
+			jIdx = i
+		}
+	}
+
+	return iIdx < jIdx
+}
+
+func flagCategory(flag cli.Flag) string {
+	for _, category := range AppHelpFlagGroups {
+		for _, flg := range category.Flags {
+			if flg.GetName() == flag.GetName() {
+				return category.Name
+			}
+		}
+	}
+	return "MISC"
+}
+
 func init() {
 	// Override the default app help template
 	cli.AppHelpTemplate = AppHelpTemplate
@@ -189,6 +244,7 @@ func init() {
 		App        interface{}
 		FlagGroups []flagGroup
 	}
+
 	// Override the default app help printer, but only for the global app help
 	originalHelpPrinter := cli.HelpPrinter
 	cli.HelpPrinter = func(w io.Writer, tmpl string, data interface{}) {
@@ -218,6 +274,27 @@ func init() {
 			}
 			// Render out custom usage screen
 			originalHelpPrinter(w, tmpl, helpData{data, AppHelpFlagGroups})
+		} else if tmpl == utils.CommandHelpTemplate {
+			// Iterate over all command specific flags and categorize them
+			categorized := make(map[string][]cli.Flag)
+			for _, flag := range data.(cli.Command).Flags {
+				if _, ok := categorized[flag.String()]; !ok {
+					categorized[flagCategory(flag)] = append(categorized[flagCategory(flag)], flag)
+				}
+			}
+
+			// sort to get a stable ordering
+			sorted := make([]flagGroup, 0, len(categorized))
+			for cat, flgs := range categorized {
+				sorted = append(sorted, flagGroup{cat, flgs})
+			}
+			sort.Sort(byCategory(sorted))
+
+			// add sorted array to data and render with default printer
+			originalHelpPrinter(w, tmpl, map[string]interface{}{
+				"cmd":              data,
+				"categorizedFlags": sorted,
+			})
 		} else {
 			originalHelpPrinter(w, tmpl, data)
 		}

cmd/gethrpctest/main.go

@@ -1,160 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// gethrpctest is a command to run the external RPC tests.
-package main
-
-import (
-	"flag"
-	"log"
-	"os"
-	"os/signal"
-
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/eth"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/logger/glog"
-	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/tests"
-	whisper "github.com/ethereum/go-ethereum/whisper/whisperv2"
-)
-
-const defaultTestKey = "b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291"
-
-var (
-	testFile = flag.String("json", "", "Path to the .json test file to load")
-	testName = flag.String("test", "", "Name of the test from the .json file to run")
-	testKey  = flag.String("key", defaultTestKey, "Private key of a test account to inject")
-)
-
-func main() {
-	flag.Parse()
-
-	// Enable logging errors, we really do want to see those
-	glog.SetV(2)
-	glog.SetToStderr(true)
-
-	// Load the test suite to run the RPC against
-	tests, err := tests.LoadBlockTests(*testFile)
-	if err != nil {
-		log.Fatalf("Failed to load test suite: %v", err)
-	}
-	test, found := tests[*testName]
-	if !found {
-		log.Fatalf("Requested test (%s) not found within suite", *testName)
-	}
-
-	stack, err := MakeSystemNode(*testKey, test)
-	if err != nil {
-		log.Fatalf("Failed to assemble test stack: %v", err)
-	}
-	if err := stack.Start(); err != nil {
-		log.Fatalf("Failed to start test node: %v", err)
-	}
-	defer stack.Stop()
-
-	log.Println("Test node started...")
-
-	// Make sure the tests contained within the suite pass
-	if err := RunTest(stack, test); err != nil {
-		log.Fatalf("Failed to run the pre-configured test: %v", err)
-	}
-	log.Println("Initial test suite passed...")
-
-	quit := make(chan os.Signal, 1)
-	signal.Notify(quit, os.Interrupt)
-	<-quit
-}
-
-// MakeSystemNode configures a protocol stack for the RPC tests based on a given
-// keystore path and initial pre-state.
-func MakeSystemNode(privkey string, test *tests.BlockTest) (*node.Node, error) {
-	// Create a networkless protocol stack
-	stack, err := node.New(&node.Config{
-		UseLightweightKDF: true,
-		IPCPath:           node.DefaultIPCEndpoint(""),
-		HTTPHost:          node.DefaultHTTPHost,
-		HTTPPort:          node.DefaultHTTPPort,
-		HTTPModules:       []string{"admin", "db", "eth", "debug", "miner", "net", "shh", "txpool", "personal", "web3"},
-		WSHost:            node.DefaultWSHost,
-		WSPort:            node.DefaultWSPort,
-		WSModules:         []string{"admin", "db", "eth", "debug", "miner", "net", "shh", "txpool", "personal", "web3"},
-		NoDiscovery:       true,
-	})
-	if err != nil {
-		return nil, err
-	}
-	// Create the keystore and inject an unlocked account if requested
-	accman := stack.AccountManager()
-	if len(privkey) > 0 {
-		key, err := crypto.HexToECDSA(privkey)
-		if err != nil {
-			return nil, err
-		}
-		a, err := accman.ImportECDSA(key, "")
-		if err != nil {
-			return nil, err
-		}
-		if err := accman.Unlock(a, ""); err != nil {
-			return nil, err
-		}
-	}
-	// Initialize and register the Ethereum protocol
-	db, _ := ethdb.NewMemDatabase()
-	if _, err := test.InsertPreState(db); err != nil {
-		return nil, err
-	}
-	ethConf := &eth.Config{
-		TestGenesisState: db,
-		TestGenesisBlock: test.Genesis,
-		ChainConfig:      &params.ChainConfig{HomesteadBlock: params.MainNetHomesteadBlock},
-	}
-	if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { return eth.New(ctx, ethConf) }); err != nil {
-		return nil, err
-	}
-	// Initialize and register the Whisper protocol
-	if err := stack.Register(func(*node.ServiceContext) (node.Service, error) { return whisper.New(), nil }); err != nil {
-		return nil, err
-	}
-	return stack, nil
-}
-
-// RunTest executes the specified test against an already pre-configured protocol
-// stack to ensure basic checks pass before running RPC tests.
-func RunTest(stack *node.Node, test *tests.BlockTest) error {
-	var ethereum *eth.Ethereum
-	stack.Service(&ethereum)
-	blockchain := ethereum.BlockChain()
-
-	// Process the blocks and verify the imported headers
-	blocks, err := test.TryBlocksInsert(blockchain)
-	if err != nil {
-		return err
-	}
-	if err := test.ValidateImportedHeaders(blockchain, blocks); err != nil {
-		return err
-	}
-	// Retrieve the assembled state and validate it
-	stateDb, err := blockchain.State()
-	if err != nil {
-		return err
-	}
-	if err := test.ValidatePostState(stateDb); err != nil {
-		return err
-	}
-	return nil
-}

cmd/internal/browser/browser.go

@@ -0,0 +1,46 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package browser provides utilities for interacting with users' browsers.
+package browser
+
+import (
+	"os"
+	"os/exec"
+	"runtime"
+)
+
+// Commands returns a list of possible commands to use to open a url.
+func Commands() [][]string {
+	var cmds [][]string
+	if exe := os.Getenv("BROWSER"); exe != "" {
+		cmds = append(cmds, []string{exe})
+	}
+	switch runtime.GOOS {
+	case "darwin":
+		cmds = append(cmds, []string{"/usr/bin/open"})
+	case "windows":
+		cmds = append(cmds, []string{"cmd", "/c", "start"})
+	default:
+		cmds = append(cmds, []string{"xdg-open"})
+	}
+	cmds = append(cmds,
+		[]string{"chrome"},
+		[]string{"google-chrome"},
+		[]string{"chromium"},
+		[]string{"firefox"},
+	)
+	return cmds
+}
+
+// Open tries to open url in a browser and reports whether it succeeded.
+func Open(url string) bool {
+	for _, args := range Commands() {
+		cmd := exec.Command(args[0], append(args[1:], url)...)
+		if cmd.Start() == nil {
+			return true
+		}
+	}
+	return false
+}

cmd/puppeth/module.go

@@ -0,0 +1,152 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+var (
+	// ErrServiceUnknown is returned when a service container doesn't exist.
+	ErrServiceUnknown = errors.New("service unknown")
+
+	// ErrServiceOffline is returned when a service container exists, but it is not
+	// running.
+	ErrServiceOffline = errors.New("service offline")
+
+	// ErrServiceUnreachable is returned when a service container is running, but
+	// seems to not respond to communication attempts.
+	ErrServiceUnreachable = errors.New("service unreachable")
+
+	// ErrNotExposed is returned if a web-service doesn't have an exposed port, nor
+	// a reverse-proxy in front of it to forward requests.
+	ErrNotExposed = errors.New("service not exposed, nor proxied")
+)
+
+// containerInfos is a heavily reduced version of the huge inspection dataset
+// returned from docker inspect, parsed into a form easily usable by puppeth.
+type containerInfos struct {
+	running bool              // Flag whether the container is running currently
+	envvars map[string]string // Collection of environmental variables set on the container
+	portmap map[string]int    // Port mapping from internal port/proto combos to host binds
+	volumes map[string]string // Volume mount points from container to host directories
+}
+
+// inspectContainer runs docker inspect against a running container
+func inspectContainer(client *sshClient, container string) (*containerInfos, error) {
+	// Check whether there's a container running for the service
+	out, err := client.Run(fmt.Sprintf("docker inspect %s", container))
+	if err != nil {
+		return nil, ErrServiceUnknown
+	}
+	// If yes, extract various configuration options
+	type inspection struct {
+		State struct {
+			Running bool
+		}
+		Mounts []struct {
+			Source      string
+			Destination string
+		}
+		Config struct {
+			Env []string
+		}
+		HostConfig struct {
+			PortBindings map[string][]map[string]string
+		}
+	}
+	var inspects []inspection
+	if err = json.Unmarshal(out, &inspects); err != nil {
+		return nil, err
+	}
+	inspect := inspects[0]
+
+	// Infos retrieved, parse the above into something meaningful
+	infos := &containerInfos{
+		running: inspect.State.Running,
+		envvars: make(map[string]string),
+		portmap: make(map[string]int),
+		volumes: make(map[string]string),
+	}
+	for _, envvar := range inspect.Config.Env {
+		if parts := strings.Split(envvar, "="); len(parts) == 2 {
+			infos.envvars[parts[0]] = parts[1]
+		}
+	}
+	for portname, details := range inspect.HostConfig.PortBindings {
+		if len(details) > 0 {
+			port, _ := strconv.Atoi(details[0]["HostPort"])
+			infos.portmap[portname] = port
+		}
+	}
+	for _, mount := range inspect.Mounts {
+		infos.volumes[mount.Destination] = mount.Source
+	}
+	return infos, err
+}
+
+// tearDown connects to a remote machine via SSH and terminates docker containers
+// running with the specified name in the specified network.
+func tearDown(client *sshClient, network string, service string, purge bool) ([]byte, error) {
+	// Tear down the running (or paused) container
+	out, err := client.Run(fmt.Sprintf("docker rm -f %s_%s_1", network, service))
+	if err != nil {
+		return out, err
+	}
+	// If requested, purge the associated docker image too
+	if purge {
+		return client.Run(fmt.Sprintf("docker rmi %s/%s", network, service))
+	}
+	return nil, nil
+}
+
+// resolve retrieves the hostname a service is running on either by returning the
+// actual server name and port, or preferably an nginx virtual host if available.
+func resolve(client *sshClient, network string, service string, port int) (string, error) {
+	// Inspect the service to get various configurations from it
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_%s_1", network, service))
+	if err != nil {
+		return "", err
+	}
+	if !infos.running {
+		return "", ErrServiceOffline
+	}
+	// Container online, extract any environmental variables
+	if vhost := infos.envvars["VIRTUAL_HOST"]; vhost != "" {
+		return vhost, nil
+	}
+	return fmt.Sprintf("%s:%d", client.server, port), nil
+}
+
+// checkPort tries to connect to a remote host on a given
+func checkPort(host string, port int) error {
+	log.Trace("Verifying remote TCP connectivity", "server", host, "port", port)
+	conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", host, port), time.Second)
+	if err != nil {
+		return err
+	}
+	conn.Close()
+	return nil
+}

cmd/puppeth/module_ethstats.go

@@ -0,0 +1,159 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// ethstatsDockerfile is the Dockerfile required to build an ethstats backend
+// and associated monitoring site.
+var ethstatsDockerfile = `
+FROM mhart/alpine-node:latest
+
+RUN \
+  apk add --update git                                         && \
+  git clone --depth=1 https://github.com/karalabe/eth-netstats && \
+	apk del git && rm -rf /var/cache/apk/*                       && \
+	\
+  cd /eth-netstats && npm install && npm install -g grunt-cli && grunt
+
+WORKDIR /eth-netstats
+EXPOSE 3000
+
+RUN echo 'module.exports = {trusted: [{{.Trusted}}], banned: []};' > lib/utils/config.js
+
+CMD ["npm", "start"]
+`
+
+// ethstatsComposefile is the docker-compose.yml file required to deploy and
+// maintain an ethstats monitoring site.
+var ethstatsComposefile = `
+version: '2'
+services:
+  ethstats:
+    build: .
+    image: {{.Network}}/ethstats{{if not .VHost}}
+    ports:
+      - "{{.Port}}:3000"{{end}}
+    environment:
+      - WS_SECRET={{.Secret}}{{if .VHost}}
+      - VIRTUAL_HOST={{.VHost}}{{end}}
+    restart: always
+`
+
+// deployEthstats deploys a new ethstats container to a remote machine via SSH,
+// docker and docker-compose. If an instance with the specified network name
+// already exists there, it will be overwritten!
+func deployEthstats(client *sshClient, network string, port int, secret string, vhost string, trusted []string) ([]byte, error) {
+	// Generate the content to upload to the server
+	workdir := fmt.Sprintf("%d", rand.Int63())
+	files := make(map[string][]byte)
+
+	for i, address := range trusted {
+		trusted[i] = fmt.Sprintf("\"%s\"", address)
+	}
+
+	dockerfile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(ethstatsDockerfile)).Execute(dockerfile, map[string]interface{}{
+		"Trusted": strings.Join(trusted, ", "),
+	})
+	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
+
+	composefile := new(bytes.Buffer)
+	template.Must(template.New("").Parse(ethstatsComposefile)).Execute(composefile, map[string]interface{}{
+		"Network": network,
+		"Port":    port,
+		"Secret":  secret,
+		"VHost":   vhost,
+	})
+	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
+
+	// Upload the deployment files to the remote server (and clean up afterwards)
+	if out, err := client.Upload(files); err != nil {
+		return out, err
+	}
+	defer client.Run("rm -rf " + workdir)
+
+	// Build and deploy the ethstats service
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build", workdir, network))
+}
+
+// ethstatsInfos is returned from an ethstats status check to allow reporting
+// various configuration parameters.
+type ethstatsInfos struct {
+	host   string
+	port   int
+	secret string
+	config string
+}
+
+// String implements the stringer interface.
+func (info *ethstatsInfos) String() string {
+	return fmt.Sprintf("host=%s, port=%d, secret=%s", info.host, info.port, info.secret)
+}
+
+// checkEthstats does a health-check against an ethstats server to verify whether
+// it's running, and if yes, gathering a collection of useful infos about it.
+func checkEthstats(client *sshClient, network string) (*ethstatsInfos, error) {
+	// Inspect a possible ethstats container on the host
+	infos, err := inspectContainer(client, fmt.Sprintf("%s_ethstats_1", network))
+	if err != nil {
+		return nil, err
+	}
+	if !infos.running {
+		return nil, ErrServiceOffline
+	}
+	// Resolve the port from the host, or the reverse proxy
+	port := infos.portmap["3000/tcp"]
+	if port == 0 {
+		if proxy, _ := checkNginx(client, network); proxy != nil {
+			port = proxy.port
+		}
+	}
+	if port == 0 {
+		return nil, ErrNotExposed
+	}
+	// Resolve the host from the reverse-proxy and configure the connection string
+	host := infos.envvars["VIRTUAL_HOST"]
+	if host == "" {
+		host = client.server
+	}
+	secret := infos.envvars["WS_SECRET"]
+	config := fmt.Sprintf("%s@%s", secret, host)
+	if port != 80 && port != 443 {
+		config += fmt.Sprintf(":%d", port)
+	}
+	// Run a sanity check to see if the port is reachable
+	if err = checkPort(host, port); err != nil {
+		log.Warn("Ethstats service seems unreachable", "server", host, "port", port, "err", err)
+	}
+	// Container available, assemble and return the useful infos
+	return &ethstatsInfos{
+		host:   host,
+		port:   port,
+		secret: secret,
+		config: config,
+	}, nil
+}