Merge pull request #3671 from karalabe/1.5.9-stable

params: 1.5.9 stable

.dockerignore

@@ -0,0 +1,3 @@
+.git
+build/_workspace
+build/_bin

.github/CONTRIBUTING.md

@@ -9,9 +9,7 @@ and help.
 
 If you'd like to contribute to go-ethereum please fork, fix, commit and
 send a pull request. Commits which do not comply with the coding standards
-are ignored (use gofmt!). If you send pull requests make absolute sure that you
-commit on the `develop` branch and that you do not merge to master.
-Commits that are directly based on master are simply ignored.
+are ignored (use gofmt!).
 
 See [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
 for more details on configuring your environment, testing, and

.travis.yml

@@ -13,15 +13,15 @@ matrix:
       go: 1.6.2
     - os: linux
       dist: trusty
-      go: 1.7
+      go: 1.7.4
     - os: osx
-      go: 1.7
+      go: 1.7.4
 
     # This builder does the Ubuntu PPA and Linux Azure uploads
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.7
+      go: 1.7.4
       env:
         - ubuntu-ppa
         - azure-linux
@@ -53,26 +53,52 @@ matrix:
         - CC=aarch64-linux-gnu-gcc go run build/ci.go install -arch arm64
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
-    # This builder does the OSX Azure, Android Maven and Azure and iOS CocoaPods and Azure uploads
+    # This builder does the Android Maven and Azure uploads
+    - os: linux
+      dist: precise # Needed for the android tools
+      addons:
+        apt:
+          packages:
+            - oracle-java8-installer
+            - oracle-java8-set-default
+      language: android
+      android:
+        components:
+          - platform-tools
+          - tools
+          - android-15
+          - android-19
+          - android-24
+      env:
+        - azure-android
+        - maven-android
+      before_install:
+        - curl https://storage.googleapis.com/golang/go1.8rc3.linux-amd64.tar.gz | tar -xz
+        - export PATH=`pwd`/go/bin:$PATH
+        - export GOROOT=`pwd`/go
+        - export GOPATH=$HOME/go # Drop post Go 1.8
+      script:
+        # Build the Android archive and upload it to Maven Central and Azure
+        - curl https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip -o android-ndk-r13b.zip
+        - unzip -q android-ndk-r13b.zip && rm android-ndk-r13b.zip
+        - mv android-ndk-r13b $HOME
+        - export ANDROID_NDK=$HOME/android-ndk-r13b
+
+        - mkdir -p $GOPATH/src/github.com/ethereum
+        - ln -s `pwd` $GOPATH/src/github.com/ethereum
+        - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
+
+    # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
     - os: osx
-      go: 1.7
+      go: 1.7.4
       env:
         - azure-osx
-        - mobile
+        - azure-ios
+        - cocoapods-ios
       script:
         - go run build/ci.go install
         - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -upload gethstore/builds
 
-        # Build the Android archive and upload it to Maven Central and Azure
-        - brew update
-        - brew install android-sdk maven gpg
-        - alias gpg="gpg2"
-
-        - export ANDROID_HOME=/usr/local/opt/android-sdk
-        - echo "y" | android update sdk --no-ui --filter `android list sdk | grep "SDK Platform Android" | grep -E 'API 15|API 19|API 24' | awk '{print $1}' | cut -d '-' -f 1 | tr '\n' ','`
-
-        - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
-
         # Build the iOS framework and upload it to CocoaPods and Azure
         - gem uninstall cocoapods -a
         - gem install cocoapods --pre
@@ -90,7 +116,7 @@ install:
   - go get golang.org/x/tools/cmd/cover
 script:
   - go run build/ci.go install
-  - go run build/ci.go test -coverage -vet
+  - go run build/ci.go test -coverage -vet -misspell
 
 notifications:
   webhooks:

Dockerfile

@@ -1,11 +1,11 @@
-FROM alpine:3.3
+FROM alpine:3.5
 
 ADD . /go-ethereum
 RUN \
-  apk add --update git go make gcc musl-dev         && \
-  (cd go-ethereum && make geth)                     && \
-  cp go-ethereum/build/bin/geth /geth               && \
-  apk del git go make gcc musl-dev                  && \
+  apk add --update git go make gcc musl-dev linux-headers && \
+  (cd go-ethereum && make geth)                           && \
+  cp go-ethereum/build/bin/geth /geth                     && \
+  apk del git go make gcc musl-dev linux-headers          && \
   rm -rf /go-ethereum && rm -rf /var/cache/apk/*
 
 EXPOSE 8545

README.md

@@ -39,9 +39,7 @@ The go-ethereum project comes with several wrappers/executables found in the `cm
 | `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow insolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
 | `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. |
 | `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). |
-| `bzzd`    | swarm daemon. This is the entrypoint for the swarm network. `bzzd --help` for command line options. See https://swarm-guide.readthedocs.io for swarm documentation. |
-| `bzzup`   | swarm command line file uploader. `bzzup --help` for command line options |
-| `bzzhash`   | command to calculate the swarm hash of a file or directory. `bzzhash --help` for command line options |
+| `swarm`    | swarm daemon and tools. This is the entrypoint for the swarm network. `swarm --help` for command line options and subcommands. See https://swarm-guide.readthedocs.io for swarm documentation. |
 
 ## Running geth
 

VERSION

@@ -1 +1 @@
-1.5.4
+1.5.9

accounts/abi/abi.go

@@ -91,8 +91,30 @@ func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
 	// first we need to create a slice of the type
 	var refSlice reflect.Value
 	switch elem.T {
-	case IntTy, UintTy, BoolTy: // int, uint, bool can all be of type big int.
-		refSlice = reflect.ValueOf([]*big.Int(nil))
+	case IntTy, UintTy, BoolTy:
+		// create a new reference slice matching the element type
+		switch t.Type.Kind {
+		case reflect.Bool:
+			refSlice = reflect.ValueOf([]bool(nil))
+		case reflect.Uint8:
+			refSlice = reflect.ValueOf([]uint8(nil))
+		case reflect.Uint16:
+			refSlice = reflect.ValueOf([]uint16(nil))
+		case reflect.Uint32:
+			refSlice = reflect.ValueOf([]uint32(nil))
+		case reflect.Uint64:
+			refSlice = reflect.ValueOf([]uint64(nil))
+		case reflect.Int8:
+			refSlice = reflect.ValueOf([]int8(nil))
+		case reflect.Int16:
+			refSlice = reflect.ValueOf([]int16(nil))
+		case reflect.Int32:
+			refSlice = reflect.ValueOf([]int32(nil))
+		case reflect.Int64:
+			refSlice = reflect.ValueOf([]int64(nil))
+		default:
+			refSlice = reflect.ValueOf([]*big.Int(nil))
+		}
 	case AddressTy: // address must be of slice Address
 		refSlice = reflect.ValueOf([]common.Address(nil))
 	case HashTy: // hash must be of slice hash
@@ -147,7 +169,27 @@ func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
 		// set inter to the correct type (cast)
 		switch elem.T {
 		case IntTy, UintTy:
-			inter = common.BytesToBig(returnOutput)
+			bigNum := common.BytesToBig(returnOutput)
+			switch t.Type.Kind {
+			case reflect.Uint8:
+				inter = uint8(bigNum.Uint64())
+			case reflect.Uint16:
+				inter = uint16(bigNum.Uint64())
+			case reflect.Uint32:
+				inter = uint32(bigNum.Uint64())
+			case reflect.Uint64:
+				inter = bigNum.Uint64()
+			case reflect.Int8:
+				inter = int8(bigNum.Int64())
+			case reflect.Int16:
+				inter = int16(bigNum.Int64())
+			case reflect.Int32:
+				inter = int32(bigNum.Int64())
+			case reflect.Int64:
+				inter = bigNum.Int64()
+			default:
+				inter = common.BytesToBig(returnOutput)
+			}
 		case BoolTy:
 			inter = common.BytesToBig(returnOutput).Uint64() > 0
 		case AddressTy:
@@ -169,7 +211,7 @@ func toGoSlice(i int, t Argument, output []byte) (interface{}, error) {
 // argument in T.
 func toGoType(i int, t Argument, output []byte) (interface{}, error) {
 	// we need to treat slices differently
-	if (t.Type.IsSlice || t.Type.IsArray) && t.Type.T != BytesTy && t.Type.T != StringTy && t.Type.T != FixedBytesTy {
+	if (t.Type.IsSlice || t.Type.IsArray) && t.Type.T != BytesTy && t.Type.T != StringTy && t.Type.T != FixedBytesTy && t.Type.T != FunctionTy {
 		return toGoSlice(i, t, output)
 	}
 
@@ -233,7 +275,7 @@ func toGoType(i int, t Argument, output []byte) (interface{}, error) {
 		return common.BytesToAddress(returnOutput), nil
 	case HashTy:
 		return common.BytesToHash(returnOutput), nil
-	case BytesTy, FixedBytesTy:
+	case BytesTy, FixedBytesTy, FunctionTy:
 		return returnOutput, nil
 	case StringTy:
 		return string(returnOutput), nil
@@ -345,12 +387,13 @@ func (abi ABI) Unpack(v interface{}, name string, output []byte) error {
 
 func (abi *ABI) UnmarshalJSON(data []byte) error {
 	var fields []struct {
-		Type     string
-		Name     string
-		Constant bool
-		Indexed  bool
-		Inputs   []Argument
-		Outputs  []Argument
+		Type      string
+		Name      string
+		Constant  bool
+		Indexed   bool
+		Anonymous bool
+		Inputs    []Argument
+		Outputs   []Argument
 	}
 
 	if err := json.Unmarshal(data, &fields); err != nil {
@@ -375,8 +418,9 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
 			}
 		case "event":
 			abi.Events[field.Name] = Event{
-				Name:   field.Name,
-				Inputs: field.Inputs,
+				Name:      field.Name,
+				Anonymous: field.Anonymous,
+				Inputs:    field.Inputs,
 			}
 		}
 	}

accounts/abi/abi_test.go

@@ -67,10 +67,10 @@ func TestTypeCheck(t *testing.T) {
 		{"uint16[3]", [4]uint16{1, 2, 3}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"},
 		{"uint16[3]", []uint16{1, 2, 3}, ""},
 		{"uint16[3]", []uint16{1, 2, 3, 4}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"},
-		{"address[]", []common.Address{common.Address{1}}, ""},
-		{"address[1]", []common.Address{common.Address{1}}, ""},
-		{"address[1]", [1]common.Address{common.Address{1}}, ""},
-		{"address[2]", [1]common.Address{common.Address{1}}, "abi: cannot use [1]array as type [2]array as argument"},
+		{"address[]", []common.Address{{1}}, ""},
+		{"address[1]", []common.Address{{1}}, ""},
+		{"address[1]", [1]common.Address{{1}}, ""},
+		{"address[2]", [1]common.Address{{1}}, "abi: cannot use [1]array as type [2]array as argument"},
 		{"bytes32", [32]byte{}, ""},
 		{"bytes32", [33]byte{}, "abi: cannot use [33]uint8 as type [32]uint8 as argument"},
 		{"bytes32", common.Hash{1}, ""},
@@ -80,7 +80,8 @@ func TestTypeCheck(t *testing.T) {
 		{"bytes", [2]byte{0, 1}, ""},
 		{"bytes", common.Hash{1}, ""},
 		{"string", "hello world", ""},
-		{"bytes32[]", [][32]byte{[32]byte{}}, ""},
+		{"bytes32[]", [][32]byte{{}}, ""},
+		{"function", [24]byte{}, ""},
 	} {
 		typ, err := NewType(test.typ)
 		if err != nil {
@@ -197,6 +198,13 @@ func TestSimpleMethodUnpack(t *testing.T) {
 			"interface",
 			"",
 		},
+		{
+			`[ { "type": "function" } ]`,
+			pad([]byte{1}, 32, false),
+			[24]byte{1},
+			"function",
+			"",
+		},
 	} {
 		abiDefinition := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def)
 		abi, err := JSON(strings.NewReader(abiDefinition))
@@ -255,6 +263,10 @@ func TestSimpleMethodUnpack(t *testing.T) {
 			var v common.Hash
 			err = abi.Unpack(&v, "method", test.marshalledOutput)
 			outvar = v
+		case "function":
+			var v [24]byte
+			err = abi.Unpack(&v, "method", test.marshalledOutput)
+			outvar = v
 		case "interface":
 			err = abi.Unpack(&outvar, "method", test.marshalledOutput)
 		default:
@@ -320,6 +332,30 @@ func TestUnpackSetInterfaceSlice(t *testing.T) {
 	}
 }
 
+func TestUnpackSetInterfaceArrayOutput(t *testing.T) {
+	var (
+		var1 = new([1]uint32)
+		var2 = new([1]uint32)
+	)
+	out := []interface{}{var1, var2}
+	abi, err := JSON(strings.NewReader(`[{"type":"function", "name":"ints", "outputs":[{"type":"uint32[1]"}, {"type":"uint32[1]"}]}]`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	marshalledReturn := append(pad([]byte{1}, 32, true), pad([]byte{2}, 32, true)...)
+	err = abi.Unpack(&out, "ints", marshalledReturn)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if *var1 != [1]uint32{1} {
+		t.Error("expected var1 to be [1], got", *var1)
+	}
+	if *var2 != [1]uint32{2} {
+		t.Error("expected var2 to be [2], got", *var2)
+	}
+}
+
 func TestPack(t *testing.T) {
 	for i, test := range []struct {
 		typ string
@@ -331,8 +367,9 @@ func TestPack(t *testing.T) {
 		{"uint16[]", []uint16{1, 2}, formatSliceOutput([]byte{1}, []byte{2})},
 		{"bytes20", [20]byte{1}, pad([]byte{1}, 32, false)},
 		{"uint256[]", []*big.Int{big.NewInt(1), big.NewInt(2)}, formatSliceOutput([]byte{1}, []byte{2})},
-		{"address[]", []common.Address{common.Address{1}, common.Address{2}}, formatSliceOutput(pad([]byte{1}, 20, false), pad([]byte{2}, 20, false))},
-		{"bytes32[]", []common.Hash{common.Hash{1}, common.Hash{2}}, formatSliceOutput(pad([]byte{1}, 32, false), pad([]byte{2}, 32, false))},
+		{"address[]", []common.Address{{1}, {2}}, formatSliceOutput(pad([]byte{1}, 20, false), pad([]byte{2}, 20, false))},
+		{"bytes32[]", []common.Hash{{1}, {2}}, formatSliceOutput(pad([]byte{1}, 32, false), pad([]byte{2}, 32, false))},
+		{"function", [24]byte{1}, pad([]byte{1}, 32, false)},
 	} {
 		typ, err := NewType(test.typ)
 		if err != nil {
@@ -445,12 +482,12 @@ func TestReader(t *testing.T) {
 	Uint256, _ := NewType("uint256")
 	exp := ABI{
 		Methods: map[string]Method{
-			"balance": Method{
+			"balance": {
 				"balance", true, nil, nil,
 			},
-			"send": Method{
+			"send": {
 				"send", false, []Argument{
-					Argument{"amount", Uint256, false},
+					{"amount", Uint256, false},
 				}, nil,
 			},
 		},
@@ -549,7 +586,7 @@ func TestTestSlice(t *testing.T) {
 
 func TestMethodSignature(t *testing.T) {
 	String, _ := NewType("string")
-	m := Method{"foo", false, []Argument{Argument{"bar", String, false}, Argument{"baz", String, false}}, nil}
+	m := Method{"foo", false, []Argument{{"bar", String, false}, {"baz", String, false}}, nil}
 	exp := "foo(string,string)"
 	if m.Sig() != exp {
 		t.Error("signature mismatch", exp, "!=", m.Sig())
@@ -561,7 +598,7 @@ func TestMethodSignature(t *testing.T) {
 	}
 
 	uintt, _ := NewType("uint")
-	m = Method{"foo", false, []Argument{Argument{"bar", uintt, false}}, nil}
+	m = Method{"foo", false, []Argument{{"bar", uintt, false}}, nil}
 	exp = "foo(uint256)"
 	if m.Sig() != exp {
 		t.Error("signature mismatch", exp, "!=", m.Sig())
@@ -752,23 +789,58 @@ func TestDefaultFunctionParsing(t *testing.T) {
 func TestBareEvents(t *testing.T) {
 	const definition = `[
 	{ "type" : "event", "name" : "balance" },
-	{ "type" : "event", "name" : "name" }]`
+	{ "type" : "event", "name" : "anon", "anonymous" : true},
+	{ "type" : "event", "name" : "args", "inputs" : [{ "indexed":false, "name":"arg0", "type":"uint256" }, { "indexed":true, "name":"arg1", "type":"address" }] }
+	]`
+
+	arg0, _ := NewType("uint256")
+	arg1, _ := NewType("address")
+
+	expectedEvents := map[string]struct {
+		Anonymous bool
+		Args      []Argument
+	}{
+		"balance": {false, nil},
+		"anon":    {true, nil},
+		"args": {false, []Argument{
+			{Name: "arg0", Type: arg0, Indexed: false},
+			{Name: "arg1", Type: arg1, Indexed: true},
+		}},
+	}
 
 	abi, err := JSON(strings.NewReader(definition))
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if len(abi.Events) != 2 {
-		t.Error("expected 2 events")
+	if len(abi.Events) != len(expectedEvents) {
+		t.Fatalf("invalid number of events after parsing, want %d, got %d", len(expectedEvents), len(abi.Events))
 	}
 
-	if _, ok := abi.Events["balance"]; !ok {
-		t.Error("expected 'balance' event to be present")
-	}
-
-	if _, ok := abi.Events["name"]; !ok {
-		t.Error("expected 'name' event to be present")
+	for name, exp := range expectedEvents {
+		got, ok := abi.Events[name]
+		if !ok {
+			t.Errorf("could not found event %s", name)
+			continue
+		}
+		if got.Anonymous != exp.Anonymous {
+			t.Errorf("invalid anonymous indication for event %s, want %v, got %v", name, exp.Anonymous, got.Anonymous)
+		}
+		if len(got.Inputs) != len(exp.Args) {
+			t.Errorf("invalid number of args, want %d, got %d", len(exp.Args), len(got.Inputs))
+			continue
+		}
+		for i, arg := range exp.Args {
+			if arg.Name != got.Inputs[i].Name {
+				t.Errorf("events[%s].Input[%d] has an invalid name, want %s, got %s", name, i, arg.Name, got.Inputs[i].Name)
+			}
+			if arg.Indexed != got.Inputs[i].Indexed {
+				t.Errorf("events[%s].Input[%d] has an invalid indexed indication, want %v, got %v", name, i, arg.Indexed, got.Inputs[i].Indexed)
+			}
+			if arg.Type.T != got.Inputs[i].Type.T {
+				t.Errorf("events[%s].Input[%d] has an invalid type, want %x, got %x", name, i, arg.Type.T, got.Inputs[i].Type.T)
+			}
+		}
 	}
 }
 

accounts/abi/argument.go

@@ -31,8 +31,9 @@ type Argument struct {
 
 func (a *Argument) UnmarshalJSON(data []byte) error {
 	var extarg struct {
-		Name string
-		Type string
+		Name    string
+		Type    string
+		Indexed bool
 	}
 	err := json.Unmarshal(data, &extarg)
 	if err != nil {
@@ -44,6 +45,7 @@ func (a *Argument) UnmarshalJSON(data []byte) error {
 		return err
 	}
 	a.Name = extarg.Name
+	a.Indexed = extarg.Indexed
 
 	return nil
 }

accounts/abi/bind/auth.go

@@ -22,7 +22,7 @@ import (
 	"io"
 	"io/ioutil"
 
-	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -35,7 +35,7 @@ func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) {
 	if err != nil {
 		return nil, err
 	}
-	key, err := accounts.DecryptKey(json, passphrase)
+	key, err := keystore.DecryptKey(json, passphrase)
 	if err != nil {
 		return nil, err
 	}
@@ -52,7 +52,7 @@ func NewKeyedTransactor(key *ecdsa.PrivateKey) *TransactOpts {
 			if address != keyAddr {
 				return nil, errors.New("not authorized to sign this account")
 			}
-			signature, err := crypto.SignEthereum(signer.Hash(tx).Bytes(), key)
+			signature, err := crypto.Sign(signer.Hash(tx).Bytes(), key)
 			if err != nil {
 				return nil, err
 			}

accounts/abi/bind/backends/simulated.go

@@ -61,7 +61,7 @@ type SimulatedBackend struct {
 func NewSimulatedBackend(accounts ...core.GenesisAccount) *SimulatedBackend {
 	database, _ := ethdb.NewMemDatabase()
 	core.WriteGenesisBlockForTesting(database, accounts...)
-	blockchain, _ := core.NewBlockChain(database, chainConfig, new(core.FakePow), new(event.TypeMux))
+	blockchain, _ := core.NewBlockChain(database, chainConfig, new(core.FakePow), new(event.TypeMux), vm.Config{})
 	backend := &SimulatedBackend{database: database, blockchain: blockchain}
 	backend.rollback()
 	return backend
@@ -201,10 +201,32 @@ func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error
 func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMsg) (*big.Int, error) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
-	defer b.pendingState.RevertToSnapshot(b.pendingState.Snapshot())
 
-	_, gas, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
-	return gas, err
+	// Binary search the gas requirement, as it may be higher than the amount used
+	var lo, hi uint64
+	if call.Gas != nil {
+		hi = call.Gas.Uint64()
+	} else {
+		hi = b.pendingBlock.GasLimit().Uint64()
+	}
+	for lo+1 < hi {
+		// Take a guess at the gas, and check transaction validity
+		mid := (hi + lo) / 2
+		call.Gas = new(big.Int).SetUint64(mid)
+
+		snapshot := b.pendingState.Snapshot()
+		_, gas, err := b.callContract(ctx, call, b.pendingBlock, b.pendingState)
+		b.pendingState.RevertToSnapshot(snapshot)
+
+		// If the transaction became invalid or used all the gas (failed), raise the gas limit
+		if err != nil || gas.Cmp(call.Gas) == 0 {
+			lo = mid
+			continue
+		}
+		// Otherwise assume the transaction succeeded, lower the gas limit
+		hi = mid
+	}
+	return new(big.Int).SetUint64(hi), nil
 }
 
 // callContract implemens common code between normal and pending contract calls.
@@ -225,7 +247,11 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 	from.SetBalance(common.MaxBig)
 	// Execute the call.
 	msg := callmsg{call}
-	vmenv := core.NewEnv(statedb, chainConfig, b.blockchain, msg, block.Header(), vm.Config{})
+
+	evmContext := core.NewEVMContext(msg, block.Header(), b.blockchain)
+	// Create a new environment which holds all relevant information
+	// about the transaction and calling mechanisms.
+	vmenv := vm.NewEVM(evmContext, statedb, chainConfig, vm.Config{})
 	gaspool := new(core.GasPool).AddGas(common.MaxBig)
 	ret, gasUsed, _, err := core.NewStateTransition(vmenv, msg, gaspool).TransitionDb()
 	return ret, gasUsed, err

accounts/abi/bind/base.go

@@ -170,7 +170,7 @@ func (c *BoundContract) transact(opts *TransactOpts, contract *common.Address, i
 	if value == nil {
 		value = new(big.Int)
 	}
-	nonce := uint64(0)
+	var nonce uint64
 	if opts.Nonce == nil {
 		nonce, err = c.transactor.PendingNonceAt(ensureContext(opts.Context), opts.From)
 		if err != nil {

accounts/abi/bind/bind.go

@@ -147,21 +147,21 @@ func bindTypeGo(kind abi.Type) string {
 
 	switch {
 	case strings.HasPrefix(stringKind, "address"):
-		parts := regexp.MustCompile("address(\\[[0-9]*\\])?").FindStringSubmatch(stringKind)
+		parts := regexp.MustCompile(`address(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
 		if len(parts) != 2 {
 			return stringKind
 		}
 		return fmt.Sprintf("%scommon.Address", parts[1])
 
 	case strings.HasPrefix(stringKind, "bytes"):
-		parts := regexp.MustCompile("bytes([0-9]*)(\\[[0-9]*\\])?").FindStringSubmatch(stringKind)
+		parts := regexp.MustCompile(`bytes([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
 		if len(parts) != 3 {
 			return stringKind
 		}
 		return fmt.Sprintf("%s[%s]byte", parts[2], parts[1])
 
 	case strings.HasPrefix(stringKind, "int") || strings.HasPrefix(stringKind, "uint"):
-		parts := regexp.MustCompile("(u)?int([0-9]*)(\\[[0-9]*\\])?").FindStringSubmatch(stringKind)
+		parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
 		if len(parts) != 4 {
 			return stringKind
 		}
@@ -172,7 +172,7 @@ func bindTypeGo(kind abi.Type) string {
 		return fmt.Sprintf("%s*big.Int", parts[3])
 
 	case strings.HasPrefix(stringKind, "bool") || strings.HasPrefix(stringKind, "string"):
-		parts := regexp.MustCompile("([a-z]+)(\\[[0-9]*\\])?").FindStringSubmatch(stringKind)
+		parts := regexp.MustCompile(`([a-z]+)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
 		if len(parts) != 3 {
 			return stringKind
 		}
@@ -191,7 +191,7 @@ func bindTypeJava(kind abi.Type) string {
 
 	switch {
 	case strings.HasPrefix(stringKind, "address"):
-		parts := regexp.MustCompile("address(\\[[0-9]*\\])?").FindStringSubmatch(stringKind)
+		parts := regexp.MustCompile(`address(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
 		if len(parts) != 2 {
 			return stringKind
 		}
@@ -201,7 +201,7 @@ func bindTypeJava(kind abi.Type) string {
 		return fmt.Sprintf("Addresses")
 
 	case strings.HasPrefix(stringKind, "bytes"):
-		parts := regexp.MustCompile("bytes([0-9]*)(\\[[0-9]*\\])?").FindStringSubmatch(stringKind)
+		parts := regexp.MustCompile(`bytes([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
 		if len(parts) != 3 {
 			return stringKind
 		}
@@ -211,7 +211,7 @@ func bindTypeJava(kind abi.Type) string {
 		return "byte[]"
 
 	case strings.HasPrefix(stringKind, "int") || strings.HasPrefix(stringKind, "uint"):
-		parts := regexp.MustCompile("(u)?int([0-9]*)(\\[[0-9]*\\])?").FindStringSubmatch(stringKind)
+		parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
 		if len(parts) != 4 {
 			return stringKind
 		}
@@ -230,7 +230,7 @@ func bindTypeJava(kind abi.Type) string {
 		return fmt.Sprintf("BigInts")
 
 	case strings.HasPrefix(stringKind, "bool"):
-		parts := regexp.MustCompile("bool(\\[[0-9]*\\])?").FindStringSubmatch(stringKind)
+		parts := regexp.MustCompile(`bool(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
 		if len(parts) != 2 {
 			return stringKind
 		}
@@ -240,7 +240,7 @@ func bindTypeJava(kind abi.Type) string {
 		return fmt.Sprintf("bool[]")
 
 	case strings.HasPrefix(stringKind, "string"):
-		parts := regexp.MustCompile("string(\\[[0-9]*\\])?").FindStringSubmatch(stringKind)
+		parts := regexp.MustCompile(`string(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
 		if len(parts) != 2 {
 			return stringKind
 		}
@@ -278,7 +278,7 @@ func namedTypeJava(javaKind string, solKind abi.Type) string {
 	case "bool[]":
 		return "Bools"
 	case "BigInt":
-		parts := regexp.MustCompile("(u)?int([0-9]*)(\\[[0-9]*\\])?").FindStringSubmatch(solKind.String())
+		parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(solKind.String())
 		if len(parts) != 4 {
 			return javaKind
 		}

accounts/abi/bind/bind_test.go

@@ -341,11 +341,11 @@ var bindTests = []struct {
 	{
 		`NonExistent`,
 		`
-		contract NonExistent {
-			function String() constant returns(string) {
-				return "I don't exist";
+			contract NonExistent {
+				function String() constant returns(string) {
+					return "I don't exist";
+				}
 			}
-		}
 		`,
 		`6060604052609f8060106000396000f3606060405260e060020a6000350463f97a60058114601a575b005b600060605260c0604052600d60809081527f4920646f6e27742065786973740000000000000000000000000000000000000060a052602060c0908152600d60e081905281906101009060a09080838184600060046012f15050815172ffffffffffffffffffffffffffffffffffffff1916909152505060405161012081900392509050f3`,
 		`[{"constant":true,"inputs":[],"name":"String","outputs":[{"name":"","type":"string"}],"type":"function"}]`,
@@ -365,6 +365,49 @@ var bindTests = []struct {
 			}
 		`,
 	},
+	// Tests that gas estimation works for contracts with	weird gas mechanics too.
+	{
+		`FunkyGasPattern`,
+		`
+			contract FunkyGasPattern {
+				string public field;
+
+				function SetField(string value) {
+					// This check will screw gas estimation! Good, good!
+					if (msg.gas < 100000) {
+						throw;
+					}
+					field = value;
+				}
+			}
+		`,
+		`606060405261021c806100126000396000f3606060405260e060020a600035046323fcf32a81146100265780634f28bf0e1461007b575b005b6040805160206004803580820135601f8101849004840285018401909552848452610024949193602493909291840191908190840183828082843750949650505050505050620186a05a101561014e57610002565b6100db60008054604080516020601f600260001961010060018816150201909516949094049384018190048102820181019092528281529291908301828280156102145780601f106101e957610100808354040283529160200191610214565b60405180806020018281038252838181518152602001915080519060200190808383829060006004602084601f0104600302600f01f150905090810190601f16801561013b5780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b505050565b8060006000509080519060200190828054600181600116156101000203166002900490600052602060002090601f016020900481019282601f106101b557805160ff19168380011785555b506101499291505b808211156101e557600081556001016101a1565b82800160010185558215610199579182015b828111156101995782518260005055916020019190600101906101c7565b5090565b820191906000526020600020905b8154815290600101906020018083116101f757829003601f168201915b50505050508156`,
+		`[{"constant":false,"inputs":[{"name":"value","type":"string"}],"name":"SetField","outputs":[],"type":"function"},{"constant":true,"inputs":[],"name":"field","outputs":[{"name":"","type":"string"}],"type":"function"}]`,
+		`
+			// Generate a new random account and a funded simulator
+			key, _ := crypto.GenerateKey()
+			auth := bind.NewKeyedTransactor(key)
+			sim := backends.NewSimulatedBackend(core.GenesisAccount{Address: auth.From, Balance: big.NewInt(10000000000)})
+
+			// Deploy a funky gas pattern contract
+			_, _, limiter, err := DeployFunkyGasPattern(auth, sim)
+			if err != nil {
+				t.Fatalf("Failed to deploy funky contract: %v", err)
+			}
+			sim.Commit()
+
+			// Set the field with automatic estimation and check that it succeeds
+			auth.GasLimit = nil
+			if _, err := limiter.SetField(auth, "automatic"); err != nil {
+				t.Fatalf("Failed to call automatically gased transaction: %v", err)
+			}
+			sim.Commit()
+
+			if field, _ := limiter.Field(nil); field != "automatic" {
+				t.Fatalf("Field mismatch: have %v, want %v", field, "automatic")
+			}
+		`,
+	},
 }
 
 // Tests that packages generated by the binder can be successfully compiled and

accounts/abi/bind/util_test.go

@@ -60,7 +60,7 @@ func TestWaitDeployed(t *testing.T) {
 
 		// Create the transaction.
 		tx := types.NewContractCreation(0, big.NewInt(0), test.gas, big.NewInt(1), common.FromHex(test.code))
-		tx, _ = tx.SignECDSA(types.HomesteadSigner{}, testKey)
+		tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey)
 
 		// Wait for it to get mined in the background.
 		var (

accounts/abi/event.go

@@ -25,10 +25,12 @@ import (
 )
 
 // Event is an event potentially triggered by the EVM's LOG mechanism. The Event
-// holds type information (inputs) about the yielded output
+// holds type information (inputs) about the yielded output. Anonymous events
+// don't get the signature canonical representation as the first LOG topic.
 type Event struct {
-	Name   string
-	Inputs []Argument
+	Name      string
+	Anonymous bool
+	Inputs    []Argument
 }
 
 // Id returns the canonical representation of the event's signature used by the

accounts/abi/packing.go

@@ -54,7 +54,7 @@ func packElement(t Type, reflectValue reflect.Value) []byte {
 			reflectValue = mustArrayToByteSlice(reflectValue)
 		}
 		return packBytesSlice(reflectValue.Bytes(), reflectValue.Len())
-	case FixedBytesTy:
+	case FixedBytesTy, FunctionTy:
 		if reflectValue.Kind() == reflect.Array {
 			reflectValue = mustArrayToByteSlice(reflectValue)
 		}

accounts/abi/type.go

@@ -33,7 +33,8 @@ const (
 	FixedBytesTy
 	BytesTy
 	HashTy
-	RealTy
+	FixedpointTy
+	FunctionTy
 )
 
 // Type is the reflection of the supported argument type
@@ -57,16 +58,16 @@ var (
 	// Types can be in the format of:
 	//
 	// 	Input  = Type [ "[" [ Number ] "]" ] Name .
-	// 	Type   = [ "u" ] "int" [ Number ] .
+	// 	Type   = [ "u" ] "int" [ Number ] [ x ] [ Number ].
 	//
 	// Examples:
 	//
-	//      string     int       uint       real
+	//      string     int       uint       fixed
 	//      string32   int8      uint8      uint[]
-	//      address    int256    uint256    real[2]
-	fullTypeRegex = regexp.MustCompile("([a-zA-Z0-9]+)(\\[([0-9]*)?\\])?")
+	//      address    int256    uint256    fixed128x128[2]
+	fullTypeRegex = regexp.MustCompile(`([a-zA-Z0-9]+)(\[([0-9]*)\])?`)
 	// typeRegex parses the abi sub types
-	typeRegex = regexp.MustCompile("([a-zA-Z]+)([0-9]*)?")
+	typeRegex = regexp.MustCompile("([a-zA-Z]+)(([0-9]+)(x([0-9]+))?)?")
 )
 
 // NewType creates a new reflection type of abi type given in t.
@@ -90,14 +91,19 @@ func NewType(t string) (typ Type, err error) {
 		}
 		typ.Elem = &sliceType
 		typ.stringKind = sliceType.stringKind + t[len(res[1]):]
-		return typ, nil
+		// Although we know that this is an array, we cannot return
+		// as we don't know the type of the element, however, if it
+		// is still an array, then don't determine the type.
+		if typ.Elem.IsArray || typ.Elem.IsSlice {
+			return typ, nil
+		}
 	}
 
 	// parse the type and size of the abi-type.
 	parsedType := typeRegex.FindAllStringSubmatch(res[1], -1)[0]
 	// varSize is the size of the variable
 	var varSize int
-	if len(parsedType[2]) > 0 {
+	if len(parsedType[3]) > 0 {
 		var err error
 		varSize, err = strconv.Atoi(parsedType[2])
 		if err != nil {
@@ -111,7 +117,12 @@ func NewType(t string) (typ Type, err error) {
 		varSize = 256
 		t += "256"
 	}
-	typ.stringKind = t
+
+	// only set stringKind if not array or slice, as for those,
+	// the correct string type has been set
+	if !(typ.IsArray || typ.IsSlice) {
+		typ.stringKind = t
+	}
 
 	switch varType {
 	case "int":
@@ -148,6 +159,12 @@ func NewType(t string) (typ Type, err error) {
 			typ.T = FixedBytesTy
 			typ.SliceSize = varSize
 		}
+	case "function":
+		sliceType, _ := NewType("uint8")
+		typ.Elem = &sliceType
+		typ.IsArray = true
+		typ.T = FunctionTy
+		typ.SliceSize = 24
 	default:
 		return Type{}, fmt.Errorf("unsupported arg type: %s", t)
 	}
@@ -168,7 +185,7 @@ func (t Type) pack(v reflect.Value) ([]byte, error) {
 		return nil, err
 	}
 
-	if (t.IsSlice || t.IsArray) && t.T != BytesTy && t.T != FixedBytesTy {
+	if (t.IsSlice || t.IsArray) && t.T != BytesTy && t.T != FixedBytesTy && t.T != FunctionTy {
 		var packed []byte
 
 		for i := 0; i < v.Len(); i++ {

accounts/abi/type_test.go

@@ -0,0 +1,78 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package abi
+
+import (
+	"reflect"
+	"testing"
+)
+
+// typeWithoutStringer is a alias for the Type type which simply doesn't implement
+// the stringer interface to allow printing type details in the tests below.
+type typeWithoutStringer Type
+
+// Tests that all allowed types get recognized by the type parser.
+func TestTypeRegexp(t *testing.T) {
+	tests := []struct {
+		blob string
+		kind Type
+	}{
+		{"int", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}},
+		{"int8", Type{Kind: reflect.Int8, Type: big_t, Size: 8, T: IntTy, stringKind: "int8"}},
+		{"int256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}},
+		{"int[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
+		{"int[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
+		{"int32[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int32, Type: big_t, Size: 32, T: IntTy, Elem: &Type{Kind: reflect.Int32, Type: big_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
+		{"int32[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int32, Type: big_t, Size: 32, T: IntTy, Elem: &Type{Kind: reflect.Int32, Type: big_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
+		{"uint", Type{Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, stringKind: "uint256"}},
+		{"uint8", Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}},
+		{"uint256", Type{Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, stringKind: "uint256"}},
+		{"uint[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, Elem: &Type{Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
+		{"uint[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, Elem: &Type{Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
+		{"uint32[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint32, Type: ubig_t, Size: 32, T: UintTy, Elem: &Type{Kind: reflect.Uint32, Type: big_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
+		{"uint32[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint32, Type: ubig_t, Size: 32, T: UintTy, Elem: &Type{Kind: reflect.Uint32, Type: big_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
+		{"bytes", Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}},
+		{"bytes32", Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}},
+		{"bytes[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
+		{"bytes[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[2]"}},
+		{"bytes32[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[]"}},
+		{"bytes32[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[2]"}},
+		{"string", Type{Kind: reflect.String, Size: -1, T: StringTy, stringKind: "string"}},
+		{"string[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[]"}},
+		{"string[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[2]"}},
+		{"address", Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}},
+		{"address[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Array, Type: address_t, T: AddressTy, Size: 20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
+		{"address[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Array, Type: address_t, T: AddressTy, Size: 20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
+
+		// TODO when fixed types are implemented properly
+		// {"fixed", Type{}},
+		// {"fixed128x128", Type{}},
+		// {"fixed[]", Type{}},
+		// {"fixed[2]", Type{}},
+		// {"fixed128x128[]", Type{}},
+		// {"fixed128x128[2]", Type{}},
+	}
+	for i, tt := range tests {
+		typ, err := NewType(tt.blob)
+		if err != nil {
+			t.Errorf("type %d: failed to parse type string: %v", i, err)
+		}
+		if !reflect.DeepEqual(typ, tt.kind) {
+			t.Errorf("type %d: parsed type mismatch:\n  have %+v\n  want %+v", i, typeWithoutStringer(typ), typeWithoutStringer(tt.kind))
+		}
+	}
+}

accounts/account_manager.go

@@ -1,363 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package accounts implements encrypted storage of secp256k1 private keys.
-//
-// Keys are stored as encrypted JSON files according to the Web3 Secret Storage specification.
-// See https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition for more information.
-package accounts
-
-import (
-	"crypto/ecdsa"
-	crand "crypto/rand"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"runtime"
-	"sync"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
-)
-
-var (
-	ErrLocked  = errors.New("account is locked")
-	ErrNoMatch = errors.New("no key for given address or file")
-	ErrDecrypt = errors.New("could not decrypt key with given passphrase")
-)
-
-// Account represents a stored key.
-// When used as an argument, it selects a unique key file to act on.
-type Account struct {
-	Address common.Address // Ethereum account address derived from the key
-
-	// File contains the key file name.
-	// When Acccount is used as an argument to select a key, File can be left blank to
-	// select just by address or set to the basename or absolute path of a file in the key
-	// directory. Accounts returned by Manager will always contain an absolute path.
-	File string
-}
-
-func (acc *Account) MarshalJSON() ([]byte, error) {
-	return []byte(`"` + acc.Address.Hex() + `"`), nil
-}
-
-func (acc *Account) UnmarshalJSON(raw []byte) error {
-	return json.Unmarshal(raw, &acc.Address)
-}
-
-// Manager manages a key storage directory on disk.
-type Manager struct {
-	cache    *addrCache
-	keyStore keyStore
-	mu       sync.RWMutex
-	unlocked map[common.Address]*unlocked
-}
-
-type unlocked struct {
-	*Key
-	abort chan struct{}
-}
-
-// NewManager creates a manager for the given directory.
-func NewManager(keydir string, scryptN, scryptP int) *Manager {
-	keydir, _ = filepath.Abs(keydir)
-	am := &Manager{keyStore: &keyStorePassphrase{keydir, scryptN, scryptP}}
-	am.init(keydir)
-	return am
-}
-
-// NewPlaintextManager creates a manager for the given directory.
-// Deprecated: Use NewManager.
-func NewPlaintextManager(keydir string) *Manager {
-	keydir, _ = filepath.Abs(keydir)
-	am := &Manager{keyStore: &keyStorePlain{keydir}}
-	am.init(keydir)
-	return am
-}
-
-func (am *Manager) init(keydir string) {
-	am.unlocked = make(map[common.Address]*unlocked)
-	am.cache = newAddrCache(keydir)
-	// TODO: In order for this finalizer to work, there must be no references
-	// to am. addrCache doesn't keep a reference but unlocked keys do,
-	// so the finalizer will not trigger until all timed unlocks have expired.
-	runtime.SetFinalizer(am, func(m *Manager) {
-		m.cache.close()
-	})
-}
-
-// HasAddress reports whether a key with the given address is present.
-func (am *Manager) HasAddress(addr common.Address) bool {
-	return am.cache.hasAddress(addr)
-}
-
-// Accounts returns all key files present in the directory.
-func (am *Manager) Accounts() []Account {
-	return am.cache.accounts()
-}
-
-// DeleteAccount deletes the key matched by account if the passphrase is correct.
-// If a contains no filename, the address must match a unique key.
-func (am *Manager) DeleteAccount(a Account, passphrase string) error {
-	// Decrypting the key isn't really necessary, but we do
-	// it anyway to check the password and zero out the key
-	// immediately afterwards.
-	a, key, err := am.getDecryptedKey(a, passphrase)
-	if key != nil {
-		zeroKey(key.PrivateKey)
-	}
-	if err != nil {
-		return err
-	}
-	// The order is crucial here. The key is dropped from the
-	// cache after the file is gone so that a reload happening in
-	// between won't insert it into the cache again.
-	err = os.Remove(a.File)
-	if err == nil {
-		am.cache.delete(a)
-	}
-	return err
-}
-
-// Sign calculates a ECDSA signature for the given hash.
-// Note, Ethereum signatures have a particular format as described in the
-// yellow paper. Use the SignEthereum function to calculate a signature
-// in Ethereum format.
-func (am *Manager) Sign(addr common.Address, hash []byte) ([]byte, error) {
-	am.mu.RLock()
-	defer am.mu.RUnlock()
-	unlockedKey, found := am.unlocked[addr]
-	if !found {
-		return nil, ErrLocked
-	}
-	return crypto.Sign(hash, unlockedKey.PrivateKey)
-}
-
-// SignEthereum calculates a ECDSA signature for the given hash.
-// The signature has the format as described in the Ethereum yellow paper.
-func (am *Manager) SignEthereum(addr common.Address, hash []byte) ([]byte, error) {
-	am.mu.RLock()
-	defer am.mu.RUnlock()
-	unlockedKey, found := am.unlocked[addr]
-	if !found {
-		return nil, ErrLocked
-	}
-	return crypto.SignEthereum(hash, unlockedKey.PrivateKey)
-}
-
-// SignWithPassphrase signs hash if the private key matching the given
-// address can be decrypted with the given passphrase.
-func (am *Manager) SignWithPassphrase(addr common.Address, passphrase string, hash []byte) (signature []byte, err error) {
-	_, key, err := am.getDecryptedKey(Account{Address: addr}, passphrase)
-	if err != nil {
-		return nil, err
-	}
-
-	defer zeroKey(key.PrivateKey)
-	return crypto.SignEthereum(hash, key.PrivateKey)
-}
-
-// Unlock unlocks the given account indefinitely.
-func (am *Manager) Unlock(a Account, passphrase string) error {
-	return am.TimedUnlock(a, passphrase, 0)
-}
-
-// Lock removes the private key with the given address from memory.
-func (am *Manager) Lock(addr common.Address) error {
-	am.mu.Lock()
-	if unl, found := am.unlocked[addr]; found {
-		am.mu.Unlock()
-		am.expire(addr, unl, time.Duration(0)*time.Nanosecond)
-	} else {
-		am.mu.Unlock()
-	}
-	return nil
-}
-
-// TimedUnlock unlocks the given account with the passphrase. The account
-// stays unlocked for the duration of timeout. A timeout of 0 unlocks the account
-// until the program exits. The account must match a unique key file.
-//
-// If the account address is already unlocked for a duration, TimedUnlock extends or
-// shortens the active unlock timeout. If the address was previously unlocked
-// indefinitely the timeout is not altered.
-func (am *Manager) TimedUnlock(a Account, passphrase string, timeout time.Duration) error {
-	a, key, err := am.getDecryptedKey(a, passphrase)
-	if err != nil {
-		return err
-	}
-
-	am.mu.Lock()
-	defer am.mu.Unlock()
-	u, found := am.unlocked[a.Address]
-	if found {
-		if u.abort == nil {
-			// The address was unlocked indefinitely, so unlocking
-			// it with a timeout would be confusing.
-			zeroKey(key.PrivateKey)
-			return nil
-		} else {
-			// Terminate the expire goroutine and replace it below.
-			close(u.abort)
-		}
-	}
-	if timeout > 0 {
-		u = &unlocked{Key: key, abort: make(chan struct{})}
-		go am.expire(a.Address, u, timeout)
-	} else {
-		u = &unlocked{Key: key}
-	}
-	am.unlocked[a.Address] = u
-	return nil
-}
-
-// Find resolves the given account into a unique entry in the keystore.
-func (am *Manager) Find(a Account) (Account, error) {
-	am.cache.maybeReload()
-	am.cache.mu.Lock()
-	a, err := am.cache.find(a)
-	am.cache.mu.Unlock()
-	return a, err
-}
-
-func (am *Manager) getDecryptedKey(a Account, auth string) (Account, *Key, error) {
-	a, err := am.Find(a)
-	if err != nil {
-		return a, nil, err
-	}
-	key, err := am.keyStore.GetKey(a.Address, a.File, auth)
-	return a, key, err
-}
-
-func (am *Manager) expire(addr common.Address, u *unlocked, timeout time.Duration) {
-	t := time.NewTimer(timeout)
-	defer t.Stop()
-	select {
-	case <-u.abort:
-		// just quit
-	case <-t.C:
-		am.mu.Lock()
-		// only drop if it's still the same key instance that dropLater
-		// was launched with. we can check that using pointer equality
-		// because the map stores a new pointer every time the key is
-		// unlocked.
-		if am.unlocked[addr] == u {
-			zeroKey(u.PrivateKey)
-			delete(am.unlocked, addr)
-		}
-		am.mu.Unlock()
-	}
-}
-
-// NewAccount generates a new key and stores it into the key directory,
-// encrypting it with the passphrase.
-func (am *Manager) NewAccount(passphrase string) (Account, error) {
-	_, account, err := storeNewKey(am.keyStore, crand.Reader, passphrase)
-	if err != nil {
-		return Account{}, err
-	}
-	// Add the account to the cache immediately rather
-	// than waiting for file system notifications to pick it up.
-	am.cache.add(account)
-	return account, nil
-}
-
-// AccountByIndex returns the ith account.
-func (am *Manager) AccountByIndex(i int) (Account, error) {
-	accounts := am.Accounts()
-	if i < 0 || i >= len(accounts) {
-		return Account{}, fmt.Errorf("account index %d out of range [0, %d]", i, len(accounts)-1)
-	}
-	return accounts[i], nil
-}
-
-// Export exports as a JSON key, encrypted with newPassphrase.
-func (am *Manager) Export(a Account, passphrase, newPassphrase string) (keyJSON []byte, err error) {
-	_, key, err := am.getDecryptedKey(a, passphrase)
-	if err != nil {
-		return nil, err
-	}
-	var N, P int
-	if store, ok := am.keyStore.(*keyStorePassphrase); ok {
-		N, P = store.scryptN, store.scryptP
-	} else {
-		N, P = StandardScryptN, StandardScryptP
-	}
-	return EncryptKey(key, newPassphrase, N, P)
-}
-
-// Import stores the given encrypted JSON key into the key directory.
-func (am *Manager) Import(keyJSON []byte, passphrase, newPassphrase string) (Account, error) {
-	key, err := DecryptKey(keyJSON, passphrase)
-	if key != nil && key.PrivateKey != nil {
-		defer zeroKey(key.PrivateKey)
-	}
-	if err != nil {
-		return Account{}, err
-	}
-	return am.importKey(key, newPassphrase)
-}
-
-// ImportECDSA stores the given key into the key directory, encrypting it with the passphrase.
-func (am *Manager) ImportECDSA(priv *ecdsa.PrivateKey, passphrase string) (Account, error) {
-	key := newKeyFromECDSA(priv)
-	if am.cache.hasAddress(key.Address) {
-		return Account{}, fmt.Errorf("account already exists")
-	}
-
-	return am.importKey(key, passphrase)
-}
-
-func (am *Manager) importKey(key *Key, passphrase string) (Account, error) {
-	a := Account{Address: key.Address, File: am.keyStore.JoinPath(keyFileName(key.Address))}
-	if err := am.keyStore.StoreKey(a.File, key, passphrase); err != nil {
-		return Account{}, err
-	}
-	am.cache.add(a)
-	return a, nil
-}
-
-// Update changes the passphrase of an existing account.
-func (am *Manager) Update(a Account, passphrase, newPassphrase string) error {
-	a, key, err := am.getDecryptedKey(a, passphrase)
-	if err != nil {
-		return err
-	}
-	return am.keyStore.StoreKey(a.File, key, newPassphrase)
-}
-
-// ImportPreSaleKey decrypts the given Ethereum presale wallet and stores
-// a key file in the key directory. The key file is encrypted with the same passphrase.
-func (am *Manager) ImportPreSaleKey(keyJSON []byte, passphrase string) (Account, error) {
-	a, _, err := importPreSaleKey(am.keyStore, keyJSON, passphrase)
-	if err != nil {
-		return a, err
-	}
-	am.cache.add(a)
-	return a, nil
-}
-
-// zeroKey zeroes a private key in memory.
-func zeroKey(k *ecdsa.PrivateKey) {
-	b := k.D.Bits()
-	for i := range b {
-		b[i] = 0
-	}
-}

accounts/accounts.go

@@ -0,0 +1,155 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package accounts implements high level Ethereum account management.
+package accounts
+
+import (
+	"math/big"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
+)
+
+// Account represents an Ethereum account located at a specific location defined
+// by the optional URL field.
+type Account struct {
+	Address common.Address `json:"address"` // Ethereum account address derived from the key
+	URL     URL            `json:"url"`     // Optional resource locator within a backend
+}
+
+// Wallet represents a software or hardware wallet that might contain one or more
+// accounts (derived from the same seed).
+type Wallet interface {
+	// URL retrieves the canonical path under which this wallet is reachable. It is
+	// user by upper layers to define a sorting order over all wallets from multiple
+	// backends.
+	URL() URL
+
+	// Status returns a textual status to aid the user in the current state of the
+	// wallet.
+	Status() string
+
+	// Open initializes access to a wallet instance. It is not meant to unlock or
+	// decrypt account keys, rather simply to establish a connection to hardware
+	// wallets and/or to access derivation seeds.
+	//
+	// The passphrase parameter may or may not be used by the implementation of a
+	// particular wallet instance. The reason there is no passwordless open method
+	// is to strive towards a uniform wallet handling, oblivious to the different
+	// backend providers.
+	//
+	// Please note, if you open a wallet, you must close it to release any allocated
+	// resources (especially important when working with hardware wallets).
+	Open(passphrase string) error
+
+	// Close releases any resources held by an open wallet instance.
+	Close() error
+
+	// Accounts retrieves the list of signing accounts the wallet is currently aware
+	// of. For hierarchical deterministic wallets, the list will not be exhaustive,
+	// rather only contain the accounts explicitly pinned during account derivation.
+	Accounts() []Account
+
+	// Contains returns whether an account is part of this particular wallet or not.
+	Contains(account Account) bool
+
+	// Derive attempts to explicitly derive a hierarchical deterministic account at
+	// the specified derivation path. If requested, the derived account will be added
+	// to the wallet's tracked account list.
+	Derive(path DerivationPath, pin bool) (Account, error)
+
+	// SelfDerive sets a base account derivation path from which the wallet attempts
+	// to discover non zero accounts and automatically add them to list of tracked
+	// accounts.
+	//
+	// Note, self derivaton will increment the last component of the specified path
+	// opposed to decending into a child path to allow discovering accounts starting
+	// from non zero components.
+	//
+	// You can disable automatic account discovery by calling SelfDerive with a nil
+	// chain state reader.
+	SelfDerive(base DerivationPath, chain ethereum.ChainStateReader)
+
+	// SignHash requests the wallet to sign the given hash.
+	//
+	// It looks up the account specified either solely via its address contained within,
+	// or optionally with the aid of any location metadata from the embedded URL field.
+	//
+	// If the wallet requires additional authentication to sign the request (e.g.
+	// a password to decrypt the account, or a PIN code o verify the transaction),
+	// an AuthNeededError instance will be returned, containing infos for the user
+	// about which fields or actions are needed. The user may retry by providing
+	// the needed details via SignHashWithPassphrase, or by other means (e.g. unlock
+	// the account in a keystore).
+	SignHash(account Account, hash []byte) ([]byte, error)
+
+	// SignTx requests the wallet to sign the given transaction.
+	//
+	// It looks up the account specified either solely via its address contained within,
+	// or optionally with the aid of any location metadata from the embedded URL field.
+	//
+	// If the wallet requires additional authentication to sign the request (e.g.
+	// a password to decrypt the account, or a PIN code o verify the transaction),
+	// an AuthNeededError instance will be returned, containing infos for the user
+	// about which fields or actions are needed. The user may retry by providing
+	// the needed details via SignTxWithPassphrase, or by other means (e.g. unlock
+	// the account in a keystore).
+	SignTx(account Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error)
+
+	// SignHashWithPassphrase requests the wallet to sign the given hash with the
+	// given passphrase as extra authentication information.
+	//
+	// It looks up the account specified either solely via its address contained within,
+	// or optionally with the aid of any location metadata from the embedded URL field.
+	SignHashWithPassphrase(account Account, passphrase string, hash []byte) ([]byte, error)
+
+	// SignTxWithPassphrase requests the wallet to sign the given transaction, with the
+	// given passphrase as extra authentication information.
+	//
+	// It looks up the account specified either solely via its address contained within,
+	// or optionally with the aid of any location metadata from the embedded URL field.
+	SignTxWithPassphrase(account Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error)
+}
+
+// Backend is a "wallet provider" that may contain a batch of accounts they can
+// sign transactions with and upon request, do so.
+type Backend interface {
+	// Wallets retrieves the list of wallets the backend is currently aware of.
+	//
+	// The returned wallets are not opened by default. For software HD wallets this
+	// means that no base seeds are decrypted, and for hardware wallets that no actual
+	// connection is established.
+	//
+	// The resulting wallet list will be sorted alphabetically based on its internal
+	// URL assigned by the backend. Since wallets (especially hardware) may come and
+	// go, the same wallet might appear at a different positions in the list during
+	// subsequent retrievals.
+	Wallets() []Wallet
+
+	// Subscribe creates an async subscription to receive notifications when the
+	// backend detects the arrival or departure of a wallet.
+	Subscribe(sink chan<- WalletEvent) event.Subscription
+}
+
+// WalletEvent is an event fired by an account backend when a wallet arrival or
+// departure is detected.
+type WalletEvent struct {
+	Wallet Wallet // Wallet instance arrived or departed
+	Arrive bool   // Whether the wallet was added or removed
+}

accounts/accounts_test.go

@@ -1,218 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package accounts
-
-import (
-	"io/ioutil"
-	"os"
-	"runtime"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-)
-
-var testSigData = make([]byte, 32)
-
-func TestManager(t *testing.T) {
-	dir, am := tmpManager(t, true)
-	defer os.RemoveAll(dir)
-
-	a, err := am.NewAccount("foo")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !strings.HasPrefix(a.File, dir) {
-		t.Errorf("account file %s doesn't have dir prefix", a.File)
-	}
-	stat, err := os.Stat(a.File)
-	if err != nil {
-		t.Fatalf("account file %s doesn't exist (%v)", a.File, err)
-	}
-	if runtime.GOOS != "windows" && stat.Mode() != 0600 {
-		t.Fatalf("account file has wrong mode: got %o, want %o", stat.Mode(), 0600)
-	}
-	if !am.HasAddress(a.Address) {
-		t.Errorf("HasAccount(%x) should've returned true", a.Address)
-	}
-	if err := am.Update(a, "foo", "bar"); err != nil {
-		t.Errorf("Update error: %v", err)
-	}
-	if err := am.DeleteAccount(a, "bar"); err != nil {
-		t.Errorf("DeleteAccount error: %v", err)
-	}
-	if common.FileExist(a.File) {
-		t.Errorf("account file %s should be gone after DeleteAccount", a.File)
-	}
-	if am.HasAddress(a.Address) {
-		t.Errorf("HasAccount(%x) should've returned true after DeleteAccount", a.Address)
-	}
-}
-
-func TestSign(t *testing.T) {
-	dir, am := tmpManager(t, true)
-	defer os.RemoveAll(dir)
-
-	pass := "" // not used but required by API
-	a1, err := am.NewAccount(pass)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := am.Unlock(a1, ""); err != nil {
-		t.Fatal(err)
-	}
-	if _, err := am.Sign(a1.Address, testSigData); err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestSignWithPassphrase(t *testing.T) {
-	dir, am := tmpManager(t, true)
-	defer os.RemoveAll(dir)
-
-	pass := "passwd"
-	acc, err := am.NewAccount(pass)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if _, unlocked := am.unlocked[acc.Address]; unlocked {
-		t.Fatal("expected account to be locked")
-	}
-
-	_, err = am.SignWithPassphrase(acc.Address, pass, testSigData)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if _, unlocked := am.unlocked[acc.Address]; unlocked {
-		t.Fatal("expected account to be locked")
-	}
-
-	if _, err = am.SignWithPassphrase(acc.Address, "invalid passwd", testSigData); err == nil {
-		t.Fatal("expected SignHash to fail with invalid password")
-	}
-}
-
-func TestTimedUnlock(t *testing.T) {
-	dir, am := tmpManager(t, true)
-	defer os.RemoveAll(dir)
-
-	pass := "foo"
-	a1, err := am.NewAccount(pass)
-
-	// Signing without passphrase fails because account is locked
-	_, err = am.Sign(a1.Address, testSigData)
-	if err != ErrLocked {
-		t.Fatal("Signing should've failed with ErrLocked before unlocking, got ", err)
-	}
-
-	// Signing with passphrase works
-	if err = am.TimedUnlock(a1, pass, 100*time.Millisecond); err != nil {
-		t.Fatal(err)
-	}
-
-	// Signing without passphrase works because account is temp unlocked
-	_, err = am.Sign(a1.Address, testSigData)
-	if err != nil {
-		t.Fatal("Signing shouldn't return an error after unlocking, got ", err)
-	}
-
-	// Signing fails again after automatic locking
-	time.Sleep(250 * time.Millisecond)
-	_, err = am.Sign(a1.Address, testSigData)
-	if err != ErrLocked {
-		t.Fatal("Signing should've failed with ErrLocked timeout expired, got ", err)
-	}
-}
-
-func TestOverrideUnlock(t *testing.T) {
-	dir, am := tmpManager(t, false)
-	defer os.RemoveAll(dir)
-
-	pass := "foo"
-	a1, err := am.NewAccount(pass)
-
-	// Unlock indefinitely.
-	if err = am.TimedUnlock(a1, pass, 5*time.Minute); err != nil {
-		t.Fatal(err)
-	}
-
-	// Signing without passphrase works because account is temp unlocked
-	_, err = am.Sign(a1.Address, testSigData)
-	if err != nil {
-		t.Fatal("Signing shouldn't return an error after unlocking, got ", err)
-	}
-
-	// reset unlock to a shorter period, invalidates the previous unlock
-	if err = am.TimedUnlock(a1, pass, 100*time.Millisecond); err != nil {
-		t.Fatal(err)
-	}
-
-	// Signing without passphrase still works because account is temp unlocked
-	_, err = am.Sign(a1.Address, testSigData)
-	if err != nil {
-		t.Fatal("Signing shouldn't return an error after unlocking, got ", err)
-	}
-
-	// Signing fails again after automatic locking
-	time.Sleep(250 * time.Millisecond)
-	_, err = am.Sign(a1.Address, testSigData)
-	if err != ErrLocked {
-		t.Fatal("Signing should've failed with ErrLocked timeout expired, got ", err)
-	}
-}
-
-// This test should fail under -race if signing races the expiration goroutine.
-func TestSignRace(t *testing.T) {
-	dir, am := tmpManager(t, false)
-	defer os.RemoveAll(dir)
-
-	// Create a test account.
-	a1, err := am.NewAccount("")
-	if err != nil {
-		t.Fatal("could not create the test account", err)
-	}
-
-	if err := am.TimedUnlock(a1, "", 15*time.Millisecond); err != nil {
-		t.Fatal("could not unlock the test account", err)
-	}
-	end := time.Now().Add(500 * time.Millisecond)
-	for time.Now().Before(end) {
-		if _, err := am.Sign(a1.Address, testSigData); err == ErrLocked {
-			return
-		} else if err != nil {
-			t.Errorf("Sign error: %v", err)
-			return
-		}
-		time.Sleep(1 * time.Millisecond)
-	}
-	t.Errorf("Account did not lock within the timeout")
-}
-
-func tmpManager(t *testing.T, encrypted bool) (string, *Manager) {
-	d, err := ioutil.TempDir("", "eth-keystore-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	new := NewPlaintextManager
-	if encrypted {
-		new = func(kd string) *Manager { return NewManager(kd, veryLightScryptN, veryLightScryptP) }
-	}
-	return d, new(d)
-}

accounts/errors.go

@@ -0,0 +1,68 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package accounts
+
+import (
+	"errors"
+	"fmt"
+)
+
+// ErrUnknownAccount is returned for any requested operation for which no backend
+// provides the specified account.
+var ErrUnknownAccount = errors.New("unknown account")
+
+// ErrUnknownWallet is returned for any requested operation for which no backend
+// provides the specified wallet.
+var ErrUnknownWallet = errors.New("unknown wallet")
+
+// ErrNotSupported is returned when an operation is requested from an account
+// backend that it does not support.
+var ErrNotSupported = errors.New("not supported")
+
+// ErrInvalidPassphrase is returned when a decryption operation receives a bad
+// passphrase.
+var ErrInvalidPassphrase = errors.New("invalid passphrase")
+
+// ErrWalletAlreadyOpen is returned if a wallet is attempted to be opened the
+// secodn time.
+var ErrWalletAlreadyOpen = errors.New("wallet already open")
+
+// ErrWalletClosed is returned if a wallet is attempted to be opened the
+// secodn time.
+var ErrWalletClosed = errors.New("wallet closed")
+
+// AuthNeededError is returned by backends for signing requests where the user
+// is required to provide further authentication before signing can succeed.
+//
+// This usually means either that a password needs to be supplied, or perhaps a
+// one time PIN code displayed by some hardware device.
+type AuthNeededError struct {
+	Needed string // Extra authentication the user needs to provide
+}
+
+// NewAuthNeededError creates a new authentication error with the extra details
+// about the needed fields set.
+func NewAuthNeededError(needed string) error {
+	return &AuthNeededError{
+		Needed: needed,
+	}
+}
+
+// Error implements the standard error interfacel.
+func (err *AuthNeededError) Error() string {
+	return fmt.Sprintf("authentication needed: %s", err.Needed)
+}

accounts/hd.go

@@ -0,0 +1,130 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package accounts
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"math/big"
+	"strings"
+)
+
+// DefaultRootDerivationPath is the root path to which custom derivation endpoints
+// are appended. As such, the first account will be at m/44'/60'/0'/0, the second
+// at m/44'/60'/0'/1, etc.
+var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0}
+
+// DefaultBaseDerivationPath is the base path from which custom derivation endpoints
+// are incremented. As such, the first account will be at m/44'/60'/0'/0, the second
+// at m/44'/60'/0'/1, etc.
+var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}
+
+// DerivationPath represents the computer friendly version of a hierarchical
+// deterministic wallet account derivaion path.
+//
+// The BIP-32 spec https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki
+// defines derivation paths to be of the form:
+//
+//   m / purpose' / coin_type' / account' / change / address_index
+//
+// The BIP-44 spec https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki
+// defines that the `purpose` be 44' (or 0x8000002C) for crypto currencies, and
+// SLIP-44 https://github.com/satoshilabs/slips/blob/master/slip-0044.md assigns
+// the `coin_type` 60' (or 0x8000003C) to Ethereum.
+//
+// The root path for Ethereum is m/44'/60'/0'/0 according to the specification
+// from https://github.com/ethereum/EIPs/issues/84, albeit it's not set in stone
+// yet whether accounts should increment the last component or the children of
+// that. We will go with the simpler approach of incrementing the last component.
+type DerivationPath []uint32
+
+// ParseDerivationPath converts a user specified derivation path string to the
+// internal binary representation.
+//
+// Full derivation paths need to start with the `m/` prefix, relative derivation
+// paths (which will get appended to the default root path) must not have prefixes
+// in front of the first element. Whitespace is ignored.
+func ParseDerivationPath(path string) (DerivationPath, error) {
+	var result DerivationPath
+
+	// Handle absolute or relative paths
+	components := strings.Split(path, "/")
+	switch {
+	case len(components) == 0:
+		return nil, errors.New("empty derivation path")
+
+	case strings.TrimSpace(components[0]) == "":
+		return nil, errors.New("ambiguous path: use 'm/' prefix for absolute paths, or no leading '/' for relative ones")
+
+	case strings.TrimSpace(components[0]) == "m":
+		components = components[1:]
+
+	default:
+		result = append(result, DefaultRootDerivationPath...)
+	}
+	// All remaining components are relative, append one by one
+	if len(components) == 0 {
+		return nil, errors.New("empty derivation path") // Empty relative paths
+	}
+	for _, component := range components {
+		// Ignore any user added whitespace
+		component = strings.TrimSpace(component)
+		var value uint32
+
+		// Handle hardened paths
+		if strings.HasSuffix(component, "'") {
+			value = 0x80000000
+			component = strings.TrimSpace(strings.TrimSuffix(component, "'"))
+		}
+		// Handle the non hardened component
+		bigval, ok := new(big.Int).SetString(component, 0)
+		if !ok {
+			return nil, fmt.Errorf("invalid component: %s", component)
+		}
+		max := math.MaxUint32 - value
+		if bigval.Sign() < 0 || bigval.Cmp(big.NewInt(int64(max))) > 0 {
+			if value == 0 {
+				return nil, fmt.Errorf("component %v out of allowed range [0, %d]", bigval, max)
+			}
+			return nil, fmt.Errorf("component %v out of allowed hardened range [0, %d]", bigval, max)
+		}
+		value += uint32(bigval.Uint64())
+
+		// Append and repeat
+		result = append(result, value)
+	}
+	return result, nil
+}
+
+// String implements the stringer interface, converting a binary derivation path
+// to its canonical representation.
+func (path DerivationPath) String() string {
+	result := "m"
+	for _, component := range path {
+		var hardened bool
+		if component >= 0x80000000 {
+			component -= 0x80000000
+			hardened = true
+		}
+		result = fmt.Sprintf("%s/%d", result, component)
+		if hardened {
+			result += "'"
+		}
+	}
+	return result
+}

accounts/hd_test.go

@@ -0,0 +1,79 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package accounts
+
+import (
+	"reflect"
+	"testing"
+)
+
+// Tests that HD derivation paths can be correctly parsed into our internal binary
+// representation.
+func TestHDPathParsing(t *testing.T) {
+	tests := []struct {
+		input  string
+		output DerivationPath
+	}{
+		// Plain absolute derivation paths
+		{"m/44'/60'/0'/0", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+		{"m/44'/60'/0'/128", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 128}},
+		{"m/44'/60'/0'/0'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+		{"m/44'/60'/0'/128'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 128}},
+		{"m/2147483692/2147483708/2147483648/0", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+		{"m/2147483692/2147483708/2147483648/2147483648", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+
+		// Plain relative derivation paths
+		{"0", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+		{"128", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 128}},
+		{"0'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+		{"128'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 128}},
+		{"2147483648", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+
+		// Hexadecimal absolute derivation paths
+		{"m/0x2C'/0x3c'/0x00'/0x00", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+		{"m/0x2C'/0x3c'/0x00'/0x80", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 128}},
+		{"m/0x2C'/0x3c'/0x00'/0x00'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+		{"m/0x2C'/0x3c'/0x00'/0x80'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 128}},
+		{"m/0x8000002C/0x8000003c/0x80000000/0x00", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+		{"m/0x8000002C/0x8000003c/0x80000000/0x80000000", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+
+		// Hexadecimal relative derivation paths
+		{"0x00", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+		{"0x80", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 128}},
+		{"0x00'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+		{"0x80'", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 128}},
+		{"0x80000000", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0x80000000 + 0}},
+
+		// Weird inputs just to ensure they work
+		{"	m  /   44			'\n/\n   60	\n\n\t'   /\n0 ' /\t\t	0", DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}},
+
+		// Invaid derivation paths
+		{"", nil},              // Empty relative derivation path
+		{"m", nil},             // Empty absolute derivation path
+		{"m/", nil},            // Missing last derivation component
+		{"/44'/60'/0'/0", nil}, // Absolute path without m prefix, might be user error
+		{"m/2147483648'", nil}, // Overflows 32 bit integer
+		{"m/-1'", nil},         // Cannot contain negative number
+	}
+	for i, tt := range tests {
+		if path, err := ParseDerivationPath(tt.input); !reflect.DeepEqual(path, tt.output) {
+			t.Errorf("test %d: parse mismatch: have %v (%v), want %v", i, path, err, tt.output)
+		} else if path == nil && err == nil {
+			t.Errorf("test %d: nil path and error: %v", i, err)
+		}
+	}
+}

accounts/keystore/account_cache.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"bufio"
@@ -28,6 +28,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
@@ -38,23 +39,23 @@ import (
 // exist yet, the code will attempt to create a watcher at most this often.
 const minReloadInterval = 2 * time.Second
 
-type accountsByFile []Account
+type accountsByURL []accounts.Account
 
-func (s accountsByFile) Len() int           { return len(s) }
-func (s accountsByFile) Less(i, j int) bool { return s[i].File < s[j].File }
-func (s accountsByFile) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s accountsByURL) Len() int           { return len(s) }
+func (s accountsByURL) Less(i, j int) bool { return s[i].URL.Cmp(s[j].URL) < 0 }
+func (s accountsByURL) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 
 // AmbiguousAddrError is returned when attempting to unlock
 // an address for which more than one file exists.
 type AmbiguousAddrError struct {
 	Addr    common.Address
-	Matches []Account
+	Matches []accounts.Account
 }
 
 func (err *AmbiguousAddrError) Error() string {
 	files := ""
 	for i, a := range err.Matches {
-		files += a.File
+		files += a.URL.Path
 		if i < len(err.Matches)-1 {
 			files += ", "
 		}
@@ -62,60 +63,63 @@ func (err *AmbiguousAddrError) Error() string {
 	return fmt.Sprintf("multiple keys match address (%s)", files)
 }
 
-// addrCache is a live index of all accounts in the keystore.
-type addrCache struct {
+// accountCache is a live index of all accounts in the keystore.
+type accountCache struct {
 	keydir   string
 	watcher  *watcher
 	mu       sync.Mutex
-	all      accountsByFile
-	byAddr   map[common.Address][]Account
+	all      accountsByURL
+	byAddr   map[common.Address][]accounts.Account
 	throttle *time.Timer
+	notify   chan struct{}
 }
 
-func newAddrCache(keydir string) *addrCache {
-	ac := &addrCache{
+func newAccountCache(keydir string) (*accountCache, chan struct{}) {
+	ac := &accountCache{
 		keydir: keydir,
-		byAddr: make(map[common.Address][]Account),
+		byAddr: make(map[common.Address][]accounts.Account),
+		notify: make(chan struct{}, 1),
 	}
 	ac.watcher = newWatcher(ac)
-	return ac
+	return ac, ac.notify
 }
 
-func (ac *addrCache) accounts() []Account {
+func (ac *accountCache) accounts() []accounts.Account {
 	ac.maybeReload()
 	ac.mu.Lock()
 	defer ac.mu.Unlock()
-	cpy := make([]Account, len(ac.all))
+	cpy := make([]accounts.Account, len(ac.all))
 	copy(cpy, ac.all)
 	return cpy
 }
 
-func (ac *addrCache) hasAddress(addr common.Address) bool {
+func (ac *accountCache) hasAddress(addr common.Address) bool {
 	ac.maybeReload()
 	ac.mu.Lock()
 	defer ac.mu.Unlock()
 	return len(ac.byAddr[addr]) > 0
 }
 
-func (ac *addrCache) add(newAccount Account) {
+func (ac *accountCache) add(newAccount accounts.Account) {
 	ac.mu.Lock()
 	defer ac.mu.Unlock()
 
-	i := sort.Search(len(ac.all), func(i int) bool { return ac.all[i].File >= newAccount.File })
+	i := sort.Search(len(ac.all), func(i int) bool { return ac.all[i].URL.Cmp(newAccount.URL) >= 0 })
 	if i < len(ac.all) && ac.all[i] == newAccount {
 		return
 	}
 	// newAccount is not in the cache.
-	ac.all = append(ac.all, Account{})
+	ac.all = append(ac.all, accounts.Account{})
 	copy(ac.all[i+1:], ac.all[i:])
 	ac.all[i] = newAccount
 	ac.byAddr[newAccount.Address] = append(ac.byAddr[newAccount.Address], newAccount)
 }
 
 // note: removed needs to be unique here (i.e. both File and Address must be set).
-func (ac *addrCache) delete(removed Account) {
+func (ac *accountCache) delete(removed accounts.Account) {
 	ac.mu.Lock()
 	defer ac.mu.Unlock()
+
 	ac.all = removeAccount(ac.all, removed)
 	if ba := removeAccount(ac.byAddr[removed.Address], removed); len(ba) == 0 {
 		delete(ac.byAddr, removed.Address)
@@ -124,7 +128,7 @@ func (ac *addrCache) delete(removed Account) {
 	}
 }
 
-func removeAccount(slice []Account, elem Account) []Account {
+func removeAccount(slice []accounts.Account, elem accounts.Account) []accounts.Account {
 	for i := range slice {
 		if slice[i] == elem {
 			return append(slice[:i], slice[i+1:]...)
@@ -134,43 +138,44 @@ func removeAccount(slice []Account, elem Account) []Account {
 }
 
 // find returns the cached account for address if there is a unique match.
-// The exact matching rules are explained by the documentation of Account.
+// The exact matching rules are explained by the documentation of accounts.Account.
 // Callers must hold ac.mu.
-func (ac *addrCache) find(a Account) (Account, error) {
+func (ac *accountCache) find(a accounts.Account) (accounts.Account, error) {
 	// Limit search to address candidates if possible.
 	matches := ac.all
 	if (a.Address != common.Address{}) {
 		matches = ac.byAddr[a.Address]
 	}
-	if a.File != "" {
+	if a.URL.Path != "" {
 		// If only the basename is specified, complete the path.
-		if !strings.ContainsRune(a.File, filepath.Separator) {
-			a.File = filepath.Join(ac.keydir, a.File)
+		if !strings.ContainsRune(a.URL.Path, filepath.Separator) {
+			a.URL.Path = filepath.Join(ac.keydir, a.URL.Path)
 		}
 		for i := range matches {
-			if matches[i].File == a.File {
+			if matches[i].URL == a.URL {
 				return matches[i], nil
 			}
 		}
 		if (a.Address == common.Address{}) {
-			return Account{}, ErrNoMatch
+			return accounts.Account{}, ErrNoMatch
 		}
 	}
 	switch len(matches) {
 	case 1:
 		return matches[0], nil
 	case 0:
-		return Account{}, ErrNoMatch
+		return accounts.Account{}, ErrNoMatch
 	default:
-		err := &AmbiguousAddrError{Addr: a.Address, Matches: make([]Account, len(matches))}
+		err := &AmbiguousAddrError{Addr: a.Address, Matches: make([]accounts.Account, len(matches))}
 		copy(err.Matches, matches)
-		return Account{}, err
+		return accounts.Account{}, err
 	}
 }
 
-func (ac *addrCache) maybeReload() {
+func (ac *accountCache) maybeReload() {
 	ac.mu.Lock()
 	defer ac.mu.Unlock()
+
 	if ac.watcher.running {
 		return // A watcher is running and will keep the cache up-to-date.
 	}
@@ -188,18 +193,22 @@ func (ac *addrCache) maybeReload() {
 	ac.throttle.Reset(minReloadInterval)
 }
 
-func (ac *addrCache) close() {
+func (ac *accountCache) close() {
 	ac.mu.Lock()
 	ac.watcher.close()
 	if ac.throttle != nil {
 		ac.throttle.Stop()
 	}
+	if ac.notify != nil {
+		close(ac.notify)
+		ac.notify = nil
+	}
 	ac.mu.Unlock()
 }
 
 // reload caches addresses of existing accounts.
 // Callers must hold ac.mu.
-func (ac *addrCache) reload() {
+func (ac *accountCache) reload() {
 	accounts, err := ac.scan()
 	if err != nil && glog.V(logger.Debug) {
 		glog.Errorf("can't load keys: %v", err)
@@ -212,10 +221,14 @@ func (ac *addrCache) reload() {
 	for _, a := range accounts {
 		ac.byAddr[a.Address] = append(ac.byAddr[a.Address], a)
 	}
+	select {
+	case ac.notify <- struct{}{}:
+	default:
+	}
 	glog.V(logger.Debug).Infof("reloaded keys, cache has %d accounts", len(ac.all))
 }
 
-func (ac *addrCache) scan() ([]Account, error) {
+func (ac *accountCache) scan() ([]accounts.Account, error) {
 	files, err := ioutil.ReadDir(ac.keydir)
 	if err != nil {
 		return nil, err
@@ -223,7 +236,7 @@ func (ac *addrCache) scan() ([]Account, error) {
 
 	var (
 		buf     = new(bufio.Reader)
-		addrs   []Account
+		addrs   []accounts.Account
 		keyJSON struct {
 			Address string `json:"address"`
 		}
@@ -250,7 +263,7 @@ func (ac *addrCache) scan() ([]Account, error) {
 		case (addr == common.Address{}):
 			glog.V(logger.Debug).Infof("can't decode key %s: missing or zero address", path)
 		default:
-			addrs = append(addrs, Account{Address: addr, File: path})
+			addrs = append(addrs, accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}})
 		}
 		fd.Close()
 	}

accounts/keystore/account_cache_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"fmt"
@@ -28,23 +28,24 @@ import (
 
 	"github.com/cespare/cp"
 	"github.com/davecgh/go-spew/spew"
+	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 )
 
 var (
 	cachetestDir, _   = filepath.Abs(filepath.Join("testdata", "keystore"))
-	cachetestAccounts = []Account{
+	cachetestAccounts = []accounts.Account{
 		{
 			Address: common.HexToAddress("7ef5a6135f1fd6a02593eedc869c6d41d934aef8"),
-			File:    filepath.Join(cachetestDir, "UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(cachetestDir, "UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8")},
 		},
 		{
 			Address: common.HexToAddress("f466859ead1932d743d622cb74fc058882e8648a"),
-			File:    filepath.Join(cachetestDir, "aaa"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(cachetestDir, "aaa")},
 		},
 		{
 			Address: common.HexToAddress("289d485d9771714cce91d3393d764e1311907acc"),
-			File:    filepath.Join(cachetestDir, "zzz"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(cachetestDir, "zzz")},
 		},
 	}
 )
@@ -52,29 +53,36 @@ var (
 func TestWatchNewFile(t *testing.T) {
 	t.Parallel()
 
-	dir, am := tmpManager(t, false)
+	dir, ks := tmpKeyStore(t, false)
 	defer os.RemoveAll(dir)
 
 	// Ensure the watcher is started before adding any files.
-	am.Accounts()
+	ks.Accounts()
 	time.Sleep(200 * time.Millisecond)
 
 	// Move in the files.
-	wantAccounts := make([]Account, len(cachetestAccounts))
+	wantAccounts := make([]accounts.Account, len(cachetestAccounts))
 	for i := range cachetestAccounts {
-		a := cachetestAccounts[i]
-		a.File = filepath.Join(dir, filepath.Base(a.File))
-		wantAccounts[i] = a
-		if err := cp.CopyFile(a.File, cachetestAccounts[i].File); err != nil {
+		wantAccounts[i] = accounts.Account{
+			Address: cachetestAccounts[i].Address,
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(dir, filepath.Base(cachetestAccounts[i].URL.Path))},
+		}
+		if err := cp.CopyFile(wantAccounts[i].URL.Path, cachetestAccounts[i].URL.Path); err != nil {
 			t.Fatal(err)
 		}
 	}
 
-	// am should see the accounts.
-	var list []Account
+	// ks should see the accounts.
+	var list []accounts.Account
 	for d := 200 * time.Millisecond; d < 5*time.Second; d *= 2 {
-		list = am.Accounts()
+		list = ks.Accounts()
 		if reflect.DeepEqual(list, wantAccounts) {
+			// ks should have also received change notifications
+			select {
+			case <-ks.changes:
+			default:
+				t.Fatalf("wasn't notified of new accounts")
+			}
 			return
 		}
 		time.Sleep(d)
@@ -85,12 +93,12 @@ func TestWatchNewFile(t *testing.T) {
 func TestWatchNoDir(t *testing.T) {
 	t.Parallel()
 
-	// Create am but not the directory that it watches.
+	// Create ks but not the directory that it watches.
 	rand.Seed(time.Now().UnixNano())
 	dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-watch-test-%d-%d", os.Getpid(), rand.Int()))
-	am := NewManager(dir, LightScryptN, LightScryptP)
+	ks := NewKeyStore(dir, LightScryptN, LightScryptP)
 
-	list := am.Accounts()
+	list := ks.Accounts()
 	if len(list) > 0 {
 		t.Error("initial account list not empty:", list)
 	}
@@ -100,16 +108,22 @@ func TestWatchNoDir(t *testing.T) {
 	os.MkdirAll(dir, 0700)
 	defer os.RemoveAll(dir)
 	file := filepath.Join(dir, "aaa")
-	if err := cp.CopyFile(file, cachetestAccounts[0].File); err != nil {
+	if err := cp.CopyFile(file, cachetestAccounts[0].URL.Path); err != nil {
 		t.Fatal(err)
 	}
 
-	// am should see the account.
-	wantAccounts := []Account{cachetestAccounts[0]}
-	wantAccounts[0].File = file
+	// ks should see the account.
+	wantAccounts := []accounts.Account{cachetestAccounts[0]}
+	wantAccounts[0].URL = accounts.URL{Scheme: KeyStoreScheme, Path: file}
 	for d := 200 * time.Millisecond; d < 8*time.Second; d *= 2 {
-		list = am.Accounts()
+		list = ks.Accounts()
 		if reflect.DeepEqual(list, wantAccounts) {
+			// ks should have also received change notifications
+			select {
+			case <-ks.changes:
+			default:
+				t.Fatalf("wasn't notified of new accounts")
+			}
 			return
 		}
 		time.Sleep(d)
@@ -118,7 +132,7 @@ func TestWatchNoDir(t *testing.T) {
 }
 
 func TestCacheInitialReload(t *testing.T) {
-	cache := newAddrCache(cachetestDir)
+	cache, _ := newAccountCache(cachetestDir)
 	accounts := cache.accounts()
 	if !reflect.DeepEqual(accounts, cachetestAccounts) {
 		t.Fatalf("got initial accounts: %swant %s", spew.Sdump(accounts), spew.Sdump(cachetestAccounts))
@@ -126,55 +140,55 @@ func TestCacheInitialReload(t *testing.T) {
 }
 
 func TestCacheAddDeleteOrder(t *testing.T) {
-	cache := newAddrCache("testdata/no-such-dir")
+	cache, _ := newAccountCache("testdata/no-such-dir")
 	cache.watcher.running = true // prevent unexpected reloads
 
-	accounts := []Account{
+	accs := []accounts.Account{
 		{
 			Address: common.HexToAddress("095e7baea6a6c7c4c2dfeb977efac326af552d87"),
-			File:    "-309830980",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "-309830980"},
 		},
 		{
 			Address: common.HexToAddress("2cac1adea150210703ba75ed097ddfe24e14f213"),
-			File:    "ggg",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "ggg"},
 		},
 		{
 			Address: common.HexToAddress("8bda78331c916a08481428e4b07c96d3e916d165"),
-			File:    "zzzzzz-the-very-last-one.keyXXX",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "zzzzzz-the-very-last-one.keyXXX"},
 		},
 		{
 			Address: common.HexToAddress("d49ff4eeb0b2686ed89c0fc0f2b6ea533ddbbd5e"),
-			File:    "SOMETHING.key",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "SOMETHING.key"},
 		},
 		{
 			Address: common.HexToAddress("7ef5a6135f1fd6a02593eedc869c6d41d934aef8"),
-			File:    "UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8"},
 		},
 		{
 			Address: common.HexToAddress("f466859ead1932d743d622cb74fc058882e8648a"),
-			File:    "aaa",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "aaa"},
 		},
 		{
 			Address: common.HexToAddress("289d485d9771714cce91d3393d764e1311907acc"),
-			File:    "zzz",
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: "zzz"},
 		},
 	}
-	for _, a := range accounts {
+	for _, a := range accs {
 		cache.add(a)
 	}
 	// Add some of them twice to check that they don't get reinserted.
-	cache.add(accounts[0])
-	cache.add(accounts[2])
+	cache.add(accs[0])
+	cache.add(accs[2])
 
 	// Check that the account list is sorted by filename.
-	wantAccounts := make([]Account, len(accounts))
-	copy(wantAccounts, accounts)
-	sort.Sort(accountsByFile(wantAccounts))
+	wantAccounts := make([]accounts.Account, len(accs))
+	copy(wantAccounts, accs)
+	sort.Sort(accountsByURL(wantAccounts))
 	list := cache.accounts()
 	if !reflect.DeepEqual(list, wantAccounts) {
-		t.Fatalf("got accounts: %s\nwant %s", spew.Sdump(accounts), spew.Sdump(wantAccounts))
+		t.Fatalf("got accounts: %s\nwant %s", spew.Sdump(accs), spew.Sdump(wantAccounts))
 	}
-	for _, a := range accounts {
+	for _, a := range accs {
 		if !cache.hasAddress(a.Address) {
 			t.Errorf("expected hasAccount(%x) to return true", a.Address)
 		}
@@ -184,13 +198,13 @@ func TestCacheAddDeleteOrder(t *testing.T) {
 	}
 
 	// Delete a few keys from the cache.
-	for i := 0; i < len(accounts); i += 2 {
+	for i := 0; i < len(accs); i += 2 {
 		cache.delete(wantAccounts[i])
 	}
-	cache.delete(Account{Address: common.HexToAddress("fd9bd350f08ee3c0c19b85a8e16114a11a60aa4e"), File: "something"})
+	cache.delete(accounts.Account{Address: common.HexToAddress("fd9bd350f08ee3c0c19b85a8e16114a11a60aa4e"), URL: accounts.URL{Scheme: KeyStoreScheme, Path: "something"}})
 
 	// Check content again after deletion.
-	wantAccountsAfterDelete := []Account{
+	wantAccountsAfterDelete := []accounts.Account{
 		wantAccounts[1],
 		wantAccounts[3],
 		wantAccounts[5],
@@ -211,63 +225,63 @@ func TestCacheAddDeleteOrder(t *testing.T) {
 
 func TestCacheFind(t *testing.T) {
 	dir := filepath.Join("testdata", "dir")
-	cache := newAddrCache(dir)
+	cache, _ := newAccountCache(dir)
 	cache.watcher.running = true // prevent unexpected reloads
 
-	accounts := []Account{
+	accs := []accounts.Account{
 		{
 			Address: common.HexToAddress("095e7baea6a6c7c4c2dfeb977efac326af552d87"),
-			File:    filepath.Join(dir, "a.key"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(dir, "a.key")},
 		},
 		{
 			Address: common.HexToAddress("2cac1adea150210703ba75ed097ddfe24e14f213"),
-			File:    filepath.Join(dir, "b.key"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(dir, "b.key")},
 		},
 		{
 			Address: common.HexToAddress("d49ff4eeb0b2686ed89c0fc0f2b6ea533ddbbd5e"),
-			File:    filepath.Join(dir, "c.key"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(dir, "c.key")},
 		},
 		{
 			Address: common.HexToAddress("d49ff4eeb0b2686ed89c0fc0f2b6ea533ddbbd5e"),
-			File:    filepath.Join(dir, "c2.key"),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(dir, "c2.key")},
 		},
 	}
-	for _, a := range accounts {
+	for _, a := range accs {
 		cache.add(a)
 	}
 
-	nomatchAccount := Account{
+	nomatchAccount := accounts.Account{
 		Address: common.HexToAddress("f466859ead1932d743d622cb74fc058882e8648a"),
-		File:    filepath.Join(dir, "something"),
+		URL:     accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Join(dir, "something")},
 	}
 	tests := []struct {
-		Query      Account
-		WantResult Account
+		Query      accounts.Account
+		WantResult accounts.Account
 		WantError  error
 	}{
 		// by address
-		{Query: Account{Address: accounts[0].Address}, WantResult: accounts[0]},
+		{Query: accounts.Account{Address: accs[0].Address}, WantResult: accs[0]},
 		// by file
-		{Query: Account{File: accounts[0].File}, WantResult: accounts[0]},
+		{Query: accounts.Account{URL: accs[0].URL}, WantResult: accs[0]},
 		// by basename
-		{Query: Account{File: filepath.Base(accounts[0].File)}, WantResult: accounts[0]},
+		{Query: accounts.Account{URL: accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Base(accs[0].URL.Path)}}, WantResult: accs[0]},
 		// by file and address
-		{Query: accounts[0], WantResult: accounts[0]},
+		{Query: accs[0], WantResult: accs[0]},
 		// ambiguous address, tie resolved by file
-		{Query: accounts[2], WantResult: accounts[2]},
+		{Query: accs[2], WantResult: accs[2]},
 		// ambiguous address error
 		{
-			Query: Account{Address: accounts[2].Address},
+			Query: accounts.Account{Address: accs[2].Address},
 			WantError: &AmbiguousAddrError{
-				Addr:    accounts[2].Address,
-				Matches: []Account{accounts[2], accounts[3]},
+				Addr:    accs[2].Address,
+				Matches: []accounts.Account{accs[2], accs[3]},
 			},
 		},
 		// no match error
 		{Query: nomatchAccount, WantError: ErrNoMatch},
-		{Query: Account{File: nomatchAccount.File}, WantError: ErrNoMatch},
-		{Query: Account{File: filepath.Base(nomatchAccount.File)}, WantError: ErrNoMatch},
-		{Query: Account{Address: nomatchAccount.Address}, WantError: ErrNoMatch},
+		{Query: accounts.Account{URL: nomatchAccount.URL}, WantError: ErrNoMatch},
+		{Query: accounts.Account{URL: accounts.URL{Scheme: KeyStoreScheme, Path: filepath.Base(nomatchAccount.URL.Path)}}, WantError: ErrNoMatch},
+		{Query: accounts.Account{Address: nomatchAccount.Address}, WantError: ErrNoMatch},
 	}
 	for i, test := range tests {
 		a, err := cache.find(test.Query)

accounts/keystore/key.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"bytes"
@@ -29,6 +29,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/crypto/secp256k1"
@@ -175,13 +176,13 @@ func newKey(rand io.Reader) (*Key, error) {
 	return newKeyFromECDSA(privateKeyECDSA), nil
 }
 
-func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, Account, error) {
+func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Account, error) {
 	key, err := newKey(rand)
 	if err != nil {
-		return nil, Account{}, err
+		return nil, accounts.Account{}, err
 	}
-	a := Account{Address: key.Address, File: ks.JoinPath(keyFileName(key.Address))}
-	if err := ks.StoreKey(a.File, key, auth); err != nil {
+	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))}}
+	if err := ks.StoreKey(a.URL.Path, key, auth); err != nil {
 		zeroKey(key.PrivateKey)
 		return nil, a, err
 	}

accounts/keystore/keystore.go

@@ -0,0 +1,494 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package keystore implements encrypted storage of secp256k1 private keys.
+//
+// Keys are stored as encrypted JSON files according to the Web3 Secret Storage specification.
+// See https://github.com/ethereum/wiki/wiki/Web3-Secret-Storage-Definition for more information.
+package keystore
+
+import (
+	"crypto/ecdsa"
+	crand "crypto/rand"
+	"errors"
+	"fmt"
+	"math/big"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/event"
+)
+
+var (
+	ErrLocked  = accounts.NewAuthNeededError("password or unlock")
+	ErrNoMatch = errors.New("no key for given address or file")
+	ErrDecrypt = errors.New("could not decrypt key with given passphrase")
+)
+
+// KeyStoreType is the reflect type of a keystore backend.
+var KeyStoreType = reflect.TypeOf(&KeyStore{})
+
+// KeyStoreScheme is the protocol scheme prefixing account and wallet URLs.
+var KeyStoreScheme = "keystore"
+
+// Maximum time between wallet refreshes (if filesystem notifications don't work).
+const walletRefreshCycle = 3 * time.Second
+
+// KeyStore manages a key storage directory on disk.
+type KeyStore struct {
+	storage  keyStore                     // Storage backend, might be cleartext or encrypted
+	cache    *accountCache                // In-memory account cache over the filesystem storage
+	changes  chan struct{}                // Channel receiving change notifications from the cache
+	unlocked map[common.Address]*unlocked // Currently unlocked account (decrypted private keys)
+
+	wallets     []accounts.Wallet       // Wallet wrappers around the individual key files
+	updateFeed  event.Feed              // Event feed to notify wallet additions/removals
+	updateScope event.SubscriptionScope // Subscription scope tracking current live listeners
+	updating    bool                    // Whether the event notification loop is running
+
+	mu sync.RWMutex
+}
+
+type unlocked struct {
+	*Key
+	abort chan struct{}
+}
+
+// NewKeyStore creates a keystore for the given directory.
+func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore {
+	keydir, _ = filepath.Abs(keydir)
+	ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP}}
+	ks.init(keydir)
+	return ks
+}
+
+// NewPlaintextKeyStore creates a keystore for the given directory.
+// Deprecated: Use NewKeyStore.
+func NewPlaintextKeyStore(keydir string) *KeyStore {
+	keydir, _ = filepath.Abs(keydir)
+	ks := &KeyStore{storage: &keyStorePlain{keydir}}
+	ks.init(keydir)
+	return ks
+}
+
+func (ks *KeyStore) init(keydir string) {
+	// Lock the mutex since the account cache might call back with events
+	ks.mu.Lock()
+	defer ks.mu.Unlock()
+
+	// Initialize the set of unlocked keys and the account cache
+	ks.unlocked = make(map[common.Address]*unlocked)
+	ks.cache, ks.changes = newAccountCache(keydir)
+
+	// TODO: In order for this finalizer to work, there must be no references
+	// to ks. addressCache doesn't keep a reference but unlocked keys do,
+	// so the finalizer will not trigger until all timed unlocks have expired.
+	runtime.SetFinalizer(ks, func(m *KeyStore) {
+		m.cache.close()
+	})
+	// Create the initial list of wallets from the cache
+	accs := ks.cache.accounts()
+	ks.wallets = make([]accounts.Wallet, len(accs))
+	for i := 0; i < len(accs); i++ {
+		ks.wallets[i] = &keystoreWallet{account: accs[i], keystore: ks}
+	}
+}
+
+// Wallets implements accounts.Backend, returning all single-key wallets from the
+// keystore directory.
+func (ks *KeyStore) Wallets() []accounts.Wallet {
+	// Make sure the list of wallets is in sync with the account cache
+	ks.refreshWallets()
+
+	ks.mu.RLock()
+	defer ks.mu.RUnlock()
+
+	cpy := make([]accounts.Wallet, len(ks.wallets))
+	copy(cpy, ks.wallets)
+	return cpy
+}
+
+// refreshWallets retrieves the current account list and based on that does any
+// necessary wallet refreshes.
+func (ks *KeyStore) refreshWallets() {
+	// Retrieve the current list of accounts
+	ks.mu.Lock()
+	accs := ks.cache.accounts()
+
+	// Transform the current list of wallets into the new one
+	wallets := make([]accounts.Wallet, 0, len(accs))
+	events := []accounts.WalletEvent{}
+
+	for _, account := range accs {
+		// Drop wallets while they were in front of the next account
+		for len(ks.wallets) > 0 && ks.wallets[0].URL().Cmp(account.URL) < 0 {
+			events = append(events, accounts.WalletEvent{Wallet: ks.wallets[0], Arrive: false})
+			ks.wallets = ks.wallets[1:]
+		}
+		// If there are no more wallets or the account is before the next, wrap new wallet
+		if len(ks.wallets) == 0 || ks.wallets[0].URL().Cmp(account.URL) > 0 {
+			wallet := &keystoreWallet{account: account, keystore: ks}
+
+			events = append(events, accounts.WalletEvent{Wallet: wallet, Arrive: true})
+			wallets = append(wallets, wallet)
+			continue
+		}
+		// If the account is the same as the first wallet, keep it
+		if ks.wallets[0].Accounts()[0] == account {
+			wallets = append(wallets, ks.wallets[0])
+			ks.wallets = ks.wallets[1:]
+			continue
+		}
+	}
+	// Drop any leftover wallets and set the new batch
+	for _, wallet := range ks.wallets {
+		events = append(events, accounts.WalletEvent{Wallet: wallet, Arrive: false})
+	}
+	ks.wallets = wallets
+	ks.mu.Unlock()
+
+	// Fire all wallet events and return
+	for _, event := range events {
+		ks.updateFeed.Send(event)
+	}
+}
+
+// Subscribe implements accounts.Backend, creating an async subscription to
+// receive notifications on the addition or removal of keystore wallets.
+func (ks *KeyStore) Subscribe(sink chan<- accounts.WalletEvent) event.Subscription {
+	// We need the mutex to reliably start/stop the update loop
+	ks.mu.Lock()
+	defer ks.mu.Unlock()
+
+	// Subscribe the caller and track the subscriber count
+	sub := ks.updateScope.Track(ks.updateFeed.Subscribe(sink))
+
+	// Subscribers require an active notification loop, start it
+	if !ks.updating {
+		ks.updating = true
+		go ks.updater()
+	}
+	return sub
+}
+
+// updater is responsible for maintaining an up-to-date list of wallets stored in
+// the keystore, and for firing wallet addition/removal events. It listens for
+// account change events from the underlying account cache, and also periodically
+// forces a manual refresh (only triggers for systems where the filesystem notifier
+// is not running).
+func (ks *KeyStore) updater() {
+	for {
+		// Wait for an account update or a refresh timeout
+		select {
+		case <-ks.changes:
+		case <-time.After(walletRefreshCycle):
+		}
+		// Run the wallet refresher
+		ks.refreshWallets()
+
+		// If all our subscribers left, stop the updater
+		ks.mu.Lock()
+		if ks.updateScope.Count() == 0 {
+			ks.updating = false
+			ks.mu.Unlock()
+			return
+		}
+		ks.mu.Unlock()
+	}
+}
+
+// HasAddress reports whether a key with the given address is present.
+func (ks *KeyStore) HasAddress(addr common.Address) bool {
+	return ks.cache.hasAddress(addr)
+}
+
+// Accounts returns all key files present in the directory.
+func (ks *KeyStore) Accounts() []accounts.Account {
+	return ks.cache.accounts()
+}
+
+// Delete deletes the key matched by account if the passphrase is correct.
+// If the account contains no filename, the address must match a unique key.
+func (ks *KeyStore) Delete(a accounts.Account, passphrase string) error {
+	// Decrypting the key isn't really necessary, but we do
+	// it anyway to check the password and zero out the key
+	// immediately afterwards.
+	a, key, err := ks.getDecryptedKey(a, passphrase)
+	if key != nil {
+		zeroKey(key.PrivateKey)
+	}
+	if err != nil {
+		return err
+	}
+	// The order is crucial here. The key is dropped from the
+	// cache after the file is gone so that a reload happening in
+	// between won't insert it into the cache again.
+	err = os.Remove(a.URL.Path)
+	if err == nil {
+		ks.cache.delete(a)
+		ks.refreshWallets()
+	}
+	return err
+}
+
+// SignHash calculates a ECDSA signature for the given hash. The produced
+// signature is in the [R || S || V] format where V is 0 or 1.
+func (ks *KeyStore) SignHash(a accounts.Account, hash []byte) ([]byte, error) {
+	// Look up the key to sign with and abort if it cannot be found
+	ks.mu.RLock()
+	defer ks.mu.RUnlock()
+
+	unlockedKey, found := ks.unlocked[a.Address]
+	if !found {
+		return nil, ErrLocked
+	}
+	// Sign the hash using plain ECDSA operations
+	return crypto.Sign(hash, unlockedKey.PrivateKey)
+}
+
+// SignTx signs the given transaction with the requested account.
+func (ks *KeyStore) SignTx(a accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	// Look up the key to sign with and abort if it cannot be found
+	ks.mu.RLock()
+	defer ks.mu.RUnlock()
+
+	unlockedKey, found := ks.unlocked[a.Address]
+	if !found {
+		return nil, ErrLocked
+	}
+	// Depending on the presence of the chain ID, sign with EIP155 or homestead
+	if chainID != nil {
+		return types.SignTx(tx, types.NewEIP155Signer(chainID), unlockedKey.PrivateKey)
+	}
+	return types.SignTx(tx, types.HomesteadSigner{}, unlockedKey.PrivateKey)
+}
+
+// SignHashWithPassphrase signs hash if the private key matching the given address
+// can be decrypted with the given passphrase. The produced signature is in the
+// [R || S || V] format where V is 0 or 1.
+func (ks *KeyStore) SignHashWithPassphrase(a accounts.Account, passphrase string, hash []byte) (signature []byte, err error) {
+	_, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return nil, err
+	}
+	defer zeroKey(key.PrivateKey)
+	return crypto.Sign(hash, key.PrivateKey)
+}
+
+// SignTxWithPassphrase signs the transaction if the private key matching the
+// given address can be decrypted with the given passphrase.
+func (ks *KeyStore) SignTxWithPassphrase(a accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	_, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return nil, err
+	}
+	defer zeroKey(key.PrivateKey)
+
+	// Depending on the presence of the chain ID, sign with EIP155 or homestead
+	if chainID != nil {
+		return types.SignTx(tx, types.NewEIP155Signer(chainID), key.PrivateKey)
+	}
+	return types.SignTx(tx, types.HomesteadSigner{}, key.PrivateKey)
+}
+
+// Unlock unlocks the given account indefinitely.
+func (ks *KeyStore) Unlock(a accounts.Account, passphrase string) error {
+	return ks.TimedUnlock(a, passphrase, 0)
+}
+
+// Lock removes the private key with the given address from memory.
+func (ks *KeyStore) Lock(addr common.Address) error {
+	ks.mu.Lock()
+	if unl, found := ks.unlocked[addr]; found {
+		ks.mu.Unlock()
+		ks.expire(addr, unl, time.Duration(0)*time.Nanosecond)
+	} else {
+		ks.mu.Unlock()
+	}
+	return nil
+}
+
+// TimedUnlock unlocks the given account with the passphrase. The account
+// stays unlocked for the duration of timeout. A timeout of 0 unlocks the account
+// until the program exits. The account must match a unique key file.
+//
+// If the account address is already unlocked for a duration, TimedUnlock extends or
+// shortens the active unlock timeout. If the address was previously unlocked
+// indefinitely the timeout is not altered.
+func (ks *KeyStore) TimedUnlock(a accounts.Account, passphrase string, timeout time.Duration) error {
+	a, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return err
+	}
+
+	ks.mu.Lock()
+	defer ks.mu.Unlock()
+	u, found := ks.unlocked[a.Address]
+	if found {
+		if u.abort == nil {
+			// The address was unlocked indefinitely, so unlocking
+			// it with a timeout would be confusing.
+			zeroKey(key.PrivateKey)
+			return nil
+		}
+		// Terminate the expire goroutine and replace it below.
+		close(u.abort)
+	}
+	if timeout > 0 {
+		u = &unlocked{Key: key, abort: make(chan struct{})}
+		go ks.expire(a.Address, u, timeout)
+	} else {
+		u = &unlocked{Key: key}
+	}
+	ks.unlocked[a.Address] = u
+	return nil
+}
+
+// Find resolves the given account into a unique entry in the keystore.
+func (ks *KeyStore) Find(a accounts.Account) (accounts.Account, error) {
+	ks.cache.maybeReload()
+	ks.cache.mu.Lock()
+	a, err := ks.cache.find(a)
+	ks.cache.mu.Unlock()
+	return a, err
+}
+
+func (ks *KeyStore) getDecryptedKey(a accounts.Account, auth string) (accounts.Account, *Key, error) {
+	a, err := ks.Find(a)
+	if err != nil {
+		return a, nil, err
+	}
+	key, err := ks.storage.GetKey(a.Address, a.URL.Path, auth)
+	return a, key, err
+}
+
+func (ks *KeyStore) expire(addr common.Address, u *unlocked, timeout time.Duration) {
+	t := time.NewTimer(timeout)
+	defer t.Stop()
+	select {
+	case <-u.abort:
+		// just quit
+	case <-t.C:
+		ks.mu.Lock()
+		// only drop if it's still the same key instance that dropLater
+		// was launched with. we can check that using pointer equality
+		// because the map stores a new pointer every time the key is
+		// unlocked.
+		if ks.unlocked[addr] == u {
+			zeroKey(u.PrivateKey)
+			delete(ks.unlocked, addr)
+		}
+		ks.mu.Unlock()
+	}
+}
+
+// NewAccount generates a new key and stores it into the key directory,
+// encrypting it with the passphrase.
+func (ks *KeyStore) NewAccount(passphrase string) (accounts.Account, error) {
+	_, account, err := storeNewKey(ks.storage, crand.Reader, passphrase)
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	// Add the account to the cache immediately rather
+	// than waiting for file system notifications to pick it up.
+	ks.cache.add(account)
+	ks.refreshWallets()
+	return account, nil
+}
+
+// Export exports as a JSON key, encrypted with newPassphrase.
+func (ks *KeyStore) Export(a accounts.Account, passphrase, newPassphrase string) (keyJSON []byte, err error) {
+	_, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return nil, err
+	}
+	var N, P int
+	if store, ok := ks.storage.(*keyStorePassphrase); ok {
+		N, P = store.scryptN, store.scryptP
+	} else {
+		N, P = StandardScryptN, StandardScryptP
+	}
+	return EncryptKey(key, newPassphrase, N, P)
+}
+
+// Import stores the given encrypted JSON key into the key directory.
+func (ks *KeyStore) Import(keyJSON []byte, passphrase, newPassphrase string) (accounts.Account, error) {
+	key, err := DecryptKey(keyJSON, passphrase)
+	if key != nil && key.PrivateKey != nil {
+		defer zeroKey(key.PrivateKey)
+	}
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	return ks.importKey(key, newPassphrase)
+}
+
+// ImportECDSA stores the given key into the key directory, encrypting it with the passphrase.
+func (ks *KeyStore) ImportECDSA(priv *ecdsa.PrivateKey, passphrase string) (accounts.Account, error) {
+	key := newKeyFromECDSA(priv)
+	if ks.cache.hasAddress(key.Address) {
+		return accounts.Account{}, fmt.Errorf("account already exists")
+	}
+
+	return ks.importKey(key, passphrase)
+}
+
+func (ks *KeyStore) importKey(key *Key, passphrase string) (accounts.Account, error) {
+	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.storage.JoinPath(keyFileName(key.Address))}}
+	if err := ks.storage.StoreKey(a.URL.Path, key, passphrase); err != nil {
+		return accounts.Account{}, err
+	}
+	ks.cache.add(a)
+	ks.refreshWallets()
+	return a, nil
+}
+
+// Update changes the passphrase of an existing account.
+func (ks *KeyStore) Update(a accounts.Account, passphrase, newPassphrase string) error {
+	a, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return err
+	}
+	return ks.storage.StoreKey(a.URL.Path, key, newPassphrase)
+}
+
+// ImportPreSaleKey decrypts the given Ethereum presale wallet and stores
+// a key file in the key directory. The key file is encrypted with the same passphrase.
+func (ks *KeyStore) ImportPreSaleKey(keyJSON []byte, passphrase string) (accounts.Account, error) {
+	a, _, err := importPreSaleKey(ks.storage, keyJSON, passphrase)
+	if err != nil {
+		return a, err
+	}
+	ks.cache.add(a)
+	ks.refreshWallets()
+	return a, nil
+}
+
+// zeroKey zeroes a private key in memory.
+func zeroKey(k *ecdsa.PrivateKey) {
+	b := k.D.Bits()
+	for i := range b {
+		b[i] = 0
+	}
+}

accounts/keystore/keystore_passphrase.go

@@ -23,7 +23,7 @@ The crypto is documented at https://github.com/ethereum/wiki/wiki/Web3-Secret-St
 
 */
 
-package accounts
+package keystore
 
 import (
 	"bytes"

accounts/keystore/keystore_passphrase_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"io/ioutil"

accounts/keystore/keystore_plain.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"encoding/json"

accounts/keystore/keystore_plain_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"crypto/rand"
@@ -30,7 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
-func tmpKeyStore(t *testing.T, encrypted bool) (dir string, ks keyStore) {
+func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) {
 	d, err := ioutil.TempDir("", "geth-keystore-test")
 	if err != nil {
 		t.Fatal(err)
@@ -44,7 +44,7 @@ func tmpKeyStore(t *testing.T, encrypted bool) (dir string, ks keyStore) {
 }
 
 func TestKeyStorePlain(t *testing.T) {
-	dir, ks := tmpKeyStore(t, false)
+	dir, ks := tmpKeyStoreIface(t, false)
 	defer os.RemoveAll(dir)
 
 	pass := "" // not used but required by API
@@ -52,7 +52,7 @@ func TestKeyStorePlain(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	k2, err := ks.GetKey(k1.Address, account.File, pass)
+	k2, err := ks.GetKey(k1.Address, account.URL.Path, pass)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -65,7 +65,7 @@ func TestKeyStorePlain(t *testing.T) {
 }
 
 func TestKeyStorePassphrase(t *testing.T) {
-	dir, ks := tmpKeyStore(t, true)
+	dir, ks := tmpKeyStoreIface(t, true)
 	defer os.RemoveAll(dir)
 
 	pass := "foo"
@@ -73,7 +73,7 @@ func TestKeyStorePassphrase(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	k2, err := ks.GetKey(k1.Address, account.File, pass)
+	k2, err := ks.GetKey(k1.Address, account.URL.Path, pass)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -86,7 +86,7 @@ func TestKeyStorePassphrase(t *testing.T) {
 }
 
 func TestKeyStorePassphraseDecryptionFail(t *testing.T) {
-	dir, ks := tmpKeyStore(t, true)
+	dir, ks := tmpKeyStoreIface(t, true)
 	defer os.RemoveAll(dir)
 
 	pass := "foo"
@@ -94,13 +94,13 @@ func TestKeyStorePassphraseDecryptionFail(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if _, err = ks.GetKey(k1.Address, account.File, "bar"); err != ErrDecrypt {
+	if _, err = ks.GetKey(k1.Address, account.URL.Path, "bar"); err != ErrDecrypt {
 		t.Fatalf("wrong error for invalid passphrase\ngot %q\nwant %q", err, ErrDecrypt)
 	}
 }
 
 func TestImportPreSaleKey(t *testing.T) {
-	dir, ks := tmpKeyStore(t, true)
+	dir, ks := tmpKeyStoreIface(t, true)
 	defer os.RemoveAll(dir)
 
 	// file content of a presale key file generated with:
@@ -115,8 +115,8 @@ func TestImportPreSaleKey(t *testing.T) {
 	if account.Address != common.HexToAddress("d4584b5f6229b7be90727b0fc8c6b91bb427821f") {
 		t.Errorf("imported account has wrong address %x", account.Address)
 	}
-	if !strings.HasPrefix(account.File, dir) {
-		t.Errorf("imported account file not in keystore directory: %q", account.File)
+	if !strings.HasPrefix(account.URL.Path, dir) {
+		t.Errorf("imported account file not in keystore directory: %q", account.URL)
 	}
 }
 
@@ -142,19 +142,19 @@ func TestV3_PBKDF2_1(t *testing.T) {
 
 func TestV3_PBKDF2_2(t *testing.T) {
 	t.Parallel()
-	tests := loadKeyStoreTestV3("../tests/files/KeyStoreTests/basic_tests.json", t)
+	tests := loadKeyStoreTestV3("../../tests/files/KeyStoreTests/basic_tests.json", t)
 	testDecryptV3(tests["test1"], t)
 }
 
 func TestV3_PBKDF2_3(t *testing.T) {
 	t.Parallel()
-	tests := loadKeyStoreTestV3("../tests/files/KeyStoreTests/basic_tests.json", t)
+	tests := loadKeyStoreTestV3("../../tests/files/KeyStoreTests/basic_tests.json", t)
 	testDecryptV3(tests["python_generated_test_with_odd_iv"], t)
 }
 
 func TestV3_PBKDF2_4(t *testing.T) {
 	t.Parallel()
-	tests := loadKeyStoreTestV3("../tests/files/KeyStoreTests/basic_tests.json", t)
+	tests := loadKeyStoreTestV3("../../tests/files/KeyStoreTests/basic_tests.json", t)
 	testDecryptV3(tests["evilnonce"], t)
 }
 
@@ -166,7 +166,7 @@ func TestV3_Scrypt_1(t *testing.T) {
 
 func TestV3_Scrypt_2(t *testing.T) {
 	t.Parallel()
-	tests := loadKeyStoreTestV3("../tests/files/KeyStoreTests/basic_tests.json", t)
+	tests := loadKeyStoreTestV3("../../tests/files/KeyStoreTests/basic_tests.json", t)
 	testDecryptV3(tests["test2"], t)
 }
 

accounts/keystore/keystore_test.go

@@ -0,0 +1,365 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package keystore
+
+import (
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"runtime"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/event"
+)
+
+var testSigData = make([]byte, 32)
+
+func TestKeyStore(t *testing.T) {
+	dir, ks := tmpKeyStore(t, true)
+	defer os.RemoveAll(dir)
+
+	a, err := ks.NewAccount("foo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(a.URL.Path, dir) {
+		t.Errorf("account file %s doesn't have dir prefix", a.URL)
+	}
+	stat, err := os.Stat(a.URL.Path)
+	if err != nil {
+		t.Fatalf("account file %s doesn't exist (%v)", a.URL, err)
+	}
+	if runtime.GOOS != "windows" && stat.Mode() != 0600 {
+		t.Fatalf("account file has wrong mode: got %o, want %o", stat.Mode(), 0600)
+	}
+	if !ks.HasAddress(a.Address) {
+		t.Errorf("HasAccount(%x) should've returned true", a.Address)
+	}
+	if err := ks.Update(a, "foo", "bar"); err != nil {
+		t.Errorf("Update error: %v", err)
+	}
+	if err := ks.Delete(a, "bar"); err != nil {
+		t.Errorf("Delete error: %v", err)
+	}
+	if common.FileExist(a.URL.Path) {
+		t.Errorf("account file %s should be gone after Delete", a.URL)
+	}
+	if ks.HasAddress(a.Address) {
+		t.Errorf("HasAccount(%x) should've returned true after Delete", a.Address)
+	}
+}
+
+func TestSign(t *testing.T) {
+	dir, ks := tmpKeyStore(t, true)
+	defer os.RemoveAll(dir)
+
+	pass := "" // not used but required by API
+	a1, err := ks.NewAccount(pass)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := ks.Unlock(a1, ""); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ks.SignHash(accounts.Account{Address: a1.Address}, testSigData); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSignWithPassphrase(t *testing.T) {
+	dir, ks := tmpKeyStore(t, true)
+	defer os.RemoveAll(dir)
+
+	pass := "passwd"
+	acc, err := ks.NewAccount(pass)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, unlocked := ks.unlocked[acc.Address]; unlocked {
+		t.Fatal("expected account to be locked")
+	}
+
+	_, err = ks.SignHashWithPassphrase(acc, pass, testSigData)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, unlocked := ks.unlocked[acc.Address]; unlocked {
+		t.Fatal("expected account to be locked")
+	}
+
+	if _, err = ks.SignHashWithPassphrase(acc, "invalid passwd", testSigData); err == nil {
+		t.Fatal("expected SignHashWithPassphrase to fail with invalid password")
+	}
+}
+
+func TestTimedUnlock(t *testing.T) {
+	dir, ks := tmpKeyStore(t, true)
+	defer os.RemoveAll(dir)
+
+	pass := "foo"
+	a1, err := ks.NewAccount(pass)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Signing without passphrase fails because account is locked
+	_, err = ks.SignHash(accounts.Account{Address: a1.Address}, testSigData)
+	if err != ErrLocked {
+		t.Fatal("Signing should've failed with ErrLocked before unlocking, got ", err)
+	}
+
+	// Signing with passphrase works
+	if err = ks.TimedUnlock(a1, pass, 100*time.Millisecond); err != nil {
+		t.Fatal(err)
+	}
+
+	// Signing without passphrase works because account is temp unlocked
+	_, err = ks.SignHash(accounts.Account{Address: a1.Address}, testSigData)
+	if err != nil {
+		t.Fatal("Signing shouldn't return an error after unlocking, got ", err)
+	}
+
+	// Signing fails again after automatic locking
+	time.Sleep(250 * time.Millisecond)
+	_, err = ks.SignHash(accounts.Account{Address: a1.Address}, testSigData)
+	if err != ErrLocked {
+		t.Fatal("Signing should've failed with ErrLocked timeout expired, got ", err)
+	}
+}
+
+func TestOverrideUnlock(t *testing.T) {
+	dir, ks := tmpKeyStore(t, false)
+	defer os.RemoveAll(dir)
+
+	pass := "foo"
+	a1, err := ks.NewAccount(pass)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Unlock indefinitely.
+	if err = ks.TimedUnlock(a1, pass, 5*time.Minute); err != nil {
+		t.Fatal(err)
+	}
+
+	// Signing without passphrase works because account is temp unlocked
+	_, err = ks.SignHash(accounts.Account{Address: a1.Address}, testSigData)
+	if err != nil {
+		t.Fatal("Signing shouldn't return an error after unlocking, got ", err)
+	}
+
+	// reset unlock to a shorter period, invalidates the previous unlock
+	if err = ks.TimedUnlock(a1, pass, 100*time.Millisecond); err != nil {
+		t.Fatal(err)
+	}
+
+	// Signing without passphrase still works because account is temp unlocked
+	_, err = ks.SignHash(accounts.Account{Address: a1.Address}, testSigData)
+	if err != nil {
+		t.Fatal("Signing shouldn't return an error after unlocking, got ", err)
+	}
+
+	// Signing fails again after automatic locking
+	time.Sleep(250 * time.Millisecond)
+	_, err = ks.SignHash(accounts.Account{Address: a1.Address}, testSigData)
+	if err != ErrLocked {
+		t.Fatal("Signing should've failed with ErrLocked timeout expired, got ", err)
+	}
+}
+
+// This test should fail under -race if signing races the expiration goroutine.
+func TestSignRace(t *testing.T) {
+	dir, ks := tmpKeyStore(t, false)
+	defer os.RemoveAll(dir)
+
+	// Create a test account.
+	a1, err := ks.NewAccount("")
+	if err != nil {
+		t.Fatal("could not create the test account", err)
+	}
+
+	if err := ks.TimedUnlock(a1, "", 15*time.Millisecond); err != nil {
+		t.Fatal("could not unlock the test account", err)
+	}
+	end := time.Now().Add(500 * time.Millisecond)
+	for time.Now().Before(end) {
+		if _, err := ks.SignHash(accounts.Account{Address: a1.Address}, testSigData); err == ErrLocked {
+			return
+		} else if err != nil {
+			t.Errorf("Sign error: %v", err)
+			return
+		}
+		time.Sleep(1 * time.Millisecond)
+	}
+	t.Errorf("Account did not lock within the timeout")
+}
+
+// Tests that the wallet notifier loop starts and stops correctly based on the
+// addition and removal of wallet event subscriptions.
+func TestWalletNotifierLifecycle(t *testing.T) {
+	// Create a temporary kesytore to test with
+	dir, ks := tmpKeyStore(t, false)
+	defer os.RemoveAll(dir)
+
+	// Ensure that the notification updater is not running yet
+	time.Sleep(250 * time.Millisecond)
+	ks.mu.RLock()
+	updating := ks.updating
+	ks.mu.RUnlock()
+
+	if updating {
+		t.Errorf("wallet notifier running without subscribers")
+	}
+	// Subscribe to the wallet feed and ensure the updater boots up
+	updates := make(chan accounts.WalletEvent)
+
+	subs := make([]event.Subscription, 2)
+	for i := 0; i < len(subs); i++ {
+		// Create a new subscription
+		subs[i] = ks.Subscribe(updates)
+
+		// Ensure the notifier comes online
+		time.Sleep(250 * time.Millisecond)
+		ks.mu.RLock()
+		updating = ks.updating
+		ks.mu.RUnlock()
+
+		if !updating {
+			t.Errorf("sub %d: wallet notifier not running after subscription", i)
+		}
+	}
+	// Unsubscribe and ensure the updater terminates eventually
+	for i := 0; i < len(subs); i++ {
+		// Close an existing subscription
+		subs[i].Unsubscribe()
+
+		// Ensure the notifier shuts down at and only at the last close
+		for k := 0; k < int(walletRefreshCycle/(250*time.Millisecond))+2; k++ {
+			ks.mu.RLock()
+			updating = ks.updating
+			ks.mu.RUnlock()
+
+			if i < len(subs)-1 && !updating {
+				t.Fatalf("sub %d: event notifier stopped prematurely", i)
+			}
+			if i == len(subs)-1 && !updating {
+				return
+			}
+			time.Sleep(250 * time.Millisecond)
+		}
+	}
+	t.Errorf("wallet notifier didn't terminate after unsubscribe")
+}
+
+// Tests that wallet notifications and correctly fired when accounts are added
+// or deleted from the keystore.
+func TestWalletNotifications(t *testing.T) {
+	// Create a temporary kesytore to test with
+	dir, ks := tmpKeyStore(t, false)
+	defer os.RemoveAll(dir)
+
+	// Subscribe to the wallet feed
+	updates := make(chan accounts.WalletEvent, 1)
+	sub := ks.Subscribe(updates)
+	defer sub.Unsubscribe()
+
+	// Randomly add and remove account and make sure events and wallets are in sync
+	live := make(map[common.Address]accounts.Account)
+	for i := 0; i < 1024; i++ {
+		// Execute a creation or deletion and ensure event arrival
+		if create := len(live) == 0 || rand.Int()%4 > 0; create {
+			// Add a new account and ensure wallet notifications arrives
+			account, err := ks.NewAccount("")
+			if err != nil {
+				t.Fatalf("failed to create test account: %v", err)
+			}
+			select {
+			case event := <-updates:
+				if !event.Arrive {
+					t.Errorf("departure event on account creation")
+				}
+				if event.Wallet.Accounts()[0] != account {
+					t.Errorf("account mismatch on created wallet: have %v, want %v", event.Wallet.Accounts()[0], account)
+				}
+			default:
+				t.Errorf("wallet arrival event not fired on account creation")
+			}
+			live[account.Address] = account
+		} else {
+			// Select a random account to delete (crude, but works)
+			var account accounts.Account
+			for _, a := range live {
+				account = a
+				break
+			}
+			// Remove an account and ensure wallet notifiaction arrives
+			if err := ks.Delete(account, ""); err != nil {
+				t.Fatalf("failed to delete test account: %v", err)
+			}
+			select {
+			case event := <-updates:
+				if event.Arrive {
+					t.Errorf("arrival event on account deletion")
+				}
+				if event.Wallet.Accounts()[0] != account {
+					t.Errorf("account mismatch on deleted wallet: have %v, want %v", event.Wallet.Accounts()[0], account)
+				}
+			default:
+				t.Errorf("wallet departure event not fired on account creation")
+			}
+			delete(live, account.Address)
+		}
+		// Retrieve the list of wallets and ensure it matches with our required live set
+		liveList := make([]accounts.Account, 0, len(live))
+		for _, account := range live {
+			liveList = append(liveList, account)
+		}
+		sort.Sort(accountsByURL(liveList))
+
+		wallets := ks.Wallets()
+		if len(liveList) != len(wallets) {
+			t.Errorf("wallet list doesn't match required accounts: have %v, want %v", wallets, liveList)
+		} else {
+			for j, wallet := range wallets {
+				if accs := wallet.Accounts(); len(accs) != 1 {
+					t.Errorf("wallet %d: contains invalid number of accounts: have %d, want 1", j, len(accs))
+				} else if accs[0] != liveList[j] {
+					t.Errorf("wallet %d: account mismatch: have %v, want %v", j, accs[0], liveList[j])
+				}
+			}
+		}
+	}
+}
+
+func tmpKeyStore(t *testing.T, encrypted bool) (string, *KeyStore) {
+	d, err := ioutil.TempDir("", "eth-keystore-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	new := NewPlaintextKeyStore
+	if encrypted {
+		new = func(kd string) *KeyStore { return NewKeyStore(kd, veryLightScryptN, veryLightScryptP) }
+	}
+	return d, new(d)
+}

accounts/keystore/keystore_wallet.go

@@ -0,0 +1,139 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package keystore
+
+import (
+	"math/big"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+// keystoreWallet implements the accounts.Wallet interface for the original
+// keystore.
+type keystoreWallet struct {
+	account  accounts.Account // Single account contained in this wallet
+	keystore *KeyStore        // Keystore where the account originates from
+}
+
+// URL implements accounts.Wallet, returning the URL of the account within.
+func (w *keystoreWallet) URL() accounts.URL {
+	return w.account.URL
+}
+
+// Status implements accounts.Wallet, always returning "open", since there is no
+// concept of open/close for plain keystore accounts.
+func (w *keystoreWallet) Status() string {
+	w.keystore.mu.RLock()
+	defer w.keystore.mu.RUnlock()
+
+	if _, ok := w.keystore.unlocked[w.account.Address]; ok {
+		return "Unlocked"
+	}
+	return "Locked"
+}
+
+// Open implements accounts.Wallet, but is a noop for plain wallets since there
+// is no connection or decryption step necessary to access the list of accounts.
+func (w *keystoreWallet) Open(passphrase string) error { return nil }
+
+// Close implements accounts.Wallet, but is a noop for plain wallets since is no
+// meaningful open operation.
+func (w *keystoreWallet) Close() error { return nil }
+
+// Accounts implements accounts.Wallet, returning an account list consisting of
+// a single account that the plain kestore wallet contains.
+func (w *keystoreWallet) Accounts() []accounts.Account {
+	return []accounts.Account{w.account}
+}
+
+// Contains implements accounts.Wallet, returning whether a particular account is
+// or is not wrapped by this wallet instance.
+func (w *keystoreWallet) Contains(account accounts.Account) bool {
+	return account.Address == w.account.Address && (account.URL == (accounts.URL{}) || account.URL == w.account.URL)
+}
+
+// Derive implements accounts.Wallet, but is a noop for plain wallets since there
+// is no notion of hierarchical account derivation for plain keystore accounts.
+func (w *keystoreWallet) Derive(path accounts.DerivationPath, pin bool) (accounts.Account, error) {
+	return accounts.Account{}, accounts.ErrNotSupported
+}
+
+// SelfDerive implements accounts.Wallet, but is a noop for plain wallets since
+// there is no notion of hierarchical account derivation for plain keystore accounts.
+func (w *keystoreWallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {}
+
+// SignHash implements accounts.Wallet, attempting to sign the given hash with
+// the given account. If the wallet does not wrap this particular account, an
+// error is returned to avoid account leakage (even though in theory we may be
+// able to sign via our shared keystore backend).
+func (w *keystoreWallet) SignHash(account accounts.Account, hash []byte) ([]byte, error) {
+	// Make sure the requested account is contained within
+	if account.Address != w.account.Address {
+		return nil, accounts.ErrUnknownAccount
+	}
+	if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignHash(account, hash)
+}
+
+// SignTx implements accounts.Wallet, attempting to sign the given transaction
+// with the given account. If the wallet does not wrap this particular account,
+// an error is returned to avoid account leakage (even though in theory we may
+// be able to sign via our shared keystore backend).
+func (w *keystoreWallet) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	// Make sure the requested account is contained within
+	if account.Address != w.account.Address {
+		return nil, accounts.ErrUnknownAccount
+	}
+	if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignTx(account, tx, chainID)
+}
+
+// SignHashWithPassphrase implements accounts.Wallet, attempting to sign the
+// given hash with the given account using passphrase as extra authentication.
+func (w *keystoreWallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
+	// Make sure the requested account is contained within
+	if account.Address != w.account.Address {
+		return nil, accounts.ErrUnknownAccount
+	}
+	if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignHashWithPassphrase(account, passphrase, hash)
+}
+
+// SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given
+// transaction with the given account using passphrase as extra authentication.
+func (w *keystoreWallet) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	// Make sure the requested account is contained within
+	if account.Address != w.account.Address {
+		return nil, accounts.ErrUnknownAccount
+	}
+	if account.URL != (accounts.URL{}) && account.URL != w.account.URL {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignTxWithPassphrase(account, passphrase, tx, chainID)
+}

accounts/keystore/presale.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package accounts
+package keystore
 
 import (
 	"crypto/aes"
@@ -22,22 +22,24 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 
+	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/pborman/uuid"
 	"golang.org/x/crypto/pbkdf2"
 )
 
 // creates a Key and stores that in the given KeyStore by decrypting a presale key JSON
-func importPreSaleKey(keyStore keyStore, keyJSON []byte, password string) (Account, *Key, error) {
+func importPreSaleKey(keyStore keyStore, keyJSON []byte, password string) (accounts.Account, *Key, error) {
 	key, err := decryptPreSaleKey(keyJSON, password)
 	if err != nil {
-		return Account{}, nil, err
+		return accounts.Account{}, nil, err
 	}
 	key.Id = uuid.NewRandom()
-	a := Account{Address: key.Address, File: keyStore.JoinPath(keyFileName(key.Address))}
-	err = keyStore.StoreKey(a.File, key, password)
+	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: keyStore.JoinPath(keyFileName(key.Address))}}
+	err = keyStore.StoreKey(a.URL.Path, key, password)
 	return a, key, err
 }
 
@@ -53,6 +55,9 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error
 		return nil, err
 	}
 	encSeedBytes, err := hex.DecodeString(preSaleKeyStruct.EncSeed)
+	if err != nil {
+		return nil, errors.New("invalid hex in encSeed")
+	}
 	iv := encSeedBytes[:16]
 	cipherText := encSeedBytes[16:]
 	/*

accounts/keystore/watch.go

@@ -16,7 +16,7 @@
 
 // +build darwin,!ios freebsd linux,!arm64 netbsd solaris
 
-package accounts
+package keystore
 
 import (
 	"time"
@@ -27,14 +27,14 @@ import (
 )
 
 type watcher struct {
-	ac       *addrCache
+	ac       *accountCache
 	starting bool
 	running  bool
 	ev       chan notify.EventInfo
 	quit     chan struct{}
 }
 
-func newWatcher(ac *addrCache) *watcher {
+func newWatcher(ac *accountCache) *watcher {
 	return &watcher{
 		ac:   ac,
 		ev:   make(chan notify.EventInfo, 10),

accounts/keystore/watch_fallback.go

@@ -19,10 +19,10 @@
 // This is the fallback implementation of directory watching.
 // It is used on unsupported platforms.
 
-package accounts
+package keystore
 
 type watcher struct{ running bool }
 
-func newWatcher(*addrCache) *watcher { return new(watcher) }
-func (*watcher) start()              {}
-func (*watcher) close()              {}
+func newWatcher(*accountCache) *watcher { return new(watcher) }
+func (*watcher) start()                 {}
+func (*watcher) close()                 {}

accounts/manager.go

@@ -0,0 +1,198 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package accounts
+
+import (
+	"reflect"
+	"sort"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/event"
+)
+
+// Manager is an overarching account manager that can communicate with various
+// backends for signing transactions.
+type Manager struct {
+	backends map[reflect.Type][]Backend // Index of backends currently registered
+	updaters []event.Subscription       // Wallet update subscriptions for all backends
+	updates  chan WalletEvent           // Subscription sink for backend wallet changes
+	wallets  []Wallet                   // Cache of all wallets from all registered backends
+
+	feed event.Feed // Wallet feed notifying of arrivals/departures
+
+	quit chan chan error
+	lock sync.RWMutex
+}
+
+// NewManager creates a generic account manager to sign transaction via various
+// supported backends.
+func NewManager(backends ...Backend) *Manager {
+	// Subscribe to wallet notifications from all backends
+	updates := make(chan WalletEvent, 4*len(backends))
+
+	subs := make([]event.Subscription, len(backends))
+	for i, backend := range backends {
+		subs[i] = backend.Subscribe(updates)
+	}
+	// Retrieve the initial list of wallets from the backends and sort by URL
+	var wallets []Wallet
+	for _, backend := range backends {
+		wallets = merge(wallets, backend.Wallets()...)
+	}
+	// Assemble the account manager and return
+	am := &Manager{
+		backends: make(map[reflect.Type][]Backend),
+		updaters: subs,
+		updates:  updates,
+		wallets:  wallets,
+		quit:     make(chan chan error),
+	}
+	for _, backend := range backends {
+		kind := reflect.TypeOf(backend)
+		am.backends[kind] = append(am.backends[kind], backend)
+	}
+	go am.update()
+
+	return am
+}
+
+// Close terminates the account manager's internal notification processes.
+func (am *Manager) Close() error {
+	errc := make(chan error)
+	am.quit <- errc
+	return <-errc
+}
+
+// update is the wallet event loop listening for notifications from the backends
+// and updating the cache of wallets.
+func (am *Manager) update() {
+	// Close all subscriptions when the manager terminates
+	defer func() {
+		am.lock.Lock()
+		for _, sub := range am.updaters {
+			sub.Unsubscribe()
+		}
+		am.updaters = nil
+		am.lock.Unlock()
+	}()
+
+	// Loop until termination
+	for {
+		select {
+		case event := <-am.updates:
+			// Wallet event arrived, update local cache
+			am.lock.Lock()
+			if event.Arrive {
+				am.wallets = merge(am.wallets, event.Wallet)
+			} else {
+				am.wallets = drop(am.wallets, event.Wallet)
+			}
+			am.lock.Unlock()
+
+			// Notify any listeners of the event
+			am.feed.Send(event)
+
+		case errc := <-am.quit:
+			// Manager terminating, return
+			errc <- nil
+			return
+		}
+	}
+}
+
+// Backends retrieves the backend(s) with the given type from the account manager.
+func (am *Manager) Backends(kind reflect.Type) []Backend {
+	return am.backends[kind]
+}
+
+// Wallets returns all signer accounts registered under this account manager.
+func (am *Manager) Wallets() []Wallet {
+	am.lock.RLock()
+	defer am.lock.RUnlock()
+
+	cpy := make([]Wallet, len(am.wallets))
+	copy(cpy, am.wallets)
+	return cpy
+}
+
+// Wallet retrieves the wallet associated with a particular URL.
+func (am *Manager) Wallet(url string) (Wallet, error) {
+	am.lock.RLock()
+	defer am.lock.RUnlock()
+
+	parsed, err := parseURL(url)
+	if err != nil {
+		return nil, err
+	}
+	for _, wallet := range am.Wallets() {
+		if wallet.URL() == parsed {
+			return wallet, nil
+		}
+	}
+	return nil, ErrUnknownWallet
+}
+
+// Find attempts to locate the wallet corresponding to a specific account. Since
+// accounts can be dynamically added to and removed from wallets, this method has
+// a linear runtime in the number of wallets.
+func (am *Manager) Find(account Account) (Wallet, error) {
+	am.lock.RLock()
+	defer am.lock.RUnlock()
+
+	for _, wallet := range am.wallets {
+		if wallet.Contains(account) {
+			return wallet, nil
+		}
+	}
+	return nil, ErrUnknownAccount
+}
+
+// Subscribe creates an async subscription to receive notifications when the
+// manager detects the arrival or departure of a wallet from any of its backends.
+func (am *Manager) Subscribe(sink chan<- WalletEvent) event.Subscription {
+	return am.feed.Subscribe(sink)
+}
+
+// merge is a sorted analogue of append for wallets, where the ordering of the
+// origin list is preserved by inserting new wallets at the correct position.
+//
+// The original slice is assumed to be already sorted by URL.
+func merge(slice []Wallet, wallets ...Wallet) []Wallet {
+	for _, wallet := range wallets {
+		n := sort.Search(len(slice), func(i int) bool { return slice[i].URL().Cmp(wallet.URL()) >= 0 })
+		if n == len(slice) {
+			slice = append(slice, wallet)
+			continue
+		}
+		slice = append(slice[:n], append([]Wallet{wallet}, slice[n:]...)...)
+	}
+	return slice
+}
+
+// drop is the couterpart of merge, which looks up wallets from within the sorted
+// cache and removes the ones specified.
+func drop(slice []Wallet, wallets ...Wallet) []Wallet {
+	for _, wallet := range wallets {
+		n := sort.Search(len(slice), func(i int) bool { return slice[i].URL().Cmp(wallet.URL()) >= 0 })
+		if n == len(slice) {
+			// Wallet not found, may happen during startup
+			continue
+		}
+		slice = append(slice[:n], slice[n+1:]...)
+	}
+	return slice
+}

accounts/url.go

@@ -0,0 +1,79 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package accounts
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+)
+
+// URL represents the canonical identification URL of a wallet or account.
+//
+// It is a simplified version of url.URL, with the important limitations (which
+// are considered features here) that it contains value-copyable components only,
+// as well as that it doesn't do any URL encoding/decoding of special characters.
+//
+// The former is important to allow an account to be copied without leaving live
+// references to the original version, whereas the latter is important to ensure
+// one single canonical form opposed to many allowed ones by the RFC 3986 spec.
+//
+// As such, these URLs should not be used outside of the scope of an Ethereum
+// wallet or account.
+type URL struct {
+	Scheme string // Protocol scheme to identify a capable account backend
+	Path   string // Path for the backend to identify a unique entity
+}
+
+// parseURL converts a user supplied URL into the accounts specific structure.
+func parseURL(url string) (URL, error) {
+	parts := strings.Split(url, "://")
+	if len(parts) != 2 || parts[0] == "" {
+		return URL{}, errors.New("protocol scheme missing")
+	}
+	return URL{
+		Scheme: parts[0],
+		Path:   parts[1],
+	}, nil
+}
+
+// String implements the stringer interface.
+func (u URL) String() string {
+	if u.Scheme != "" {
+		return fmt.Sprintf("%s://%s", u.Scheme, u.Path)
+	}
+	return u.Path
+}
+
+// MarshalJSON implements the json.Marshaller interface.
+func (u URL) MarshalJSON() ([]byte, error) {
+	return json.Marshal(u.String())
+}
+
+// Cmp compares x and y and returns:
+//
+//   -1 if x <  y
+//    0 if x == y
+//   +1 if x >  y
+//
+func (u URL) Cmp(url URL) int {
+	if u.Scheme == url.Scheme {
+		return strings.Compare(u.Path, url.Path)
+	}
+	return strings.Compare(u.Scheme, url.Scheme)
+}

accounts/usbwallet/ledger_hub.go

@@ -0,0 +1,209 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// This file contains the implementation for interacting with the Ledger hardware
+// wallets. The wire protocol spec can be found in the Ledger Blue GitHub repo:
+// https://raw.githubusercontent.com/LedgerHQ/blue-app-eth/master/doc/ethapp.asc
+
+// +build !ios
+
+package usbwallet
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/karalabe/gousb/usb"
+)
+
+// LedgerScheme is the protocol scheme prefixing account and wallet URLs.
+var LedgerScheme = "ledger"
+
+// ledgerDeviceIDs are the known device IDs that Ledger wallets use.
+var ledgerDeviceIDs = []deviceID{
+	{Vendor: 0x2c97, Product: 0x0000}, // Ledger Blue
+	{Vendor: 0x2c97, Product: 0x0001}, // Ledger Nano S
+}
+
+// Maximum time between wallet refreshes (if USB hotplug notifications don't work).
+const ledgerRefreshCycle = time.Second
+
+// Minimum time between wallet refreshes to avoid USB trashing.
+const ledgerRefreshThrottling = 500 * time.Millisecond
+
+// LedgerHub is a accounts.Backend that can find and handle Ledger hardware wallets.
+type LedgerHub struct {
+	ctx *usb.Context // Context interfacing with a libusb instance
+
+	refreshed   time.Time               // Time instance when the list of wallets was last refreshed
+	wallets     []accounts.Wallet       // List of Ledger devices currently tracking
+	updateFeed  event.Feed              // Event feed to notify wallet additions/removals
+	updateScope event.SubscriptionScope // Subscription scope tracking current live listeners
+	updating    bool                    // Whether the event notification loop is running
+
+	quit chan chan error
+	lock sync.RWMutex
+}
+
+// NewLedgerHub creates a new hardware wallet manager for Ledger devices.
+func NewLedgerHub() (*LedgerHub, error) {
+	// Initialize the USB library to access Ledgers through
+	ctx, err := usb.NewContext()
+	if err != nil {
+		return nil, err
+	}
+	// Create the USB hub, start and return it
+	hub := &LedgerHub{
+		ctx:  ctx,
+		quit: make(chan chan error),
+	}
+	hub.refreshWallets()
+
+	return hub, nil
+}
+
+// Wallets implements accounts.Backend, returning all the currently tracked USB
+// devices that appear to be Ledger hardware wallets.
+func (hub *LedgerHub) Wallets() []accounts.Wallet {
+	// Make sure the list of wallets is up to date
+	hub.refreshWallets()
+
+	hub.lock.RLock()
+	defer hub.lock.RUnlock()
+
+	cpy := make([]accounts.Wallet, len(hub.wallets))
+	copy(cpy, hub.wallets)
+	return cpy
+}
+
+// refreshWallets scans the USB devices attached to the machine and updates the
+// list of wallets based on the found devices.
+func (hub *LedgerHub) refreshWallets() {
+	// Don't scan the USB like crazy it the user fetches wallets in a loop
+	hub.lock.RLock()
+	elapsed := time.Since(hub.refreshed)
+	hub.lock.RUnlock()
+
+	if elapsed < ledgerRefreshThrottling {
+		return
+	}
+	// Retrieve the current list of Ledger devices
+	var devIDs []deviceID
+	var busIDs []uint16
+
+	hub.ctx.ListDevices(func(desc *usb.Descriptor) bool {
+		// Gather Ledger devices, don't connect any just yet
+		for _, id := range ledgerDeviceIDs {
+			if desc.Vendor == id.Vendor && desc.Product == id.Product {
+				devIDs = append(devIDs, deviceID{Vendor: desc.Vendor, Product: desc.Product})
+				busIDs = append(busIDs, uint16(desc.Bus)<<8+uint16(desc.Address))
+				return false
+			}
+		}
+		// Not ledger, ignore and don't connect either
+		return false
+	})
+	// Transform the current list of wallets into the new one
+	hub.lock.Lock()
+
+	wallets := make([]accounts.Wallet, 0, len(devIDs))
+	events := []accounts.WalletEvent{}
+
+	for i := 0; i < len(devIDs); i++ {
+		devID, busID := devIDs[i], busIDs[i]
+
+		url := accounts.URL{Scheme: LedgerScheme, Path: fmt.Sprintf("%03d:%03d", busID>>8, busID&0xff)}
+
+		// Drop wallets in front of the next device or those that failed for some reason
+		for len(hub.wallets) > 0 && (hub.wallets[0].URL().Cmp(url) < 0 || hub.wallets[0].(*ledgerWallet).failed()) {
+			events = append(events, accounts.WalletEvent{Wallet: hub.wallets[0], Arrive: false})
+			hub.wallets = hub.wallets[1:]
+		}
+		// If there are no more wallets or the device is before the next, wrap new wallet
+		if len(hub.wallets) == 0 || hub.wallets[0].URL().Cmp(url) > 0 {
+			wallet := &ledgerWallet{context: hub.ctx, hardwareID: devID, locationID: busID, url: &url}
+
+			events = append(events, accounts.WalletEvent{Wallet: wallet, Arrive: true})
+			wallets = append(wallets, wallet)
+			continue
+		}
+		// If the device is the same as the first wallet, keep it
+		if hub.wallets[0].URL().Cmp(url) == 0 {
+			wallets = append(wallets, hub.wallets[0])
+			hub.wallets = hub.wallets[1:]
+			continue
+		}
+	}
+	// Drop any leftover wallets and set the new batch
+	for _, wallet := range hub.wallets {
+		events = append(events, accounts.WalletEvent{Wallet: wallet, Arrive: false})
+	}
+	hub.refreshed = time.Now()
+	hub.wallets = wallets
+	hub.lock.Unlock()
+
+	// Fire all wallet events and return
+	for _, event := range events {
+		hub.updateFeed.Send(event)
+	}
+}
+
+// Subscribe implements accounts.Backend, creating an async subscription to
+// receive notifications on the addition or removal of Ledger wallets.
+func (hub *LedgerHub) Subscribe(sink chan<- accounts.WalletEvent) event.Subscription {
+	// We need the mutex to reliably start/stop the update loop
+	hub.lock.Lock()
+	defer hub.lock.Unlock()
+
+	// Subscribe the caller and track the subscriber count
+	sub := hub.updateScope.Track(hub.updateFeed.Subscribe(sink))
+
+	// Subscribers require an active notification loop, start it
+	if !hub.updating {
+		hub.updating = true
+		go hub.updater()
+	}
+	return sub
+}
+
+// updater is responsible for maintaining an up-to-date list of wallets stored in
+// the keystore, and for firing wallet addition/removal events. It listens for
+// account change events from the underlying account cache, and also periodically
+// forces a manual refresh (only triggers for systems where the filesystem notifier
+// is not running).
+func (hub *LedgerHub) updater() {
+	for {
+		// Wait for a USB hotplug event (not supported yet) or a refresh timeout
+		select {
+		//case <-hub.changes: // reenable on hutplug implementation
+		case <-time.After(ledgerRefreshCycle):
+		}
+		// Run the wallet refresher
+		hub.refreshWallets()
+
+		// If all our subscribers left, stop the updater
+		hub.lock.Lock()
+		if hub.updateScope.Count() == 0 {
+			hub.updating = false
+			hub.lock.Unlock()
+			return
+		}
+		hub.lock.Unlock()
+	}
+}

accounts/usbwallet/ledger_wallet.go

@@ -0,0 +1,945 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// This file contains the implementation for interacting with the Ledger hardware
+// wallets. The wire protocol spec can be found in the Ledger Blue GitHub repo:
+// https://raw.githubusercontent.com/LedgerHQ/blue-app-eth/master/doc/ethapp.asc
+
+// +build !ios
+
+package usbwallet
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"math/big"
+	"sync"
+	"time"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/karalabe/gousb/usb"
+	"golang.org/x/net/context"
+)
+
+// Maximum time between wallet health checks to detect USB unplugs.
+const ledgerHeartbeatCycle = time.Second
+
+// Minimum time to wait between self derivation attempts, even it the user is
+// requesting accounts like crazy.
+const ledgerSelfDeriveThrottling = time.Second
+
+// ledgerOpcode is an enumeration encoding the supported Ledger opcodes.
+type ledgerOpcode byte
+
+// ledgerParam1 is an enumeration encoding the supported Ledger parameters for
+// specific opcodes. The same parameter values may be reused between opcodes.
+type ledgerParam1 byte
+
+// ledgerParam2 is an enumeration encoding the supported Ledger parameters for
+// specific opcodes. The same parameter values may be reused between opcodes.
+type ledgerParam2 byte
+
+const (
+	ledgerOpRetrieveAddress  ledgerOpcode = 0x02 // Returns the public key and Ethereum address for a given BIP 32 path
+	ledgerOpSignTransaction  ledgerOpcode = 0x04 // Signs an Ethereum transaction after having the user validate the parameters
+	ledgerOpGetConfiguration ledgerOpcode = 0x06 // Returns specific wallet application configuration
+
+	ledgerP1DirectlyFetchAddress    ledgerParam1 = 0x00 // Return address directly from the wallet
+	ledgerP1ConfirmFetchAddress     ledgerParam1 = 0x01 // Require a user confirmation before returning the address
+	ledgerP1InitTransactionData     ledgerParam1 = 0x00 // First transaction data block for signing
+	ledgerP1ContTransactionData     ledgerParam1 = 0x80 // Subsequent transaction data block for signing
+	ledgerP2DiscardAddressChainCode ledgerParam2 = 0x00 // Do not return the chain code along with the address
+	ledgerP2ReturnAddressChainCode  ledgerParam2 = 0x01 // Require a user confirmation before returning the address
+)
+
+// errReplyInvalidHeader is the error message returned by a Ledfer data exchange
+// if the device replies with a mismatching header. This usually means the device
+// is in browser mode.
+var errReplyInvalidHeader = errors.New("invalid reply header")
+
+// ledgerWallet represents a live USB Ledger hardware wallet.
+type ledgerWallet struct {
+	context    *usb.Context  // USB context to interface libusb through
+	hardwareID deviceID      // USB identifiers to identify this device type
+	locationID uint16        // USB bus and address to identify this device instance
+	url        *accounts.URL // Textual URL uniquely identifying this wallet
+
+	device  *usb.Device  // USB device advertising itself as a Ledger wallet
+	input   usb.Endpoint // Input endpoint to send data to this device
+	output  usb.Endpoint // Output endpoint to receive data from this device
+	failure error        // Any failure that would make the device unusable
+
+	version  [3]byte                                    // Current version of the Ledger Ethereum app (zero if app is offline)
+	browser  bool                                       // Flag whether the Ledger is in browser mode (reply channel mismatch)
+	accounts []accounts.Account                         // List of derive accounts pinned on the Ledger
+	paths    map[common.Address]accounts.DerivationPath // Known derivation paths for signing operations
+
+	deriveNextPath accounts.DerivationPath   // Next derivation path for account auto-discovery
+	deriveNextAddr common.Address            // Next derived account address for auto-discovery
+	deriveChain    ethereum.ChainStateReader // Blockchain state reader to discover used account with
+	deriveReq      chan chan struct{}        // Channel to request a self-derivation on
+	deriveQuit     chan chan error           // Channel to terminate the self-deriver with
+
+	healthQuit chan chan error
+
+	// Locking a hardware wallet is a bit special. Since hardware devices are lower
+	// performing, any communication with them might take a non negligible amount of
+	// time. Worse still, waiting for user confirmation can take arbitrarily long,
+	// but exclusive communication must be upheld during. Locking the entire wallet
+	// in the mean time however would stall any parts of the system that don't want
+	// to communicate, just read some state (e.g. list the accounts).
+	//
+	// As such, a hardware wallet needs two locks to function correctly. A state
+	// lock can be used to protect the wallet's software-side internal state, which
+	// must not be held exlusively during hardware communication. A communication
+	// lock can be used to achieve exclusive access to the device itself, this one
+	// however should allow "skipping" waiting for operations that might want to
+	// use the device, but can live without too (e.g. account self-derivation).
+	//
+	// Since we have two locks, it's important to know how to properly use them:
+	//   - Communication requires the `device` to not change, so obtaining the
+	//     commsLock should be done after having a stateLock.
+	//   - Communication must not disable read access to the wallet state, so it
+	//     must only ever hold a *read* lock to stateLock.
+	commsLock chan struct{} // Mutex (buf=1) for the USB comms without keeping the state locked
+	stateLock sync.RWMutex  // Protects read and write access to the wallet struct fields
+}
+
+// URL implements accounts.Wallet, returning the URL of the Ledger device.
+func (w *ledgerWallet) URL() accounts.URL {
+	return *w.url // Immutable, no need for a lock
+}
+
+// Status implements accounts.Wallet, always whether the Ledger is opened, closed
+// or whether the Ethereum app was not started on it.
+func (w *ledgerWallet) Status() string {
+	w.stateLock.RLock() // No device communication, state lock is enough
+	defer w.stateLock.RUnlock()
+
+	if w.failure != nil {
+		return fmt.Sprintf("Failed: %v", w.failure)
+	}
+	if w.device == nil {
+		return "Closed"
+	}
+	if w.browser {
+		return "Ethereum app in browser mode"
+	}
+	if w.offline() {
+		return "Ethereum app offline"
+	}
+	return fmt.Sprintf("Ethereum app v%d.%d.%d online", w.version[0], w.version[1], w.version[2])
+}
+
+// offline returns whether the wallet and the Ethereum app is offline or not.
+//
+// The method assumes that the state lock is held!
+func (w *ledgerWallet) offline() bool {
+	return w.version == [3]byte{0, 0, 0}
+}
+
+// failed returns if the USB device wrapped by the wallet failed for some reason.
+// This is used by the device scanner to report failed wallets as departed.
+//
+// The method assumes that the state lock is *not* held!
+func (w *ledgerWallet) failed() bool {
+	w.stateLock.RLock() // No device communication, state lock is enough
+	defer w.stateLock.RUnlock()
+
+	return w.failure != nil
+}
+
+// Open implements accounts.Wallet, attempting to open a USB connection to the
+// Ledger hardware wallet. The Ledger does not require a user passphrase, so that
+// parameter is silently discarded.
+func (w *ledgerWallet) Open(passphrase string) error {
+	w.stateLock.Lock() // State lock is enough since there's no connection yet at this point
+	defer w.stateLock.Unlock()
+
+	// If the wallet was already opened, don't try to open again
+	if w.device != nil {
+		return accounts.ErrWalletAlreadyOpen
+	}
+	// Otherwise iterate over all USB devices and find this again (no way to directly do this)
+	// Iterate over all attached devices and fetch those seemingly Ledger
+	devices, err := w.context.ListDevices(func(desc *usb.Descriptor) bool {
+		// Only open this single specific device
+		return desc.Vendor == w.hardwareID.Vendor && desc.Product == w.hardwareID.Product &&
+			uint16(desc.Bus)<<8+uint16(desc.Address) == w.locationID
+	})
+	if err != nil {
+		return err
+	}
+	if len(devices) == 0 {
+		return accounts.ErrUnknownWallet
+	}
+	// Device opened, attach to the input and output endpoints
+	device := devices[0]
+
+	var invalid string
+	switch {
+	case len(device.Descriptor.Configs) == 0:
+		invalid = "no endpoint config available"
+	case len(device.Descriptor.Configs[0].Interfaces) == 0:
+		invalid = "no endpoint interface available"
+	case len(device.Descriptor.Configs[0].Interfaces[0].Setups) == 0:
+		invalid = "no endpoint setup available"
+	case len(device.Descriptor.Configs[0].Interfaces[0].Setups[0].Endpoints) < 2:
+		invalid = "not enough IO endpoints available"
+	}
+	if invalid != "" {
+		device.Close()
+		return fmt.Errorf("ledger wallet [%s] invalid: %s", w.url, invalid)
+	}
+	// Open the input and output endpoints to the device
+	input, err := device.OpenEndpoint(
+		device.Descriptor.Configs[0].Config,
+		device.Descriptor.Configs[0].Interfaces[0].Number,
+		device.Descriptor.Configs[0].Interfaces[0].Setups[0].Number,
+		device.Descriptor.Configs[0].Interfaces[0].Setups[0].Endpoints[1].Address,
+	)
+	if err != nil {
+		device.Close()
+		return fmt.Errorf("ledger wallet [%s] input open failed: %v", w.url, err)
+	}
+	output, err := device.OpenEndpoint(
+		device.Descriptor.Configs[0].Config,
+		device.Descriptor.Configs[0].Interfaces[0].Number,
+		device.Descriptor.Configs[0].Interfaces[0].Setups[0].Number,
+		device.Descriptor.Configs[0].Interfaces[0].Setups[0].Endpoints[0].Address,
+	)
+	if err != nil {
+		device.Close()
+		return fmt.Errorf("ledger wallet [%s] output open failed: %v", w.url, err)
+	}
+	// Wallet seems to be successfully opened, guess if the Ethereum app is running
+	w.device, w.input, w.output = device, input, output
+	w.commsLock = make(chan struct{}, 1)
+	w.commsLock <- struct{}{} // Enable lock
+
+	w.paths = make(map[common.Address]accounts.DerivationPath)
+
+	w.deriveReq = make(chan chan struct{})
+	w.deriveQuit = make(chan chan error)
+	w.healthQuit = make(chan chan error)
+
+	defer func() {
+		go w.heartbeat()
+		go w.selfDerive()
+	}()
+
+	if _, err = w.ledgerDerive(accounts.DefaultBaseDerivationPath); err != nil {
+		// Ethereum app is not running or in browser mode, nothing more to do, return
+		if err == errReplyInvalidHeader {
+			w.browser = true
+		}
+		return nil
+	}
+	// Try to resolve the Ethereum app's version, will fail prior to v1.0.2
+	if w.version, err = w.ledgerVersion(); err != nil {
+		w.version = [3]byte{1, 0, 0} // Assume worst case, can't verify if v1.0.0 or v1.0.1
+	}
+	return nil
+}
+
+// heartbeat is a health check loop for the Ledger wallets to periodically verify
+// whether they are still present or if they malfunctioned. It is needed because:
+//  - libusb on Windows doesn't support hotplug, so we can't detect USB unplugs
+//  - communication timeout on the Ledger requires a device power cycle to fix
+func (w *ledgerWallet) heartbeat() {
+	glog.V(logger.Debug).Infof("%s health-check started", w.url.String())
+	defer glog.V(logger.Debug).Infof("%s health-check stopped", w.url.String())
+
+	// Execute heartbeat checks until termination or error
+	var (
+		errc chan error
+		err  error
+	)
+	for errc == nil && err == nil {
+		// Wait until termination is requested or the heartbeat cycle arrives
+		select {
+		case errc = <-w.healthQuit:
+			// Termination requested
+			continue
+		case <-time.After(ledgerHeartbeatCycle):
+			// Heartbeat time
+		}
+		// Execute a tiny data exchange to see responsiveness
+		w.stateLock.RLock()
+		if w.device == nil {
+			// Terminated while waiting for the lock
+			w.stateLock.RUnlock()
+			continue
+		}
+		<-w.commsLock // Don't lock state while resolving version
+		_, err = w.ledgerVersion()
+		w.commsLock <- struct{}{}
+		w.stateLock.RUnlock()
+
+		if err == usb.ERROR_IO || err == usb.ERROR_NO_DEVICE {
+			w.stateLock.Lock() // Lock state to tear the wallet down
+			w.failure = err
+			w.close()
+			w.stateLock.Unlock()
+		}
+		// Ignore uninteresting errors
+		err = nil
+	}
+	// In case of error, wait for termination
+	if err != nil {
+		glog.V(logger.Debug).Infof("%s health-check failed: %v", w.url.String(), err)
+		errc = <-w.healthQuit
+	}
+	errc <- err
+}
+
+// Close implements accounts.Wallet, closing the USB connection to the Ledger.
+func (w *ledgerWallet) Close() error {
+	// Ensure the wallet was opened
+	w.stateLock.RLock()
+	hQuit, dQuit := w.healthQuit, w.deriveQuit
+	w.stateLock.RUnlock()
+
+	// Terminate the health checks
+	var herr error
+	if hQuit != nil {
+		errc := make(chan error)
+		hQuit <- errc
+		herr = <-errc // Save for later, we *must* close the USB
+	}
+	// Terminate the self-derivations
+	var derr error
+	if dQuit != nil {
+		errc := make(chan error)
+		dQuit <- errc
+		derr = <-errc // Save for later, we *must* close the USB
+	}
+	// Terminate the device connection
+	w.stateLock.Lock()
+	defer w.stateLock.Unlock()
+
+	w.healthQuit = nil
+	w.deriveQuit = nil
+	w.deriveReq = nil
+
+	if err := w.close(); err != nil {
+		return err
+	}
+	if herr != nil {
+		return herr
+	}
+	return derr
+}
+
+// close is the internal wallet closer that terminates the USB connection and
+// resets all the fields to their defaults.
+//
+// Note, close assumes the state lock is held!
+func (w *ledgerWallet) close() error {
+	// Allow duplicate closes, especially for health-check failures
+	if w.device == nil {
+		return nil
+	}
+	// Close the device, clear everything, then return
+	err := w.device.Close()
+
+	w.device, w.input, w.output = nil, nil, nil
+	w.browser, w.version = false, [3]byte{}
+	w.accounts, w.paths = nil, nil
+
+	return err
+}
+
+// Accounts implements accounts.Wallet, returning the list of accounts pinned to
+// the Ledger hardware wallet. If self-derivation was enabled, the account list
+// is periodically expanded based on current chain state.
+func (w *ledgerWallet) Accounts() []accounts.Account {
+	// Attempt self-derivation if it's running
+	reqc := make(chan struct{}, 1)
+	select {
+	case w.deriveReq <- reqc:
+		// Self-derivation request accepted, wait for it
+		<-reqc
+	default:
+		// Self-derivation offline, throttled or busy, skip
+	}
+	// Return whatever account list we ended up with
+	w.stateLock.RLock()
+	defer w.stateLock.RUnlock()
+
+	cpy := make([]accounts.Account, len(w.accounts))
+	copy(cpy, w.accounts)
+	return cpy
+}
+
+// selfDerive is an account derivation loop that upon request attempts to find
+// new non-zero accounts.
+func (w *ledgerWallet) selfDerive() {
+	glog.V(logger.Debug).Infof("%s self-derivation started", w.url.String())
+	defer glog.V(logger.Debug).Infof("%s self-derivation stopped", w.url.String())
+
+	// Execute self-derivations until termination or error
+	var (
+		reqc chan struct{}
+		errc chan error
+		err  error
+	)
+	for errc == nil && err == nil {
+		// Wait until either derivation or termination is requested
+		select {
+		case errc = <-w.deriveQuit:
+			// Termination requested
+			continue
+		case reqc = <-w.deriveReq:
+			// Account discovery requested
+		}
+		// Derivation needs a chain and device access, skip if either unavailable
+		w.stateLock.RLock()
+		if w.device == nil || w.deriveChain == nil || w.offline() {
+			w.stateLock.RUnlock()
+			reqc <- struct{}{}
+			continue
+		}
+		select {
+		case <-w.commsLock:
+		default:
+			w.stateLock.RUnlock()
+			reqc <- struct{}{}
+			continue
+		}
+		// Device lock obtained, derive the next batch of accounts
+		var (
+			accs  []accounts.Account
+			paths []accounts.DerivationPath
+
+			nextAddr = w.deriveNextAddr
+			nextPath = w.deriveNextPath
+
+			context = context.Background()
+		)
+		for empty := false; !empty; {
+			// Retrieve the next derived Ethereum account
+			if nextAddr == (common.Address{}) {
+				if nextAddr, err = w.ledgerDerive(nextPath); err != nil {
+					glog.V(logger.Warn).Infof("%s self-derivation failed: %v", w.url.String(), err)
+					break
+				}
+			}
+			// Check the account's status against the current chain state
+			var (
+				balance *big.Int
+				nonce   uint64
+			)
+			balance, err = w.deriveChain.BalanceAt(context, nextAddr, nil)
+			if err != nil {
+				glog.V(logger.Warn).Infof("%s self-derivation balance retrieval failed: %v", w.url.String(), err)
+				break
+			}
+			nonce, err = w.deriveChain.NonceAt(context, nextAddr, nil)
+			if err != nil {
+				glog.V(logger.Warn).Infof("%s self-derivation nonce retrieval failed: %v", w.url.String(), err)
+				break
+			}
+			// If the next account is empty, stop self-derivation, but add it nonetheless
+			if balance.BitLen() == 0 && nonce == 0 {
+				empty = true
+			}
+			// We've just self-derived a new account, start tracking it locally
+			path := make(accounts.DerivationPath, len(nextPath))
+			copy(path[:], nextPath[:])
+			paths = append(paths, path)
+
+			account := accounts.Account{
+				Address: nextAddr,
+				URL:     accounts.URL{Scheme: w.url.Scheme, Path: fmt.Sprintf("%s/%s", w.url.Path, path)},
+			}
+			accs = append(accs, account)
+
+			// Display a log message to the user for new (or previously empty accounts)
+			if _, known := w.paths[nextAddr]; !known || (!empty && nextAddr == w.deriveNextAddr) {
+				glog.V(logger.Info).Infof("%s discovered %s (balance %22v, nonce %4d) at %s", w.url.String(), nextAddr.Hex(), balance, nonce, path)
+			}
+			// Fetch the next potential account
+			if !empty {
+				nextAddr = common.Address{}
+				nextPath[len(nextPath)-1]++
+			}
+		}
+		// Self derivation complete, release device lock
+		w.commsLock <- struct{}{}
+		w.stateLock.RUnlock()
+
+		// Insert any accounts successfully derived
+		w.stateLock.Lock()
+		for i := 0; i < len(accs); i++ {
+			if _, ok := w.paths[accs[i].Address]; !ok {
+				w.accounts = append(w.accounts, accs[i])
+				w.paths[accs[i].Address] = paths[i]
+			}
+		}
+		// Shift the self-derivation forward
+		// TODO(karalabe): don't overwrite changes from wallet.SelfDerive
+		w.deriveNextAddr = nextAddr
+		w.deriveNextPath = nextPath
+		w.stateLock.Unlock()
+
+		// Notify the user of termination and loop after a bit of time (to avoid trashing)
+		reqc <- struct{}{}
+		if err == nil {
+			select {
+			case errc = <-w.deriveQuit:
+				// Termination requested, abort
+			case <-time.After(ledgerSelfDeriveThrottling):
+				// Waited enough, willing to self-derive again
+			}
+		}
+	}
+	// In case of error, wait for termination
+	if err != nil {
+		glog.V(logger.Debug).Infof("%s self-derivation failed: %s", w.url.String(), err)
+		errc = <-w.deriveQuit
+	}
+	errc <- err
+}
+
+// Contains implements accounts.Wallet, returning whether a particular account is
+// or is not pinned into this Ledger instance. Although we could attempt to resolve
+// unpinned accounts, that would be an non-negligible hardware operation.
+func (w *ledgerWallet) Contains(account accounts.Account) bool {
+	w.stateLock.RLock()
+	defer w.stateLock.RUnlock()
+
+	_, exists := w.paths[account.Address]
+	return exists
+}
+
+// Derive implements accounts.Wallet, deriving a new account at the specific
+// derivation path. If pin is set to true, the account will be added to the list
+// of tracked accounts.
+func (w *ledgerWallet) Derive(path accounts.DerivationPath, pin bool) (accounts.Account, error) {
+	// Try to derive the actual account and update its URL if successful
+	w.stateLock.RLock() // Avoid device disappearing during derivation
+
+	if w.device == nil || w.offline() {
+		w.stateLock.RUnlock()
+		return accounts.Account{}, accounts.ErrWalletClosed
+	}
+	<-w.commsLock // Avoid concurrent hardware access
+	address, err := w.ledgerDerive(path)
+	w.commsLock <- struct{}{}
+
+	w.stateLock.RUnlock()
+
+	// If an error occurred or no pinning was requested, return
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	account := accounts.Account{
+		Address: address,
+		URL:     accounts.URL{Scheme: w.url.Scheme, Path: fmt.Sprintf("%s/%s", w.url.Path, path)},
+	}
+	if !pin {
+		return account, nil
+	}
+	// Pinning needs to modify the state
+	w.stateLock.Lock()
+	defer w.stateLock.Unlock()
+
+	if _, ok := w.paths[address]; !ok {
+		w.accounts = append(w.accounts, account)
+		w.paths[address] = path
+	}
+	return account, nil
+}
+
+// SelfDerive implements accounts.Wallet, trying to discover accounts that the
+// user used previously (based on the chain state), but ones that he/she did not
+// explicitly pin to the wallet manually. To avoid chain head monitoring, self
+// derivation only runs during account listing (and even then throttled).
+func (w *ledgerWallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {
+	w.stateLock.Lock()
+	defer w.stateLock.Unlock()
+
+	w.deriveNextPath = make(accounts.DerivationPath, len(base))
+	copy(w.deriveNextPath[:], base[:])
+
+	w.deriveNextAddr = common.Address{}
+	w.deriveChain = chain
+}
+
+// SignHash implements accounts.Wallet, however signing arbitrary data is not
+// supported for Ledger wallets, so this method will always return an error.
+func (w *ledgerWallet) SignHash(acc accounts.Account, hash []byte) ([]byte, error) {
+	return nil, accounts.ErrNotSupported
+}
+
+// SignTx implements accounts.Wallet. It sends the transaction over to the Ledger
+// wallet to request a confirmation from the user. It returns either the signed
+// transaction or a failure if the user denied the transaction.
+//
+// Note, if the version of the Ethereum application running on the Ledger wallet is
+// too old to sign EIP-155 transactions, but such is requested nonetheless, an error
+// will be returned opposed to silently signing in Homestead mode.
+func (w *ledgerWallet) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	w.stateLock.RLock() // Comms have own mutex, this is for the state fields
+	defer w.stateLock.RUnlock()
+
+	// If the wallet is closed, or the Ethereum app doesn't run, abort
+	if w.device == nil || w.offline() {
+		return nil, accounts.ErrWalletClosed
+	}
+	// Make sure the requested account is contained within
+	path, ok := w.paths[account.Address]
+	if !ok {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Ensure the wallet is capable of signing the given transaction
+	if chainID != nil && w.version[0] <= 1 && w.version[1] <= 0 && w.version[2] <= 2 {
+		return nil, fmt.Errorf("Ledger v%d.%d.%d doesn't support signing this transaction, please update to v1.0.3 at least", w.version[0], w.version[1], w.version[2])
+	}
+	// All infos gathered and metadata checks out, request signing
+	<-w.commsLock
+	defer func() { w.commsLock <- struct{}{} }()
+
+	return w.ledgerSign(path, account.Address, tx, chainID)
+}
+
+// SignHashWithPassphrase implements accounts.Wallet, however signing arbitrary
+// data is not supported for Ledger wallets, so this method will always return
+// an error.
+func (w *ledgerWallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
+	return nil, accounts.ErrNotSupported
+}
+
+// SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given
+// transaction with the given account using passphrase as extra authentication.
+// Since the Ledger does not support extra passphrases, it is silently ignored.
+func (w *ledgerWallet) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	return w.SignTx(account, tx, chainID)
+}
+
+// ledgerVersion retrieves the current version of the Ethereum wallet app running
+// on the Ledger wallet.
+//
+// The version retrieval protocol is defined as follows:
+//
+//   CLA | INS | P1 | P2 | Lc | Le
+//   ----+-----+----+----+----+---
+//    E0 | 06  | 00 | 00 | 00 | 04
+//
+// With no input data, and the output data being:
+//
+//   Description                                        | Length
+//   ---------------------------------------------------+--------
+//   Flags 01: arbitrary data signature enabled by user | 1 byte
+//   Application major version                          | 1 byte
+//   Application minor version                          | 1 byte
+//   Application patch version                          | 1 byte
+func (w *ledgerWallet) ledgerVersion() ([3]byte, error) {
+	// Send the request and wait for the response
+	reply, err := w.ledgerExchange(ledgerOpGetConfiguration, 0, 0, nil)
+	if err != nil {
+		return [3]byte{}, err
+	}
+	if len(reply) != 4 {
+		return [3]byte{}, errors.New("reply not of correct size")
+	}
+	// Cache the version for future reference
+	var version [3]byte
+	copy(version[:], reply[1:])
+	return version, nil
+}
+
+// ledgerDerive retrieves the currently active Ethereum address from a Ledger
+// wallet at the specified derivation path.
+//
+// The address derivation protocol is defined as follows:
+//
+//   CLA | INS | P1 | P2 | Lc  | Le
+//   ----+-----+----+----+-----+---
+//    E0 | 02  | 00 return address
+//               01 display address and confirm before returning
+//                  | 00: do not return the chain code
+//                  | 01: return the chain code
+//                       | var | 00
+//
+// Where the input data is:
+//
+//   Description                                      | Length
+//   -------------------------------------------------+--------
+//   Number of BIP 32 derivations to perform (max 10) | 1 byte
+//   First derivation index (big endian)              | 4 bytes
+//   ...                                              | 4 bytes
+//   Last derivation index (big endian)               | 4 bytes
+//
+// And the output data is:
+//
+//   Description             | Length
+//   ------------------------+-------------------
+//   Public Key length       | 1 byte
+//   Uncompressed Public Key | arbitrary
+//   Ethereum address length | 1 byte
+//   Ethereum address        | 40 bytes hex ascii
+//   Chain code if requested | 32 bytes
+func (w *ledgerWallet) ledgerDerive(derivationPath []uint32) (common.Address, error) {
+	// Flatten the derivation path into the Ledger request
+	path := make([]byte, 1+4*len(derivationPath))
+	path[0] = byte(len(derivationPath))
+	for i, component := range derivationPath {
+		binary.BigEndian.PutUint32(path[1+4*i:], component)
+	}
+	// Send the request and wait for the response
+	reply, err := w.ledgerExchange(ledgerOpRetrieveAddress, ledgerP1DirectlyFetchAddress, ledgerP2DiscardAddressChainCode, path)
+	if err != nil {
+		return common.Address{}, err
+	}
+	// Discard the public key, we don't need that for now
+	if len(reply) < 1 || len(reply) < 1+int(reply[0]) {
+		return common.Address{}, errors.New("reply lacks public key entry")
+	}
+	reply = reply[1+int(reply[0]):]
+
+	// Extract the Ethereum hex address string
+	if len(reply) < 1 || len(reply) < 1+int(reply[0]) {
+		return common.Address{}, errors.New("reply lacks address entry")
+	}
+	hexstr := reply[1 : 1+int(reply[0])]
+
+	// Decode the hex sting into an Ethereum address and return
+	var address common.Address
+	hex.Decode(address[:], hexstr)
+	return address, nil
+}
+
+// ledgerSign sends the transaction to the Ledger wallet, and waits for the user
+// to confirm or deny the transaction.
+//
+// The transaction signing protocol is defined as follows:
+//
+//   CLA | INS | P1 | P2 | Lc  | Le
+//   ----+-----+----+----+-----+---
+//    E0 | 04  | 00: first transaction data block
+//               80: subsequent transaction data block
+//                  | 00 | variable | variable
+//
+// Where the input for the first transaction block (first 255 bytes) is:
+//
+//   Description                                      | Length
+//   -------------------------------------------------+----------
+//   Number of BIP 32 derivations to perform (max 10) | 1 byte
+//   First derivation index (big endian)              | 4 bytes
+//   ...                                              | 4 bytes
+//   Last derivation index (big endian)               | 4 bytes
+//   RLP transaction chunk                            | arbitrary
+//
+// And the input for subsequent transaction blocks (first 255 bytes) are:
+//
+//   Description           | Length
+//   ----------------------+----------
+//   RLP transaction chunk | arbitrary
+//
+// And the output data is:
+//
+//   Description | Length
+//   ------------+---------
+//   signature V | 1 byte
+//   signature R | 32 bytes
+//   signature S | 32 bytes
+func (w *ledgerWallet) ledgerSign(derivationPath []uint32, address common.Address, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	// We need to modify the timeouts to account for user feedback
+	defer func(old time.Duration) { w.device.ReadTimeout = old }(w.device.ReadTimeout)
+	w.device.ReadTimeout = time.Hour * 24 * 30 // Timeout requires a Ledger power cycle, only if you must
+
+	// Flatten the derivation path into the Ledger request
+	path := make([]byte, 1+4*len(derivationPath))
+	path[0] = byte(len(derivationPath))
+	for i, component := range derivationPath {
+		binary.BigEndian.PutUint32(path[1+4*i:], component)
+	}
+	// Create the transaction RLP based on whether legacy or EIP155 signing was requeste
+	var (
+		txrlp []byte
+		err   error
+	)
+	if chainID == nil {
+		if txrlp, err = rlp.EncodeToBytes([]interface{}{tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data()}); err != nil {
+			return nil, err
+		}
+	} else {
+		if txrlp, err = rlp.EncodeToBytes([]interface{}{tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data(), chainID, big.NewInt(0), big.NewInt(0)}); err != nil {
+			return nil, err
+		}
+	}
+	payload := append(path, txrlp...)
+
+	// Send the request and wait for the response
+	var (
+		op    = ledgerP1InitTransactionData
+		reply []byte
+	)
+	for len(payload) > 0 {
+		// Calculate the size of the next data chunk
+		chunk := 255
+		if chunk > len(payload) {
+			chunk = len(payload)
+		}
+		// Send the chunk over, ensuring it's processed correctly
+		reply, err = w.ledgerExchange(ledgerOpSignTransaction, op, 0, payload[:chunk])
+		if err != nil {
+			return nil, err
+		}
+		// Shift the payload and ensure subsequent chunks are marked as such
+		payload = payload[chunk:]
+		op = ledgerP1ContTransactionData
+	}
+	// Extract the Ethereum signature and do a sanity validation
+	if len(reply) != 65 {
+		return nil, errors.New("reply lacks signature")
+	}
+	signature := append(reply[1:], reply[0])
+
+	// Create the correct signer and signature transform based on the chain ID
+	var signer types.Signer
+	if chainID == nil {
+		signer = new(types.HomesteadSigner)
+	} else {
+		signer = types.NewEIP155Signer(chainID)
+		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+	}
+	// Inject the final signature into the transaction and sanity check the sender
+	signed, err := tx.WithSignature(signer, signature)
+	if err != nil {
+		return nil, err
+	}
+	sender, err := types.Sender(signer, signed)
+	if err != nil {
+		return nil, err
+	}
+	if sender != address {
+		return nil, fmt.Errorf("signer mismatch: expected %s, got %s", address.Hex(), sender.Hex())
+	}
+	return signed, nil
+}
+
+// ledgerExchange performs a data exchange with the Ledger wallet, sending it a
+// message and retrieving the response.
+//
+// The common transport header is defined as follows:
+//
+//  Description                           | Length
+//  --------------------------------------+----------
+//  Communication channel ID (big endian) | 2 bytes
+//  Command tag                           | 1 byte
+//  Packet sequence index (big endian)    | 2 bytes
+//  Payload                               | arbitrary
+//
+// The Communication channel ID allows commands multiplexing over the same
+// physical link. It is not used for the time being, and should be set to 0101
+// to avoid compatibility issues with implementations ignoring a leading 00 byte.
+//
+// The Command tag describes the message content. Use TAG_APDU (0x05) for standard
+// APDU payloads, or TAG_PING (0x02) for a simple link test.
+//
+// The Packet sequence index describes the current sequence for fragmented payloads.
+// The first fragment index is 0x00.
+//
+// APDU Command payloads are encoded as follows:
+//
+//  Description              | Length
+//  -----------------------------------
+//  APDU length (big endian) | 2 bytes
+//  APDU CLA                 | 1 byte
+//  APDU INS                 | 1 byte
+//  APDU P1                  | 1 byte
+//  APDU P2                  | 1 byte
+//  APDU length              | 1 byte
+//  Optional APDU data       | arbitrary
+func (w *ledgerWallet) ledgerExchange(opcode ledgerOpcode, p1 ledgerParam1, p2 ledgerParam2, data []byte) ([]byte, error) {
+	// Construct the message payload, possibly split into multiple chunks
+	apdu := make([]byte, 2, 7+len(data))
+
+	binary.BigEndian.PutUint16(apdu, uint16(5+len(data)))
+	apdu = append(apdu, []byte{0xe0, byte(opcode), byte(p1), byte(p2), byte(len(data))}...)
+	apdu = append(apdu, data...)
+
+	// Stream all the chunks to the device
+	header := []byte{0x01, 0x01, 0x05, 0x00, 0x00} // Channel ID and command tag appended
+	chunk := make([]byte, 64)
+	space := len(chunk) - len(header)
+
+	for i := 0; len(apdu) > 0; i++ {
+		// Construct the new message to stream
+		chunk = append(chunk[:0], header...)
+		binary.BigEndian.PutUint16(chunk[3:], uint16(i))
+
+		if len(apdu) > space {
+			chunk = append(chunk, apdu[:space]...)
+			apdu = apdu[space:]
+		} else {
+			chunk = append(chunk, apdu...)
+			apdu = nil
+		}
+		// Send over to the device
+		if glog.V(logger.Detail) {
+			glog.Infof("-> %03d.%03d: %x", w.device.Bus, w.device.Address, chunk)
+		}
+		if _, err := w.input.Write(chunk); err != nil {
+			return nil, err
+		}
+	}
+	// Stream the reply back from the wallet in 64 byte chunks
+	var reply []byte
+	chunk = chunk[:64] // Yeah, we surely have enough space
+	for {
+		// Read the next chunk from the Ledger wallet
+		if _, err := io.ReadFull(w.output, chunk); err != nil {
+			return nil, err
+		}
+		if glog.V(logger.Detail) {
+			glog.Infof("<- %03d.%03d: %x", w.device.Bus, w.device.Address, chunk)
+		}
+		// Make sure the transport header matches
+		if chunk[0] != 0x01 || chunk[1] != 0x01 || chunk[2] != 0x05 {
+			return nil, errReplyInvalidHeader
+		}
+		// If it's the first chunk, retrieve the total message length
+		var payload []byte
+
+		if chunk[3] == 0x00 && chunk[4] == 0x00 {
+			reply = make([]byte, 0, int(binary.BigEndian.Uint16(chunk[5:7])))
+			payload = chunk[7:]
+		} else {
+			payload = chunk[5:]
+		}
+		// Append to the reply and stop when filled up
+		if left := cap(reply) - len(reply); left > len(payload) {
+			reply = append(reply, payload...)
+		} else {
+			reply = append(reply, payload[:left]...)
+			break
+		}
+	}
+	return reply[:len(reply)-2], nil
+}

accounts/usbwallet/usbwallet.go

@@ -1,4 +1,4 @@
-// Copyright 2014 The go-ethereum Authors
+// Copyright 2017 The go-ethereum Authors
 // This file is part of the go-ethereum library.
 //
 // The go-ethereum library is free software: you can redistribute it and/or modify
@@ -14,22 +14,16 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package dagger
+// +build !ios
 
-import (
-	"math/big"
-	"testing"
+// Package usbwallet implements support for USB hardware wallets.
+package usbwallet
 
-	"github.com/ethereum/go-ethereum/common"
-)
+import "github.com/karalabe/gousb/usb"
 
-func BenchmarkDaggerSearch(b *testing.B) {
-	hash := big.NewInt(0)
-	diff := common.BigPow(2, 36)
-	o := big.NewInt(0) // nonce doesn't matter. We're only testing against speed, not validity
-
-	// Reset timer so the big generation isn't included in the benchmark
-	b.ResetTimer()
-	// Validate
-	DaggerVerify(hash, diff, o)
+// deviceID is a combined vendor/product identifier to uniquely identify a USB
+// hardware device.
+type deviceID struct {
+	Vendor  usb.ID // The Vendor identifer
+	Product usb.ID // The Product identifier
 }

accounts/usbwallet/usbwallet_ios.go

@@ -0,0 +1,38 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// This file contains the implementation for interacting with the Ledger hardware
+// wallets. The wire protocol spec can be found in the Ledger Blue GitHub repo:
+// https://raw.githubusercontent.com/LedgerHQ/blue-app-eth/master/doc/ethapp.asc
+
+// +build ios
+
+package usbwallet
+
+import (
+	"errors"
+
+	"github.com/ethereum/go-ethereum/accounts"
+)
+
+// Here be dragons! There is no USB support on iOS.
+
+// ErrIOSNotSupported is returned for all USB hardware backends on iOS.
+var ErrIOSNotSupported = errors.New("no USB support on iOS")
+
+func NewLedgerHub() (accounts.Backend, error) {
+	return nil, ErrIOSNotSupported
+}

appveyor.yml

@@ -22,8 +22,8 @@ environment:
 
 install:
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.3.windows-%GETH_ARCH%.zip
-  - 7z x go1.7.3.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.4.windows-%GETH_ARCH%.zip
+  - 7z x go1.7.4.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

build/ci.go

@@ -24,7 +24,7 @@ Usage: go run ci.go <command> <command flags/arguments>
 Available commands are:
 
    install    [-arch architecture] [ packages... ]                                           -- builds packages and executables
-   test       [ -coverage ] [ -vet ] [ packages... ]                                         -- runs the tests
+   test       [ -coverage ] [ -vet ] [ -misspell ] [ packages... ]                           -- runs the tests
    archive    [-arch architecture] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
    importkeys                                                                                -- imports signing keys from env
    debsrc     [ -signer key-id ] [ -upload dest ]                                            -- creates a debian source package
@@ -72,9 +72,7 @@ var (
 		executablePath("abigen"),
 		executablePath("evm"),
 		executablePath("geth"),
-		executablePath("bzzd"),
-		executablePath("bzzhash"),
-		executablePath("bzzup"),
+		executablePath("swarm"),
 		executablePath("rlpdump"),
 	}
 
@@ -93,16 +91,8 @@ var (
 			Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.",
 		},
 		{
-			Name:        "bzzd",
-			Description: "Ethereum Swarm daemon",
-		},
-		{
-			Name:        "bzzup",
-			Description: "Ethereum Swarm command line file/directory uploader",
-		},
-		{
-			Name:        "bzzhash",
-			Description: "Ethereum Swarm file/directory hash calculator",
+			Name:        "swarm",
+			Description: "Ethereum Swarm daemon and tools",
 		},
 		{
 			Name:        "abigen",
@@ -112,7 +102,8 @@ var (
 
 	// Distros for which packages are created.
 	// Note: vivid is unsupported because there is no golang-1.6 package for it.
-	debDistros = []string{"trusty", "wily", "xenial", "yakkety"}
+	// Note: wily is unsupported because it was officially deprecated on lanchpad.
+	debDistros = []string{"trusty", "xenial", "yakkety"}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -204,7 +195,7 @@ func doInstall(cmdline []string) {
 			if err != nil {
 				log.Fatal(err)
 			}
-			for name, _ := range pkgs {
+			for name := range pkgs {
 				if name == "main" {
 					gobuild := goToolArch(*arch, "build", buildFlags(env)...)
 					gobuild.Args = append(gobuild.Args, "-v")
@@ -245,6 +236,14 @@ func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd {
 	gocmd := filepath.Join(runtime.GOROOT(), "bin", "go")
 	cmd := exec.Command(gocmd, subcmd)
 	cmd.Args = append(cmd.Args, args...)
+
+	if subcmd == "build" || subcmd == "install" || subcmd == "test" {
+		// Go CGO has a Windows linker error prior to 1.8 (https://github.com/golang/go/issues/8756).
+		// Work around issue by allowing multiple definitions for <1.8 builds.
+		if runtime.GOOS == "windows" && runtime.Version() < "go1.8" {
+			cmd.Args = append(cmd.Args, []string{"-ldflags", "-extldflags -Wl,--allow-multiple-definition"}...)
+		}
+	}
 	cmd.Env = []string{
 		"GO15VENDOREXPERIMENT=1",
 		"GOPATH=" + build.GOPATH(),
@@ -271,6 +270,7 @@ func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd {
 func doTest(cmdline []string) {
 	var (
 		vet      = flag.Bool("vet", false, "Whether to run go vet")
+		misspell = flag.Bool("misspell", false, "Whether to run the spell checker")
 		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
 	)
 	flag.CommandLine.Parse(cmdline)
@@ -296,7 +296,9 @@ func doTest(cmdline []string) {
 	if *vet {
 		build.MustRun(goTool("vet", packages...))
 	}
-
+	if *misspell {
+		spellcheck(packages)
+	}
 	// Run the actual tests.
 	gotest := goTool("test")
 	// Test a single package at a time. CI builders are slow
@@ -309,6 +311,34 @@ func doTest(cmdline []string) {
 	build.MustRun(gotest)
 }
 
+// spellcheck runs the client9/misspell spellchecker package on all Go, Cgo and
+// test files in the requested packages.
+func spellcheck(packages []string) {
+	// Ensure the spellchecker is available
+	build.MustRun(goTool("get", "github.com/client9/misspell/cmd/misspell"))
+
+	// Windows chokes on long argument lists, check packages individually
+	for _, pkg := range packages {
+		// The spell checker doesn't work on packages, gather all .go files for it
+		out, err := goTool("list", "-f", "{{.Dir}}{{range .GoFiles}}\n{{.}}{{end}}{{range .CgoFiles}}\n{{.}}{{end}}{{range .TestGoFiles}}\n{{.}}{{end}}", pkg).CombinedOutput()
+		if err != nil {
+			log.Fatalf("source file listing failed: %v\n%s", err, string(out))
+		}
+		// Retrieve the folder and assemble the source list
+		lines := strings.Split(string(out), "\n")
+		root := lines[0]
+
+		sources := make([]string, 0, len(lines)-1)
+		for _, line := range lines[1:] {
+			if line = strings.TrimSpace(line); line != "" {
+				sources = append(sources, filepath.Join(root, line))
+			}
+		}
+		// Run the spell checker for this particular package
+		build.MustRunCommand(filepath.Join(GOBIN, "misspell"), append([]string{"-error"}, sources...)...)
+	}
+}
+
 // Release Packaging
 
 func doArchive(cmdline []string) {
@@ -638,6 +668,7 @@ func doWindowsInstaller(cmdline []string) {
 	build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil)
 	build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData)
 	build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools)
+	build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil)
 	build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil)
 	build.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll", 0755)
 	build.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING", 0755)
@@ -677,9 +708,16 @@ func doAndroidArchive(cmdline []string) {
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
 
+	// Sanity check that the SDK and NDK are installed and set
+	if os.Getenv("ANDROID_HOME") == "" {
+		log.Fatal("Please ensure ANDROID_HOME points to your Android SDK")
+	}
+	if os.Getenv("ANDROID_NDK") == "" {
+		log.Fatal("Please ensure ANDROID_NDK points to your Android NDK")
+	}
 	// Build the Android archive and Maven resources
 	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile"))
-	build.MustRun(gomobileTool("init"))
+	build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK")))
 	build.MustRun(gomobileTool("bind", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile"))
 
 	if *local {
@@ -800,7 +838,7 @@ func doXCodeFramework(cmdline []string) {
 	// Build the iOS XCode framework
 	build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile"))
 	build.MustRun(gomobileTool("init"))
-	bind := gomobileTool("bind", "--target", "ios", "--tags", "ios", "--prefix", "GE", "-v", "github.com/ethereum/go-ethereum/mobile")
+	bind := gomobileTool("bind", "--target", "ios", "--tags", "ios", "-v", "github.com/ethereum/go-ethereum/mobile")
 
 	if *local {
 		// If we're building locally, use the build folder and stop afterwards

build/nsis.geth.nsi

@@ -17,8 +17,12 @@
 #
 # Requirements:
 # - NSIS, http://nsis.sourceforge.net/Main_Page
+# - NSIS Large Strings build, http://nsis.sourceforge.net/Special_Builds
 # - SFP, http://nsis.sourceforge.net/NSIS_Simple_Firewall_Plugin (put dll in NSIS\Plugins\x86-ansi)
 #
+# After intalling NSIS extra the NSIS Large Strings build zip and replace the makensis.exe and the
+# files found in Stub.
+#
 # based on: http://nsis.sourceforge.net/A_simple_installer_with_start_menu_shortcut_and_uninstaller
 #
 # TODO:
@@ -37,6 +41,7 @@ RequestExecutionLevel admin
 SetCompressor /SOLID lzma
 
 !include LogicLib.nsh
+!include PathUpdate.nsh
 !include EnvVarUpdate.nsh
 
 !macro VerifyUserIsAdmin

build/nsis.install.nsh

@@ -37,8 +37,9 @@ Section "Geth" GETH_IDX
   ${EnvVarUpdate} $0 "ETHEREUM_SOCKET" "R" "HKLM" "\\.\pipe\geth.ipc"
   ${EnvVarUpdate} $0 "ETHEREUM_SOCKET" "A" "HKLM" "\\.\pipe\geth.ipc"
 
-  # Add geth to PATH
-  ${EnvVarUpdate} $0 "PATH" "A" "HKLM" $INSTDIR
+  # Add instdir to PATH
+  Push "$INSTDIR"
+  Call AddToPath
 SectionEnd
 
 # Install optional develop tools.

build/nsis.pathupdate.nsh

@@ -0,0 +1,153 @@
+!include "WinMessages.nsh"
+
+; see https://support.microsoft.com/en-us/kb/104011
+!define Environ 'HKLM "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"'
+; HKEY_LOCAL_MACHINE = 0x80000002
+
+; AddToPath - Appends dir to PATH
+;   (does not work on Win9x/ME)
+;
+; Usage:
+;   Push "dir"
+;   Call AddToPath
+Function AddToPath
+  Exch $0
+  Push $1
+  Push $2
+  Push $3
+  Push $4
+
+  ; NSIS ReadRegStr returns empty string on string overflow
+  ; Native calls are used here to check actual length of PATH
+  ; $4 = RegOpenKey(HKEY_LOCAL_MACHINE, "SYSTEM\CurrentControlSet\Control\Session Manager\Environment", &$3)
+  System::Call "advapi32::RegOpenKey(i 0x80000002, t'SYSTEM\CurrentControlSet\Control\Session Manager\Environment', *i.r3) i.r4"
+  IntCmp $4 0 0 done done
+
+  ; $4 = RegQueryValueEx($3, "PATH", (DWORD*)0, (DWORD*)0, &$1, ($2=NSIS_MAX_STRLEN, &$2))
+  ; RegCloseKey($3)
+  System::Call "advapi32::RegQueryValueEx(i $3, t'PATH', i 0, i 0, t.r1, *i ${NSIS_MAX_STRLEN} r2) i.r4"
+  System::Call "advapi32::RegCloseKey(i $3)"
+
+  IntCmp $4 234 0 +4 +4 ; $4 == ERROR_MORE_DATA
+    DetailPrint "AddToPath: original length $2 > ${NSIS_MAX_STRLEN}"
+    MessageBox MB_OK "PATH not updated, original length $2 > ${NSIS_MAX_STRLEN}"
+    Goto done
+
+  IntCmp $4 0 +5 ; $4 != NO_ERROR
+    IntCmp $4 2 +3 ; $4 != ERROR_FILE_NOT_FOUND
+      DetailPrint "AddToPath: unexpected error code $4"
+      Goto done
+    StrCpy $1 ""
+
+  ; Check if already in PATH
+  Push "$1;"
+  Push "$0;"
+  Call StrStr
+  Pop $2
+  StrCmp $2 "" 0 done
+  Push "$1;"
+  Push "$0\;"
+  Call StrStr
+  Pop $2
+  StrCmp $2 "" 0 done
+
+  ; Prevent NSIS string overflow
+  StrLen $2 $0
+  StrLen $3 $1
+  IntOp $2 $2 + $3
+  IntOp $2 $2 + 2 ; $2 = strlen(dir) + strlen(PATH) + sizeof(";")
+  IntCmp $2 ${NSIS_MAX_STRLEN} +4 +4 0
+    DetailPrint "AddToPath: new length $2 > ${NSIS_MAX_STRLEN}"
+    MessageBox MB_OK "PATH not updated, new length $2 > ${NSIS_MAX_STRLEN}."
+    Goto done
+
+  ; Append dir to PATH
+  DetailPrint "Add to PATH: $0"
+  StrCpy $2 $1 1 -1
+  StrCmp $2 ";" 0 +2
+    StrCpy $1 $1 -1 ; remove trailing ';'
+  StrCmp $1 "" +2   ; no leading ';'
+    StrCpy $0 "$1;$0"
+  
+  WriteRegExpandStr ${Environ} "PATH" $0
+  SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000
+
+done:
+  Pop $4
+  Pop $3
+  Pop $2
+  Pop $1
+  Pop $0
+FunctionEnd
+
+
+; RemoveFromPath - Removes dir from PATH
+;
+; Usage:
+;   Push "dir"
+;   Call RemoveFromPath
+Function un.RemoveFromPath
+  Exch $0
+  Push $1
+  Push $2
+  Push $3
+  Push $4
+  Push $5
+  Push $6
+
+  ; NSIS ReadRegStr returns empty string on string overflow
+  ; Native calls are used here to check actual length of PATH
+  ; $4 = RegOpenKey(HKEY_LOCAL_MACHINE, "SYSTEM\CurrentControlSet\Control\Session Manager\Environment", &$3)
+  System::Call "advapi32::RegOpenKey(i 0x80000002, t'SYSTEM\CurrentControlSet\Control\Session Manager\Environment', *i.r3) i.r4"
+  IntCmp $4 0 0 done done
+
+  ; $4 = RegQueryValueEx($3, "PATH", (DWORD*)0, (DWORD*)0, &$1, ($2=NSIS_MAX_STRLEN, &$2))
+  ; RegCloseKey($3)
+  System::Call "advapi32::RegQueryValueEx(i $3, t'PATH', i 0, i 0, t.r1, *i ${NSIS_MAX_STRLEN} r2) i.r4"
+  System::Call "advapi32::RegCloseKey(i $3)"
+
+  IntCmp $4 234 0 +4 +4 ; $4 == ERROR_MORE_DATA
+    DetailPrint "RemoveFromPath: original length $2 > ${NSIS_MAX_STRLEN}"
+    MessageBox MB_OK "PATH not updated, original length $2 > ${NSIS_MAX_STRLEN}"
+    Goto done
+
+  IntCmp $4 0 +5 ; $4 != NO_ERROR
+    IntCmp $4 2 +3 ; $4 != ERROR_FILE_NOT_FOUND
+      DetailPrint "RemoveFromPath: unexpected error code $4"
+      Goto done
+    StrCpy $1 ""
+
+  ; length < ${NSIS_MAX_STRLEN} -> ReadRegStr can be used
+  ReadRegStr $1 ${Environ} "PATH"
+  StrCpy $5 $1 1 -1
+  StrCmp $5 ";" +2
+  StrCpy $1 "$1;" ; ensure trailing ';'
+  Push $1
+  Push "$0;"
+  Call un.StrStr
+  Pop $2 ; pos of our dir
+  StrCmp $2 "" done
+
+  DetailPrint "Remove from PATH: $0"
+  StrLen $3 "$0;"
+  StrLen $4 $2
+  StrCpy $5 $1 -$4 ; $5 is now the part before the path to remove
+  StrCpy $6 $2 "" $3 ; $6 is now the part after the path to remove
+  StrCpy $3 "$5$6"
+  StrCpy $5 $3 1 -1
+  StrCmp $5 ";" 0 +2
+    StrCpy $3 $3 -1 ; remove trailing ';'
+  WriteRegExpandStr ${Environ} "PATH" $3
+  SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000
+
+done:
+  Pop $6
+  Pop $5
+  Pop $4
+  Pop $3
+  Pop $2
+  Pop $1
+  Pop $0
+FunctionEnd
+ 
+

build/nsis.uninstall.nsh

@@ -25,7 +25,8 @@ Section "Uninstall"
   ${un.EnvVarUpdate} $0 "ETHEREUM_SOCKET" "R" "HKLM" "\\.\pipe\geth.ipc"
 
   # Remove install directory from PATH
-  ${un.EnvVarUpdate} $0 "PATH" "R" "HKLM" $INSTDIR
+  Push "$INSTDIR"
+  Call un.RemoveFromPath
 
   # Cleanup registry (deletes all sub keys)
   DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\${GROUPNAME} ${APPNAME}"

build/update-license.go

@@ -185,7 +185,7 @@ func getFiles() []string {
 		files = append(files, line)
 	})
 	if err != nil {
-		log.Fatalf("error getting files:", err)
+		log.Fatal("error getting files:", err)
 	}
 	return files
 }
@@ -294,7 +294,7 @@ func getInfo(files <-chan string, out chan<- *info, wg *sync.WaitGroup) {
 	wg.Done()
 }
 
-// fileInfo finds the lowest year in which the given file was commited.
+// fileInfo finds the lowest year in which the given file was committed.
 func fileInfo(file string) (*info, error) {
 	info := &info{file: file, Year: int64(time.Now().Year())}
 	cmd := exec.Command("git", "log", "--follow", "--find-renames=80", "--find-copies=80", "--pretty=format:%ai", "--", file)

cmd/abigen/main.go

@@ -94,7 +94,9 @@ func main() {
 			abi, _ := json.Marshal(contract.Info.AbiDefinition) // Flatten the compiler parse
 			abis = append(abis, string(abi))
 			bins = append(bins, contract.Code)
-			types = append(types, name)
+
+			nameParts := strings.Split(name, ":")
+			types = append(types, nameParts[len(nameParts)-1])
 		}
 	} else {
 		// Otherwise load up the ABI, optional bytecode and type name from the parameters

cmd/bzzup/main.go

@@ -1,161 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// Command bzzup uploads files to the swarm HTTP API.
-package main
-
-import (
-	"bytes"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"log"
-	"mime"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-func main() {
-	var (
-		bzzapiFlag    = flag.String("bzzapi", "http://127.0.0.1:8500", "Swarm HTTP endpoint")
-		recursiveFlag = flag.Bool("recursive", false, "Upload directories recursively")
-		manifestFlag  = flag.Bool("manifest", true, "Skip automatic manifest upload")
-	)
-	log.SetOutput(os.Stderr)
-	log.SetFlags(0)
-	flag.Parse()
-	if flag.NArg() != 1 {
-		log.Fatal("need filename as the first and only argument")
-	}
-
-	var (
-		file   = flag.Arg(0)
-		client = &client{api: *bzzapiFlag}
-		mroot  manifest
-	)
-	fi, err := os.Stat(file)
-	if err != nil {
-		log.Fatal(err)
-	}
-	if fi.IsDir() {
-		if !*recursiveFlag {
-			log.Fatal("argument is a directory and recursive upload is disabled")
-		}
-		mroot, err = client.uploadDirectory(file)
-	} else {
-		mroot, err = client.uploadFile(file, fi)
-		if *manifestFlag {
-			// Wrap the raw file entry in a proper manifest so both hashes get printed.
-			mroot = manifest{Entries: []manifest{mroot}}
-		}
-	}
-	if err != nil {
-		log.Fatalln("upload failed:", err)
-	}
-	if *manifestFlag {
-		hash, err := client.uploadManifest(mroot)
-		if err != nil {
-			log.Fatalln("manifest upload failed:", err)
-		}
-		mroot.Hash = hash
-	}
-
-	// Print the manifest. This is the only output to stdout.
-	mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
-	fmt.Println(string(mrootJSON))
-}
-
-// client wraps interaction with the swarm HTTP gateway.
-type client struct {
-	api string
-}
-
-// manifest is the JSON representation of a swarm manifest.
-type manifest struct {
-	Hash        string     `json:"hash,omitempty"`
-	ContentType string     `json:"contentType,omitempty"`
-	Path        string     `json:"path,omitempty"`
-	Entries     []manifest `json:"entries,omitempty"`
-}
-
-func (c *client) uploadFile(file string, fi os.FileInfo) (manifest, error) {
-	hash, err := c.uploadFileContent(file, fi)
-	m := manifest{
-		Hash:        hash,
-		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
-	}
-	return m, err
-}
-
-func (c *client) uploadDirectory(dir string) (manifest, error) {
-	dirm := manifest{}
-	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
-	err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
-		if err != nil || fi.IsDir() {
-			return err
-		}
-		if !strings.HasPrefix(path, dir) {
-			return fmt.Errorf("path %s outside directory %s", path, dir)
-		}
-		entry, err := c.uploadFile(path, fi)
-		entry.Path = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
-		dirm.Entries = append(dirm.Entries, entry)
-		return err
-	})
-	return dirm, err
-}
-
-func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(file)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Printf("uploading file %s (%d bytes)", file, fi.Size())
-	return c.postRaw("application/octet-stream", fi.Size(), fd)
-}
-
-func (c *client) uploadManifest(m manifest) (string, error) {
-	jsm, err := json.Marshal(m)
-	if err != nil {
-		panic(err)
-	}
-	log.Println("uploading manifest")
-	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
-}
-
-func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
-	req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Set("content-type", mimetype)
-	req.ContentLength = size
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
-}

cmd/disasm/main.go

@@ -18,11 +18,12 @@
 package main
 
 import (
+	"encoding/hex"
 	"fmt"
 	"io/ioutil"
 	"os"
+	"strings"
 
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/vm"
 )
 
@@ -32,20 +33,28 @@ func main() {
 		fmt.Println(err)
 		os.Exit(1)
 	}
-	code = common.Hex2Bytes(string(code[:len(code)-1]))
+	code, err = hex.DecodeString(strings.TrimSpace(string(code[:])))
+	if err != nil {
+		fmt.Printf("Error: %v\n", err)
+		return
+	}
 	fmt.Printf("%x\n", code)
 
 	for pc := uint64(0); pc < uint64(len(code)); pc++ {
 		op := vm.OpCode(code[pc])
-		fmt.Printf("%-5d  %v", pc, op)
 
 		switch op {
 		case vm.PUSH1, vm.PUSH2, vm.PUSH3, vm.PUSH4, vm.PUSH5, vm.PUSH6, vm.PUSH7, vm.PUSH8, vm.PUSH9, vm.PUSH10, vm.PUSH11, vm.PUSH12, vm.PUSH13, vm.PUSH14, vm.PUSH15, vm.PUSH16, vm.PUSH17, vm.PUSH18, vm.PUSH19, vm.PUSH20, vm.PUSH21, vm.PUSH22, vm.PUSH23, vm.PUSH24, vm.PUSH25, vm.PUSH26, vm.PUSH27, vm.PUSH28, vm.PUSH29, vm.PUSH30, vm.PUSH31, vm.PUSH32:
 			a := uint64(op) - uint64(vm.PUSH1) + 1
-			fmt.Printf("  => %x", code[pc+1:pc+1+a])
-
+			u := pc + 1 + a
+			if uint64(len(code)) <= pc || uint64(len(code)) < u {
+				fmt.Printf("Error: incomplete push instruction at %v\n", pc)
+				return
+			}
+			fmt.Printf("%-5d  %v  => %x\n", pc, op, code[pc+1:u])
 			pc += a
+		default:
+			fmt.Printf("%-5d  %v\n", pc, op)
 		}
-		fmt.Println()
 	}
 }

cmd/ethtest/main.go

@@ -88,12 +88,7 @@ func runTestWithReader(test string, r io.Reader) error {
 	default:
 		err = fmt.Errorf("Invalid test type specified: %v", test)
 	}
-
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return err
 }
 
 func getFiles(path string) ([]string, error) {

cmd/evm/main.go

@@ -20,21 +20,18 @@ package main
 import (
 	"fmt"
 	"io/ioutil"
-	"math/big"
 	"os"
-	"runtime"
+	goruntime "runtime"
 	"time"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/core/vm/runtime"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/logger/glog"
-	"github.com/ethereum/go-ethereum/params"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -47,14 +44,6 @@ var (
 		Name:  "debug",
 		Usage: "output full trace logs",
 	}
-	ForceJitFlag = cli.BoolFlag{
-		Name:  "forcejit",
-		Usage: "forces jit compilation",
-	}
-	DisableJitFlag = cli.BoolFlag{
-		Name:  "nojit",
-		Usage: "disabled jit compilation",
-	}
 	CodeFlag = cli.StringFlag{
 		Name:  "code",
 		Usage: "EVM code",
@@ -98,6 +87,10 @@ var (
 		Name:  "create",
 		Usage: "indicates the action should be create rather than call",
 	}
+	DisableGasMeteringFlag = cli.BoolFlag{
+		Name:  "nogasmetering",
+		Usage: "disable gas metering",
+	}
 )
 
 func init() {
@@ -105,8 +98,6 @@ func init() {
 		CreateFlag,
 		DebugFlag,
 		VerbosityFlag,
-		ForceJitFlag,
-		DisableJitFlag,
 		SysStatFlag,
 		CodeFlag,
 		CodeFileFlag,
@@ -115,6 +106,7 @@ func init() {
 		ValueFlag,
 		DumpFlag,
 		InputFlag,
+		DisableGasMeteringFlag,
 	}
 	app.Action = run
 }
@@ -129,13 +121,6 @@ func run(ctx *cli.Context) error {
 
 	logger := vm.NewStructLogger(nil)
 
-	vmenv := NewEnv(statedb, common.StringToAddress("evmuser"), common.Big(ctx.GlobalString(ValueFlag.Name)), vm.Config{
-		Debug:     ctx.GlobalBool(DebugFlag.Name),
-		ForceJit:  ctx.GlobalBool(ForceJitFlag.Name),
-		EnableJit: !ctx.GlobalBool(DisableJitFlag.Name),
-		Tracer:    logger,
-	})
-
 	tstart := time.Now()
 
 	var (
@@ -168,25 +153,34 @@ func run(ctx *cli.Context) error {
 
 	if ctx.GlobalBool(CreateFlag.Name) {
 		input := append(code, common.Hex2Bytes(ctx.GlobalString(InputFlag.Name))...)
-		ret, _, err = vmenv.Create(
-			sender,
-			input,
-			common.Big(ctx.GlobalString(GasFlag.Name)),
-			common.Big(ctx.GlobalString(PriceFlag.Name)),
-			common.Big(ctx.GlobalString(ValueFlag.Name)),
-		)
+		ret, _, err = runtime.Create(input, &runtime.Config{
+			Origin:   sender.Address(),
+			State:    statedb,
+			GasLimit: common.Big(ctx.GlobalString(GasFlag.Name)),
+			GasPrice: common.Big(ctx.GlobalString(PriceFlag.Name)),
+			Value:    common.Big(ctx.GlobalString(ValueFlag.Name)),
+			EVMConfig: vm.Config{
+				Tracer:             logger,
+				Debug:              ctx.GlobalBool(DebugFlag.Name),
+				DisableGasMetering: ctx.GlobalBool(DisableGasMeteringFlag.Name),
+			},
+		})
 	} else {
 		receiver := statedb.CreateAccount(common.StringToAddress("receiver"))
-
 		receiver.SetCode(crypto.Keccak256Hash(code), code)
-		ret, err = vmenv.Call(
-			sender,
-			receiver.Address(),
-			common.Hex2Bytes(ctx.GlobalString(InputFlag.Name)),
-			common.Big(ctx.GlobalString(GasFlag.Name)),
-			common.Big(ctx.GlobalString(PriceFlag.Name)),
-			common.Big(ctx.GlobalString(ValueFlag.Name)),
-		)
+
+		ret, err = runtime.Call(receiver.Address(), common.Hex2Bytes(ctx.GlobalString(InputFlag.Name)), &runtime.Config{
+			Origin:   sender.Address(),
+			State:    statedb,
+			GasLimit: common.Big(ctx.GlobalString(GasFlag.Name)),
+			GasPrice: common.Big(ctx.GlobalString(PriceFlag.Name)),
+			Value:    common.Big(ctx.GlobalString(ValueFlag.Name)),
+			EVMConfig: vm.Config{
+				Tracer:             logger,
+				Debug:              ctx.GlobalBool(DebugFlag.Name),
+				DisableGasMetering: ctx.GlobalBool(DisableGasMeteringFlag.Name),
+			},
+		})
 	}
 	vmdone := time.Since(tstart)
 
@@ -197,8 +191,8 @@ func run(ctx *cli.Context) error {
 	vm.StdErrFormat(logger.StructLogs())
 
 	if ctx.GlobalBool(SysStatFlag.Name) {
-		var mem runtime.MemStats
-		runtime.ReadMemStats(&mem)
+		var mem goruntime.MemStats
+		goruntime.ReadMemStats(&mem)
 		fmt.Printf("vm took %v\n", vmdone)
 		fmt.Printf(`alloc:      %d
 tot alloc:  %d
@@ -223,87 +217,3 @@ func main() {
 		os.Exit(1)
 	}
 }
-
-type VMEnv struct {
-	state *state.StateDB
-	block *types.Block
-
-	transactor *common.Address
-	value      *big.Int
-
-	depth int
-	Gas   *big.Int
-	time  *big.Int
-	logs  []vm.StructLog
-
-	evm *vm.EVM
-}
-
-func NewEnv(state *state.StateDB, transactor common.Address, value *big.Int, cfg vm.Config) *VMEnv {
-	env := &VMEnv{
-		state:      state,
-		transactor: &transactor,
-		value:      value,
-		time:       big.NewInt(time.Now().Unix()),
-	}
-
-	env.evm = vm.New(env, cfg)
-	return env
-}
-
-// ruleSet implements vm.ChainConfig and will always default to the homestead rule set.
-type ruleSet struct{}
-
-func (ruleSet) IsHomestead(*big.Int) bool { return true }
-func (ruleSet) GasTable(*big.Int) params.GasTable {
-	return params.GasTableHomesteadGasRepriceFork
-}
-
-func (self *VMEnv) ChainConfig() *params.ChainConfig { return params.TestChainConfig }
-func (self *VMEnv) Vm() vm.Vm                        { return self.evm }
-func (self *VMEnv) Db() vm.Database                  { return self.state }
-func (self *VMEnv) SnapshotDatabase() int            { return self.state.Snapshot() }
-func (self *VMEnv) RevertToSnapshot(snap int)        { self.state.RevertToSnapshot(snap) }
-func (self *VMEnv) Origin() common.Address           { return *self.transactor }
-func (self *VMEnv) BlockNumber() *big.Int            { return common.Big0 }
-func (self *VMEnv) Coinbase() common.Address         { return *self.transactor }
-func (self *VMEnv) Time() *big.Int                   { return self.time }
-func (self *VMEnv) Difficulty() *big.Int             { return common.Big1 }
-func (self *VMEnv) BlockHash() []byte                { return make([]byte, 32) }
-func (self *VMEnv) Value() *big.Int                  { return self.value }
-func (self *VMEnv) GasLimit() *big.Int               { return big.NewInt(1000000000) }
-func (self *VMEnv) VmType() vm.Type                  { return vm.StdVmTy }
-func (self *VMEnv) Depth() int                       { return 0 }
-func (self *VMEnv) SetDepth(i int)                   { self.depth = i }
-func (self *VMEnv) GetHash(n uint64) common.Hash {
-	if self.block.Number().Cmp(big.NewInt(int64(n))) == 0 {
-		return self.block.Hash()
-	}
-	return common.Hash{}
-}
-func (self *VMEnv) AddLog(log *vm.Log) {
-	self.state.AddLog(log)
-}
-func (self *VMEnv) CanTransfer(from common.Address, balance *big.Int) bool {
-	return self.state.GetBalance(from).Cmp(balance) >= 0
-}
-func (self *VMEnv) Transfer(from, to vm.Account, amount *big.Int) {
-	core.Transfer(from, to, amount)
-}
-
-func (self *VMEnv) Call(caller vm.ContractRef, addr common.Address, data []byte, gas, price, value *big.Int) ([]byte, error) {
-	self.Gas = gas
-	return core.Call(self, caller, addr, data, gas, price, value)
-}
-
-func (self *VMEnv) CallCode(caller vm.ContractRef, addr common.Address, data []byte, gas, price, value *big.Int) ([]byte, error) {
-	return core.CallCode(self, caller, addr, data, gas, price, value)
-}
-
-func (self *VMEnv) DelegateCall(caller vm.ContractRef, addr common.Address, data []byte, gas, price *big.Int) ([]byte, error) {
-	return core.DelegateCall(self, caller, addr, data, gas, price)
-}
-
-func (self *VMEnv) Create(caller vm.ContractRef, data []byte, gas, price, value *big.Int) ([]byte, common.Address, error) {
-	return core.Create(self, caller, data, gas, price, value)
-}

cmd/geth/accountcmd.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 
 	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -180,31 +181,36 @@ nodes.
 
 func accountList(ctx *cli.Context) error {
 	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-	for i, acct := range stack.AccountManager().Accounts() {
-		fmt.Printf("Account #%d: {%x} %s\n", i, acct.Address, acct.File)
+
+	var index int
+	for _, wallet := range stack.AccountManager().Wallets() {
+		for _, account := range wallet.Accounts() {
+			fmt.Printf("Account #%d: {%x} %s\n", index, account.Address, &account.URL)
+			index++
+		}
 	}
 	return nil
 }
 
 // tries unlocking the specified account a few times.
-func unlockAccount(ctx *cli.Context, accman *accounts.Manager, address string, i int, passwords []string) (accounts.Account, string) {
-	account, err := utils.MakeAddress(accman, address)
+func unlockAccount(ctx *cli.Context, ks *keystore.KeyStore, address string, i int, passwords []string) (accounts.Account, string) {
+	account, err := utils.MakeAddress(ks, address)
 	if err != nil {
 		utils.Fatalf("Could not list accounts: %v", err)
 	}
 	for trials := 0; trials < 3; trials++ {
 		prompt := fmt.Sprintf("Unlocking account %s | Attempt %d/%d", address, trials+1, 3)
 		password := getPassPhrase(prompt, false, i, passwords)
-		err = accman.Unlock(account, password)
+		err = ks.Unlock(account, password)
 		if err == nil {
 			glog.V(logger.Info).Infof("Unlocked account %x", account.Address)
 			return account, password
 		}
-		if err, ok := err.(*accounts.AmbiguousAddrError); ok {
+		if err, ok := err.(*keystore.AmbiguousAddrError); ok {
 			glog.V(logger.Info).Infof("Unlocked account %x", account.Address)
-			return ambiguousAddrRecovery(accman, err, password), password
+			return ambiguousAddrRecovery(ks, err, password), password
 		}
-		if err != accounts.ErrDecrypt {
+		if err != keystore.ErrDecrypt {
 			// No need to prompt again if the error is not decryption-related.
 			break
 		}
@@ -244,15 +250,15 @@ func getPassPhrase(prompt string, confirmation bool, i int, passwords []string)
 	return password
 }
 
-func ambiguousAddrRecovery(am *accounts.Manager, err *accounts.AmbiguousAddrError, auth string) accounts.Account {
+func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrError, auth string) accounts.Account {
 	fmt.Printf("Multiple key files exist for address %x:\n", err.Addr)
 	for _, a := range err.Matches {
-		fmt.Println("  ", a.File)
+		fmt.Println("  ", a.URL)
 	}
 	fmt.Println("Testing your passphrase against all of them...")
 	var match *accounts.Account
 	for _, a := range err.Matches {
-		if err := am.Unlock(a, auth); err == nil {
+		if err := ks.Unlock(a, auth); err == nil {
 			match = &a
 			break
 		}
@@ -260,11 +266,11 @@ func ambiguousAddrRecovery(am *accounts.Manager, err *accounts.AmbiguousAddrErro
 	if match == nil {
 		utils.Fatalf("None of the listed files could be unlocked.")
 	}
-	fmt.Printf("Your passphrase unlocked %s\n", match.File)
+	fmt.Printf("Your passphrase unlocked %s\n", match.URL)
 	fmt.Println("In order to avoid this warning, you need to remove the following duplicate key files:")
 	for _, a := range err.Matches {
 		if a != *match {
-			fmt.Println("  ", a.File)
+			fmt.Println("  ", a.URL)
 		}
 	}
 	return *match
@@ -275,7 +281,8 @@ func accountCreate(ctx *cli.Context) error {
 	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
 	password := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx))
 
-	account, err := stack.AccountManager().NewAccount(password)
+	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+	account, err := ks.NewAccount(password)
 	if err != nil {
 		utils.Fatalf("Failed to create account: %v", err)
 	}
@@ -290,9 +297,11 @@ func accountUpdate(ctx *cli.Context) error {
 		utils.Fatalf("No accounts specified to update")
 	}
 	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
-	account, oldPassword := unlockAccount(ctx, stack.AccountManager(), ctx.Args().First(), 0, nil)
+	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+
+	account, oldPassword := unlockAccount(ctx, ks, ctx.Args().First(), 0, nil)
 	newPassword := getPassPhrase("Please give a new password. Do not forget this password.", true, 0, nil)
-	if err := stack.AccountManager().Update(account, oldPassword, newPassword); err != nil {
+	if err := ks.Update(account, oldPassword, newPassword); err != nil {
 		utils.Fatalf("Could not update the account: %v", err)
 	}
 	return nil
@@ -310,7 +319,9 @@ func importWallet(ctx *cli.Context) error {
 
 	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
 	passphrase := getPassPhrase("", false, 0, utils.MakePasswordList(ctx))
-	acct, err := stack.AccountManager().ImportPreSaleKey(keyJson, passphrase)
+
+	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+	acct, err := ks.ImportPreSaleKey(keyJson, passphrase)
 	if err != nil {
 		utils.Fatalf("%v", err)
 	}
@@ -329,7 +340,9 @@ func accountImport(ctx *cli.Context) error {
 	}
 	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
 	passphrase := getPassPhrase("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx))
-	acct, err := stack.AccountManager().ImportECDSA(key, passphrase)
+
+	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+	acct, err := ks.ImportECDSA(key, passphrase)
 	if err != nil {
 		utils.Fatalf("Could not create the account: %v", err)
 	}

cmd/geth/accountcmd_test.go

@@ -35,7 +35,7 @@ import (
 func tmpDatadirWithKeystore(t *testing.T) string {
 	datadir := tmpdir(t)
 	keystore := filepath.Join(datadir, "keystore")
-	source := filepath.Join("..", "..", "accounts", "testdata", "keystore")
+	source := filepath.Join("..", "..", "accounts", "keystore", "testdata", "keystore")
 	if err := cp.CopyAll(keystore, source); err != nil {
 		t.Fatal(err)
 	}
@@ -53,15 +53,15 @@ func TestAccountList(t *testing.T) {
 	defer geth.expectExit()
 	if runtime.GOOS == "windows" {
 		geth.expect(`
-Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} {{.Datadir}}\keystore\UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8
-Account #1: {f466859ead1932d743d622cb74fc058882e8648a} {{.Datadir}}\keystore\aaa
-Account #2: {289d485d9771714cce91d3393d764e1311907acc} {{.Datadir}}\keystore\zzz
+Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} keystore://{{.Datadir}}\keystore\UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8
+Account #1: {f466859ead1932d743d622cb74fc058882e8648a} keystore://{{.Datadir}}\keystore\aaa
+Account #2: {289d485d9771714cce91d3393d764e1311907acc} keystore://{{.Datadir}}\keystore\zzz
 `)
 	} else {
 		geth.expect(`
-Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} {{.Datadir}}/keystore/UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8
-Account #1: {f466859ead1932d743d622cb74fc058882e8648a} {{.Datadir}}/keystore/aaa
-Account #2: {289d485d9771714cce91d3393d764e1311907acc} {{.Datadir}}/keystore/zzz
+Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} keystore://{{.Datadir}}/keystore/UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8
+Account #1: {f466859ead1932d743d622cb74fc058882e8648a} keystore://{{.Datadir}}/keystore/aaa
+Account #2: {289d485d9771714cce91d3393d764e1311907acc} keystore://{{.Datadir}}/keystore/zzz
 `)
 	}
 }
@@ -148,7 +148,7 @@ Passphrase: {{.InputLine "foobar"}}
 		"Unlocked account f466859ead1932d743d622cb74fc058882e8648a",
 	}
 	for _, m := range wantMessages {
-		if strings.Index(geth.stderrText(), m) == -1 {
+		if !strings.Contains(geth.stderrText(), m) {
 			t.Errorf("stderr text does not contain %q", m)
 		}
 	}
@@ -193,7 +193,7 @@ Passphrase: {{.InputLine "foobar"}}
 		"Unlocked account 289d485d9771714cce91d3393d764e1311907acc",
 	}
 	for _, m := range wantMessages {
-		if strings.Index(geth.stderrText(), m) == -1 {
+		if !strings.Contains(geth.stderrText(), m) {
 			t.Errorf("stderr text does not contain %q", m)
 		}
 	}
@@ -212,7 +212,7 @@ func TestUnlockFlagPasswordFile(t *testing.T) {
 		"Unlocked account 289d485d9771714cce91d3393d764e1311907acc",
 	}
 	for _, m := range wantMessages {
-		if strings.Index(geth.stderrText(), m) == -1 {
+		if !strings.Contains(geth.stderrText(), m) {
 			t.Errorf("stderr text does not contain %q", m)
 		}
 	}
@@ -230,7 +230,7 @@ Fatal: Failed to unlock account 0 (could not decrypt key with given passphrase)
 }
 
 func TestUnlockFlagAmbiguous(t *testing.T) {
-	store := filepath.Join("..", "..", "accounts", "testdata", "dupes")
+	store := filepath.Join("..", "..", "accounts", "keystore", "testdata", "dupes")
 	geth := runGeth(t,
 		"--keystore", store, "--nat", "none", "--nodiscover", "--dev",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a",
@@ -247,12 +247,12 @@ Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
 Passphrase: {{.InputLine "foobar"}}
 Multiple key files exist for address f466859ead1932d743d622cb74fc058882e8648a:
-   {{keypath "1"}}
-   {{keypath "2"}}
+   keystore://{{keypath "1"}}
+   keystore://{{keypath "2"}}
 Testing your passphrase against all of them...
-Your passphrase unlocked {{keypath "1"}}
+Your passphrase unlocked keystore://{{keypath "1"}}
 In order to avoid this warning, you need to remove the following duplicate key files:
-   {{keypath "2"}}
+   keystore://{{keypath "2"}}
 `)
 	geth.expectExit()
 
@@ -260,14 +260,14 @@ In order to avoid this warning, you need to remove the following duplicate key f
 		"Unlocked account f466859ead1932d743d622cb74fc058882e8648a",
 	}
 	for _, m := range wantMessages {
-		if strings.Index(geth.stderrText(), m) == -1 {
+		if !strings.Contains(geth.stderrText(), m) {
 			t.Errorf("stderr text does not contain %q", m)
 		}
 	}
 }
 
 func TestUnlockFlagAmbiguousWrongPassword(t *testing.T) {
-	store := filepath.Join("..", "..", "accounts", "testdata", "dupes")
+	store := filepath.Join("..", "..", "accounts", "keystore", "testdata", "dupes")
 	geth := runGeth(t,
 		"--keystore", store, "--nat", "none", "--nodiscover", "--dev",
 		"--unlock", "f466859ead1932d743d622cb74fc058882e8648a")
@@ -283,8 +283,8 @@ Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3
 !! Unsupported terminal, password will be echoed.
 Passphrase: {{.InputLine "wrong"}}
 Multiple key files exist for address f466859ead1932d743d622cb74fc058882e8648a:
-   {{keypath "1"}}
-   {{keypath "2"}}
+   keystore://{{keypath "1"}}
+   keystore://{{keypath "2"}}
 Testing your passphrase against all of them...
 Fatal: None of the listed files could be unlocked.
 `)

cmd/geth/chaincmd.go

@@ -32,6 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/syndtr/goleveldb/leveldb/util"
@@ -39,6 +40,18 @@ import (
 )
 
 var (
+	initCommand = cli.Command{
+		Action:    initGenesis,
+		Name:      "init",
+		Usage:     "Bootstrap and initialize a new genesis block",
+		ArgsUsage: "<genesisPath>",
+		Category:  "BLOCKCHAIN COMMANDS",
+		Description: `
+The init command initializes a new genesis block and definition for the network.
+This is a destructive action and changes the network in which you will be
+participating.
+`,
+	}
 	importCommand = cli.Command{
 		Action:    importChain,
 		Name:      "import",
@@ -95,6 +108,31 @@ Use "ethereum dump 0" to dump the genesis block.
 	}
 )
 
+// initGenesis will initialise the given JSON format genesis file and writes it as
+// the zero'd block (i.e. genesis) or will fail hard if it can't succeed.
+func initGenesis(ctx *cli.Context) error {
+	genesisPath := ctx.Args().First()
+	if len(genesisPath) == 0 {
+		utils.Fatalf("must supply path to genesis JSON file")
+	}
+
+	stack := makeFullNode(ctx)
+	chaindb := utils.MakeChainDatabase(ctx, stack)
+
+	genesisFile, err := os.Open(genesisPath)
+	if err != nil {
+		utils.Fatalf("failed to read genesis file: %v", err)
+	}
+	defer genesisFile.Close()
+
+	block, err := core.WriteGenesisBlock(chaindb, genesisFile)
+	if err != nil {
+		utils.Fatalf("failed to write genesis block: %v", err)
+	}
+	glog.V(logger.Info).Infof("successfully wrote genesis block and/or chain rule set: %x", block.Hash())
+	return nil
+}
+
 func importChain(ctx *cli.Context) error {
 	if len(ctx.Args()) != 1 {
 		utils.Fatalf("This command requires an argument.")

cmd/geth/dao_test.go

@@ -83,75 +83,28 @@ var daoGenesisForkBlock = big.NewInt(314)
 // TestDAOForkBlockNewChain tests that the DAO hard-fork number and the nodes support/opposition is correctly
 // set in the database after various initialization procedures and invocations.
 func TestDAOForkBlockNewChain(t *testing.T) {
-	for _, arg := range []struct {
+	for i, arg := range []struct {
 		testnet     bool
 		genesis     string
-		votes       [][2]bool
 		expectBlock *big.Int
 		expectVote  bool
 	}{
 		// Test DAO Default Mainnet
-		{false, "", [][2]bool{{false, false}}, params.MainNetDAOForkBlock, true},
-		// test DAO Support Mainnet
-		{false, "", [][2]bool{{true, false}}, params.MainNetDAOForkBlock, true},
-		// test DAO Oppose Mainnet
-		{false, "", [][2]bool{{false, true}}, params.MainNetDAOForkBlock, false},
-		// test DAO Switch To Support Mainnet
-		{false, "", [][2]bool{{false, true}, {true, false}}, params.MainNetDAOForkBlock, true},
-		// test DAO Switch To Oppose Mainnet
-		{false, "", [][2]bool{{true, false}, {false, true}}, params.MainNetDAOForkBlock, false},
+		{false, "", params.MainNetDAOForkBlock, true},
 		// test DAO Default Testnet
-		{true, "", [][2]bool{{false, false}}, params.TestNetDAOForkBlock, true},
-		// test DAO Support Testnet
-		{true, "", [][2]bool{{true, false}}, params.TestNetDAOForkBlock, true},
-		// test DAO Oppose Testnet
-		{true, "", [][2]bool{{false, true}}, params.TestNetDAOForkBlock, false},
-		// test DAO Switch To Support Testnet
-		{true, "", [][2]bool{{false, true}, {true, false}}, params.TestNetDAOForkBlock, true},
-		// test DAO Switch To Oppose Testnet
-		{true, "", [][2]bool{{true, false}, {false, true}}, params.TestNetDAOForkBlock, false},
+		{true, "", params.TestNetDAOForkBlock, true},
 		// test DAO Init Old Privnet
-		{false, daoOldGenesis, [][2]bool{}, nil, false},
-		// test DAO Default Old Privnet
-		{false, daoOldGenesis, [][2]bool{{false, false}}, nil, false},
-		// test DAO Support Old Privnet
-		{false, daoOldGenesis, [][2]bool{{true, false}}, nil, true},
-		// test DAO Oppose Old Privnet
-		{false, daoOldGenesis, [][2]bool{{false, true}}, nil, false},
-		// test DAO Switch To Support Old Privnet
-		{false, daoOldGenesis, [][2]bool{{false, true}, {true, false}}, nil, true},
-		// test DAO Switch To Oppose Old Privnet
-		{false, daoOldGenesis, [][2]bool{{true, false}, {false, true}}, nil, false},
-		// test DAO Init No Fork Privnet
-		{false, daoNoForkGenesis, [][2]bool{}, daoGenesisForkBlock, false},
+		{false, daoOldGenesis, nil, false},
 		// test DAO Default No Fork Privnet
-		{false, daoNoForkGenesis, [][2]bool{{false, false}}, daoGenesisForkBlock, false},
-		// test DAO Support No Fork Privnet
-		{false, daoNoForkGenesis, [][2]bool{{true, false}}, daoGenesisForkBlock, true},
-		// test DAO Oppose No Fork Privnet
-		{false, daoNoForkGenesis, [][2]bool{{false, true}}, daoGenesisForkBlock, false},
-		// test DAO Switch To Support No Fork Privnet
-		{false, daoNoForkGenesis, [][2]bool{{false, true}, {true, false}}, daoGenesisForkBlock, true},
-		// test DAO Switch To Oppose No Fork Privnet
-		{false, daoNoForkGenesis, [][2]bool{{true, false}, {false, true}}, daoGenesisForkBlock, false},
-		// test DAO Init Pro Fork Privnet
-		{false, daoProForkGenesis, [][2]bool{}, daoGenesisForkBlock, true},
+		{false, daoNoForkGenesis, daoGenesisForkBlock, false},
 		// test DAO Default Pro Fork Privnet
-		{false, daoProForkGenesis, [][2]bool{{false, false}}, daoGenesisForkBlock, true},
-		// test DAO Support Pro Fork Privnet
-		{false, daoProForkGenesis, [][2]bool{{true, false}}, daoGenesisForkBlock, true},
-		// test DAO Oppose Pro Fork Privnet
-		{false, daoProForkGenesis, [][2]bool{{false, true}}, daoGenesisForkBlock, false},
-		// test DAO Switch To Support Pro Fork Privnet
-		{false, daoProForkGenesis, [][2]bool{{false, true}, {true, false}}, daoGenesisForkBlock, true},
-		// test DAO Switch To Oppose Pro Fork Privnet
-		{false, daoProForkGenesis, [][2]bool{{true, false}, {false, true}}, daoGenesisForkBlock, false},
+		{false, daoProForkGenesis, daoGenesisForkBlock, true},
 	} {
-		testDAOForkBlockNewChain(t, arg.testnet, arg.genesis, arg.votes, arg.expectBlock, arg.expectVote)
+		testDAOForkBlockNewChain(t, i, arg.testnet, arg.genesis, arg.expectBlock, arg.expectVote)
 	}
 }
 
-func testDAOForkBlockNewChain(t *testing.T, testnet bool, genesis string, votes [][2]bool, expectBlock *big.Int, expectVote bool) {
+func testDAOForkBlockNewChain(t *testing.T, test int, testnet bool, genesis string, expectBlock *big.Int, expectVote bool) {
 	// Create a temporary data directory to use and inspect later
 	datadir := tmpdir(t)
 	defer os.RemoveAll(datadir)
@@ -160,21 +113,15 @@ func testDAOForkBlockNewChain(t *testing.T, testnet bool, genesis string, votes
 	if genesis != "" {
 		json := filepath.Join(datadir, "genesis.json")
 		if err := ioutil.WriteFile(json, []byte(genesis), 0600); err != nil {
-			t.Fatalf("failed to write genesis file: %v", err)
+			t.Fatalf("test %d: failed to write genesis file: %v", test, err)
 		}
 		runGeth(t, "--datadir", datadir, "init", json).cmd.Wait()
-	}
-	for _, vote := range votes {
+	} else {
+		// Force chain initialization
 		args := []string{"--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--datadir", datadir}
 		if testnet {
 			args = append(args, "--testnet")
 		}
-		if vote[0] {
-			args = append(args, "--support-dao-fork")
-		}
-		if vote[1] {
-			args = append(args, "--oppose-dao-fork")
-		}
 		geth := runGeth(t, append(args, []string{"--exec", "2+2", "console"}...)...)
 		geth.cmd.Wait()
 	}
@@ -185,7 +132,7 @@ func testDAOForkBlockNewChain(t *testing.T, testnet bool, genesis string, votes
 	}
 	db, err := ethdb.NewLDBDatabase(path, 0, 0)
 	if err != nil {
-		t.Fatalf("failed to open test database: %v", err)
+		t.Fatalf("test %d: failed to open test database: %v", test, err)
 	}
 	defer db.Close()
 
@@ -198,20 +145,20 @@ func testDAOForkBlockNewChain(t *testing.T, testnet bool, genesis string, votes
 	}
 	config, err := core.GetChainConfig(db, genesisHash)
 	if err != nil {
-		t.Errorf("failed to retrieve chain config: %v", err)
+		t.Errorf("test %d: failed to retrieve chain config: %v", test, err)
 		return // we want to return here, the other checks can't make it past this point (nil panic).
 	}
 	// Validate the DAO hard-fork block number against the expected value
 	if config.DAOForkBlock == nil {
 		if expectBlock != nil {
-			t.Errorf("dao hard-fork block mismatch: have nil, want %v", expectBlock)
+			t.Errorf("test %d: dao hard-fork block mismatch: have nil, want %v", test, expectBlock)
 		}
 	} else if expectBlock == nil {
-		t.Errorf("dao hard-fork block mismatch: have %v, want nil", config.DAOForkBlock)
+		t.Errorf("test %d: dao hard-fork block mismatch: have %v, want nil", test, config.DAOForkBlock)
 	} else if config.DAOForkBlock.Cmp(expectBlock) != 0 {
-		t.Errorf("dao hard-fork block mismatch: have %v, want %v", config.DAOForkBlock, expectBlock)
+		t.Errorf("test %d: dao hard-fork block mismatch: have %v, want %v", test, config.DAOForkBlock, expectBlock)
 	}
 	if config.DAOForkSupport != expectVote {
-		t.Errorf("dao hard-fork support mismatch: have %v, want %v", config.DAOForkSupport, expectVote)
+		t.Errorf("test %d: dao hard-fork support mismatch: have %v, want %v", test, config.DAOForkSupport, expectVote)
 	}
 }

cmd/geth/library.c

@@ -1,24 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// Simple wrapper to translate the API exposed methods and types to inthernal
-// Go versions of the same types.
-
-#include "_cgo_export.h"
-
-int run(const char* args) {
- return doRun((char*)args);
-}

cmd/geth/library.go

@@ -1,46 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// Contains a simple library definition to allow creating a Geth instance from
-// straight C code.
-
-package main
-
-// #ifdef __cplusplus
-// extern "C" {
-// #endif
-//
-// extern int run(const char*);
-//
-// #ifdef __cplusplus
-// }
-// #endif
-import "C"
-import (
-	"fmt"
-	"os"
-	"strings"
-)
-
-//export doRun
-func doRun(args *C.char) C.int {
-	// This is equivalent to geth.main, just modified to handle the function arg passing
-	if err := app.Run(strings.Split("geth "+C.GoString(args), " ")); err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		return -1
-	}
-	return 0
-}

cmd/geth/library_android.go

@@ -1,56 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// Contains specialized code for running Geth on Android.
-
-package main
-
-// #include <android/log.h>
-// #cgo LDFLAGS: -llog
-import "C"
-import (
-	"bufio"
-	"os"
-)
-
-func init() {
-	// Redirect the standard output and error to logcat
-	oldStdout, oldStderr := os.Stdout, os.Stderr
-
-	outRead, outWrite, _ := os.Pipe()
-	errRead, errWrite, _ := os.Pipe()
-
-	os.Stdout = outWrite
-	os.Stderr = errWrite
-
-	go func() {
-		scanner := bufio.NewScanner(outRead)
-		for scanner.Scan() {
-			line := scanner.Text()
-			C.__android_log_write(C.ANDROID_LOG_INFO, C.CString("Stdout"), C.CString(line))
-			oldStdout.WriteString(line + "\n")
-		}
-	}()
-
-	go func() {
-		scanner := bufio.NewScanner(errRead)
-		for scanner.Scan() {
-			line := scanner.Text()
-			C.__android_log_write(C.ANDROID_LOG_INFO, C.CString("Stderr"), C.CString(line))
-			oldStderr.WriteString(line + "\n")
-		}
-	}()
-}

cmd/geth/main.go

@@ -20,21 +20,19 @@ package main
 import (
 	"encoding/hex"
 	"fmt"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
 	"time"
 
-	"github.com/ethereum/ethash"
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/contracts/release"
-	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
@@ -64,59 +62,26 @@ func init() {
 	app.HideVersion = true // we have a command to print the version
 	app.Copyright = "Copyright 2013-2016 The go-ethereum Authors"
 	app.Commands = []cli.Command{
+		// See chaincmd.go:
+		initCommand,
 		importCommand,
 		exportCommand,
 		upgradedbCommand,
 		removedbCommand,
 		dumpCommand,
+		// See monitorcmd.go:
 		monitorCommand,
+		// See accountcmd.go:
 		accountCommand,
 		walletCommand,
+		// See consolecmd.go:
 		consoleCommand,
 		attachCommand,
 		javascriptCommand,
-		{
-			Action:    makedag,
-			Name:      "makedag",
-			Usage:     "Generate ethash DAG (for testing)",
-			ArgsUsage: "<blockNum> <outputDir>",
-			Category:  "MISCELLANEOUS COMMANDS",
-			Description: `
-The makedag command generates an ethash DAG in /tmp/dag.
-
-This command exists to support the system testing project.
-Regular users do not need to execute it.
-`,
-		},
-		{
-			Action:    version,
-			Name:      "version",
-			Usage:     "Print version numbers",
-			ArgsUsage: " ",
-			Category:  "MISCELLANEOUS COMMANDS",
-			Description: `
-The output of this command is supposed to be machine-readable.
-`,
-		},
-		{
-			Action:    initGenesis,
-			Name:      "init",
-			Usage:     "Bootstrap and initialize a new genesis block",
-			ArgsUsage: "<genesisPath>",
-			Category:  "BLOCKCHAIN COMMANDS",
-			Description: `
-The init command initializes a new genesis block and definition for the network.
-This is a destructive action and changes the network in which you will be
-participating.
-`,
-		},
-		{
-			Action:    license,
-			Name:      "license",
-			Usage:     "Display license information",
-			ArgsUsage: " ",
-			Category:  "MISCELLANEOUS COMMANDS",
-		},
+		// See misccmd.go:
+		makedagCommand,
+		versionCommand,
+		licenseCommand,
 	}
 
 	app.Flags = []cli.Flag{
@@ -126,7 +91,6 @@ participating.
 		utils.BootnodesFlag,
 		utils.DataDirFlag,
 		utils.KeyStoreDirFlag,
-		utils.OlympicFlag,
 		utils.FastSyncFlag,
 		utils.LightModeFlag,
 		utils.LightServFlag,
@@ -140,14 +104,11 @@ participating.
 		utils.MaxPendingPeersFlag,
 		utils.EtherbaseFlag,
 		utils.GasPriceFlag,
-		utils.SupportDAOFork,
-		utils.OpposeDAOFork,
 		utils.MinerThreadsFlag,
 		utils.MiningEnabledFlag,
 		utils.AutoDAGFlag,
 		utils.TargetGasLimitFlag,
 		utils.NATFlag,
-		utils.NatspecEnabledFlag,
 		utils.NoDiscoverFlag,
 		utils.DiscoveryV5Flag,
 		utils.NetrestrictFlag,
@@ -173,6 +134,7 @@ participating.
 		utils.VMForceJitFlag,
 		utils.VMJitCacheFlag,
 		utils.VMEnableJitFlag,
+		utils.VMEnableDebugFlag,
 		utils.NetworkIdFlag,
 		utils.RPCCORSDomainFlag,
 		utils.EthStatsURLFlag,
@@ -208,7 +170,6 @@ participating.
 	}
 
 	app.After = func(ctx *cli.Context) error {
-		logger.Flush()
 		debug.Exit()
 		console.Stdin.Close() // Resets terminal mode.
 		return nil
@@ -232,30 +193,6 @@ func geth(ctx *cli.Context) error {
 	return nil
 }
 
-// initGenesis will initialise the given JSON format genesis file and writes it as
-// the zero'd block (i.e. genesis) or will fail hard if it can't succeed.
-func initGenesis(ctx *cli.Context) error {
-	genesisPath := ctx.Args().First()
-	if len(genesisPath) == 0 {
-		utils.Fatalf("must supply path to genesis JSON file")
-	}
-
-	stack := makeFullNode(ctx)
-	chaindb := utils.MakeChainDatabase(ctx, stack)
-
-	genesisFile, err := os.Open(genesisPath)
-	if err != nil {
-		utils.Fatalf("failed to read genesis file: %v", err)
-	}
-
-	block, err := core.WriteGenesisBlock(chaindb, genesisFile)
-	if err != nil {
-		utils.Fatalf("failed to write genesis block: %v", err)
-	}
-	glog.V(logger.Info).Infof("successfully wrote genesis block and/or chain rule set: %x", block.Hash())
-	return nil
-}
-
 func makeFullNode(ctx *cli.Context) *node.Node {
 	// Create the default extradata and construct the base node
 	var clientInfo = struct {
@@ -311,14 +248,50 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 	utils.StartNode(stack)
 
 	// Unlock any account specifically requested
-	accman := stack.AccountManager()
+	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+
 	passwords := utils.MakePasswordList(ctx)
-	accounts := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",")
-	for i, account := range accounts {
+	unlocks := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",")
+	for i, account := range unlocks {
 		if trimmed := strings.TrimSpace(account); trimmed != "" {
-			unlockAccount(ctx, accman, trimmed, i, passwords)
+			unlockAccount(ctx, ks, trimmed, i, passwords)
 		}
 	}
+	// Register wallet event handlers to open and auto-derive wallets
+	events := make(chan accounts.WalletEvent, 16)
+	stack.AccountManager().Subscribe(events)
+
+	go func() {
+		// Create an chain state reader for self-derivation
+		rpcClient, err := stack.Attach()
+		if err != nil {
+			utils.Fatalf("Failed to attach to self: %v", err)
+		}
+		stateReader := ethclient.NewClient(rpcClient)
+
+		// Open and self derive any wallets already attached
+		for _, wallet := range stack.AccountManager().Wallets() {
+			if err := wallet.Open(""); err != nil {
+				glog.V(logger.Warn).Infof("Failed to open wallet %s: %v", wallet.URL(), err)
+			} else {
+				wallet.SelfDerive(accounts.DefaultBaseDerivationPath, stateReader)
+			}
+		}
+		// Listen for wallet event till termination
+		for event := range events {
+			if event.Arrive {
+				if err := event.Wallet.Open(""); err != nil {
+					glog.V(logger.Info).Infof("New wallet appeared: %s, failed to open: %s", event.Wallet.URL(), err)
+				} else {
+					glog.V(logger.Info).Infof("New wallet appeared: %s, %s", event.Wallet.URL(), event.Wallet.Status())
+					event.Wallet.SelfDerive(accounts.DefaultBaseDerivationPath, stateReader)
+				}
+			} else {
+				glog.V(logger.Info).Infof("Old wallet dropped:  %s", event.Wallet.URL())
+				event.Wallet.Close()
+			}
+		}
+	}()
 	// Start auxiliary services if enabled
 	if ctx.GlobalBool(utils.MiningEnabledFlag.Name) {
 		var ethereum *eth.Ethereum
@@ -330,65 +303,3 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 		}
 	}
 }
-
-func makedag(ctx *cli.Context) error {
-	args := ctx.Args()
-	wrongArgs := func() {
-		utils.Fatalf(`Usage: geth makedag <block number> <outputdir>`)
-	}
-	switch {
-	case len(args) == 2:
-		blockNum, err := strconv.ParseUint(args[0], 0, 64)
-		dir := args[1]
-		if err != nil {
-			wrongArgs()
-		} else {
-			dir = filepath.Clean(dir)
-			// seems to require a trailing slash
-			if !strings.HasSuffix(dir, "/") {
-				dir = dir + "/"
-			}
-			_, err = ioutil.ReadDir(dir)
-			if err != nil {
-				utils.Fatalf("Can't find dir")
-			}
-			fmt.Println("making DAG, this could take awhile...")
-			ethash.MakeDAG(blockNum, dir)
-		}
-	default:
-		wrongArgs()
-	}
-	return nil
-}
-
-func version(ctx *cli.Context) error {
-	fmt.Println(strings.Title(clientIdentifier))
-	fmt.Println("Version:", params.Version)
-	if gitCommit != "" {
-		fmt.Println("Git Commit:", gitCommit)
-	}
-	fmt.Println("Protocol Versions:", eth.ProtocolVersions)
-	fmt.Println("Network Id:", ctx.GlobalInt(utils.NetworkIdFlag.Name))
-	fmt.Println("Go Version:", runtime.Version())
-	fmt.Println("OS:", runtime.GOOS)
-	fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH"))
-	fmt.Printf("GOROOT=%s\n", runtime.GOROOT())
-	return nil
-}
-
-func license(_ *cli.Context) error {
-	fmt.Println(`Geth is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-Geth is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with geth. If not, see <http://www.gnu.org/licenses/>.
-`)
-	return nil
-}

cmd/geth/misccmd.go

@@ -0,0 +1,128 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/ethereum/ethash"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/params"
+	"gopkg.in/urfave/cli.v1"
+)
+
+var (
+	makedagCommand = cli.Command{
+		Action:    makedag,
+		Name:      "makedag",
+		Usage:     "Generate ethash DAG (for testing)",
+		ArgsUsage: "<blockNum> <outputDir>",
+		Category:  "MISCELLANEOUS COMMANDS",
+		Description: `
+The makedag command generates an ethash DAG in /tmp/dag.
+
+This command exists to support the system testing project.
+Regular users do not need to execute it.
+`,
+	}
+	versionCommand = cli.Command{
+		Action:    version,
+		Name:      "version",
+		Usage:     "Print version numbers",
+		ArgsUsage: " ",
+		Category:  "MISCELLANEOUS COMMANDS",
+		Description: `
+The output of this command is supposed to be machine-readable.
+`,
+	}
+	licenseCommand = cli.Command{
+		Action:    license,
+		Name:      "license",
+		Usage:     "Display license information",
+		ArgsUsage: " ",
+		Category:  "MISCELLANEOUS COMMANDS",
+	}
+)
+
+func makedag(ctx *cli.Context) error {
+	args := ctx.Args()
+	wrongArgs := func() {
+		utils.Fatalf(`Usage: geth makedag <block number> <outputdir>`)
+	}
+	switch {
+	case len(args) == 2:
+		blockNum, err := strconv.ParseUint(args[0], 0, 64)
+		dir := args[1]
+		if err != nil {
+			wrongArgs()
+		} else {
+			dir = filepath.Clean(dir)
+			// seems to require a trailing slash
+			if !strings.HasSuffix(dir, "/") {
+				dir = dir + "/"
+			}
+			_, err = ioutil.ReadDir(dir)
+			if err != nil {
+				utils.Fatalf("Can't find dir")
+			}
+			fmt.Println("making DAG, this could take awhile...")
+			ethash.MakeDAG(blockNum, dir)
+		}
+	default:
+		wrongArgs()
+	}
+	return nil
+}
+
+func version(ctx *cli.Context) error {
+	fmt.Println(strings.Title(clientIdentifier))
+	fmt.Println("Version:", params.Version)
+	if gitCommit != "" {
+		fmt.Println("Git Commit:", gitCommit)
+	}
+	fmt.Println("Protocol Versions:", eth.ProtocolVersions)
+	fmt.Println("Network Id:", ctx.GlobalInt(utils.NetworkIdFlag.Name))
+	fmt.Println("Go Version:", runtime.Version())
+	fmt.Println("OS:", runtime.GOOS)
+	fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH"))
+	fmt.Printf("GOROOT=%s\n", runtime.GOROOT())
+	return nil
+}
+
+func license(_ *cli.Context) error {
+	fmt.Println(`Geth is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+Geth is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with geth. If not, see <http://www.gnu.org/licenses/>.
+`)
+	return nil
+}

cmd/geth/monitorcmd.go

@@ -236,8 +236,9 @@ func expandMetrics(metrics map[string]interface{}, path string) []string {
 
 // fetchMetric iterates over the metrics map and retrieves a specific one.
 func fetchMetric(metrics map[string]interface{}, metric string) float64 {
-	parts, found := strings.Split(metric, "/"), true
+	parts := strings.Split(metric, "/")
 	for _, part := range parts[:len(parts)-1] {
+		var found bool
 		metrics, found = metrics[part].(map[string]interface{})
 		if !found {
 			return 0

cmd/geth/usage.go

@@ -67,7 +67,6 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.DataDirFlag,
 			utils.KeyStoreDirFlag,
 			utils.NetworkIdFlag,
-			utils.OlympicFlag,
 			utils.TestNetFlag,
 			utils.DevModeFlag,
 			utils.IdentityFlag,
@@ -156,6 +155,7 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.VMEnableJitFlag,
 			utils.VMForceJitFlag,
 			utils.VMJitCacheFlag,
+			utils.VMEnableDebugFlag,
 		},
 	},
 	{
@@ -170,7 +170,6 @@ var AppHelpFlagGroups = []flagGroup{
 		Name: "EXPERIMENTAL",
 		Flags: []cli.Flag{
 			utils.WhisperEnabledFlag,
-			utils.NatspecEnabledFlag,
 		},
 	},
 	{

cmd/gethrpctest/main.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"os/signal"
 
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -99,17 +100,18 @@ func MakeSystemNode(privkey string, test *tests.BlockTest) (*node.Node, error) {
 		return nil, err
 	}
 	// Create the keystore and inject an unlocked account if requested
-	accman := stack.AccountManager()
+	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+
 	if len(privkey) > 0 {
 		key, err := crypto.HexToECDSA(privkey)
 		if err != nil {
 			return nil, err
 		}
-		a, err := accman.ImportECDSA(key, "")
+		a, err := ks.ImportECDSA(key, "")
 		if err != nil {
 			return nil, err
 		}
-		if err := accman.Unlock(a, ""); err != nil {
+		if err := ks.Unlock(a, ""); err != nil {
 			return nil, err
 		}
 	}

cmd/swarm/cleandb.go

@@ -14,25 +14,26 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package vm
+package main
 
 import (
-	"math/big"
-	"testing"
+	"log"
 
-	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+	"gopkg.in/urfave/cli.v1"
 )
 
-func TestInit(t *testing.T) {
-	jumpTable := newJumpTable(&params.ChainConfig{HomesteadBlock: big.NewInt(1)}, big.NewInt(0))
-	if jumpTable[DELEGATECALL].valid {
-		t.Error("Expected DELEGATECALL not to be present")
+func cleandb(ctx *cli.Context) {
+	args := ctx.Args()
+	if len(args) != 1 {
+		log.Fatal("need path to chunks database as the first and only argument")
 	}
 
-	for _, n := range []int64{1, 2, 100} {
-		jumpTable := newJumpTable(&params.ChainConfig{HomesteadBlock: big.NewInt(1)}, big.NewInt(n))
-		if !jumpTable[DELEGATECALL].valid {
-			t.Error("Expected DELEGATECALL to be present for block", n)
-		}
+	chunkDbPath := args[0]
+	hash := storage.MakeHashFunc("SHA3")
+	dbStore, err := storage.NewDbStore(chunkDbPath, hash, 10000000, 0)
+	if err != nil {
+		log.Fatalf("cannot initialise dbstore: %v", err)
 	}
+	dbStore.Cleanup()
 }

cmd/swarm/hash.go

@@ -19,30 +19,30 @@ package main
 
 import (
 	"fmt"
+	"log"
 	"os"
-	"runtime"
 
 	"github.com/ethereum/go-ethereum/swarm/storage"
+	"gopkg.in/urfave/cli.v1"
 )
 
-func main() {
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
-	if len(os.Args) < 2 {
-		fmt.Println("Usage: bzzhash <file name>")
-		os.Exit(0)
+func hash(ctx *cli.Context) {
+	args := ctx.Args()
+	if len(args) < 1 {
+		log.Fatal("Usage: swarm hash <file name>")
 	}
-	f, err := os.Open(os.Args[1])
+	f, err := os.Open(args[0])
 	if err != nil {
-		fmt.Println("Error opening file " + os.Args[1])
+		fmt.Println("Error opening file " + args[1])
 		os.Exit(1)
 	}
+	defer f.Close()
 
 	stat, _ := f.Stat()
 	chunker := storage.NewTreeChunker(storage.NewChunkerParams())
 	key, err := chunker.Split(f, stat.Size(), nil, nil, nil)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
+		log.Fatalf("%v\n", err)
 	} else {
 		fmt.Printf("%v\n", key)
 	}

cmd/swarm/main.go

@@ -21,11 +21,14 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"os/signal"
 	"runtime"
 	"strconv"
 	"strings"
+	"syscall"
 
 	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
@@ -39,15 +42,24 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/swarm"
 	bzzapi "github.com/ethereum/go-ethereum/swarm/api"
-	"github.com/ethereum/go-ethereum/swarm/network"
 	"gopkg.in/urfave/cli.v1"
 )
 
-const clientIdentifier = "bzzd"
+const (
+	clientIdentifier = "swarm"
+	versionString    = "0.2"
+)
 
 var (
-	gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
-	app       = utils.NewApp(gitCommit, "Ethereum Swarm server daemon")
+	gitCommit        string // Git SHA1 commit hash of the release (set via linker flags)
+	app              = utils.NewApp(gitCommit, "Ethereum Swarm")
+	testbetBootNodes = []string{
+		"enode://ec8ae764f7cb0417bdfb009b9d0f18ab3818a3a4e8e7c67dd5f18971a93510a2e6f43cd0b69a27e439a9629457ea804104f37c85e41eed057d3faabbf7744cdf@13.74.157.139:30429",
+		"enode://c2e1fceb3bf3be19dff71eec6cccf19f2dbf7567ee017d130240c670be8594bc9163353ca55dd8df7a4f161dd94b36d0615c17418b5a3cdcbb4e9d99dfa4de37@13.74.157.139:30430",
+		"enode://fe29b82319b734ce1ec68b84657d57145fee237387e63273989d354486731e59f78858e452ef800a020559da22dcca759536e6aa5517c53930d29ce0b1029286@13.74.157.139:30431",
+		"enode://1d7187e7bde45cf0bee489ce9852dd6d1a0d9aa67a33a6b8e6db8a4fbc6fcfa6f0f1a5419343671521b863b187d1c73bad3603bae66421d157ffef357669ddb8@13.74.157.139:30432",
+		"enode://0e4cba800f7b1ee73673afa6a4acead4018f0149d2e3216be3f133318fd165b324cd71b81fbe1e80deac8dbf56e57a49db7be67f8b9bc81bd2b7ee496434fb5d@13.74.157.139:30433",
+	}
 )
 
 var (
@@ -65,30 +77,48 @@ var (
 	}
 	SwarmNetworkIdFlag = cli.IntFlag{
 		Name:  "bzznetworkid",
-		Usage: "Network identifier (integer, default 322=swarm testnet)",
-		Value: network.NetworkId,
+		Usage: "Network identifier (integer, default 3=swarm testnet)",
 	}
 	SwarmConfigPathFlag = cli.StringFlag{
 		Name:  "bzzconfig",
 		Usage: "Swarm config file path (datadir/bzz)",
 	}
-	SwarmSwapEnabled = cli.BoolFlag{
+	SwarmSwapEnabledFlag = cli.BoolFlag{
 		Name:  "swap",
 		Usage: "Swarm SWAP enabled (default false)",
 	}
-	SwarmSyncEnabled = cli.BoolTFlag{
+	SwarmSyncEnabledFlag = cli.BoolTFlag{
 		Name:  "sync",
 		Usage: "Swarm Syncing enabled (default true)",
 	}
-	EthAPI = cli.StringFlag{
+	EthAPIFlag = cli.StringFlag{
 		Name:  "ethapi",
 		Usage: "URL of the Ethereum API provider",
 		Value: node.DefaultIPCEndpoint("geth"),
 	}
+	SwarmApiFlag = cli.StringFlag{
+		Name:  "bzzapi",
+		Usage: "Swarm HTTP endpoint",
+		Value: "http://127.0.0.1:8500",
+	}
+	SwarmRecursiveUploadFlag = cli.BoolFlag{
+		Name:  "recursive",
+		Usage: "Upload directories recursively",
+	}
+	SwarmWantManifestFlag = cli.BoolTFlag{
+		Name:  "manifest",
+		Usage: "Automatic manifest upload",
+	}
+	SwarmUploadDefaultPath = cli.StringFlag{
+		Name:  "defaultpath",
+		Usage: "path to file served for empty url path (none)",
+	}
+	CorsStringFlag = cli.StringFlag{
+		Name:  "corsdomain",
+		Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
+	}
 )
 
-var defaultBootnodes = []string{}
-
 func init() {
 	// Override flag defaults so bzzd can run alongside geth.
 	utils.ListenPortFlag.Value = 30399
@@ -96,8 +126,85 @@ func init() {
 	utils.IPCApiFlag.Value = "admin, bzz, chequebook, debug, rpc, web3"
 
 	// Set up the cli app.
-	app.Commands = nil
 	app.Action = bzzd
+	app.HideVersion = true // we have a command to print the version
+	app.Copyright = "Copyright 2013-2016 The go-ethereum Authors"
+	app.Commands = []cli.Command{
+		{
+			Action:    version,
+			Name:      "version",
+			Usage:     "Print version numbers",
+			ArgsUsage: " ",
+			Description: `
+The output of this command is supposed to be machine-readable.
+`,
+		},
+		{
+			Action:    upload,
+			Name:      "up",
+			Usage:     "upload a file or directory to swarm using the HTTP API",
+			ArgsUsage: " <file>",
+			Description: `
+"upload a file or directory to swarm using the HTTP API and prints the root hash",
+`,
+		},
+		{
+			Action:    hash,
+			Name:      "hash",
+			Usage:     "print the swarm hash of a file or directory",
+			ArgsUsage: " <file>",
+			Description: `
+Prints the swarm hash of file or directory.
+`,
+		},
+		{
+			Name:      "manifest",
+			Usage:     "update a MANIFEST",
+			ArgsUsage: "manifest COMMAND",
+			Description: `
+Updates a MANIFEST by adding/removing/updating the hash of a path.
+`,
+			Subcommands: []cli.Command{
+				{
+					Action:    add,
+					Name:      "add",
+					Usage:     "add a new path to the manifest",
+					ArgsUsage: "<MANIFEST> <path> <hash> [<content-type>]",
+					Description: `
+Adds a new path to the manifest
+`,
+				},
+				{
+					Action:    update,
+					Name:      "update",
+					Usage:     "update the hash for an already existing path in the manifest",
+					ArgsUsage: "<MANIFEST> <path> <newhash> [<newcontent-type>]",
+					Description: `
+Update the hash for an already existing path in the manifest
+`,
+				},
+				{
+					Action:    remove,
+					Name:      "remove",
+					Usage:     "removes a path from the manifest",
+					ArgsUsage: "<MANIFEST> <path>",
+					Description: `
+Removes a path from the manifest
+`,
+				},
+			},
+		},
+		{
+			Action:    cleandb,
+			Name:      "cleandb",
+			Usage:     "Cleans database of corrupted entries",
+			ArgsUsage: " ",
+			Description: `
+Cleans database of corrupted entries.
+`,
+		},
+	}
+
 	app.Flags = []cli.Flag{
 		utils.IdentityFlag,
 		utils.DataDirFlag,
@@ -115,14 +222,20 @@ func init() {
 		utils.IPCApiFlag,
 		utils.IPCPathFlag,
 		// bzzd-specific flags
-		EthAPI,
+		CorsStringFlag,
+		EthAPIFlag,
 		SwarmConfigPathFlag,
-		SwarmSwapEnabled,
-		SwarmSyncEnabled,
+		SwarmSwapEnabledFlag,
+		SwarmSyncEnabledFlag,
 		SwarmPortFlag,
 		SwarmAccountFlag,
 		SwarmNetworkIdFlag,
 		ChequebookAddrFlag,
+		// upload flags
+		SwarmApiFlag,
+		SwarmRecursiveUploadFlag,
+		SwarmWantManifestFlag,
+		SwarmUploadDefaultPath,
 	}
 	app.Flags = append(app.Flags, debug.Flags...)
 	app.Before = func(ctx *cli.Context) error {
@@ -142,17 +255,41 @@ func main() {
 	}
 }
 
+func version(ctx *cli.Context) error {
+	fmt.Println(strings.Title(clientIdentifier))
+	fmt.Println("Version:", versionString)
+	if gitCommit != "" {
+		fmt.Println("Git Commit:", gitCommit)
+	}
+	fmt.Println("Network Id:", ctx.GlobalInt(utils.NetworkIdFlag.Name))
+	fmt.Println("Go Version:", runtime.Version())
+	fmt.Println("OS:", runtime.GOOS)
+	fmt.Printf("GOPATH=%s\n", os.Getenv("GOPATH"))
+	fmt.Printf("GOROOT=%s\n", runtime.GOROOT())
+	return nil
+}
+
 func bzzd(ctx *cli.Context) error {
 	stack := utils.MakeNode(ctx, clientIdentifier, gitCommit)
 	registerBzzService(ctx, stack)
 	utils.StartNode(stack)
-
+	go func() {
+		sigc := make(chan os.Signal, 1)
+		signal.Notify(sigc, syscall.SIGTERM)
+		defer signal.Stop(sigc)
+		<-sigc
+		glog.V(logger.Info).Infoln("Got sigterm, shutting down...")
+		stack.Stop()
+	}()
+	networkId := ctx.GlobalUint64(SwarmNetworkIdFlag.Name)
 	// Add bootnodes as initial peers.
 	if ctx.GlobalIsSet(utils.BootnodesFlag.Name) {
 		bootnodes := strings.Split(ctx.GlobalString(utils.BootnodesFlag.Name), ",")
 		injectBootnodes(stack.Server(), bootnodes)
 	} else {
-		injectBootnodes(stack.Server(), defaultBootnodes)
+		if networkId == 3 {
+			injectBootnodes(stack.Server(), testbetBootNodes)
+		}
 	}
 
 	stack.Wait()
@@ -160,6 +297,7 @@ func bzzd(ctx *cli.Context) error {
 }
 
 func registerBzzService(ctx *cli.Context, stack *node.Node) {
+
 	prvkey := getAccount(ctx, stack)
 
 	chbookaddr := common.HexToAddress(ctx.GlobalString(ChequebookAddrFlag.Name))
@@ -167,6 +305,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
 	if bzzdir == "" {
 		bzzdir = stack.InstanceDir()
 	}
+
 	bzzconfig, err := bzzapi.NewConfig(bzzdir, chbookaddr, prvkey, ctx.GlobalUint64(SwarmNetworkIdFlag.Name))
 	if err != nil {
 		utils.Fatalf("unable to configure swarm: %v", err)
@@ -175,22 +314,21 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
 	if len(bzzport) > 0 {
 		bzzconfig.Port = bzzport
 	}
-	swapEnabled := ctx.GlobalBool(SwarmSwapEnabled.Name)
-	syncEnabled := ctx.GlobalBoolT(SwarmSyncEnabled.Name)
+	swapEnabled := ctx.GlobalBool(SwarmSwapEnabledFlag.Name)
+	syncEnabled := ctx.GlobalBoolT(SwarmSyncEnabledFlag.Name)
 
-	ethapi := ctx.GlobalString(EthAPI.Name)
+	ethapi := ctx.GlobalString(EthAPIFlag.Name)
+	cors := ctx.GlobalString(CorsStringFlag.Name)
 
 	boot := func(ctx *node.ServiceContext) (node.Service, error) {
 		var client *ethclient.Client
-		if ethapi == "" {
-			err = fmt.Errorf("use ethapi flag to connect to a an eth client and talk to the blockchain")
-		} else {
+		if len(ethapi) > 0 {
 			client, err = ethclient.Dial(ethapi)
+			if err != nil {
+				utils.Fatalf("Can't connect: %v", err)
+			}
 		}
-		if err != nil {
-			utils.Fatalf("Can't connect: %v", err)
-		}
-		return swarm.NewSwarm(ctx, client, bzzconfig, swapEnabled, syncEnabled)
+		return swarm.NewSwarm(ctx, client, bzzconfig, swapEnabled, syncEnabled, cors)
 	}
 	if err := stack.Register(boot); err != nil {
 		utils.Fatalf("Failed to register the Swarm service: %v", err)
@@ -199,6 +337,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
 
 func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey {
 	keyid := ctx.GlobalString(SwarmAccountFlag.Name)
+
 	if keyid == "" {
 		utils.Fatalf("Option %q is required", SwarmAccountFlag.Name)
 	}
@@ -208,29 +347,36 @@ func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey {
 		return key
 	}
 	// Otherwise try getting it from the keystore.
-	return decryptStoreAccount(stack.AccountManager(), keyid)
+	am := stack.AccountManager()
+	ks := am.Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
+
+	return decryptStoreAccount(ks, keyid)
 }
 
-func decryptStoreAccount(accman *accounts.Manager, account string) *ecdsa.PrivateKey {
+func decryptStoreAccount(ks *keystore.KeyStore, account string) *ecdsa.PrivateKey {
 	var a accounts.Account
 	var err error
 	if common.IsHexAddress(account) {
-		a, err = accman.Find(accounts.Account{Address: common.HexToAddress(account)})
-	} else if ix, ixerr := strconv.Atoi(account); ixerr == nil {
-		a, err = accman.AccountByIndex(ix)
+		a, err = ks.Find(accounts.Account{Address: common.HexToAddress(account)})
+	} else if ix, ixerr := strconv.Atoi(account); ixerr == nil && ix > 0 {
+		if accounts := ks.Accounts(); len(accounts) > ix {
+			a = accounts[ix]
+		} else {
+			err = fmt.Errorf("index %d higher than number of accounts %d", ix, len(accounts))
+		}
 	} else {
 		utils.Fatalf("Can't find swarm account key %s", account)
 	}
 	if err != nil {
 		utils.Fatalf("Can't find swarm account key: %v", err)
 	}
-	keyjson, err := ioutil.ReadFile(a.File)
+	keyjson, err := ioutil.ReadFile(a.URL.Path)
 	if err != nil {
 		utils.Fatalf("Can't load swarm account key: %v", err)
 	}
 	for i := 1; i <= 3; i++ {
 		passphrase := promptPassphrase(fmt.Sprintf("Unlocking swarm account %s [%d/3]", a.Address.Hex(), i))
-		key, err := accounts.DecryptKey(keyjson, passphrase)
+		key, err := keystore.DecryptKey(keyjson, passphrase)
 		if err == nil {
 			return key.PrivateKey
 		}

cmd/swarm/manifest.go

@@ -0,0 +1,360 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// Command  MANIFEST update
+package main
+
+import (
+	"gopkg.in/urfave/cli.v1"
+	"log"
+	"mime"
+	"path/filepath"
+	"strings"
+	"fmt"
+	"encoding/json"
+)
+
+func add(ctx *cli.Context) {
+
+	args := ctx.Args()
+	if len(args) < 3 {
+		log.Fatal("need atleast three arguments <MHASH> <path> <HASH> [<content-type>]")
+	}
+
+	var (
+		mhash  = args[0]
+		path   = args[1]
+		hash   = args[2]
+
+		ctype  string
+		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
+		mroot  manifest
+	)
+
+
+	if len(args) > 3 {
+		ctype = args[3]
+	} else {
+		ctype = mime.TypeByExtension(filepath.Ext(path))
+	}
+
+	newManifest := addEntryToManifest (ctx, mhash, path, hash, ctype)
+	fmt.Println(newManifest)
+
+	if !wantManifest {
+		// Print the manifest. This is the only output to stdout.
+		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
+		fmt.Println(string(mrootJSON))
+		return
+	}
+}
+
+func update(ctx *cli.Context) {
+
+	args := ctx.Args()
+	if len(args) < 3 {
+		log.Fatal("need atleast three arguments <MHASH> <path> <HASH>")
+	}
+
+	var (
+		mhash  = args[0]
+		path   = args[1]
+		hash   = args[2]
+
+		ctype  string
+		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
+		mroot  manifest
+	)
+	if len(args) > 3 {
+		ctype = args[3]
+	} else {
+		ctype = mime.TypeByExtension(filepath.Ext(path))
+	}
+
+	newManifest := updateEntryInManifest (ctx, mhash, path, hash, ctype)
+	fmt.Println(newManifest)
+
+	if !wantManifest {
+		// Print the manifest. This is the only output to stdout.
+		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
+		fmt.Println(string(mrootJSON))
+		return
+	}
+}
+
+func remove(ctx *cli.Context) {
+	args := ctx.Args()
+	if len(args) < 2 {
+		log.Fatal("need atleast two arguments <MHASH> <path>")
+	}
+
+	var (
+		mhash  = args[0]
+		path   = args[1]
+
+		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
+		mroot  manifest
+	)
+
+	newManifest := removeEntryFromManifest (ctx, mhash, path)
+	fmt.Println(newManifest)
+
+	if !wantManifest {
+		// Print the manifest. This is the only output to stdout.
+		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
+		fmt.Println(string(mrootJSON))
+		return
+	}
+}
+
+func addEntryToManifest(ctx *cli.Context, mhash , path, hash , ctype string)  string {
+
+	var (
+		bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client = &client{api: bzzapi}
+		longestPathEntry = manifestEntry{
+			Path:        "",
+			Hash:        "",
+			ContentType:  "",
+		}
+	)
+
+	mroot, err := client.downloadManifest(mhash)
+	if err != nil {
+		log.Fatalln("manifest download failed:", err)
+	}
+
+	//TODO: check if the "hash" to add is valid and present in swarm
+	_, err = client.downloadManifest(hash)
+	if err != nil {
+		log.Fatalln("hash to add is not present:", err)
+	}
+
+
+	// See if we path is in this Manifest or do we have to dig deeper
+	for _, entry := range mroot.Entries {
+		if path == entry.Path {
+			log.Fatal(path, "Already present, not adding anything")
+		}else {
+			if entry.ContentType == "application/bzz-manifest+json" {
+				prfxlen := strings.HasPrefix(path, entry.Path)
+				if prfxlen && len(path) > len(longestPathEntry.Path) {
+					longestPathEntry = entry
+				}
+			}
+		}
+	}
+
+	if longestPathEntry.Path != "" {
+		// Load the child Manifest add the entry there
+		newPath := path[len(longestPathEntry.Path):]
+		newHash := addEntryToManifest (ctx, longestPathEntry.Hash, newPath, hash, ctype)
+
+		// Replace the hash for parent Manifests
+		newMRoot := manifest{}
+		for _, entry := range mroot.Entries {
+			if longestPathEntry.Path == entry.Path {
+				entry.Hash = newHash
+			}
+			newMRoot.Entries = append(newMRoot.Entries, entry)
+		}
+		mroot = newMRoot
+	} else {
+		// Add the entry in the leaf Manifest
+		newEntry := manifestEntry{
+			Path:        path,
+			Hash:        hash,
+			ContentType: ctype,
+		}
+		mroot.Entries = append(mroot.Entries, newEntry)
+	}
+
+
+	newManifestHash, err := client.uploadManifest(mroot)
+	if err != nil {
+		log.Fatalln("manifest upload failed:", err)
+	}
+	return newManifestHash
+
+
+
+}
+
+func updateEntryInManifest(ctx *cli.Context, mhash , path, hash , ctype string) string {
+
+	var (
+		bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client = &client{api: bzzapi}
+		newEntry = manifestEntry{
+			Path:        "",
+			Hash:        "",
+			ContentType:  "",
+		}
+		longestPathEntry = manifestEntry{
+			Path:        "",
+			Hash:        "",
+			ContentType:  "",
+		}
+	)
+
+	mroot, err := client.downloadManifest(mhash)
+	if err != nil {
+		log.Fatalln("manifest download failed:", err)
+	}
+
+	//TODO: check if the "hash" with which to update is valid and present in swarm
+
+
+	// See if we path is in this Manifest or do we have to dig deeper
+	for _, entry := range mroot.Entries {
+		if path == entry.Path {
+			newEntry = entry
+		}else {
+			if entry.ContentType == "application/bzz-manifest+json" {
+				prfxlen := strings.HasPrefix(path, entry.Path)
+				if prfxlen && len(path) > len(longestPathEntry.Path) {
+					longestPathEntry = entry
+				}
+			}
+		}
+	}
+
+	if longestPathEntry.Path == "" && newEntry.Path == "" {
+		log.Fatal(path, " Path not present in the Manifest, not setting anything")
+	}
+
+	if longestPathEntry.Path != "" {
+		// Load the child Manifest add the entry there
+		newPath := path[len(longestPathEntry.Path):]
+		newHash := updateEntryInManifest (ctx, longestPathEntry.Hash, newPath, hash, ctype)
+
+		// Replace the hash for parent Manifests
+		newMRoot := manifest{}
+		for _, entry := range mroot.Entries {
+			if longestPathEntry.Path == entry.Path {
+				entry.Hash = newHash
+			}
+			newMRoot.Entries = append(newMRoot.Entries, entry)
+
+		}
+		mroot = newMRoot
+	}
+
+	if newEntry.Path != "" {
+		// Replace the hash for leaf Manifest
+		newMRoot := manifest{}
+		for _, entry := range mroot.Entries {
+			if newEntry.Path == entry.Path {
+				myEntry := manifestEntry{
+					Path:        entry.Path,
+					Hash:        hash,
+					ContentType: ctype,
+				}
+				newMRoot.Entries = append(newMRoot.Entries, myEntry)
+			} else {
+				newMRoot.Entries = append(newMRoot.Entries, entry)
+			}
+		}
+		mroot = newMRoot
+	}
+
+
+	newManifestHash, err := client.uploadManifest(mroot)
+	if err != nil {
+		log.Fatalln("manifest upload failed:", err)
+	}
+	return newManifestHash
+}
+
+func removeEntryFromManifest(ctx *cli.Context, mhash , path string) string {
+
+	var (
+		bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client = &client{api: bzzapi}
+		entryToRemove = manifestEntry{
+			Path:        "",
+			Hash:        "",
+			ContentType:  "",
+		}
+		longestPathEntry = manifestEntry{
+			Path:        "",
+			Hash:        "",
+			ContentType:  "",
+		}
+	)
+
+	mroot, err := client.downloadManifest(mhash)
+	if err != nil {
+		log.Fatalln("manifest download failed:", err)
+	}
+
+
+
+	// See if we path is in this Manifest or do we have to dig deeper
+	for _, entry := range mroot.Entries {
+		if path == entry.Path {
+			entryToRemove = entry
+		}else {
+			if entry.ContentType == "application/bzz-manifest+json" {
+				prfxlen := strings.HasPrefix(path, entry.Path)
+				if prfxlen && len(path) > len(longestPathEntry.Path) {
+					longestPathEntry = entry
+				}
+			}
+		}
+	}
+
+	if longestPathEntry.Path == "" && entryToRemove.Path == "" {
+		log.Fatal(path, "Path not present in the Manifest, not removing anything")
+	}
+
+	if longestPathEntry.Path != "" {
+		// Load the child Manifest remove the entry there
+		newPath := path[len(longestPathEntry.Path):]
+		newHash := removeEntryFromManifest (ctx, longestPathEntry.Hash, newPath)
+
+		// Replace the hash for parent Manifests
+		newMRoot := manifest{}
+		for _, entry := range mroot.Entries {
+			if longestPathEntry.Path == entry.Path {
+				entry.Hash = newHash
+			}
+			newMRoot.Entries = append(newMRoot.Entries, entry)
+		}
+		mroot = newMRoot
+	}
+
+	if entryToRemove.Path != "" {
+		// remove the entry in this Manifest
+		newMRoot := manifest{}
+		for _, entry := range mroot.Entries {
+			if entryToRemove.Path != entry.Path {
+				newMRoot.Entries = append(newMRoot.Entries, entry)
+			}
+		}
+		mroot = newMRoot
+	}
+
+
+	newManifestHash, err := client.uploadManifest(mroot)
+	if err != nil {
+		log.Fatalln("manifest upload failed:", err)
+	}
+	return newManifestHash
+
+
+}
+

cmd/swarm/upload.go

@@ -0,0 +1,257 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// Command bzzup uploads files to the swarm HTTP API.
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"mime"
+	"net/http"
+	"os"
+	"os/user"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"gopkg.in/urfave/cli.v1"
+)
+
+func upload(ctx *cli.Context) {
+	args := ctx.Args()
+	var (
+		bzzapi       = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		recursive    = ctx.GlobalBool(SwarmRecursiveUploadFlag.Name)
+		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
+		defaultPath  = ctx.GlobalString(SwarmUploadDefaultPath.Name)
+	)
+	if len(args) != 1 {
+		log.Fatal("need filename as the first and only argument")
+	}
+
+	var (
+		file   = args[0]
+		client = &client{api: bzzapi}
+	)
+	fi, err := os.Stat(expandPath(file))
+	if err != nil {
+		log.Fatal(err)
+	}
+	if fi.IsDir() {
+		if !recursive {
+			log.Fatal("argument is a directory and recursive upload is disabled")
+		}
+		if !wantManifest {
+			log.Fatal("manifest is required for directory uploads")
+		}
+		mhash, err := client.uploadDirectory(file, defaultPath)
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Println(mhash)
+		return
+	}
+	entry, err := client.uploadFile(file, fi)
+	if err != nil {
+		log.Fatalln("upload failed:", err)
+	}
+	mroot := manifest{[]manifestEntry{entry}}
+	if !wantManifest {
+		// Print the manifest. This is the only output to stdout.
+		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
+		fmt.Println(string(mrootJSON))
+		return
+	}
+	hash, err := client.uploadManifest(mroot)
+	if err != nil {
+		log.Fatalln("manifest upload failed:", err)
+	}
+	fmt.Println(hash)
+}
+
+// Expands a file path
+// 1. replace tilde with users home dir
+// 2. expands embedded environment variables
+// 3. cleans the path, e.g. /a/b/../c -> /a/c
+// Note, it has limitations, e.g. ~someuser/tmp will not be expanded
+func expandPath(p string) string {
+	if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") {
+		if home := homeDir(); home != "" {
+			p = home + p[1:]
+		}
+	}
+	return path.Clean(os.ExpandEnv(p))
+}
+
+func homeDir() string {
+	if home := os.Getenv("HOME"); home != "" {
+		return home
+	}
+	if usr, err := user.Current(); err == nil {
+		return usr.HomeDir
+	}
+	return ""
+}
+
+// client wraps interaction with the swarm HTTP gateway.
+type client struct {
+	api string
+}
+
+// manifest is the JSON representation of a swarm manifest.
+type manifestEntry struct {
+	Hash        string `json:"hash,omitempty"`
+	ContentType string `json:"contentType,omitempty"`
+	Path        string `json:"path,omitempty"`
+}
+
+// manifest is the JSON representation of a swarm manifest.
+type manifest struct {
+	Entries []manifestEntry `json:"entries,omitempty"`
+}
+
+func (c *client) uploadDirectory(dir string, defaultPath string) (string, error) {
+	mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
+	if err != nil {
+		return "", fmt.Errorf("failed to upload empty manifest")
+	}
+	if len(defaultPath) > 0 {
+		fi, err := os.Stat(defaultPath)
+		if err != nil {
+			return "", err
+		}
+		mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
+		if err != nil {
+			return "", err
+		}
+	}
+	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
+	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
+		if err != nil || fi.IsDir() {
+			return err
+		}
+		if !strings.HasPrefix(path, dir) {
+			return fmt.Errorf("path %s outside directory %s", path, dir)
+		}
+		uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
+		mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
+		return err
+	})
+	return mhash, err
+}
+
+func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) {
+	hash, err := c.uploadFileContent(file, fi)
+	m := manifestEntry{
+		Hash:        hash,
+		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
+	}
+	return m, err
+}
+
+func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
+	fd, err := os.Open(file)
+	if err != nil {
+		return "", err
+	}
+	defer fd.Close()
+	log.Printf("uploading file %s (%d bytes)", file, fi.Size())
+	return c.postRaw("application/octet-stream", fi.Size(), fd)
+}
+
+func (c *client) uploadManifest(m manifest) (string, error) {
+	jsm, err := json.Marshal(m)
+	if err != nil {
+		panic(err)
+	}
+	log.Println("uploading manifest")
+	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
+}
+
+func (c *client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
+	fd, err := os.Open(fpath)
+	if err != nil {
+		return "", err
+	}
+	defer fd.Close()
+	log.Printf("uploading file %s (%d bytes) and adding path %v", fpath, fi.Size(), path)
+	req, err := http.NewRequest("PUT", c.api+"/bzz:/"+mhash+"/"+path, fd)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
+	req.ContentLength = fi.Size()
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode >= 400 {
+		return "", fmt.Errorf("bad status: %s", resp.Status)
+	}
+	content, err := ioutil.ReadAll(resp.Body)
+	return string(content), err
+}
+
+func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
+	req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("content-type", mimetype)
+	req.ContentLength = size
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode >= 400 {
+		return "", fmt.Errorf("bad status: %s", resp.Status)
+	}
+	content, err := ioutil.ReadAll(resp.Body)
+	return string(content), err
+}
+
+func (c *client) downloadManifest(mhash string) (manifest, error) {
+
+	mroot := manifest{}
+	req, err := http.NewRequest("GET", c.api + "/bzzr:/" + mhash, nil)
+	if err != nil {
+		return mroot, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return mroot, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return mroot, fmt.Errorf("bad status: %s", resp.Status)
+
+	}
+	content, err := ioutil.ReadAll(resp.Body)
+
+	err = json.Unmarshal(content, &mroot)
+	if err != nil {
+		return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
+	}
+	return mroot, err
+}

cmd/utils/cmd.go

@@ -18,12 +18,14 @@
 package utils
 
 import (
+	"compress/gzip"
 	"fmt"
 	"io"
 	"os"
 	"os/signal"
 	"regexp"
 	"runtime"
+	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
@@ -65,7 +67,6 @@ func Fatalf(format string, args ...interface{}) {
 		}
 	}
 	fmt.Fprintf(w, "Fatal: "+format+"\n", args...)
-	logger.Flush()
 	os.Exit(1)
 }
 
@@ -93,7 +94,7 @@ func StartNode(stack *node.Node) {
 
 func FormatTransactionData(data string) []byte {
 	d := common.StringToByteFunc(data, func(s string) (ret []byte) {
-		slice := regexp.MustCompile("\\n|\\s").Split(s, 1000000000)
+		slice := regexp.MustCompile(`\n|\s`).Split(s, 1000000000)
 		for _, dataItem := range slice {
 			d := common.FormatData(dataItem)
 			ret = append(ret, d...)
@@ -133,7 +134,15 @@ func ImportChain(chain *core.BlockChain, fn string) error {
 		return err
 	}
 	defer fh.Close()
-	stream := rlp.NewStream(fh, 0)
+
+	var reader io.Reader = fh
+	if strings.HasSuffix(fn, ".gz") {
+		if reader, err = gzip.NewReader(reader); err != nil {
+			return err
+		}
+	}
+
+	stream := rlp.NewStream(reader, 0)
 
 	// Run actual the import.
 	blocks := make(types.Blocks, importBatchSize)
@@ -195,10 +204,18 @@ func ExportChain(blockchain *core.BlockChain, fn string) error {
 		return err
 	}
 	defer fh.Close()
-	if err := blockchain.Export(fh); err != nil {
+
+	var writer io.Writer = fh
+	if strings.HasSuffix(fn, ".gz") {
+		writer = gzip.NewWriter(writer)
+		defer writer.(*gzip.Writer).Close()
+	}
+
+	if err := blockchain.Export(writer); err != nil {
 		return err
 	}
 	glog.Infoln("Exported blockchain to ", fn)
+
 	return nil
 }
 
@@ -210,7 +227,14 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las
 		return err
 	}
 	defer fh.Close()
-	if err := blockchain.ExportN(fh, first, last); err != nil {
+
+	var writer io.Writer = fh
+	if strings.HasSuffix(fn, ".gz") {
+		writer = gzip.NewWriter(writer)
+		defer writer.(*gzip.Writer).Close()
+	}
+
+	if err := blockchain.ExportN(writer, first, last); err != nil {
 		return err
 	}
 	glog.Infoln("Exported blockchain to ", fn)

cmd/utils/customflags.go

@@ -23,8 +23,6 @@ import (
 	"os/user"
 	"path"
 	"strings"
-
-	"gopkg.in/urfave/cli.v1"
 )
 
 // Custom type which is registered in the flags library which cli uses for
@@ -46,7 +44,6 @@ func (self *DirectoryString) Set(value string) error {
 // Custom cli.Flag type which expand the received string to an absolute path.
 // e.g. ~/.ethereum -> /home/username/.ethereum
 type DirectoryFlag struct {
-	cli.GenericFlag
 	Name   string
 	Value  DirectoryString
 	Usage  string
@@ -54,15 +51,10 @@ type DirectoryFlag struct {
 }
 
 func (self DirectoryFlag) String() string {
-	var fmtString string
-	fmtString = "%s %v\t%v"
-
+	fmtString := "%s %v\t%v"
 	if len(self.Value.Value) > 0 {
 		fmtString = "%s \"%v\"\t%v"
-	} else {
-		fmtString = "%s %v\t%v"
 	}
-
 	return withEnvHint(self.EnvVar, fmt.Sprintf(fmtString, prefixedNames(self.Name), self.Value.Value, self.Usage))
 }
 
@@ -122,7 +114,7 @@ func withEnvHint(envVar, str string) string {
 	return str + envText
 }
 
-func (self DirectoryFlag) getName() string {
+func (self DirectoryFlag) GetName() string {
 	return self.Name
 }
 

cmd/utils/flags.go

@@ -21,7 +21,6 @@ import (
 	"crypto/ecdsa"
 	"fmt"
 	"io/ioutil"
-	"math"
 	"math/big"
 	"os"
 	"path/filepath"
@@ -31,9 +30,11 @@ import (
 
 	"github.com/ethereum/ethash"
 	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -116,16 +117,12 @@ var (
 	}
 	NetworkIdFlag = cli.IntFlag{
 		Name:  "networkid",
-		Usage: "Network identifier (integer, 0=Olympic (disused), 1=Frontier, 2=Morden (disused), 3=Ropsten)",
+		Usage: "Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten)",
 		Value: eth.NetworkId,
 	}
-	OlympicFlag = cli.BoolFlag{
-		Name:  "olympic",
-		Usage: "Olympic network: pre-configured pre-release test network",
-	}
 	TestNetFlag = cli.BoolFlag{
 		Name:  "testnet",
-		Usage: "Morden network: pre-configured test network with modified starting nonces (replay protection)",
+		Usage: "Ropsten network: pre-configured test network",
 	}
 	DevModeFlag = cli.BoolFlag{
 		Name:  "dev",
@@ -135,10 +132,6 @@ var (
 		Name:  "identity",
 		Usage: "Custom node name",
 	}
-	NatspecEnabledFlag = cli.BoolFlag{
-		Name:  "natspec",
-		Usage: "Enable NatSpec confirmation notice",
-	}
 	DocRootFlag = DirectoryFlag{
 		Name:  "docroot",
 		Usage: "Document Root for HTTPClient file scheme",
@@ -177,15 +170,6 @@ var (
 		Usage: "Number of trie node generations to keep in memory",
 		Value: int(state.MaxTrieCacheGen),
 	}
-	// Fork settings
-	SupportDAOFork = cli.BoolFlag{
-		Name:  "support-dao-fork",
-		Usage: "Updates the chain rules to support the DAO hard-fork",
-	}
-	OpposeDAOFork = cli.BoolFlag{
-		Name:  "oppose-dao-fork",
-		Usage: "Updates the chain rules to oppose the DAO hard-fork",
-	}
 	// Miner settings
 	MiningEnabledFlag = cli.BoolFlag{
 		Name:  "mine",
@@ -244,6 +228,10 @@ var (
 		Name:  "jitvm",
 		Usage: "Enable the JIT VM",
 	}
+	VMEnableDebugFlag = cli.BoolFlag{
+		Name:  "vmdebug",
+		Usage: "Record information useful for VM and contract debugging",
+	}
 	// Logging and debug settings
 	EthStatsURLFlag = cli.StringFlag{
 		Name:  "ethstats",
@@ -494,17 +482,15 @@ func makeNodeUserIdent(ctx *cli.Context) string {
 // MakeBootstrapNodes creates a list of bootstrap nodes from the command line
 // flags, reverting to pre-configured ones if none have been specified.
 func MakeBootstrapNodes(ctx *cli.Context) []*discover.Node {
-	// Return pre-configured nodes if none were manually requested
-	if !ctx.GlobalIsSet(BootnodesFlag.Name) {
-		if ctx.GlobalBool(TestNetFlag.Name) {
-			return params.TestnetBootnodes
-		}
-		return params.MainnetBootnodes
+	urls := params.MainnetBootnodes
+	if ctx.GlobalIsSet(BootnodesFlag.Name) {
+		urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",")
+	} else if ctx.GlobalBool(TestNetFlag.Name) {
+		urls = params.TestnetBootnodes
 	}
-	// Otherwise parse and use the CLI bootstrap nodes
-	bootnodes := []*discover.Node{}
 
-	for _, url := range strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",") {
+	bootnodes := make([]*discover.Node, 0, len(urls))
+	for _, url := range urls {
 		node, err := discover.ParseNode(url)
 		if err != nil {
 			glog.V(logger.Error).Infof("Bootstrap URL %s: %v\n", url, err)
@@ -518,14 +504,13 @@ func MakeBootstrapNodes(ctx *cli.Context) []*discover.Node {
 // MakeBootstrapNodesV5 creates a list of bootstrap nodes from the command line
 // flags, reverting to pre-configured ones if none have been specified.
 func MakeBootstrapNodesV5(ctx *cli.Context) []*discv5.Node {
-	// Return pre-configured nodes if none were manually requested
-	if !ctx.GlobalIsSet(BootnodesFlag.Name) {
-		return params.DiscoveryV5Bootnodes
+	urls := params.DiscoveryV5Bootnodes
+	if ctx.GlobalIsSet(BootnodesFlag.Name) {
+		urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",")
 	}
-	// Otherwise parse and use the CLI bootstrap nodes
-	bootnodes := []*discv5.Node{}
 
-	for _, url := range strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",") {
+	bootnodes := make([]*discv5.Node, 0, len(urls))
+	for _, url := range urls {
 		node, err := discv5.ParseNode(url)
 		if err != nil {
 			glog.V(logger.Error).Infof("Bootstrap URL %s: %v\n", url, err)
@@ -603,23 +588,27 @@ func MakeDatabaseHandles() int {
 
 // MakeAddress converts an account specified directly as a hex encoded string or
 // a key index in the key store to an internal account representation.
-func MakeAddress(accman *accounts.Manager, account string) (accounts.Account, error) {
+func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error) {
 	// If the specified account is a valid address, return it
 	if common.IsHexAddress(account) {
 		return accounts.Account{Address: common.HexToAddress(account)}, nil
 	}
 	// Otherwise try to interpret the account as a keystore index
 	index, err := strconv.Atoi(account)
-	if err != nil {
+	if err != nil || index < 0 {
 		return accounts.Account{}, fmt.Errorf("invalid account address or index %q", account)
 	}
-	return accman.AccountByIndex(index)
+	accs := ks.Accounts()
+	if len(accs) <= index {
+		return accounts.Account{}, fmt.Errorf("index %d higher than number of accounts %d", index, len(accs))
+	}
+	return accs[index], nil
 }
 
 // MakeEtherbase retrieves the etherbase either from the directly specified
 // command line flags or from the keystore if CLI indexed.
-func MakeEtherbase(accman *accounts.Manager, ctx *cli.Context) common.Address {
-	accounts := accman.Accounts()
+func MakeEtherbase(ks *keystore.KeyStore, ctx *cli.Context) common.Address {
+	accounts := ks.Accounts()
 	if !ctx.GlobalIsSet(EtherbaseFlag.Name) && len(accounts) == 0 {
 		glog.V(logger.Error).Infoln("WARNING: No etherbase set and no accounts found as default")
 		return common.Address{}
@@ -629,7 +618,7 @@ func MakeEtherbase(accman *accounts.Manager, ctx *cli.Context) common.Address {
 		return common.Address{}
 	}
 	// If the specified etherbase is a valid address, return it
-	account, err := MakeAddress(accman, etherbase)
+	account, err := MakeAddress(ks, etherbase)
 	if err != nil {
 		Fatalf("Option %q: %v", EtherbaseFlag.Name, err)
 	}
@@ -670,6 +659,10 @@ func MakeNode(ctx *cli.Context, name, gitCommit string) *node.Node {
 		vsn += "-" + gitCommit[:8]
 	}
 
+	// if we're running a light client or server, force enable the v5 peer discovery unless it is explicitly disabled with --nodiscover
+	// note that explicitly specifying --v5disc overrides --nodiscover, in which case the later only disables v4 discovery
+	forceV5Discovery := (ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalInt(LightServFlag.Name) > 0) && !ctx.GlobalBool(NoDiscoverFlag.Name)
+
 	config := &node.Config{
 		DataDir:           MakeDataDir(ctx),
 		KeyStoreDir:       ctx.GlobalString(KeyStoreDirFlag.Name),
@@ -678,8 +671,8 @@ func MakeNode(ctx *cli.Context, name, gitCommit string) *node.Node {
 		Name:              name,
 		Version:           vsn,
 		UserIdent:         makeNodeUserIdent(ctx),
-		NoDiscovery:       ctx.GlobalBool(NoDiscoverFlag.Name) || ctx.GlobalBool(LightModeFlag.Name),
-		DiscoveryV5:       ctx.GlobalBool(DiscoveryV5Flag.Name) || ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalInt(LightServFlag.Name) > 0,
+		NoDiscovery:       ctx.GlobalBool(NoDiscoverFlag.Name) || ctx.GlobalBool(LightModeFlag.Name), // always disable v4 discovery in light client mode
+		DiscoveryV5:       ctx.GlobalBool(DiscoveryV5Flag.Name) || forceV5Discovery,
 		DiscoveryV5Addr:   MakeDiscoveryV5Address(ctx),
 		BootstrapNodes:    MakeBootstrapNodes(ctx),
 		BootstrapNodesV5:  MakeBootstrapNodesV5(ctx),
@@ -724,7 +717,7 @@ func MakeNode(ctx *cli.Context, name, gitCommit string) *node.Node {
 // given node.
 func RegisterEthService(ctx *cli.Context, stack *node.Node, extra []byte) {
 	// Avoid conflicting network flags
-	networks, netFlags := 0, []cli.BoolFlag{DevModeFlag, TestNetFlag, OlympicFlag}
+	networks, netFlags := 0, []cli.BoolFlag{DevModeFlag, TestNetFlag}
 	for _, flag := range netFlags {
 		if ctx.GlobalBool(flag.Name) {
 			networks++
@@ -733,9 +726,10 @@ func RegisterEthService(ctx *cli.Context, stack *node.Node, extra []byte) {
 	if networks > 1 {
 		Fatalf("The %v flags are mutually exclusive", netFlags)
 	}
+	ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
 
 	ethConf := &eth.Config{
-		Etherbase:               MakeEtherbase(stack.AccountManager(), ctx),
+		Etherbase:               MakeEtherbase(ks, ctx),
 		ChainConfig:             MakeChainConfig(ctx, stack),
 		FastSync:                ctx.GlobalBool(FastSyncFlag.Name),
 		LightMode:               ctx.GlobalBool(LightModeFlag.Name),
@@ -747,7 +741,6 @@ func RegisterEthService(ctx *cli.Context, stack *node.Node, extra []byte) {
 		NetworkId:               ctx.GlobalInt(NetworkIdFlag.Name),
 		MinerThreads:            ctx.GlobalInt(MinerThreadsFlag.Name),
 		ExtraData:               MakeMinerExtra(extra, ctx),
-		NatSpec:                 ctx.GlobalBool(NatspecEnabledFlag.Name),
 		DocRoot:                 ctx.GlobalString(DocRootFlag.Name),
 		GasPrice:                common.String2Big(ctx.GlobalString(GasPriceFlag.Name)),
 		GpoMinGasPrice:          common.String2Big(ctx.GlobalString(GpoMinGasPriceFlag.Name)),
@@ -758,16 +751,11 @@ func RegisterEthService(ctx *cli.Context, stack *node.Node, extra []byte) {
 		GpobaseCorrectionFactor: ctx.GlobalInt(GpobaseCorrectionFactorFlag.Name),
 		SolcPath:                ctx.GlobalString(SolcPathFlag.Name),
 		AutoDAG:                 ctx.GlobalBool(AutoDAGFlag.Name) || ctx.GlobalBool(MiningEnabledFlag.Name),
+		EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name),
 	}
 
 	// Override any default configs in dev mode or the test net
 	switch {
-	case ctx.GlobalBool(OlympicFlag.Name):
-		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
-			ethConf.NetworkId = 1
-		}
-		ethConf.Genesis = core.OlympicGenesisBlock()
-
 	case ctx.GlobalBool(TestNetFlag.Name):
 		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
 			ethConf.NetworkId = 3
@@ -775,7 +763,7 @@ func RegisterEthService(ctx *cli.Context, stack *node.Node, extra []byte) {
 		ethConf.Genesis = core.DefaultTestnetGenesisBlock()
 
 	case ctx.GlobalBool(DevModeFlag.Name):
-		ethConf.Genesis = core.OlympicGenesisBlock()
+		ethConf.Genesis = core.DevGenesisBlock()
 		if !ctx.GlobalIsSet(GasPriceFlag.Name) {
 			ethConf.GasPrice = new(big.Int)
 		}
@@ -832,16 +820,6 @@ func RegisterEthStatsService(stack *node.Node, url string) {
 
 // SetupNetwork configures the system for either the main net or some test network.
 func SetupNetwork(ctx *cli.Context) {
-	switch {
-	case ctx.GlobalBool(OlympicFlag.Name):
-		params.DurationLimit = big.NewInt(8)
-		params.GenesisGasLimit = big.NewInt(3141592)
-		params.MinGasLimit = big.NewInt(125000)
-		params.MaximumExtraDataSize = big.NewInt(1024)
-		NetworkIdFlag.Value = 0
-		core.BlockReward = big.NewInt(1.5e+18)
-		core.ExpDiffPeriod = big.NewInt(math.MaxInt64)
-	}
 	params.TargetGasLimit = common.String2Big(ctx.GlobalString(TargetGasLimitFlag.Name))
 }
 
@@ -901,13 +879,6 @@ func MakeChainConfigFromDb(ctx *cli.Context, db ethdb.Database) *params.ChainCon
 			config.ChainId = params.MainNetChainID
 		}
 	}
-	// Force override any existing configs if explicitly requested
-	switch {
-	case ctx.GlobalBool(SupportDAOFork.Name):
-		config.DAOForkSupport = true
-	case ctx.GlobalBool(OpposeDAOFork.Name):
-		config.DAOForkSupport = false
-	}
 	return config
 }
 
@@ -939,19 +910,20 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 	var err error
 	chainDb = MakeChainDatabase(ctx, stack)
 
-	if ctx.GlobalBool(OlympicFlag.Name) {
+	if ctx.GlobalBool(TestNetFlag.Name) {
 		_, err := core.WriteTestNetGenesisBlock(chainDb)
 		if err != nil {
 			glog.Fatalln(err)
 		}
 	}
+
 	chainConfig := MakeChainConfigFromDb(ctx, chainDb)
 
 	pow := pow.PoW(core.FakePow{})
 	if !ctx.GlobalBool(FakePoWFlag.Name) {
 		pow = ethash.New()
 	}
-	chain, err = core.NewBlockChain(chainDb, chainConfig, pow, new(event.TypeMux))
+	chain, err = core.NewBlockChain(chainDb, chainConfig, pow, new(event.TypeMux), vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)})
 	if err != nil {
 		Fatalf("Could not start chainmanager: %v", err)
 	}

cmd/wnode/main.go

@@ -0,0 +1,542 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// This is a simple Whisper node. It could be used as a stand-alone bootstrap node.
+// Also, could be used for different test and diagnostics purposes.
+
+package main
+
+import (
+	"bufio"
+	"crypto/ecdsa"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/binary"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/console"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/logger"
+	"github.com/ethereum/go-ethereum/logger/glog"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/nat"
+	"github.com/ethereum/go-ethereum/whisper/mailserver"
+	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
+	"golang.org/x/crypto/pbkdf2"
+)
+
+const quitCommand = "~Q"
+
+// singletons
+var (
+	server     *p2p.Server
+	shh        *whisper.Whisper
+	done       chan struct{}
+	mailServer mailserver.WMailServer
+
+	input = bufio.NewReader(os.Stdin)
+)
+
+// encryption
+var (
+	symKey     []byte
+	pub        *ecdsa.PublicKey
+	asymKey    *ecdsa.PrivateKey
+	nodeid     *ecdsa.PrivateKey
+	topic      whisper.TopicType
+	filterID   uint32
+	msPassword string
+)
+
+// cmd arguments
+var (
+	echoMode       = flag.Bool("e", false, "echo mode: prints some arguments for diagnostics")
+	bootstrapMode  = flag.Bool("b", false, "boostrap node: don't actively connect to peers, wait for incoming connections")
+	forwarderMode  = flag.Bool("f", false, "forwarder mode: only forward messages, neither send nor decrypt messages")
+	mailServerMode = flag.Bool("s", false, "mail server mode: delivers expired messages on demand")
+	requestMail    = flag.Bool("r", false, "request expired messages from the bootstrap server")
+	asymmetricMode = flag.Bool("a", false, "use asymmetric encryption")
+	testMode       = flag.Bool("t", false, "use of predefined parameters for diagnostics")
+	generateKey    = flag.Bool("k", false, "generate and show the private key")
+
+	argTTL       = flag.Uint("ttl", 30, "time-to-live for messages in seconds")
+	argWorkTime  = flag.Uint("work", 5, "work time in seconds")
+	argPoW       = flag.Float64("pow", whisper.MinimumPoW, "PoW for normal messages in float format (e.g. 2.7)")
+	argServerPoW = flag.Float64("mspow", whisper.MinimumPoW, "PoW requirement for Mail Server request")
+
+	argIP     = flag.String("ip", "", "IP address and port of this node (e.g. 127.0.0.1:30303)")
+	argSalt   = flag.String("salt", "", "salt (for topic and key derivation)")
+	argPub    = flag.String("pub", "", "public key for asymmetric encryption")
+	argDBPath = flag.String("dbpath", "", "path to the server's DB directory")
+	argIDFile = flag.String("idfile", "", "file name with node id (private key)")
+	argEnode  = flag.String("boot", "", "bootstrap node you want to connect to (e.g. enode://e454......08d50@52.176.211.200:16428)")
+	argTopic  = flag.String("topic", "", "topic in hexadecimal format (e.g. 70a4beef)")
+)
+
+func main() {
+	processArgs()
+	initialize()
+	run()
+}
+
+func processArgs() {
+	flag.Parse()
+
+	if len(*argIDFile) > 0 {
+		var err error
+		nodeid, err = crypto.LoadECDSA(*argIDFile)
+		if err != nil {
+			utils.Fatalf("Failed to load file [%s]: %s.", *argIDFile, err)
+		}
+	}
+
+	const enodePrefix = "enode://"
+	if len(*argEnode) > 0 {
+		if (*argEnode)[:len(enodePrefix)] != enodePrefix {
+			*argEnode = enodePrefix + *argEnode
+		}
+	}
+
+	if len(*argTopic) > 0 {
+		x, err := hex.DecodeString(*argTopic)
+		if err != nil {
+			utils.Fatalf("Failed to parse the topic: %s", err)
+		}
+		topic = whisper.BytesToTopic(x)
+	}
+
+	if *asymmetricMode && len(*argPub) > 0 {
+		pub = crypto.ToECDSAPub(common.FromHex(*argPub))
+		if !isKeyValid(pub) {
+			utils.Fatalf("invalid public key")
+		}
+	}
+
+	if *echoMode {
+		echo()
+	}
+}
+
+func echo() {
+	fmt.Printf("ttl = %d \n", *argTTL)
+	fmt.Printf("workTime = %d \n", *argWorkTime)
+	fmt.Printf("pow = %f \n", *argPoW)
+	fmt.Printf("mspow = %f \n", *argServerPoW)
+	fmt.Printf("ip = %s \n", *argIP)
+	fmt.Printf("salt = %s \n", *argSalt)
+	fmt.Printf("pub = %s \n", common.ToHex(crypto.FromECDSAPub(pub)))
+	fmt.Printf("idfile = %s \n", *argIDFile)
+	fmt.Printf("dbpath = %s \n", *argDBPath)
+	fmt.Printf("boot = %s \n", *argEnode)
+}
+
+func initialize() {
+	glog.SetV(logger.Warn)
+	glog.SetToStderr(true)
+
+	done = make(chan struct{})
+	var peers []*discover.Node
+	var err error
+
+	if *generateKey {
+		key, err := crypto.GenerateKey()
+		if err != nil {
+			utils.Fatalf("Failed to generate private key: %s", err)
+		}
+		k := hex.EncodeToString(crypto.FromECDSA(key))
+		fmt.Printf("Random private key: %s \n", k)
+		os.Exit(0)
+	}
+
+	if *testMode {
+		password := []byte("test password for symmetric encryption")
+		salt := []byte("test salt for symmetric encryption")
+		symKey = pbkdf2.Key(password, salt, 64, 32, sha256.New)
+		topic = whisper.TopicType{0xFF, 0xFF, 0xFF, 0xFF}
+		msPassword = "mail server test password"
+	}
+
+	if *bootstrapMode {
+		if len(*argIP) == 0 {
+			argIP = scanLineA("Please enter your IP and port (e.g. 127.0.0.1:30348): ")
+		}
+	} else {
+		if len(*argEnode) == 0 {
+			argEnode = scanLineA("Please enter the peer's enode: ")
+		}
+		peer := discover.MustParseNode(*argEnode)
+		peers = append(peers, peer)
+	}
+
+	if *mailServerMode {
+		if len(msPassword) == 0 {
+			msPassword, err = console.Stdin.PromptPassword("Please enter the Mail Server password: ")
+			if err != nil {
+				utils.Fatalf("Failed to read Mail Server password: %s", err)
+			}
+		}
+		shh = whisper.NewWhisper(&mailServer)
+		mailServer.Init(shh, *argDBPath, msPassword, *argServerPoW)
+	} else {
+		shh = whisper.NewWhisper(nil)
+	}
+
+	asymKey = shh.NewIdentity()
+	if nodeid == nil {
+		nodeid = shh.NewIdentity()
+	}
+
+	maxPeers := 80
+	if *bootstrapMode {
+		maxPeers = 800
+	}
+
+	server = &p2p.Server{
+		Config: p2p.Config{
+			PrivateKey:     nodeid,
+			MaxPeers:       maxPeers,
+			Name:           common.MakeName("whisper-go", "5.0"),
+			Protocols:      shh.Protocols(),
+			ListenAddr:     *argIP,
+			NAT:            nat.Any(),
+			BootstrapNodes: peers,
+			StaticNodes:    peers,
+			TrustedNodes:   peers,
+		},
+	}
+}
+
+func startServer() {
+	err := server.Start()
+	if err != nil {
+		utils.Fatalf("Failed to start Whisper peer: %s.", err)
+	}
+
+	fmt.Printf("my public key: %s \n", common.ToHex(crypto.FromECDSAPub(&asymKey.PublicKey)))
+	fmt.Println(server.NodeInfo().Enode)
+
+	if *bootstrapMode {
+		configureNode()
+		fmt.Println("Bootstrap Whisper node started")
+	} else {
+		fmt.Println("Whisper node started")
+		// first see if we can establish connection, then ask for user input
+		waitForConnection(true)
+		configureNode()
+	}
+
+	if !*forwarderMode {
+		fmt.Printf("Please type the message. To quit type: '%s'\n", quitCommand)
+	}
+}
+
+func isKeyValid(k *ecdsa.PublicKey) bool {
+	return k.X != nil && k.Y != nil
+}
+
+func configureNode() {
+	var err error
+	var p2pAccept bool
+
+	if *forwarderMode {
+		return
+	}
+
+	if *asymmetricMode {
+		if len(*argPub) == 0 {
+			s := scanLine("Please enter the peer's public key: ")
+			pub = crypto.ToECDSAPub(common.FromHex(s))
+			if !isKeyValid(pub) {
+				utils.Fatalf("Error: invalid public key")
+			}
+		}
+	}
+
+	if *requestMail {
+		p2pAccept = true
+		if len(msPassword) == 0 {
+			msPassword, err = console.Stdin.PromptPassword("Please enter the Mail Server password: ")
+			if err != nil {
+				utils.Fatalf("Failed to read Mail Server password: %s", err)
+			}
+		}
+	}
+
+	if !*asymmetricMode && !*forwarderMode && !*testMode {
+		pass, err := console.Stdin.PromptPassword("Please enter the password: ")
+		if err != nil {
+			utils.Fatalf("Failed to read passphrase: %v", err)
+		}
+
+		if len(*argSalt) == 0 {
+			argSalt = scanLineA("Please enter the salt: ")
+		}
+
+		symKey = pbkdf2.Key([]byte(pass), []byte(*argSalt), 65356, 32, sha256.New)
+
+		if len(*argTopic) == 0 {
+			generateTopic([]byte(pass), []byte(*argSalt))
+		}
+	}
+
+	if *mailServerMode {
+		if len(*argDBPath) == 0 {
+			argDBPath = scanLineA("Please enter the path to DB file: ")
+		}
+	}
+
+	filter := whisper.Filter{
+		KeySym:    symKey,
+		KeyAsym:   asymKey,
+		Topics:    []whisper.TopicType{topic},
+		AcceptP2P: p2pAccept,
+	}
+	filterID = shh.Watch(&filter)
+	fmt.Printf("Filter is configured for the topic: %x \n", topic)
+}
+
+func generateTopic(password, salt []byte) {
+	const rounds = 4000
+	const size = 128
+	x1 := pbkdf2.Key(password, salt, rounds, size, sha512.New)
+	x2 := pbkdf2.Key(password, salt, rounds, size, sha1.New)
+	x3 := pbkdf2.Key(x1, x2, rounds, size, sha256.New)
+
+	for i := 0; i < size; i++ {
+		topic[i%whisper.TopicLength] ^= x3[i]
+	}
+}
+
+func waitForConnection(timeout bool) {
+	var cnt int
+	var connected bool
+	for !connected {
+		time.Sleep(time.Millisecond * 50)
+		connected = server.PeerCount() > 0
+		if timeout {
+			cnt++
+			if cnt > 1000 {
+				utils.Fatalf("Timeout expired, failed to connect")
+			}
+		}
+	}
+
+	fmt.Println("Connected to peer.")
+}
+
+func run() {
+	defer mailServer.Close()
+	startServer()
+	defer server.Stop()
+	shh.Start(nil)
+	defer shh.Stop()
+
+	if !*forwarderMode {
+		go messageLoop()
+	}
+
+	if *requestMail {
+		requestExpiredMessagesLoop()
+	} else {
+		sendLoop()
+	}
+}
+
+func sendLoop() {
+	for {
+		s := scanLine("")
+		if s == quitCommand {
+			fmt.Println("Quit command received")
+			close(done)
+			break
+		}
+		sendMsg([]byte(s))
+
+		if *asymmetricMode {
+			// print your own message for convenience,
+			// because in asymmetric mode it is impossible to decrypt it
+			hour, min, sec := time.Now().Clock()
+			from := crypto.PubkeyToAddress(asymKey.PublicKey)
+			fmt.Printf("\n%02d:%02d:%02d <%x>: %s\n", hour, min, sec, from, s)
+		}
+	}
+}
+
+func scanLine(prompt string) string {
+	if len(prompt) > 0 {
+		fmt.Print(prompt)
+	}
+	txt, err := input.ReadString('\n')
+	if err != nil {
+		utils.Fatalf("input error: %s", err)
+	}
+	txt = strings.TrimRight(txt, "\n\r")
+	return txt
+}
+
+func scanLineA(prompt string) *string {
+	s := scanLine(prompt)
+	return &s
+}
+
+func scanUint(prompt string) uint32 {
+	s := scanLine(prompt)
+	i, err := strconv.Atoi(s)
+	if err != nil {
+		utils.Fatalf("Fail to parse the lower time limit: %s", err)
+	}
+	return uint32(i)
+}
+
+func sendMsg(payload []byte) {
+	params := whisper.MessageParams{
+		Src:      asymKey,
+		Dst:      pub,
+		KeySym:   symKey,
+		Payload:  payload,
+		Topic:    topic,
+		TTL:      uint32(*argTTL),
+		PoW:      *argPoW,
+		WorkTime: uint32(*argWorkTime),
+	}
+
+	msg := whisper.NewSentMessage(&params)
+	envelope, err := msg.Wrap(&params)
+	if err != nil {
+		fmt.Printf("failed to seal message: %v \n", err)
+		return
+	}
+
+	err = shh.Send(envelope)
+	if err != nil {
+		fmt.Printf("failed to send message: %v \n", err)
+	}
+}
+
+func messageLoop() {
+	f := shh.GetFilter(filterID)
+	if f == nil {
+		utils.Fatalf("filter is not installed")
+	}
+
+	ticker := time.NewTicker(time.Millisecond * 50)
+
+	for {
+		select {
+		case <-ticker.C:
+			messages := f.Retrieve()
+			for _, msg := range messages {
+				printMessageInfo(msg)
+			}
+		case <-done:
+			return
+		}
+	}
+}
+
+func printMessageInfo(msg *whisper.ReceivedMessage) {
+	timestamp := fmt.Sprintf("%d", msg.Sent) // unix timestamp for diagnostics
+	text := string(msg.Payload)
+
+	var address common.Address
+	if msg.Src != nil {
+		address = crypto.PubkeyToAddress(*msg.Src)
+	}
+
+	if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) {
+		fmt.Printf("\n%s <%x>: %s\n", timestamp, address, text) // message from myself
+	} else {
+		fmt.Printf("\n%s [%x]: %s\n", timestamp, address, text) // message from a peer
+	}
+}
+
+func requestExpiredMessagesLoop() {
+	var key, peerID []byte
+	var timeLow, timeUpp uint32
+	var t string
+	var xt, empty whisper.TopicType
+
+	err := shh.AddSymKey(mailserver.MailServerKeyName, []byte(msPassword))
+	if err != nil {
+		utils.Fatalf("Failed to create symmetric key for mail request: %s", err)
+	}
+	key = shh.GetSymKey(mailserver.MailServerKeyName)
+	peerID = extractIdFromEnode(*argEnode)
+	shh.MarkPeerTrusted(peerID)
+
+	for {
+		timeLow = scanUint("Please enter the lower limit of the time range (unix timestamp): ")
+		timeUpp = scanUint("Please enter the upper limit of the time range (unix timestamp): ")
+		t = scanLine("Please enter the topic (hexadecimal): ")
+		if len(t) >= whisper.TopicLength*2 {
+			x, err := hex.DecodeString(t)
+			if err != nil {
+				utils.Fatalf("Failed to parse the topic: %s", err)
+			}
+			xt = whisper.BytesToTopic(x)
+		}
+		if timeUpp == 0 {
+			timeUpp = 0xFFFFFFFF
+		}
+
+		data := make([]byte, 8+whisper.TopicLength)
+		binary.BigEndian.PutUint32(data, timeLow)
+		binary.BigEndian.PutUint32(data[4:], timeUpp)
+		copy(data[8:], xt[:])
+		if xt == empty {
+			data = data[:8]
+		}
+
+		var params whisper.MessageParams
+		params.PoW = *argServerPoW
+		params.Payload = data
+		params.KeySym = key
+		params.Src = nodeid
+		params.WorkTime = 5
+
+		msg := whisper.NewSentMessage(&params)
+		env, err := msg.Wrap(&params)
+		if err != nil {
+			utils.Fatalf("Wrap failed: %s", err)
+		}
+
+		err = shh.RequestHistoricMessages(peerID, env)
+		if err != nil {
+			utils.Fatalf("Failed to send P2P message: %s", err)
+		}
+
+		time.Sleep(time.Second * 5)
+	}
+}
+
+func extractIdFromEnode(s string) []byte {
+	n, err := discover.ParseNode(s)
+	if err != nil {
+		utils.Fatalf("Failed to parse enode: %s", err)
+		return nil
+	}
+	return n.ID[:]
+}

common/big_test.go

@@ -27,7 +27,7 @@ func TestMisc(t *testing.T) {
 	c := []byte{1, 2, 3, 4}
 	z := BitTest(a, 1)
 
-	if z != true {
+	if !z {
 		t.Error("Expected true got", z)
 	}
 
@@ -79,11 +79,11 @@ func TestBigCopy(t *testing.T) {
 	z := BigToBytes(c, 16)
 	zbytes := []byte{232, 212, 165, 16, 0}
 
-	if bytes.Compare(y, ybytes) != 0 {
+	if !bytes.Equal(y, ybytes) {
 		t.Error("Got", ybytes)
 	}
 
-	if bytes.Compare(z, zbytes) != 0 {
+	if !bytes.Equal(z, zbytes) {
 		t.Error("Got", zbytes)
 	}
 }

common/bytes.go

@@ -143,7 +143,7 @@ func Hex2BytesFixed(str string, flen int) []byte {
 		return h
 	} else {
 		if len(h) > flen {
-			return h[len(h)-flen : len(h)]
+			return h[len(h)-flen:]
 		} else {
 			hh := make([]byte, flen)
 			copy(hh[flen-len(h):flen], h[:])

common/bytes_test.go

@@ -181,7 +181,7 @@ func TestFromHex(t *testing.T) {
 	input := "0x01"
 	expected := []byte{1}
 	result := FromHex(input)
-	if bytes.Compare(expected, result) != 0 {
+	if !bytes.Equal(expected, result) {
 		t.Errorf("Expected % x got % x", expected, result)
 	}
 }
@@ -190,7 +190,7 @@ func TestFromHexOddLength(t *testing.T) {
 	input := "0x1"
 	expected := []byte{1}
 	result := FromHex(input)
-	if bytes.Compare(expected, result) != 0 {
+	if !bytes.Equal(expected, result) {
 		t.Errorf("Expected % x got % x", expected, result)
 	}
 }

common/compiler/solidity.go

@@ -22,9 +22,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
-	"os"
 	"os/exec"
 	"regexp"
 	"strings"
@@ -34,7 +32,7 @@ import (
 )
 
 var (
-	versionRegexp = regexp.MustCompile("[0-9]+\\.[0-9]+\\.[0-9]+")
+	versionRegexp = regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+`)
 	solcParams    = []string{
 		"--combined-json", "bin,abi,userdoc,devdoc",
 		"--add-std",  // include standard lib contracts
@@ -96,27 +94,16 @@ func CompileSolidityString(solc, source string) (map[string]*Contract, error) {
 	if solc == "" {
 		solc = "solc"
 	}
-	// Write source to a temporary file. Compiling stdin used to be supported
-	// but seems to produce an exception with solc 0.3.5.
-	infile, err := ioutil.TempFile("", "geth-compile-solidity")
-	if err != nil {
-		return nil, err
-	}
-	defer os.Remove(infile.Name())
-	if _, err := io.WriteString(infile, source); err != nil {
-		return nil, err
-	}
-	if err := infile.Close(); err != nil {
-		return nil, err
-	}
-
-	return CompileSolidity(solc, infile.Name())
+	args := append(solcParams, "--")
+	cmd := exec.Command(solc, append(args, "-")...)
+	cmd.Stdin = strings.NewReader(source)
+	return runsolc(cmd, source)
 }
 
 // CompileSolidity compiles all given Solidity source files.
 func CompileSolidity(solc string, sourcefiles ...string) (map[string]*Contract, error) {
 	if len(sourcefiles) == 0 {
-		return nil, errors.New("solc: no source ")
+		return nil, errors.New("solc: no source files")
 	}
 	source, err := slurpFiles(sourcefiles)
 	if err != nil {
@@ -125,10 +112,13 @@ func CompileSolidity(solc string, sourcefiles ...string) (map[string]*Contract,
 	if solc == "" {
 		solc = "solc"
 	}
-
-	var stderr, stdout bytes.Buffer
 	args := append(solcParams, "--")
 	cmd := exec.Command(solc, append(args, sourcefiles...)...)
+	return runsolc(cmd, source)
+}
+
+func runsolc(cmd *exec.Cmd, source string) (map[string]*Contract, error) {
+	var stderr, stdout bytes.Buffer
 	cmd.Stderr = &stderr
 	cmd.Stdout = &stdout
 	if err := cmd.Run(); err != nil {

common/compiler/solidity_test.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"io/ioutil"
 	"os"
+	"os/exec"
 	"path"
 	"testing"
 
@@ -27,8 +28,7 @@ import (
 )
 
 const (
-	supportedSolcVersion = "0.3.5"
-	testSource           = `
+	testSource = `
 contract test {
    /// @notice Will multiply ` + "`a`" + ` by 7.
    function multiply(uint a) returns(uint d) {
@@ -36,23 +36,18 @@ contract test {
    }
 }
 `
-	testCode = "0x6060604052602a8060106000396000f3606060405260e060020a6000350463c6888fa18114601a575b005b6007600435026060908152602090f3"
-	testInfo = `{"source":"\ncontract test {\n   /// @notice Will multiply ` + "`a`" + ` by 7.\n   function multiply(uint a) returns(uint d) {\n       return a * 7;\n   }\n}\n","language":"Solidity","languageVersion":"0.1.1","compilerVersion":"0.1.1","compilerOptions":"--binary file --json-abi file --natspec-user file --natspec-dev file --add-std 1","abiDefinition":[{"constant":false,"inputs":[{"name":"a","type":"uint256"}],"name":"multiply","outputs":[{"name":"d","type":"uint256"}],"type":"function"}],"userDoc":{"methods":{"multiply(uint256)":{"notice":"Will multiply ` + "`a`" + ` by 7."}}},"developerDoc":{"methods":{}}}`
+	testInfo = `{"source":"\ncontract test {\n   /// @notice Will multiply ` + "`a`" + ` by 7.\n   function multiply(uint a) returns(uint d) {\n       return a * 7;\n   }\n}\n","language":"Solidity","languageVersion":"0.1.1","compilerVersion":"0.1.1","compilerOptions":"--binary file --json-abi file --add-std 1","abiDefinition":[{"constant":false,"inputs":[{"name":"a","type":"uint256"}],"name":"multiply","outputs":[{"name":"d","type":"uint256"}],"type":"function"}],"userDoc":{"methods":{"multiply(uint256)":{"notice":"Will multiply ` + "`a`" + ` by 7."}}},"developerDoc":{"methods":{}}}`
 )
 
-func skipUnsupported(t *testing.T) {
-	sol, err := SolidityVersion("")
-	if err != nil {
+func skipWithoutSolc(t *testing.T) {
+	if _, err := exec.LookPath("solc"); err != nil {
 		t.Skip(err)
-		return
-	}
-	if sol.Version != supportedSolcVersion {
-		t.Skipf("unsupported version of solc found (%v, expect %v)", sol.Version, supportedSolcVersion)
 	}
 }
 
 func TestCompiler(t *testing.T) {
-	skipUnsupported(t)
+	skipWithoutSolc(t)
+
 	contracts, err := CompileSolidityString("", testSource)
 	if err != nil {
 		t.Fatalf("error compiling source. result %v: %v", contracts, err)
@@ -64,19 +59,20 @@ func TestCompiler(t *testing.T) {
 	if !ok {
 		t.Fatal("info for contract 'test' not present in result")
 	}
-	if c.Code != testCode {
-		t.Errorf("wrong code: expected\n%s, got\n%s", testCode, c.Code)
+	if c.Code == "" {
+		t.Error("empty code")
 	}
 	if c.Info.Source != testSource {
 		t.Error("wrong source")
 	}
-	if c.Info.CompilerVersion != supportedSolcVersion {
-		t.Errorf("wrong version: expected %q, got %q", supportedSolcVersion, c.Info.CompilerVersion)
+	if c.Info.CompilerVersion == "" {
+		t.Error("empty version")
 	}
 }
 
 func TestCompileError(t *testing.T) {
-	skipUnsupported(t)
+	skipWithoutSolc(t)
+
 	contracts, err := CompileSolidityString("", testSource[4:])
 	if err == nil {
 		t.Errorf("error expected compiling source. got none. result %v", contracts)
@@ -103,7 +99,7 @@ func TestSaveInfo(t *testing.T) {
 	if string(got) != testInfo {
 		t.Errorf("incorrect info.json extracted, expected:\n%s\ngot\n%s", testInfo, string(got))
 	}
-	wantHash := common.HexToHash("0x9f3803735e7f16120c5a140ab3f02121fd3533a9655c69b33a10e78752cc49b0")
+	wantHash := common.HexToHash("0x22450a77f0c3ff7a395948d07bc1456881226a1b6325f4189cb5f1254a824080")
 	if cinfohash != wantHash {
 		t.Errorf("content hash for info is incorrect. expected %v, got %v", wantHash.Hex(), cinfohash.Hex())
 	}

common/format.go

@@ -27,7 +27,7 @@ import (
 // the unnecessary precision off from the formatted textual representation.
 type PrettyDuration time.Duration
 
-var prettyDurationRe = regexp.MustCompile("\\.[0-9]+")
+var prettyDurationRe = regexp.MustCompile(`\.[0-9]+`)
 
 // String implements the Stringer interface, allowing pretty printing of duration
 // values rounded to three decimals.