params, swarm: release Geth v1.8.17 and Swar v0.3.5

* swarm/network/stream: introduced visualized snapshot sync test

* swarm/network/stream: non-existing hash visualization sim

* swarm/network/stream: fixed retrieval tests; new backend for visualization

* swarm/network/stream: cleanup of visualized_snapshot_sync_sim_test.go

* swarm/network/stream: rebased PR on master

* swarm/network/stream: fixed loop logic in retrieval tests

* swarm/network/stream: fixed iterations for snapshot tests

* swarm/network/stream: address PR comments

* swarm/network/stream: addressed PR comments

.github/CODEOWNERS

@@ -27,6 +27,6 @@ swarm/services                  @zelig
 swarm/state                     @justelad
 swarm/storage/encryption        @gbalint @zelig @nagydani
 swarm/storage/mock              @janos
-swarm/storage/mru               @nolash
+swarm/storage/feed              @nolash @jpeletier
 swarm/testutil                  @lmars
 whisper/                        @gballet @gluk256

.github/CONTRIBUTING.md

@@ -1,16 +1,40 @@
+# Contributing
+
+Thank you for considering to help out with the source code! We welcome 
+contributions from anyone on the internet, and are grateful for even the 
+smallest of fixes!
+
+If you'd like to contribute to go-ethereum, please fork, fix, commit and send a 
+pull request for the maintainers to review and merge into the main code base. If
+you wish to submit more complex changes though, please check up with the core 
+devs first on [our gitter channel](https://gitter.im/ethereum/go-ethereum) to 
+ensure those changes are in line with the general philosophy of the project 
+and/or get some early feedback which can make both your efforts much lighter as
+well as our review and merge procedures quick and simple.
+
+## Coding guidelines
+
+Please make sure your contributions adhere to our coding guidelines:
+
+ * Code must adhere to the official Go 
+[formatting](https://golang.org/doc/effective_go.html#formatting) guidelines 
+(i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
+ * Code must be documented adhering to the official Go 
+[commentary](https://golang.org/doc/effective_go.html#commentary) guidelines.
+ * Pull requests need to be based on and opened against the `master` branch.
+ * Commit messages should be prefixed with the package(s) they modify.
+   * E.g. "eth, rpc: make trace configs optional"
+
 ## Can I have feature X
 
-Before you do a feature request please check and make sure that it isn't possible
-through some other means. The JavaScript enabled console is a powerful feature
-in the right hands. Please check our [Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
+Before you submit a feature request, please check and make sure that it isn't 
+possible through some other means. The JavaScript-enabled console is a powerful 
+feature in the right hands. Please check our 
+[Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
 and help.
 
-## Contributing
+## Configuration, dependencies, and tests
 
-If you'd like to contribute to go-ethereum please fork, fix, commit and
-send a pull request. Commits which do not comply with the coding standards
-are ignored (use gofmt!).
-
-See [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
-for more details on configuring your environment, testing, and
-dependency management.
+Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
+for more details on configuring your environment, managing project dependencies
+and testing procedures.

.github/no-response.yml

@@ -7,5 +7,5 @@ closeComment: >
   This issue has been automatically closed because there has been no response
   to our request for more information from the original author. With only the
   information that is currently in the issue, we don't have enough information
-  to take action. Please reach out if you have or find the answers we need so
-  that we can investigate further.
+  to take action. Please reach out if you have more relevant information or
+  answers to our questions so that we can investigate further.

.travis.yml

@@ -6,19 +6,19 @@ matrix:
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.9.x
+      go: 1.10.x
       script:
-        - sudo modprobe fuse
-        - sudo chmod 666 /dev/fuse
-        - sudo chown root:$USER /etc/fuse.conf
-        - go run build/ci.go install
-        - go run build/ci.go test -coverage $TEST_PACKAGES
+      - sudo modprobe fuse
+      - sudo chmod 666 /dev/fuse
+      - sudo chown root:$USER /etc/fuse.conf
+      - go run build/ci.go install
+      - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # These are the latest Go versions.
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.10.x
+      go: 1.11.x
       script:
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
@@ -27,7 +27,7 @@ matrix:
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
     - os: osx
-      go: 1.10.x
+      go: 1.11.x
       script:
         - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
         - go run build/ci.go install
@@ -36,7 +36,7 @@ matrix:
     # This builder only tests code linters on latest version of Go
     - os: linux
       dist: trusty
-      go: 1.10.x
+      go: 1.11.x
       env:
         - lint
       git:
@@ -45,9 +45,10 @@ matrix:
         - go run build/ci.go lint
 
     # This builder does the Ubuntu PPA upload
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
-      go: 1.10.x
+      go: 1.11.x
       env:
         - ubuntu-ppa
       git:
@@ -63,10 +64,11 @@ matrix:
         - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
 
     # This builder does the Linux Azure uploads
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
       sudo: required
-      go: 1.10.x
+      go: 1.11.x
       env:
         - azure-linux
       git:
@@ -96,11 +98,12 @@ matrix:
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Linux Azure MIPS xgo uploads
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
       services:
         - docker
-      go: 1.10.x
+      go: 1.11.x
       env:
         - azure-linux-mips
       git:
@@ -123,7 +126,8 @@ matrix:
         - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Android Maven and Azure uploads
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
       addons:
         apt:
@@ -144,7 +148,7 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.10.3.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.11.1.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
@@ -160,8 +164,9 @@ matrix:
         - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
 
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
-    - os: osx
-      go: 1.10.x
+    - if: type = push
+      os: osx
+      go: 1.11.x
       env:
         - azure-osx
         - azure-ios
@@ -188,19 +193,13 @@ matrix:
         - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
 
     # This builder does the Azure archive purges to avoid accumulating junk
-    - os: linux
+    - if: type = cron
+      os: linux
       dist: trusty
-      go: 1.10.x
+      go: 1.11.x
       env:
         - azure-purge
       git:
         submodules: false # avoid cloning ethereum/tests
       script:
         - go run build/ci.go purge -store gethstore/builds -days 14
-
-notifications:
-  webhooks:
-    urls:
-      - https://webhooks.gitter.im/e/e09ccdce1048c5e03445
-    on_success: change
-    on_failure: always

Dockerfile

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.10-alpine as builder
+FROM golang:1.11-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 

Dockerfile.alltools

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.10-alpine as builder
+FROM golang:1.11-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 

Makefile

@@ -57,6 +57,9 @@ devtools:
 	@type "solc" 2> /dev/null || echo 'Please install solc'
 	@type "protoc" 2> /dev/null || echo 'Please install protoc'
 
+swarm-devtools:
+	env GOBIN= go install ./cmd/swarm/mimegen
+
 # Cross Compilation Targets (xgo)
 
 geth-cross: geth-linux geth-darwin geth-windows geth-android geth-ios

README.md

@@ -7,7 +7,7 @@ https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/6874
 )](https://godoc.org/github.com/ethereum/go-ethereum)
 [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum)
 [![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
-[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethereum/go-ethereum?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+[![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv)
 
 Automated builds are available for stable releases and the unstable master branch.
 Binary archives are published at https://geth.ethereum.org/downloads/.

accounts/abi/abi.go

@@ -137,6 +137,9 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
 // MethodById looks up a method by the 4-byte id
 // returns nil if none found
 func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
+	if len(sigdata) < 4 {
+		return nil, fmt.Errorf("data too short (% bytes) for abi method lookup", len(sigdata))
+	}
 	for _, method := range abi.Methods {
 		if bytes.Equal(method.Id(), sigdata[:4]) {
 			return &method, nil

accounts/abi/abi_test.go

@@ -711,5 +711,14 @@ func TestABI_MethodById(t *testing.T) {
 			t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.Id()))
 		}
 	}
-
+	// Also test empty
+	if _, err := abi.MethodById([]byte{0x00}); err == nil {
+		t.Errorf("Expected error, too short to decode data")
+	}
+	if _, err := abi.MethodById([]byte{}); err == nil {
+		t.Errorf("Expected error, too short to decode data")
+	}
+	if _, err := abi.MethodById(nil); err == nil {
+		t.Errorf("Expected error, nil is short to decode data")
+	}
 }

accounts/abi/bind/backends/simulated.go

@@ -69,7 +69,7 @@ func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBac
 	database := ethdb.NewMemDatabase()
 	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
 	genesis.MustCommit(database)
-	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil)
 
 	backend := &SimulatedBackend{
 		database:   database,
@@ -208,7 +208,7 @@ func (b *SimulatedBackend) PendingNonceAt(ctx context.Context, account common.Ad
 }
 
 // SuggestGasPrice implements ContractTransactor.SuggestGasPrice. Since the simulated
-// chain doens't have miners, we just return a gas price of 1 for any call.
+// chain doesn't have miners, we just return a gas price of 1 for any call.
 func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
 	return big.NewInt(1), nil
 }

accounts/abi/bind/bind.go

@@ -23,13 +23,13 @@ package bind
 import (
 	"bytes"
 	"fmt"
+	"go/format"
 	"regexp"
 	"strings"
 	"text/template"
 	"unicode"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
-	"golang.org/x/tools/imports"
 )
 
 // Lang is a target programming language selector to generate bindings for.
@@ -145,9 +145,9 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 	if err := tmpl.Execute(buffer, data); err != nil {
 		return "", err
 	}
-	// For Go bindings pass the code through goimports to clean it up and double check
+	// For Go bindings pass the code through gofmt to clean it up
 	if lang == LangGo {
-		code, err := imports.Process(".", buffer.Bytes(), nil)
+		code, err := format.Source(buffer.Bytes())
 		if err != nil {
 			return "", fmt.Errorf("%v\n%s", err, buffer)
 		}
@@ -207,7 +207,7 @@ func bindTypeGo(kind abi.Type) string {
 
 // The inner function of bindTypeGo, this finds the inner type of stringKind.
 // (Or just the type itself if it is not an array or slice)
-// The length of the matched part is returned, with the the translated type.
+// The length of the matched part is returned, with the translated type.
 func bindUnnestedTypeGo(stringKind string) (int, string) {
 
 	switch {
@@ -255,7 +255,7 @@ func bindTypeJava(kind abi.Type) string {
 
 // The inner function of bindTypeJava, this finds the inner type of stringKind.
 // (Or just the type itself if it is not an array or slice)
-// The length of the matched part is returned, with the the translated type.
+// The length of the matched part is returned, with the translated type.
 func bindUnnestedTypeJava(stringKind string) (int, string) {
 
 	switch {

accounts/abi/bind/template.go

@@ -64,6 +64,30 @@ const tmplSourceGo = `
 
 package {{.Package}}
 
+import (
+	"math/big"
+	"strings"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var (
+	_ = big.NewInt
+	_ = strings.NewReader
+	_ = ethereum.NotFound
+	_ = abi.U256
+	_ = bind.Bind
+	_ = common.Big1
+	_ = types.BloomLookup
+	_ = event.NewSubscription
+)
+
 {{range $contract := .Contracts}}
 	// {{.Type}}ABI is the input ABI used to generate the binding from.
 	const {{.Type}}ABI = "{{.InputABI}}"

accounts/abi/type.go

@@ -103,7 +103,12 @@ func NewType(t string) (typ Type, err error) {
 		return typ, err
 	}
 	// parse the type and size of the abi-type.
-	parsedType := typeRegex.FindAllStringSubmatch(t, -1)[0]
+	matches := typeRegex.FindAllStringSubmatch(t, -1)
+	if len(matches) == 0 {
+		return Type{}, fmt.Errorf("invalid type '%v'", t)
+	}
+	parsedType := matches[0]
+
 	// varSize is the size of the variable
 	var varSize int
 	if len(parsedType[3]) > 0 {

accounts/abi/unpack.go

@@ -25,8 +25,17 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
+var (
+	maxUint256 = big.NewInt(0).Add(
+		big.NewInt(0).Exp(big.NewInt(2), big.NewInt(256), nil),
+		big.NewInt(-1))
+	maxInt256 = big.NewInt(0).Add(
+		big.NewInt(0).Exp(big.NewInt(2), big.NewInt(255), nil),
+		big.NewInt(-1))
+)
+
 // reads the integer based on its kind
-func readInteger(kind reflect.Kind, b []byte) interface{} {
+func readInteger(typ byte, kind reflect.Kind, b []byte) interface{} {
 	switch kind {
 	case reflect.Uint8:
 		return b[len(b)-1]
@@ -45,7 +54,20 @@ func readInteger(kind reflect.Kind, b []byte) interface{} {
 	case reflect.Int64:
 		return int64(binary.BigEndian.Uint64(b[len(b)-8:]))
 	default:
-		return new(big.Int).SetBytes(b)
+		// the only case lefts for integer is int256/uint256.
+		// big.SetBytes can't tell if a number is negative, positive on itself.
+		// On EVM, if the returned number > max int256, it is negative.
+		ret := new(big.Int).SetBytes(b)
+		if typ == UintTy {
+			return ret
+		}
+
+		if ret.Cmp(maxInt256) > 0 {
+			ret.Add(maxUint256, big.NewInt(0).Neg(ret))
+			ret.Add(ret, big.NewInt(1))
+			ret.Neg(ret)
+		}
+		return ret
 	}
 }
 
@@ -179,7 +201,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) {
 	case StringTy: // variable arrays are written at the end of the return bytes
 		return string(output[begin : begin+end]), nil
 	case IntTy, UintTy:
-		return readInteger(t.Kind, returnOutput), nil
+		return readInteger(t.T, t.Kind, returnOutput), nil
 	case BoolTy:
 		return readBool(returnOutput)
 	case AddressTy:

accounts/abi/unpack_test.go

@@ -117,6 +117,11 @@ var unpackTests = []unpackTest{
 		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
 		want: big.NewInt(1),
 	},
+	{
+		def:  `[{"type": "int256"}]`,
+		enc:  "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+		want: big.NewInt(-1),
+	},
 	{
 		def:  `[{"type": "address"}]`,
 		enc:  "0000000000000000000000000100000000000000000000000000000000000000",

accounts/keystore/key.go

@@ -179,26 +179,34 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou
 	return key, a, err
 }
 
-func writeKeyFile(file string, content []byte) error {
+func writeTemporaryKeyFile(file string, content []byte) (string, error) {
 	// Create the keystore directory with appropriate permissions
 	// in case it is not present yet.
 	const dirPerm = 0700
 	if err := os.MkdirAll(filepath.Dir(file), dirPerm); err != nil {
-		return err
+		return "", err
 	}
 	// Atomic write: create a temporary hidden file first
 	// then move it into place. TempFile assigns mode 0600.
 	f, err := ioutil.TempFile(filepath.Dir(file), "."+filepath.Base(file)+".tmp")
 	if err != nil {
-		return err
+		return "", err
 	}
 	if _, err := f.Write(content); err != nil {
 		f.Close()
 		os.Remove(f.Name())
-		return err
+		return "", err
 	}
 	f.Close()
-	return os.Rename(f.Name(), file)
+	return f.Name(), nil
+}
+
+func writeKeyFile(file string, content []byte) error {
+	name, err := writeTemporaryKeyFile(file, content)
+	if err != nil {
+		return err
+	}
+	return os.Rename(name, file)
 }
 
 // keyFileName implements the naming convention for keyfiles:

accounts/keystore/keystore.go

@@ -78,7 +78,7 @@ type unlocked struct {
 // NewKeyStore creates a keystore for the given directory.
 func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore {
 	keydir, _ = filepath.Abs(keydir)
-	ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP}}
+	ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP, false}}
 	ks.init(keydir)
 	return ks
 }

accounts/keystore/keystore_passphrase.go

@@ -35,6 +35,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -72,6 +73,10 @@ type keyStorePassphrase struct {
 	keysDirPath string
 	scryptN     int
 	scryptP     int
+	// skipKeyFileVerification disables the security-feature which does
+	// reads and decrypts any newly created keyfiles. This should be 'false' in all
+	// cases except tests -- setting this to 'true' is not recommended.
+	skipKeyFileVerification bool
 }
 
 func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string) (*Key, error) {
@@ -93,7 +98,7 @@ func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string)
 
 // StoreKey generates a key, encrypts with 'auth' and stores in the given directory
 func StoreKey(dir, auth string, scryptN, scryptP int) (common.Address, error) {
-	_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP}, rand.Reader, auth)
+	_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP, false}, rand.Reader, auth)
 	return a.Address, err
 }
 
@@ -102,7 +107,25 @@ func (ks keyStorePassphrase) StoreKey(filename string, key *Key, auth string) er
 	if err != nil {
 		return err
 	}
-	return writeKeyFile(filename, keyjson)
+	// Write into temporary file
+	tmpName, err := writeTemporaryKeyFile(filename, keyjson)
+	if err != nil {
+		return err
+	}
+	if !ks.skipKeyFileVerification {
+		// Verify that we can decrypt the file with the given password.
+		_, err = ks.GetKey(key.Address, tmpName, auth)
+		if err != nil {
+			msg := "An error was encountered when saving and verifying the keystore file. \n" +
+				"This indicates that the keystore is corrupted. \n" +
+				"The corrupted file is stored at \n%v\n" +
+				"Please file a ticket at:\n\n" +
+				"https://github.com/ethereum/go-ethereum/issues." +
+				"The error was : %s"
+			return fmt.Errorf(msg, tmpName, err)
+		}
+	}
+	return os.Rename(tmpName, filename)
 }
 
 func (ks keyStorePassphrase) JoinPath(filename string) string {

accounts/keystore/keystore_plain_test.go

@@ -37,7 +37,7 @@ func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) {
 		t.Fatal(err)
 	}
 	if encrypted {
-		ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP}
+		ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP, true}
 	} else {
 		ks = &keyStorePlain{d}
 	}
@@ -191,7 +191,7 @@ func TestV1_1(t *testing.T) {
 
 func TestV1_2(t *testing.T) {
 	t.Parallel()
-	ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP}
+	ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP, true}
 	addr := common.HexToAddress("cb61d5a9c4896fb9658090b597ef0e7be6f7b67e")
 	file := "testdata/v1/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e"
 	k, err := ks.GetKey(addr, file, "g")

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.3.windows-%GETH_ARCH%.zip
-  - 7z x go1.10.3.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.1.windows-%GETH_ARCH%.zip
+  - 7z x go1.11.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

build/ci.go

@@ -320,9 +320,7 @@ func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd
 // "tests" also includes static analysis tools such as vet.
 
 func doTest(cmdline []string) {
-	var (
-		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
-	)
+	coverage := flag.Bool("coverage", false, "Whether to record code coverage")
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
 
@@ -332,14 +330,11 @@ func doTest(cmdline []string) {
 	}
 	packages = build.ExpandPackagesNoVendor(packages)
 
-	// Run analysis tools before the tests.
-	build.MustRun(goTool("vet", packages...))
-
 	// Run the actual tests.
-	gotest := goTool("test", buildFlags(env)...)
 	// Test a single package at a time. CI builders are slow
 	// and some tests run into timeouts under load.
-	gotest.Args = append(gotest.Args, "-p", "1")
+	gotest := goTool("test", buildFlags(env)...)
+	gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m")
 	if *coverage {
 		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
 	}
@@ -1040,7 +1035,7 @@ func xgoTool(args []string) *exec.Cmd {
 func doPurge(cmdline []string) {
 	var (
 		store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`)
-		limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`)
+		limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`)
 	)
 	flag.CommandLine.Parse(cmdline)
 

cmd/abigen/main.go

@@ -75,7 +75,7 @@ func main() {
 		bins  []string
 		types []string
 	)
-	if *solFlag != "" || *abiFlag == "-" {
+	if *solFlag != "" || (*abiFlag == "-" && *pkgFlag == "") {
 		// Generate the list of types to exclude from binding
 		exclude := make(map[string]bool)
 		for _, kind := range strings.Split(*excFlag, ",") {
@@ -111,7 +111,13 @@ func main() {
 		}
 	} else {
 		// Otherwise load up the ABI, optional bytecode and type name from the parameters
-		abi, err := ioutil.ReadFile(*abiFlag)
+		var abi []byte
+		var err error
+		if *abiFlag == "-" {
+			abi, err = ioutil.ReadAll(os.Stdin)
+		} else {
+			abi, err = ioutil.ReadFile(*abiFlag)
+		}
 		if err != nil {
 			fmt.Printf("Failed to read input ABI: %v\n", err)
 			os.Exit(-1)
@@ -155,6 +161,5 @@ func contractsFromStdin() (map[string]*compiler.Contract, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	return compiler.ParseCombinedJSON(bytes, "", "", "", "")
 }

cmd/bootnode/main.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 )
@@ -85,7 +86,7 @@ func main() {
 	}
 
 	if *writeAddr {
-		fmt.Printf("%v\n", discover.PubkeyID(&nodeKey.PublicKey))
+		fmt.Printf("%v\n", enode.PubkeyToIDV4(&nodeKey.PublicKey))
 		os.Exit(0)
 	}
 

cmd/clef/README.md

@@ -875,3 +875,4 @@ There are a couple of implementation for a UI. We'll try to keep this list up to
 | QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: |  :+1: (partially)|
 | GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: |  :x: |
 | Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: |  :x: |
+| Clef UI| https://github.com/kyokan/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: |  :+1: (approve tx only)|

cmd/clef/extapi_changelog.md

@@ -1,6 +1,13 @@
 ### Changelog for external API
 
+#### 4.0.0
 
+* The external `account_Ecrecover`-method was removed. 
+* The external `account_Import`-method was removed.
+
+#### 3.0.0
+
+* The external `account_List`-method was changed to not expose `url`, which contained info about the local filesystem. It now returns only a list of addresses. 
 
 #### 2.0.0
 

cmd/clef/intapi_changelog.md

@@ -1,5 +1,21 @@
 ### Changelog for internal API (ui-api)
 
+### 2.1.0
+
+* Add `OnInputRequired(info UserInputRequest)` to internal API. This method is used when Clef needs user input, e.g. passwords.
+
+The following structures are used:
+```golang
+       UserInputRequest struct {
+               Prompt     string `json:"prompt"`
+               Title      string `json:"title"`
+               IsPassword bool   `json:"isPassword"`
+       }
+       UserInputResponse struct {
+               Text string `json:"text"`
+       }
+```
+
 ### 2.0.0
 
 * Modify how `call_info` on a transaction is conveyed. New format:

cmd/clef/main.go

@@ -48,7 +48,7 @@ import (
 )
 
 // ExternalAPIVersion -- see extapi_changelog.md
-const ExternalAPIVersion = "2.0.0"
+const ExternalAPIVersion = "3.0.0"
 
 // InternalAPIVersion -- see intapi_changelog.md
 const InternalAPIVersion = "2.0.0"
@@ -70,6 +70,10 @@ var (
 		Value: 4,
 		Usage: "log level to emit to the screen",
 	}
+	advancedMode = cli.BoolFlag{
+		Name:  "advanced",
+		Usage: "If enabled, issues warnings instead of rejections for suspicious requests. Default off",
+	}
 	keystoreFlag = cli.StringFlag{
 		Name:  "keystore",
 		Value: filepath.Join(node.DefaultDataDir(), "keystore"),
@@ -191,6 +195,7 @@ func init() {
 		ruleFlag,
 		stdiouiFlag,
 		testFlag,
+		advancedMode,
 	}
 	app.Action = signer
 	app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand}
@@ -225,7 +230,7 @@ func initializeSecrets(c *cli.Context) error {
 	if _, err := os.Stat(location); err == nil {
 		return fmt.Errorf("file %v already exists, will not overwrite", location)
 	}
-	err = ioutil.WriteFile(location, masterSeed, 0700)
+	err = ioutil.WriteFile(location, masterSeed, 0400)
 	if err != nil {
 		return err
 	}
@@ -384,7 +389,8 @@ func signer(c *cli.Context) error {
 		c.String(keystoreFlag.Name),
 		c.Bool(utils.NoUSBFlag.Name),
 		ui, db,
-		c.Bool(utils.LightKDFFlag.Name))
+		c.Bool(utils.LightKDFFlag.Name),
+		c.Bool(advancedMode.Name))
 
 	api = apiImpl
 
@@ -540,14 +546,14 @@ func readMasterKey(ctx *cli.Context) ([]byte, error) {
 
 // checkFile is a convenience function to check if a file
 // * exists
-// * is mode 0600
+// * is mode 0400
 func checkFile(filename string) error {
 	info, err := os.Stat(filename)
 	if err != nil {
 		return fmt.Errorf("failed stat on %s: %v", filename, err)
 	}
 	// Check the unix permission bits
-	if info.Mode().Perm()&077 != 0 {
+	if info.Mode().Perm()&0377 != 0 {
 		return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String())
 	}
 	return nil

cmd/clef/tutorial.md

@@ -31,43 +31,51 @@ NOTE: This file does not contain your accounts. Those need to be backed up separ
 
 ## Creating rules
 
-Now, you can create a rule-file.
+Now, you can create a rule-file. Note that it is not mandatory to use predefined rules, but it's really handy.
 
 ```javascript
 function ApproveListing(){
     return "Approve"
 }
 ```
-Get the `sha256` hash....
+
+Get the `sha256` hash. If you have openssl, you can do `openssl sha256 rules.js`...
 ```text
 #sha256sum rules.js
 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72  rules.js
 ```
-...And then `attest` the file:
+...now `attest` the file...
 ```text
 #./signer attest 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
 
 INFO [02-21|12:14:38] Ruleset attestation updated              sha256=6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
 ```
-At this point, we then start the signer with the rule-file:
 
+...and (this is required only for non-production versions) load a mock-up `4byte.json` by copying the file from the source to your current working directory:
 ```text
-#./signer --rules rules.json
+#cp $GOPATH/src/github.com/ethereum/go-ethereum/cmd/clef/4byte.json $PWD
+```
 
-INFO [02-21|12:15:18] Using CLI as UI-channel
-INFO [02-21|12:15:18] Loaded 4byte db                          signatures=5509 file=./4byte.json
-INFO [02-21|12:15:18] Could not load rulefile, rules not enabled file=rulefile
-DEBUG[02-21|12:15:18] FS scan times                            list=35.335µs set=5.536µs diff=5.073µs
-DEBUG[02-21|12:15:18] Ledger support enabled
-DEBUG[02-21|12:15:18] Trezor support enabled
-INFO [02-21|12:15:18] Audit logs configured                    file=audit.log
-INFO [02-21|12:15:18] HTTP endpoint opened                     url=http://localhost:8550
+At this point, we can start the signer with the rule-file:
+```text
+#./signer --rules rules.js --rpc
+
+INFO [09-25|20:28:11.866] Using CLI as UI-channel 
+INFO [09-25|20:28:11.876] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [09-25|20:28:11.877] Rule engine configured                   file=./rules.js
+DEBUG[09-25|20:28:11.877] FS scan times                            list=100.781µs set=13.253µs diff=5.761µs
+DEBUG[09-25|20:28:11.884] Ledger support enabled 
+DEBUG[09-25|20:28:11.888] Trezor support enabled 
+INFO [09-25|20:28:11.888] Audit logs configured                    file=audit.log
+DEBUG[09-25|20:28:11.888] HTTP registered                          namespace=account
+INFO [09-25|20:28:11.890] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|20:28:11.890] IPC registered                           namespace=account
+INFO [09-25|20:28:11.890] IPC endpoint opened                      url=<nil>
 ------- Signer info -------
+* extapi_version : 2.0.0
+* intapi_version : 2.0.0
 * extapi_http : http://localhost:8550
 * extapi_ipc : <nil>
-* extapi_version : 2.0.0
-* intapi_version : 1.2.0
-
 ```
 
 Any list-requests will now be auto-approved by our rule-file.
@@ -107,16 +115,16 @@ The `master_seed` was then used to derive a few other things:
 
 ## Adding credentials
 
-In order to make more useful rules; sign transactions, the signer needs access to the passwords needed to unlock keystores.
+In order to make more useful rules like signing transactions, the signer needs access to the passwords needed to unlock keystores.
 
 ```text
-#./signer addpw 0x694267f14675d7e1b9494fd8d72fefe1755710fa test
+#./signer addpw "0x694267f14675d7e1b9494fd8d72fefe1755710fa" "test_password"
 
 INFO [02-21|13:43:21] Credential store updated                 key=0x694267f14675d7e1b9494fd8d72fefe1755710fa
 ```
 ## More advanced rules
 
-Now let's update the rules to make use of credentials
+Now let's update the rules to make use of credentials:
 
 ```javascript
 function ApproveListing(){
@@ -134,13 +142,15 @@ function ApproveSignData(r){
 }
 
 ```
-In this example,
-* any requests to sign data with the account `0x694...` will be
-    * auto-approved if the message contains with `bazonk`,
-    * and auto-rejected if it does not.
-    * Any other signing-requests will be passed along for manual approve/reject.
+In this example:
+* Any requests to sign data with the account `0x694...` will be
+    * auto-approved if the message contains with `bazonk`
+    * auto-rejected if it does not.
+* Any other signing-requests will be passed along for manual approve/reject.
 
-..attest the new file
+_Note: make sure that `0x694...` is an account you have access to. You can create it either via the clef or the traditional account cli tool. If the latter was chosen, make sure both clef and geth use the same keystore by specifing `--keystore path/to/your/keystore` when running clef._
+
+Attest the new file...
 ```text
 #sha256sum rules.js
 2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f  rules.js
@@ -153,23 +163,26 @@ INFO [02-21|14:36:30] Ruleset attestation updated              sha256=2a0cb661da
 And start the signer:
 
 ```
-#./signer --rules rules.js
+#./signer --rules rules.js --rpc
 
-INFO [02-21|14:41:56] Using CLI as UI-channel
-INFO [02-21|14:41:56] Loaded 4byte db                          signatures=5509 file=./4byte.json
-INFO [02-21|14:41:56] Rule engine configured                   file=rules.js
-DEBUG[02-21|14:41:56] FS scan times                            list=34.607µs set=4.509µs diff=4.87µs
-DEBUG[02-21|14:41:56] Ledger support enabled
-DEBUG[02-21|14:41:56] Trezor support enabled
-INFO [02-21|14:41:56] Audit logs configured                    file=audit.log
-INFO [02-21|14:41:56] HTTP endpoint opened                     url=http://localhost:8550
+INFO [09-25|21:02:16.450] Using CLI as UI-channel 
+INFO [09-25|21:02:16.466] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [09-25|21:02:16.467] Rule engine configured                   file=./rules.js
+DEBUG[09-25|21:02:16.468] FS scan times                            list=1.45262ms set=21.926µs diff=6.944µs
+DEBUG[09-25|21:02:16.473] Ledger support enabled 
+DEBUG[09-25|21:02:16.475] Trezor support enabled 
+INFO [09-25|21:02:16.476] Audit logs configured                    file=audit.log
+DEBUG[09-25|21:02:16.476] HTTP registered                          namespace=account
+INFO [09-25|21:02:16.478] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|21:02:16.478] IPC registered                           namespace=account
+INFO [09-25|21:02:16.478] IPC endpoint opened                      url=<nil>
 ------- Signer info -------
 * extapi_version : 2.0.0
-* intapi_version : 1.2.0
+* intapi_version : 2.0.0
 * extapi_http : http://localhost:8550
 * extapi_ipc : <nil>
-INFO [02-21|14:41:56] error occurred during execution          error="ReferenceError: 'OnSignerStartup' is not defined"
 ```
+
 And then test signing, once with `bazonk` and once without:
 
 ```
@@ -190,9 +203,9 @@ INFO [02-21|14:42:56] Op rejected
 The signer also stores all traffic over the external API in a log file. The last 4 lines shows the two requests and their responses:
 
 ```text
-#tail audit.log -n 4
+#tail -n 4 audit.log
 t=2018-02-21T14:42:41+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49706\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=202062617a6f6e6b2062617a2067617a0a
 t=2018-02-21T14:42:42+0100 lvl=info msg=Sign       api=signer type=response data=93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c error=nil
 t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49708\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=2020626f6e6b2062617a2067617a0a
 t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=response data=                                                                                                                                   error="Request denied"
-```
+```

cmd/ethkey/README.md

@@ -21,21 +21,33 @@ Private key information can be printed by using the `--private` flag;
 make sure to use this feature with great caution!
 
 
-### `ethkey sign <keyfile> <message/file>`
+### `ethkey signmessage <keyfile> <message/file>`
 
 Sign the message with a keyfile.
 It is possible to refer to a file containing the message.
+To sign a message contained in a file, use the `--msgfile` flag.
 
 
-### `ethkey verify <address> <signature> <message/file>`
+### `ethkey verifymessage <address> <signature> <message/file>`
 
 Verify the signature of the message.
 It is possible to refer to a file containing the message.
+To sign a message contained in a file, use the --msgfile flag.
+
+
+### `ethkey changepassphrase <keyfile>`
+
+Change the passphrase of a keyfile.
+use the `--newpasswordfile` to point to the new password file.
 
 
 ## Passphrases
 
 For every command that uses a keyfile, you will be prompted to provide the 
 passphrase for decrypting the keyfile.  To avoid this message, it is possible
-to pass the passphrase by using the `--passphrase` flag pointing to a file that
+to pass the passphrase by using the `--passwordfile` flag pointing to a file that
 contains the passphrase.
+
+## JSON
+
+In case you need to output the result in a JSON format, you shall by using the `--json` flag.

cmd/evm/disasm.go

@@ -44,7 +44,7 @@ func disasmCmd(ctx *cli.Context) error {
 		return err
 	}
 
-	code := strings.TrimSpace(string(in[:]))
+	code := strings.TrimSpace(string(in))
 	fmt.Printf("%v\n", code)
 	return asm.PrintDisassembled(code)
 }

cmd/evm/runner.go

@@ -80,13 +80,13 @@ func runCmd(ctx *cli.Context) error {
 	}
 
 	var (
-		tracer      vm.Tracer
-		debugLogger *vm.StructLogger
-		statedb     *state.StateDB
-		chainConfig *params.ChainConfig
-		sender      = common.BytesToAddress([]byte("sender"))
-		receiver    = common.BytesToAddress([]byte("receiver"))
-		blockNumber uint64
+		tracer        vm.Tracer
+		debugLogger   *vm.StructLogger
+		statedb       *state.StateDB
+		chainConfig   *params.ChainConfig
+		sender        = common.BytesToAddress([]byte("sender"))
+		receiver      = common.BytesToAddress([]byte("receiver"))
+		genesisConfig *core.Genesis
 	)
 	if ctx.GlobalBool(MachineFlag.Name) {
 		tracer = NewJSONLogger(logconfig, os.Stdout)
@@ -98,13 +98,14 @@ func runCmd(ctx *cli.Context) error {
 	}
 	if ctx.GlobalString(GenesisFlag.Name) != "" {
 		gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
+		genesisConfig = gen
 		db := ethdb.NewMemDatabase()
 		genesis := gen.ToBlock(db)
 		statedb, _ = state.New(genesis.Root(), state.NewDatabase(db))
 		chainConfig = gen.Config
-		blockNumber = gen.Number
 	} else {
 		statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+		genesisConfig = new(core.Genesis)
 	}
 	if ctx.GlobalString(SenderFlag.Name) != "" {
 		sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name))
@@ -156,13 +157,19 @@ func runCmd(ctx *cli.Context) error {
 	}
 
 	initialGas := ctx.GlobalUint64(GasFlag.Name)
+	if genesisConfig.GasLimit != 0 {
+		initialGas = genesisConfig.GasLimit
+	}
 	runtimeConfig := runtime.Config{
 		Origin:      sender,
 		State:       statedb,
 		GasLimit:    initialGas,
 		GasPrice:    utils.GlobalBig(ctx, PriceFlag.Name),
 		Value:       utils.GlobalBig(ctx, ValueFlag.Name),
-		BlockNumber: new(big.Int).SetUint64(blockNumber),
+		Difficulty:  genesisConfig.Difficulty,
+		Time:        new(big.Int).SetUint64(genesisConfig.Timestamp),
+		Coinbase:    genesisConfig.Coinbase,
+		BlockNumber: new(big.Int).SetUint64(genesisConfig.Number),
 		EVMConfig: vm.Config{
 			Tracer: tracer,
 			Debug:  ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name),

cmd/evm/staterunner.go

@@ -97,6 +97,10 @@ func stateTestCmd(ctx *cli.Context) error {
 			// Run the test and aggregate the result
 			result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
 			state, err := test.Run(st, cfg)
+			// print state root for evmlab tracing
+			if ctx.GlobalBool(MachineFlag.Name) && state != nil {
+				fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
+			}
 			if err != nil {
 				// Test failed, mark as so and dump any state to aid debugging
 				result.Pass, result.Error = false, err.Error()
@@ -105,10 +109,6 @@ func stateTestCmd(ctx *cli.Context) error {
 					result.State = &dump
 				}
 			}
-			// print state root for evmlab tracing (already committed above, so no need to delete objects again
-			if ctx.GlobalBool(MachineFlag.Name) && state != nil {
-				fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
-			}
 
 			results = append(results, *result)
 

cmd/faucet/faucet.go

@@ -54,8 +54,8 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/params"
 	"golang.org/x/net/websocket"
@@ -157,7 +157,8 @@ func main() {
 	if blob, err = ioutil.ReadFile(*accPassFlag); err != nil {
 		log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err)
 	}
-	pass := string(blob)
+	// Delete trailing newline in password
+	pass := strings.TrimSuffix(string(blob), "\n")
 
 	ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP)
 	if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil {
@@ -198,6 +199,8 @@ type faucet struct {
 
 	keystore *keystore.KeyStore // Keystore containing the single signer
 	account  accounts.Account   // Account funding user faucet requests
+	head     *types.Header      // Current head header of the faucet
+	balance  *big.Int           // Current balance of the faucet
 	nonce    uint64             // Current pending nonce of the faucet
 	price    *big.Int           // Current gas price to issue funds with
 
@@ -252,8 +255,10 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
 		return nil, err
 	}
 	for _, boot := range enodes {
-		old, _ := discover.ParseNode(boot.String())
-		stack.Server().AddPeer(old)
+		old, err := enode.ParseV4(boot.String())
+		if err != nil {
+			stack.Server().AddPeer(old)
+		}
 	}
 	// Attach to the client and retrieve and interesting metadatas
 	api, err := stack.Attach()
@@ -323,33 +328,30 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 		nonce   uint64
 		err     error
 	)
-	for {
-		// Attempt to retrieve the stats, may error on no faucet connectivity
-		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
-		head, err = f.client.HeaderByNumber(ctx, nil)
-		if err == nil {
-			balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
-			if err == nil {
-				nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
-			}
+	for head == nil || balance == nil {
+		// Retrieve the current stats cached by the faucet
+		f.lock.RLock()
+		if f.head != nil {
+			head = types.CopyHeader(f.head)
 		}
-		cancel()
+		if f.balance != nil {
+			balance = new(big.Int).Set(f.balance)
+		}
+		nonce = f.nonce
+		f.lock.RUnlock()
 
-		// If stats retrieval failed, wait a bit and retry
-		if err != nil {
-			if err = sendError(conn, errors.New("Faucet offline: "+err.Error())); err != nil {
+		if head == nil || balance == nil {
+			// Report the faucet offline until initial stats are ready
+			if err = sendError(conn, errors.New("Faucet offline")); err != nil {
 				log.Warn("Failed to send faucet error to client", "err", err)
 				return
 			}
 			time.Sleep(3 * time.Second)
-			continue
 		}
-		// Initial stats reported successfully, proceed with user interaction
-		break
 	}
 	// Send over the initial stats and the latest header
 	if err = send(conn, map[string]interface{}{
-		"funds":    balance.Div(balance, ether),
+		"funds":    new(big.Int).Div(balance, ether),
 		"funded":   nonce,
 		"peers":    f.stack.Server().PeerCount(),
 		"requests": f.reqs,
@@ -519,6 +521,47 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 	}
 }
 
+// refresh attempts to retrieve the latest header from the chain and extract the
+// associated faucet balance and nonce for connectivity caching.
+func (f *faucet) refresh(head *types.Header) error {
+	// Ensure a state update does not run for too long
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	// If no header was specified, use the current chain head
+	var err error
+	if head == nil {
+		if head, err = f.client.HeaderByNumber(ctx, nil); err != nil {
+			return err
+		}
+	}
+	// Retrieve the balance, nonce and gas price from the current head
+	var (
+		balance *big.Int
+		nonce   uint64
+		price   *big.Int
+	)
+	if balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number); err != nil {
+		return err
+	}
+	if nonce, err = f.client.NonceAt(ctx, f.account.Address, head.Number); err != nil {
+		return err
+	}
+	if price, err = f.client.SuggestGasPrice(ctx); err != nil {
+		return err
+	}
+	// Everything succeeded, update the cached stats and eject old requests
+	f.lock.Lock()
+	f.head, f.balance = head, balance
+	f.price, f.nonce = price, nonce
+	for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
+		f.reqs = f.reqs[1:]
+	}
+	f.lock.Unlock()
+
+	return nil
+}
+
 // loop keeps waiting for interesting events and pushes them out to connected
 // websockets.
 func (f *faucet) loop() {
@@ -536,45 +579,27 @@ func (f *faucet) loop() {
 	go func() {
 		for head := range update {
 			// New chain head arrived, query the current stats and stream to clients
-			var (
-				balance *big.Int
-				nonce   uint64
-				price   *big.Int
-				err     error
-			)
-			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-			balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
-			if err == nil {
-				nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
-				if err == nil {
-					price, err = f.client.SuggestGasPrice(ctx)
-				}
+			timestamp := time.Unix(head.Time.Int64(), 0)
+			if time.Since(timestamp) > time.Hour {
+				log.Warn("Skipping faucet refresh, head too old", "number", head.Number, "hash", head.Hash(), "age", common.PrettyAge(timestamp))
+				continue
 			}
-			cancel()
-
-			// If querying the data failed, try for the next block
-			if err != nil {
+			if err := f.refresh(head); err != nil {
 				log.Warn("Failed to update faucet state", "block", head.Number, "hash", head.Hash(), "err", err)
 				continue
-			} else {
-				log.Info("Updated faucet state", "block", head.Number, "hash", head.Hash(), "balance", balance, "nonce", nonce, "price", price)
 			}
 			// Faucet state retrieved, update locally and send to clients
-			balance = new(big.Int).Div(balance, ether)
-
-			f.lock.Lock()
-			f.price, f.nonce = price, nonce
-			for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
-				f.reqs = f.reqs[1:]
-			}
-			f.lock.Unlock()
-
 			f.lock.RLock()
+			log.Info("Updated faucet state", "number", head.Number, "hash", head.Hash(), "age", common.PrettyAge(timestamp), "balance", f.balance, "nonce", f.nonce, "price", f.price)
+
+			balance := new(big.Int).Div(f.balance, ether)
+			peers := f.stack.Server().PeerCount()
+
 			for _, conn := range f.conns {
 				if err := send(conn, map[string]interface{}{
 					"funds":    balance,
 					"funded":   f.nonce,
-					"peers":    f.stack.Server().PeerCount(),
+					"peers":    peers,
 					"requests": f.reqs,
 				}, time.Second); err != nil {
 					log.Warn("Failed to send stats to client", "err", err)

cmd/geth/chaincmd.go

@@ -340,9 +340,9 @@ func importPreimages(ctx *cli.Context) error {
 
 	start := time.Now()
 	if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil {
-		utils.Fatalf("Export error: %v\n", err)
+		utils.Fatalf("Import error: %v\n", err)
 	}
-	fmt.Printf("Export done in %v\n", time.Since(start))
+	fmt.Printf("Import done in %v\n", time.Since(start))
 	return nil
 }
 

cmd/geth/config.go

@@ -168,6 +168,9 @@ func makeFullNode(ctx *cli.Context) *node.Node {
 		if ctx.GlobalIsSet(utils.WhisperMinPOWFlag.Name) {
 			cfg.Shh.MinimumAcceptedPOW = ctx.Float64(utils.WhisperMinPOWFlag.Name)
 		}
+		if ctx.GlobalIsSet(utils.WhisperRestrictConnectionBetweenLightClientsFlag.Name) {
+			cfg.Shh.RestrictConnectionBetweenLightClients = true
+		}
 		utils.RegisterShhService(stack, &cfg.Shh)
 	}
 

cmd/geth/main.go

@@ -101,6 +101,7 @@ var (
 		utils.MinerNotifyFlag,
 		utils.MinerGasTargetFlag,
 		utils.MinerLegacyGasTargetFlag,
+		utils.MinerGasLimitFlag,
 		utils.MinerGasPriceFlag,
 		utils.MinerLegacyGasPriceFlag,
 		utils.MinerEtherbaseFlag,
@@ -108,6 +109,7 @@ var (
 		utils.MinerExtraDataFlag,
 		utils.MinerLegacyExtraDataFlag,
 		utils.MinerRecommitIntervalFlag,
+		utils.MinerNoVerfiyFlag,
 		utils.NATFlag,
 		utils.NoDiscoverFlag,
 		utils.DiscoveryV5Flag,
@@ -128,6 +130,8 @@ var (
 		utils.NoCompactionFlag,
 		utils.GpoBlocksFlag,
 		utils.GpoPercentileFlag,
+		utils.EWASMInterpreterFlag,
+		utils.EVMInterpreterFlag,
 		configFileFlag,
 	}
 
@@ -149,6 +153,7 @@ var (
 		utils.WhisperEnabledFlag,
 		utils.WhisperMaxMessageSizeFlag,
 		utils.WhisperMinPOWFlag,
+		utils.WhisperRestrictConnectionBetweenLightClientsFlag,
 	}
 
 	metricsFlags = []cli.Flag{
@@ -235,7 +240,6 @@ func init() {
 		// Start system runtime metrics collection
 		go metrics.CollectProcessMetrics(3 * time.Second)
 
-		utils.SetupNetwork(ctx)
 		return nil
 	}
 
@@ -336,26 +340,18 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 		if err := stack.Service(&ethereum); err != nil {
 			utils.Fatalf("Ethereum service not running: %v", err)
 		}
-		// Use a reduced number of threads if requested
-		threads := ctx.GlobalInt(utils.MinerLegacyThreadsFlag.Name)
-		if ctx.GlobalIsSet(utils.MinerThreadsFlag.Name) {
-			threads = ctx.GlobalInt(utils.MinerThreadsFlag.Name)
-		}
-		if threads > 0 {
-			type threaded interface {
-				SetThreads(threads int)
-			}
-			if th, ok := ethereum.Engine().(threaded); ok {
-				th.SetThreads(threads)
-			}
-		}
 		// Set the gas price to the limits from the CLI and start mining
 		gasprice := utils.GlobalBig(ctx, utils.MinerLegacyGasPriceFlag.Name)
 		if ctx.IsSet(utils.MinerGasPriceFlag.Name) {
 			gasprice = utils.GlobalBig(ctx, utils.MinerGasPriceFlag.Name)
 		}
 		ethereum.TxPool().SetGasPrice(gasprice)
-		if err := ethereum.StartMining(true); err != nil {
+
+		threads := ctx.GlobalInt(utils.MinerLegacyThreadsFlag.Name)
+		if ctx.GlobalIsSet(utils.MinerThreadsFlag.Name) {
+			threads = ctx.GlobalInt(utils.MinerThreadsFlag.Name)
+		}
+		if err := ethereum.StartMining(threads); err != nil {
 			utils.Fatalf("Failed to start mining: %v", err)
 		}
 	}

cmd/geth/usage.go

@@ -189,9 +189,11 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.MinerNotifyFlag,
 			utils.MinerGasPriceFlag,
 			utils.MinerGasTargetFlag,
+			utils.MinerGasLimitFlag,
 			utils.MinerEtherbaseFlag,
 			utils.MinerExtraDataFlag,
 			utils.MinerRecommitIntervalFlag,
+			utils.MinerNoVerfiyFlag,
 		},
 	},
 	{
@@ -205,6 +207,8 @@ var AppHelpFlagGroups = []flagGroup{
 		Name: "VIRTUAL MACHINE",
 		Flags: []cli.Flag{
 			utils.VMEnableDebugFlag,
+			utils.EVMInterpreterFlag,
+			utils.EWASMInterpreterFlag,
 		},
 	},
 	{

cmd/p2psim/main.go

@@ -47,7 +47,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/simulations"
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -285,7 +285,7 @@ func createNode(ctx *cli.Context) error {
 		if err != nil {
 			return err
 		}
-		config.ID = discover.PubkeyID(&privKey.PublicKey)
+		config.ID = enode.PubkeyToIDV4(&privKey.PublicKey)
 		config.PrivateKey = privKey
 	}
 	if services := ctx.String("services"); services != "" {

cmd/puppeth/module_node.go

@@ -42,7 +42,7 @@ ADD genesis.json /genesis.json
 RUN \
   echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
 	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
-	echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gasprice {{.GasPrice}}' >> geth.sh
+	echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --nat extip:{{.IP}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh
 
 ENTRYPOINT ["/bin/sh", "geth.sh"]
 `
@@ -68,6 +68,7 @@ services:
       - STATS_NAME={{.Ethstats}}
       - MINER_NAME={{.Etherbase}}
       - GAS_TARGET={{.GasTarget}}
+      - GAS_LIMIT={{.GasLimit}}
       - GAS_PRICE={{.GasPrice}}
     logging:
       driver: "json-file"
@@ -98,12 +99,14 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n
 	template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{
 		"NetworkID": config.network,
 		"Port":      config.port,
+		"IP":        client.address,
 		"Peers":     config.peersTotal,
 		"LightFlag": lightFlag,
 		"Bootnodes": strings.Join(bootnodes, ","),
 		"Ethstats":  config.ethstats,
 		"Etherbase": config.etherbase,
 		"GasTarget": uint64(1000000 * config.gasTarget),
+		"GasLimit":  uint64(1000000 * config.gasLimit),
 		"GasPrice":  uint64(1000000000 * config.gasPrice),
 		"Unlock":    config.keyJSON != "",
 	})
@@ -122,6 +125,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n
 		"Ethstats":   config.ethstats[:strings.Index(config.ethstats, ":")],
 		"Etherbase":  config.etherbase,
 		"GasTarget":  config.gasTarget,
+		"GasLimit":   config.gasLimit,
 		"GasPrice":   config.gasPrice,
 	})
 	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
@@ -160,6 +164,7 @@ type nodeInfos struct {
 	keyJSON    string
 	keyPass    string
 	gasTarget  float64
+	gasLimit   float64
 	gasPrice   float64
 }
 
@@ -175,8 +180,9 @@ func (info *nodeInfos) Report() map[string]string {
 	}
 	if info.gasTarget > 0 {
 		// Miner or signer node
-		report["Gas limit (baseline target)"] = fmt.Sprintf("%0.3f MGas", info.gasTarget)
 		report["Gas price (minimum accepted)"] = fmt.Sprintf("%0.3f GWei", info.gasPrice)
+		report["Gas floor (baseline target)"] = fmt.Sprintf("%0.3f MGas", info.gasTarget)
+		report["Gas ceil  (target maximum)"] = fmt.Sprintf("%0.3f MGas", info.gasLimit)
 
 		if info.etherbase != "" {
 			// Ethash proof-of-work miner
@@ -217,14 +223,15 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 	totalPeers, _ := strconv.Atoi(infos.envvars["TOTAL_PEERS"])
 	lightPeers, _ := strconv.Atoi(infos.envvars["LIGHT_PEERS"])
 	gasTarget, _ := strconv.ParseFloat(infos.envvars["GAS_TARGET"], 64)
+	gasLimit, _ := strconv.ParseFloat(infos.envvars["GAS_LIMIT"], 64)
 	gasPrice, _ := strconv.ParseFloat(infos.envvars["GAS_PRICE"], 64)
 
 	// Container available, retrieve its node ID and its genesis json
 	var out []byte
-	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.id --cache=16 attach", network, kind)); err != nil {
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.enode --cache=16 attach", network, kind)); err != nil {
 		return nil, ErrServiceUnreachable
 	}
-	id := bytes.Trim(bytes.TrimSpace(out), "\"")
+	enode := bytes.Trim(bytes.TrimSpace(out), "\"")
 
 	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /genesis.json", network, kind)); err != nil {
 		return nil, ErrServiceUnreachable
@@ -256,9 +263,10 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 		keyJSON:    keyJSON,
 		keyPass:    keyPass,
 		gasTarget:  gasTarget,
+		gasLimit:   gasLimit,
 		gasPrice:   gasPrice,
 	}
-	stats.enode = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.port)
+	stats.enode = string(enode)
 
 	return stats, nil
 }

cmd/puppeth/wizard_dashboard.go

@@ -92,7 +92,7 @@ func (w *wizard) deployDashboard() {
 				pages = append(pages, page)
 			}
 		}
-		// Promt the user to chose one, enter manually or simply not list this service
+		// Prompt the user to chose one, enter manually or simply not list this service
 		defLabel, defChoice := "don't list", len(pages)+2
 		if len(pages) > 0 {
 			defLabel, defChoice = pages[0], 1

cmd/puppeth/wizard_network.go

@@ -87,7 +87,7 @@ func (w *wizard) makeServer() string {
 	return input
 }
 
-// selectServer lists the user all the currnetly known servers to choose from,
+// selectServer lists the user all the currently known servers to choose from,
 // also granting the option to add a new one.
 func (w *wizard) selectServer() string {
 	// List the available server to the user and wait for a choice
@@ -115,7 +115,7 @@ func (w *wizard) selectServer() string {
 // manageComponents displays a list of network components the user can tear down
 // and an option
 func (w *wizard) manageComponents() {
-	// List all the componens we can tear down, along with an entry to deploy a new one
+	// List all the components we can tear down, along with an entry to deploy a new one
 	fmt.Println()
 
 	var serviceHosts, serviceNames []string

cmd/puppeth/wizard_node.go

@@ -50,7 +50,7 @@ func (w *wizard) deployNode(boot bool) {
 		if boot {
 			infos = &nodeInfos{port: 30303, peersTotal: 512, peersLight: 256}
 		} else {
-			infos = &nodeInfos{port: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18}
+			infos = &nodeInfos{port: 30303, peersTotal: 50, peersLight: 0, gasTarget: 7.5, gasLimit: 10, gasPrice: 1}
 		}
 	}
 	existed := err == nil
@@ -152,6 +152,10 @@ func (w *wizard) deployNode(boot bool) {
 		fmt.Printf("What gas limit should empty blocks target (MGas)? (default = %0.3f)\n", infos.gasTarget)
 		infos.gasTarget = w.readDefaultFloat(infos.gasTarget)
 
+		fmt.Println()
+		fmt.Printf("What gas limit should full blocks target (MGas)? (default = %0.3f)\n", infos.gasLimit)
+		infos.gasLimit = w.readDefaultFloat(infos.gasLimit)
+
 		fmt.Println()
 		fmt.Printf("What gas price should the signer require (GWei)? (default = %0.3f)\n", infos.gasPrice)
 		infos.gasPrice = w.readDefaultFloat(infos.gasPrice)

cmd/swarm/access.go

@@ -51,7 +51,7 @@ func accessNewPass(ctx *cli.Context) {
 		password  = getPassPhrase("", 0, makePasswordList(ctx))
 		dryRun    = ctx.Bool(SwarmDryRunFlag.Name)
 	)
-	accessKey, ae, err = api.DoPasswordNew(ctx, password, salt)
+	accessKey, ae, err = api.DoPassword(ctx, password, salt)
 	if err != nil {
 		utils.Fatalf("error getting session key: %v", err)
 	}
@@ -62,7 +62,6 @@ func accessNewPass(ctx *cli.Context) {
 			utils.Fatalf("had an error printing the manifests: %v", err)
 		}
 	} else {
-		utils.Fatalf("uploading manifests")
 		err = uploadManifests(ctx, m, nil)
 		if err != nil {
 			utils.Fatalf("had an error uploading the manifests: %v", err)
@@ -85,7 +84,7 @@ func accessNewPK(ctx *cli.Context) {
 		granteePublicKey = ctx.String(SwarmAccessGrantKeyFlag.Name)
 		dryRun           = ctx.Bool(SwarmDryRunFlag.Name)
 	)
-	sessionKey, ae, err = api.DoPKNew(ctx, privateKey, granteePublicKey, salt)
+	sessionKey, ae, err = api.DoPK(ctx, privateKey, granteePublicKey, salt)
 	if err != nil {
 		utils.Fatalf("error getting session key: %v", err)
 	}
@@ -110,23 +109,38 @@ func accessNewACT(ctx *cli.Context) {
 	}
 
 	var (
-		ae          *api.AccessEntry
-		actManifest *api.Manifest
-		accessKey   []byte
-		err         error
-		ref         = args[0]
-		grantees    = []string{}
-		actFilename = ctx.String(SwarmAccessGrantKeysFlag.Name)
-		privateKey  = getPrivKey(ctx)
-		dryRun      = ctx.Bool(SwarmDryRunFlag.Name)
+		ae                   *api.AccessEntry
+		actManifest          *api.Manifest
+		accessKey            []byte
+		err                  error
+		ref                  = args[0]
+		pkGrantees           = []string{}
+		passGrantees         = []string{}
+		pkGranteesFilename   = ctx.String(SwarmAccessGrantKeysFlag.Name)
+		passGranteesFilename = ctx.String(utils.PasswordFileFlag.Name)
+		privateKey           = getPrivKey(ctx)
+		dryRun               = ctx.Bool(SwarmDryRunFlag.Name)
 	)
-
-	bytes, err := ioutil.ReadFile(actFilename)
-	if err != nil {
-		utils.Fatalf("had an error reading the grantee public key list")
+	if pkGranteesFilename == "" && passGranteesFilename == "" {
+		utils.Fatalf("you have to provide either a grantee public-keys file or an encryption passwords file (or both)")
 	}
-	grantees = strings.Split(string(bytes), "\n")
-	accessKey, ae, actManifest, err = api.DoACTNew(ctx, privateKey, salt, grantees)
+
+	if pkGranteesFilename != "" {
+		bytes, err := ioutil.ReadFile(pkGranteesFilename)
+		if err != nil {
+			utils.Fatalf("had an error reading the grantee public key list")
+		}
+		pkGrantees = strings.Split(strings.Trim(string(bytes), "\n"), "\n")
+	}
+
+	if passGranteesFilename != "" {
+		bytes, err := ioutil.ReadFile(passGranteesFilename)
+		if err != nil {
+			utils.Fatalf("could not read password filename: %v", err)
+		}
+		passGrantees = strings.Split(strings.Trim(string(bytes), "\n"), "\n")
+	}
+	accessKey, ae, actManifest, err = api.DoACT(ctx, privateKey, salt, pkGrantees, passGrantees)
 	if err != nil {
 		utils.Fatalf("error generating ACT manifest: %v", err)
 	}

cmd/swarm/access_test.go

@@ -13,6 +13,9 @@
 //
 // You should have received a copy of the GNU General Public License
 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// +build !windows
+
 package main
 
 import (
@@ -25,18 +28,26 @@ import (
 	gorand "math/rand"
 	"net/http"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/crypto/ecies"
 	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
+const (
+	hashRegexp = `[a-f\d]{128}`
+	data       = "notsorandomdata"
+)
+
+var DefaultCurve = crypto.S256()
+
 // TestAccessPassword tests for the correct creation of an ACT manifest protected by a password.
 // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
 // The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded
@@ -47,23 +58,8 @@ func TestAccessPassword(t *testing.T) {
 	defer cluster.Shutdown()
 	proxyNode := cluster.Nodes[0]
 
-	// create a tmp file
-	tmp, err := ioutil.TempDir("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmp)
-
-	// write data to file
-	data := "notsorandomdata"
-	dataFilename := filepath.Join(tmp, "data.txt")
-
-	err = ioutil.WriteFile(dataFilename, []byte(data), 0666)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hashRegexp := `[a-f\d]{128}`
+	dataFilename := testutil.TempFileWithContent(t, data)
+	defer os.RemoveAll(dataFilename)
 
 	// upload the file with 'swarm up' and expect a hash
 	up := runSwarm(t,
@@ -80,14 +76,14 @@ func TestAccessPassword(t *testing.T) {
 	}
 
 	ref := matches[0]
-
-	password := "smth"
-	passwordFilename := filepath.Join(tmp, "password.txt")
-
-	err = ioutil.WriteFile(passwordFilename, []byte(password), 0666)
+	tmp, err := ioutil.TempDir("", "swarm-test")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer os.RemoveAll(tmp)
+	password := "smth"
+	passwordFilename := testutil.TempFileWithContent(t, "smth")
+	defer os.RemoveAll(passwordFilename)
 
 	up = runSwarm(t,
 		"access",
@@ -139,7 +135,9 @@ func TestAccessPassword(t *testing.T) {
 	if a.KdfParams == nil {
 		t.Fatal("manifest access kdf params is nil")
 	}
-
+	if a.Publisher != "" {
+		t.Fatal("should be empty")
+	}
 	client := swarm.NewClient(cluster.Nodes[0].URL)
 
 	hash, err := client.UploadManifest(&m, false)
@@ -185,12 +183,8 @@ func TestAccessPassword(t *testing.T) {
 		t.Errorf("expected decrypted data %q, got %q", data, string(d))
 	}
 
-	wrongPasswordFilename := filepath.Join(tmp, "password-wrong.txt")
-
-	err = ioutil.WriteFile(wrongPasswordFilename, []byte("just wr0ng"), 0666)
-	if err != nil {
-		t.Fatal(err)
-	}
+	wrongPasswordFilename := testutil.TempFileWithContent(t, "just wr0ng")
+	defer os.RemoveAll(wrongPasswordFilename)
 
 	//download file with 'swarm down' with wrong password
 	up = runSwarm(t,
@@ -216,25 +210,11 @@ func TestAccessPassword(t *testing.T) {
 // the test will fail if the proxy's given private key is not granted on the ACT.
 func TestAccessPK(t *testing.T) {
 	// Setup Swarm and upload a test file to it
-	cluster := newTestCluster(t, 1)
+	cluster := newTestCluster(t, 2)
 	defer cluster.Shutdown()
 
-	// create a tmp file
-	tmp, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer tmp.Close()
-	defer os.Remove(tmp.Name())
-
-	// write data to file
-	data := "notsorandomdata"
-	_, err = io.WriteString(tmp, data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hashRegexp := `[a-f\d]{128}`
+	dataFilename := testutil.TempFileWithContent(t, data)
+	defer os.RemoveAll(dataFilename)
 
 	// upload the file with 'swarm up' and expect a hash
 	up := runSwarm(t,
@@ -242,7 +222,7 @@ func TestAccessPK(t *testing.T) {
 		cluster.Nodes[0].URL,
 		"up",
 		"--encrypt",
-		tmp.Name())
+		dataFilename)
 	_, matches := up.ExpectRegexp(hashRegexp)
 	up.ExpectExit()
 
@@ -251,7 +231,6 @@ func TestAccessPK(t *testing.T) {
 	}
 
 	ref := matches[0]
-
 	pk := cluster.Nodes[0].PrivateKey
 	granteePubKey := crypto.CompressPubkey(&pk.PublicKey)
 
@@ -260,22 +239,15 @@ func TestAccessPK(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	passFile, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer passFile.Close()
-	defer os.Remove(passFile.Name())
-	_, err = io.WriteString(passFile, testPassphrase)
-	if err != nil {
-		t.Fatal(err)
-	}
+	passwordFilename := testutil.TempFileWithContent(t, testPassphrase)
+	defer os.RemoveAll(passwordFilename)
+
 	_, publisherAccount := getTestAccount(t, publisherDir)
 	up = runSwarm(t,
 		"--bzzaccount",
 		publisherAccount.Address.String(),
 		"--password",
-		passFile.Name(),
+		passwordFilename,
 		"--datadir",
 		publisherDir,
 		"--bzzapi",
@@ -296,6 +268,20 @@ func TestAccessPK(t *testing.T) {
 		t.Fatalf("stdout not matched")
 	}
 
+	//get the public key from the publisher directory
+	publicKeyFromDataDir := runSwarm(t,
+		"--bzzaccount",
+		publisherAccount.Address.String(),
+		"--password",
+		passwordFilename,
+		"--datadir",
+		publisherDir,
+		"print-keys",
+		"--compressed",
+	)
+	_, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+")
+	publicKeyFromDataDir.ExpectExit()
+	pkComp := strings.Split(publicKeyString[0], "=")[1]
 	var m api.Manifest
 
 	err = json.Unmarshal([]byte(matches[0]), &m)
@@ -329,7 +315,9 @@ func TestAccessPK(t *testing.T) {
 	if a.KdfParams != nil {
 		t.Fatal("manifest access kdf params should be nil")
 	}
-
+	if a.Publisher != pkComp {
+		t.Fatal("publisher key did not match")
+	}
 	client := swarm.NewClient(cluster.Nodes[0].URL)
 
 	hash, err := client.UploadManifest(&m, false)
@@ -356,36 +344,34 @@ func TestAccessPK(t *testing.T) {
 	}
 }
 
-// TestAccessACT tests the e2e creation, uploading and downloading of an ACT type access control
-// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
-// set. the third node should fail decoding the reference as it will not be granted access. the publisher uploads through
-// one of the nodes then disappears.
+// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
 func TestAccessACT(t *testing.T) {
+	testAccessACT(t, 0)
+}
+
+// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest)
+func TestAccessACTScale(t *testing.T) {
+	testAccessACT(t, 1000)
+}
+
+// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
+// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
+// set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access.
+// the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails.
+// the publisher uploads through one of the nodes then disappears.
+func testAccessACT(t *testing.T, bogusEntries int) {
 	// Setup Swarm and upload a test file to it
-	cluster := newTestCluster(t, 3)
+	const clusterSize = 3
+	cluster := newTestCluster(t, clusterSize)
 	defer cluster.Shutdown()
 
 	var uploadThroughNode = cluster.Nodes[0]
 	client := swarm.NewClient(uploadThroughNode.URL)
 
 	r1 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
-	nodeToSkip := r1.Intn(3) // a number between 0 and 2 (node indices in `cluster`)
-	// create a tmp file
-	tmp, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer tmp.Close()
-	defer os.Remove(tmp.Name())
-
-	// write data to file
-	data := "notsorandomdata"
-	_, err = io.WriteString(tmp, data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hashRegexp := `[a-f\d]{128}`
+	nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`)
+	dataFilename := testutil.TempFileWithContent(t, data)
+	defer os.RemoveAll(dataFilename)
 
 	// upload the file with 'swarm up' and expect a hash
 	up := runSwarm(t,
@@ -393,7 +379,7 @@ func TestAccessACT(t *testing.T) {
 		cluster.Nodes[0].URL,
 		"up",
 		"--encrypt",
-		tmp.Name())
+		dataFilename)
 	_, matches := up.ExpectRegexp(hashRegexp)
 	up.ExpectExit()
 
@@ -412,41 +398,42 @@ func TestAccessACT(t *testing.T) {
 		grantees = append(grantees, hex.EncodeToString(granteePubKey))
 	}
 
-	granteesPubkeyListFile, err := ioutil.TempFile("", "grantees-pubkey-list.csv")
-	if err != nil {
-		t.Fatal(err)
-	}
+	if bogusEntries > 0 {
+		bogusGrantees := []string{}
 
-	_, err = granteesPubkeyListFile.WriteString(strings.Join(grantees, "\n"))
-	if err != nil {
-		t.Fatal(err)
+		for i := 0; i < bogusEntries; i++ {
+			prv, err := ecies.GenerateKey(rand.Reader, DefaultCurve, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			bogusGrantees = append(bogusGrantees, hex.EncodeToString(crypto.CompressPubkey(&prv.ExportECDSA().PublicKey)))
+		}
+		r2 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
+		for i := 0; i < len(grantees); i++ {
+			insertAtIdx := r2.Intn(len(bogusGrantees))
+			bogusGrantees = append(bogusGrantees[:insertAtIdx], append([]string{grantees[i]}, bogusGrantees[insertAtIdx:]...)...)
+		}
+		grantees = bogusGrantees
 	}
-
-	defer granteesPubkeyListFile.Close()
-	defer os.Remove(granteesPubkeyListFile.Name())
+	granteesPubkeyListFile := testutil.TempFileWithContent(t, strings.Join(grantees, "\n"))
+	defer os.RemoveAll(granteesPubkeyListFile)
 
 	publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp")
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer os.RemoveAll(publisherDir)
 
-	passFile, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer passFile.Close()
-	defer os.Remove(passFile.Name())
-	_, err = io.WriteString(passFile, testPassphrase)
-	if err != nil {
-		t.Fatal(err)
-	}
-
+	passwordFilename := testutil.TempFileWithContent(t, testPassphrase)
+	defer os.RemoveAll(passwordFilename)
+	actPasswordFilename := testutil.TempFileWithContent(t, "smth")
+	defer os.RemoveAll(actPasswordFilename)
 	_, publisherAccount := getTestAccount(t, publisherDir)
 	up = runSwarm(t,
 		"--bzzaccount",
 		publisherAccount.Address.String(),
 		"--password",
-		passFile.Name(),
+		passwordFilename,
 		"--datadir",
 		publisherDir,
 		"--bzzapi",
@@ -455,7 +442,9 @@ func TestAccessACT(t *testing.T) {
 		"new",
 		"act",
 		"--grant-keys",
-		granteesPubkeyListFile.Name(),
+		granteesPubkeyListFile,
+		"--password",
+		actPasswordFilename,
 		ref,
 	)
 
@@ -465,6 +454,22 @@ func TestAccessACT(t *testing.T) {
 	if len(matches) == 0 {
 		t.Fatalf("stdout not matched")
 	}
+
+	//get the public key from the publisher directory
+	publicKeyFromDataDir := runSwarm(t,
+		"--bzzaccount",
+		publisherAccount.Address.String(),
+		"--password",
+		passwordFilename,
+		"--datadir",
+		publisherDir,
+		"print-keys",
+		"--compressed",
+	)
+	_, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+")
+	publicKeyFromDataDir.ExpectExit()
+	pkComp := strings.Split(publicKeyString[0], "=")[1]
+
 	hash := matches[0]
 	m, _, err := client.DownloadManifest(hash)
 	if err != nil {
@@ -494,10 +499,10 @@ func TestAccessACT(t *testing.T) {
 	if len(a.Salt) < 32 {
 		t.Fatalf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
 	}
-	if a.KdfParams != nil {
-		t.Fatal("manifest access kdf params should be nil")
-	}
 
+	if a.Publisher != pkComp {
+		t.Fatal("publisher key did not match")
+	}
 	httpClient := &http.Client{}
 
 	// all nodes except the skipped node should be able to decrypt the content
@@ -518,6 +523,25 @@ func TestAccessACT(t *testing.T) {
 				t.Fatalf("should be a 401")
 			}
 
+			// try downloading using a password instead, using the unauthorized node
+			passwordUrl := strings.Replace(url, "http://", "http://:smth@", -1)
+			response, err = httpClient.Get(passwordUrl)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if response.StatusCode != http.StatusOK {
+				t.Fatal("should be a 200")
+			}
+
+			// now try with the wrong password, expect 401
+			passwordUrl = strings.Replace(url, "http://", "http://:smthWrong@", -1)
+			response, err = httpClient.Get(passwordUrl)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if response.StatusCode != http.StatusUnauthorized {
+				t.Fatal("should be a 401")
+			}
 			continue
 		}
 

cmd/swarm/config.go

@@ -59,27 +59,28 @@ var (
 
 //constants for environment variables
 const (
-	SWARM_ENV_CHEQUEBOOK_ADDR      = "SWARM_CHEQUEBOOK_ADDR"
-	SWARM_ENV_ACCOUNT              = "SWARM_ACCOUNT"
-	SWARM_ENV_LISTEN_ADDR          = "SWARM_LISTEN_ADDR"
-	SWARM_ENV_PORT                 = "SWARM_PORT"
-	SWARM_ENV_NETWORK_ID           = "SWARM_NETWORK_ID"
-	SWARM_ENV_SWAP_ENABLE          = "SWARM_SWAP_ENABLE"
-	SWARM_ENV_SWAP_API             = "SWARM_SWAP_API"
-	SWARM_ENV_SYNC_DISABLE         = "SWARM_SYNC_DISABLE"
-	SWARM_ENV_SYNC_UPDATE_DELAY    = "SWARM_ENV_SYNC_UPDATE_DELAY"
-	SWARM_ENV_LIGHT_NODE_ENABLE    = "SWARM_LIGHT_NODE_ENABLE"
-	SWARM_ENV_DELIVERY_SKIP_CHECK  = "SWARM_DELIVERY_SKIP_CHECK"
-	SWARM_ENV_ENS_API              = "SWARM_ENS_API"
-	SWARM_ENV_ENS_ADDR             = "SWARM_ENS_ADDR"
-	SWARM_ENV_CORS                 = "SWARM_CORS"
-	SWARM_ENV_BOOTNODES            = "SWARM_BOOTNODES"
-	SWARM_ENV_PSS_ENABLE           = "SWARM_PSS_ENABLE"
-	SWARM_ENV_STORE_PATH           = "SWARM_STORE_PATH"
-	SWARM_ENV_STORE_CAPACITY       = "SWARM_STORE_CAPACITY"
-	SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY"
-	SWARM_ACCESS_PASSWORD          = "SWARM_ACCESS_PASSWORD"
-	GETH_ENV_DATADIR               = "GETH_DATADIR"
+	SWARM_ENV_CHEQUEBOOK_ADDR         = "SWARM_CHEQUEBOOK_ADDR"
+	SWARM_ENV_ACCOUNT                 = "SWARM_ACCOUNT"
+	SWARM_ENV_LISTEN_ADDR             = "SWARM_LISTEN_ADDR"
+	SWARM_ENV_PORT                    = "SWARM_PORT"
+	SWARM_ENV_NETWORK_ID              = "SWARM_NETWORK_ID"
+	SWARM_ENV_SWAP_ENABLE             = "SWARM_SWAP_ENABLE"
+	SWARM_ENV_SWAP_API                = "SWARM_SWAP_API"
+	SWARM_ENV_SYNC_DISABLE            = "SWARM_SYNC_DISABLE"
+	SWARM_ENV_SYNC_UPDATE_DELAY       = "SWARM_ENV_SYNC_UPDATE_DELAY"
+	SWARM_ENV_MAX_STREAM_PEER_SERVERS = "SWARM_ENV_MAX_STREAM_PEER_SERVERS"
+	SWARM_ENV_LIGHT_NODE_ENABLE       = "SWARM_LIGHT_NODE_ENABLE"
+	SWARM_ENV_DELIVERY_SKIP_CHECK     = "SWARM_DELIVERY_SKIP_CHECK"
+	SWARM_ENV_ENS_API                 = "SWARM_ENS_API"
+	SWARM_ENV_ENS_ADDR                = "SWARM_ENS_ADDR"
+	SWARM_ENV_CORS                    = "SWARM_CORS"
+	SWARM_ENV_BOOTNODES               = "SWARM_BOOTNODES"
+	SWARM_ENV_PSS_ENABLE              = "SWARM_PSS_ENABLE"
+	SWARM_ENV_STORE_PATH              = "SWARM_STORE_PATH"
+	SWARM_ENV_STORE_CAPACITY          = "SWARM_STORE_CAPACITY"
+	SWARM_ENV_STORE_CACHE_CAPACITY    = "SWARM_STORE_CACHE_CAPACITY"
+	SWARM_ACCESS_PASSWORD             = "SWARM_ACCESS_PASSWORD"
+	GETH_ENV_DATADIR                  = "GETH_DATADIR"
 )
 
 // These settings ensure that TOML keys use the same names as Go struct fields.
@@ -124,7 +125,7 @@ func initSwarmNode(config *bzzapi.Config, stack *node.Node, ctx *cli.Context) {
 	//get the account for the provided swarm account
 	prvkey := getAccount(config.BzzAccount, ctx, stack)
 	//set the resolved config path (geth --datadir)
-	config.Path = stack.InstanceDir()
+	config.Path = expandPath(stack.InstanceDir())
 	//finally, initialize the configuration
 	config.Init(prvkey)
 	//configuration phase completed here
@@ -175,14 +176,18 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 	}
 
 	if networkid := ctx.GlobalString(SwarmNetworkIdFlag.Name); networkid != "" {
-		if id, _ := strconv.Atoi(networkid); id != 0 {
-			currentConfig.NetworkID = uint64(id)
+		id, err := strconv.ParseUint(networkid, 10, 64)
+		if err != nil {
+			utils.Fatalf("invalid cli flag %s: %v", SwarmNetworkIdFlag.Name, err)
+		}
+		if id != 0 {
+			currentConfig.NetworkID = id
 		}
 	}
 
 	if ctx.GlobalIsSet(utils.DataDirFlag.Name) {
 		if datadir := ctx.GlobalString(utils.DataDirFlag.Name); datadir != "" {
-			currentConfig.Path = datadir
+			currentConfig.Path = expandPath(datadir)
 		}
 	}
 
@@ -207,6 +212,9 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		currentConfig.SyncUpdateDelay = d
 	}
 
+	// any value including 0 is acceptable
+	currentConfig.MaxStreamPeerServers = ctx.GlobalInt(SwarmMaxStreamPeerServersFlag.Name)
+
 	if ctx.GlobalIsSet(SwarmLightNodeEnabled.Name) {
 		currentConfig.LightNodeEnabled = true
 	}
@@ -226,6 +234,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		if len(ensAPIs) == 1 && ensAPIs[0] == "" {
 			ensAPIs = nil
 		}
+		for i := range ensAPIs {
+			ensAPIs[i] = expandPath(ensAPIs[i])
+		}
+
 		currentConfig.EnsAPIs = ensAPIs
 	}
 
@@ -262,13 +274,17 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 	}
 
 	if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" {
-		if id, _ := strconv.Atoi(networkid); id != 0 {
-			currentConfig.NetworkID = uint64(id)
+		id, err := strconv.ParseUint(networkid, 10, 64)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_NETWORK_ID, err)
+		}
+		if id != 0 {
+			currentConfig.NetworkID = id
 		}
 	}
 
 	if datadir := os.Getenv(GETH_ENV_DATADIR); datadir != "" {
-		currentConfig.Path = datadir
+		currentConfig.Path = expandPath(datadir)
 	}
 
 	bzzport := os.Getenv(SWARM_ENV_PORT)
@@ -281,33 +297,50 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 	}
 
 	if swapenable := os.Getenv(SWARM_ENV_SWAP_ENABLE); swapenable != "" {
-		if swap, err := strconv.ParseBool(swapenable); err != nil {
-			currentConfig.SwapEnabled = swap
+		swap, err := strconv.ParseBool(swapenable)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SWAP_ENABLE, err)
 		}
+		currentConfig.SwapEnabled = swap
 	}
 
 	if syncdisable := os.Getenv(SWARM_ENV_SYNC_DISABLE); syncdisable != "" {
-		if sync, err := strconv.ParseBool(syncdisable); err != nil {
-			currentConfig.SyncEnabled = !sync
+		sync, err := strconv.ParseBool(syncdisable)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_DISABLE, err)
 		}
+		currentConfig.SyncEnabled = !sync
 	}
 
 	if v := os.Getenv(SWARM_ENV_DELIVERY_SKIP_CHECK); v != "" {
-		if skipCheck, err := strconv.ParseBool(v); err != nil {
+		skipCheck, err := strconv.ParseBool(v)
+		if err != nil {
 			currentConfig.DeliverySkipCheck = skipCheck
 		}
 	}
 
 	if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" {
-		if d, err := time.ParseDuration(v); err != nil {
-			currentConfig.SyncUpdateDelay = d
+		d, err := time.ParseDuration(v)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_UPDATE_DELAY, err)
 		}
+		currentConfig.SyncUpdateDelay = d
+	}
+
+	if max := os.Getenv(SWARM_ENV_MAX_STREAM_PEER_SERVERS); max != "" {
+		m, err := strconv.Atoi(max)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_MAX_STREAM_PEER_SERVERS, err)
+		}
+		currentConfig.MaxStreamPeerServers = m
 	}
 
 	if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" {
-		if lightnode, err := strconv.ParseBool(lne); err != nil {
-			currentConfig.LightNodeEnabled = lightnode
+		lightnode, err := strconv.ParseBool(lne)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_LIGHT_NODE_ENABLE, err)
 		}
+		currentConfig.LightNodeEnabled = lightnode
 	}
 
 	if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {

cmd/swarm/db.go

@@ -93,21 +93,6 @@ func dbImport(ctx *cli.Context) {
 	log.Info(fmt.Sprintf("successfully imported %d chunks", count))
 }
 
-func dbClean(ctx *cli.Context) {
-	args := ctx.Args()
-	if len(args) != 2 {
-		utils.Fatalf("invalid arguments, please specify <chunkdb> (path to a local chunk database) and the base key")
-	}
-
-	store, err := openLDBStore(args[0], common.Hex2Bytes(args[1]))
-	if err != nil {
-		utils.Fatalf("error opening local chunk database: %s", err)
-	}
-	defer store.Close()
-
-	store.Cleanup()
-}
-
 func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) {
 	if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
 		return nil, fmt.Errorf("invalid chunkdb path: %s", err)

cmd/swarm/export_test.go

@@ -24,6 +24,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"runtime"
 	"strings"
 	"testing"
 
@@ -38,6 +39,9 @@ import (
 // 5. imports the exported datastore
 // 6. fetches the uploaded random file from the second node
 func TestCLISwarmExportImport(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
 	cluster := newTestCluster(t, 1)
 
 	// generate random 10mb file

cmd/swarm/feeds.go

@@ -0,0 +1,172 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// Command feed allows the user to create and update signed Swarm feeds
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed"
+	"gopkg.in/urfave/cli.v1"
+)
+
+func NewGenericSigner(ctx *cli.Context) feed.Signer {
+	return feed.NewGenericSigner(getPrivKey(ctx))
+}
+
+func getTopic(ctx *cli.Context) (topic feed.Topic) {
+	var name = ctx.String(SwarmFeedNameFlag.Name)
+	var relatedTopic = ctx.String(SwarmFeedTopicFlag.Name)
+	var relatedTopicBytes []byte
+	var err error
+
+	if relatedTopic != "" {
+		relatedTopicBytes, err = hexutil.Decode(relatedTopic)
+		if err != nil {
+			utils.Fatalf("Error parsing topic: %s", err)
+		}
+	}
+
+	topic, err = feed.NewTopic(name, relatedTopicBytes)
+	if err != nil {
+		utils.Fatalf("Error parsing topic: %s", err)
+	}
+	return topic
+}
+
+// swarm feed create <frequency> [--name <name>] [--data <0x Hexdata> [--multihash=false]]
+// swarm feed update <Manifest Address or ENS domain> <0x Hexdata> [--multihash=false]
+// swarm feed info <Manifest Address or ENS domain>
+
+func feedCreateManifest(ctx *cli.Context) {
+	var (
+		bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client = swarm.NewClient(bzzapi)
+	)
+
+	newFeedUpdateRequest := feed.NewFirstRequest(getTopic(ctx))
+	newFeedUpdateRequest.Feed.User = feedGetUser(ctx)
+
+	manifestAddress, err := client.CreateFeedWithManifest(newFeedUpdateRequest)
+	if err != nil {
+		utils.Fatalf("Error creating feed manifest: %s", err.Error())
+		return
+	}
+	fmt.Println(manifestAddress) // output manifest address to the user in a single line (useful for other commands to pick up)
+
+}
+
+func feedUpdate(ctx *cli.Context) {
+	args := ctx.Args()
+
+	var (
+		bzzapi                  = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client                  = swarm.NewClient(bzzapi)
+		manifestAddressOrDomain = ctx.String(SwarmFeedManifestFlag.Name)
+	)
+
+	if len(args) < 1 {
+		fmt.Println("Incorrect number of arguments")
+		cli.ShowCommandHelpAndExit(ctx, "update", 1)
+		return
+	}
+
+	signer := NewGenericSigner(ctx)
+
+	data, err := hexutil.Decode(args[0])
+	if err != nil {
+		utils.Fatalf("Error parsing data: %s", err.Error())
+		return
+	}
+
+	var updateRequest *feed.Request
+	var query *feed.Query
+
+	if manifestAddressOrDomain == "" {
+		query = new(feed.Query)
+		query.User = signer.Address()
+		query.Topic = getTopic(ctx)
+
+	}
+
+	// Retrieve a feed update request
+	updateRequest, err = client.GetFeedRequest(query, manifestAddressOrDomain)
+	if err != nil {
+		utils.Fatalf("Error retrieving feed status: %s", err.Error())
+	}
+
+	// set the new data
+	updateRequest.SetData(data)
+
+	// sign update
+	if err = updateRequest.Sign(signer); err != nil {
+		utils.Fatalf("Error signing feed update: %s", err.Error())
+	}
+
+	// post update
+	err = client.UpdateFeed(updateRequest)
+	if err != nil {
+		utils.Fatalf("Error updating feed: %s", err.Error())
+		return
+	}
+}
+
+func feedInfo(ctx *cli.Context) {
+	var (
+		bzzapi                  = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client                  = swarm.NewClient(bzzapi)
+		manifestAddressOrDomain = ctx.String(SwarmFeedManifestFlag.Name)
+	)
+
+	var query *feed.Query
+	if manifestAddressOrDomain == "" {
+		query = new(feed.Query)
+		query.Topic = getTopic(ctx)
+		query.User = feedGetUser(ctx)
+	}
+
+	metadata, err := client.GetFeedRequest(query, manifestAddressOrDomain)
+	if err != nil {
+		utils.Fatalf("Error retrieving feed metadata: %s", err.Error())
+		return
+	}
+	encodedMetadata, err := metadata.MarshalJSON()
+	if err != nil {
+		utils.Fatalf("Error encoding metadata to JSON for display:%s", err)
+	}
+	fmt.Println(string(encodedMetadata))
+}
+
+func feedGetUser(ctx *cli.Context) common.Address {
+	var user = ctx.String(SwarmFeedUserFlag.Name)
+	if user != "" {
+		return common.HexToAddress(user)
+	}
+	pk := getPrivKey(ctx)
+	if pk == nil {
+		utils.Fatalf("Cannot read private key. Must specify --user or --bzzaccount")
+	}
+	return crypto.PubkeyToAddress(pk.PublicKey)
+
+}

cmd/swarm/feeds_test.go

@@ -0,0 +1,182 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/log"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
+)
+
+func TestCLIFeedUpdate(t *testing.T) {
+
+	srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer {
+		return swarmhttp.NewServer(api, "")
+	}, nil)
+	log.Info("starting a test swarm server")
+	defer srv.Close()
+
+	// create a private key file for signing
+	pkfile, err := ioutil.TempFile("", "swarm-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pkfile.Close()
+	defer os.Remove(pkfile.Name())
+
+	privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979"
+	privKey, _ := crypto.HexToECDSA(privkeyHex)
+	address := crypto.PubkeyToAddress(privKey.PublicKey)
+
+	// save the private key to a file
+	_, err = io.WriteString(pkfile, privkeyHex)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// compose a topic. We'll be doing quotes about Miguel de Cervantes
+	var topic feed.Topic
+	subject := []byte("Miguel de Cervantes")
+	copy(topic[:], subject[:])
+	name := "quotes"
+
+	// prepare some data for the update
+	data := []byte("En boca cerrada no entran moscas")
+	hexData := hexutil.Encode(data)
+
+	flags := []string{
+		"--bzzapi", srv.URL,
+		"--bzzaccount", pkfile.Name(),
+		"feed", "update",
+		"--topic", topic.Hex(),
+		"--name", name,
+		hexData}
+
+	// create an update and expect an exit without errors
+	log.Info(fmt.Sprintf("updating a feed with 'swarm feed update'"))
+	cmd := runSwarm(t, flags...)
+	cmd.ExpectExit()
+
+	// now try to get the update using the client
+	client := swarm.NewClient(srv.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// build the same topic as before, this time
+	// we use NewTopic to create a topic automatically.
+	topic, err = feed.NewTopic(name, subject)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Feed configures whose updates we will be looking up.
+	fd := feed.Feed{
+		Topic: topic,
+		User:  address,
+	}
+
+	// Build a query to get the latest update
+	query := feed.NewQueryLatest(&fd, lookup.NoClue)
+
+	// retrieve content!
+	reader, err := client.QueryFeed(query, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	retrieved, err := ioutil.ReadAll(reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check we retrieved the sent information
+	if !bytes.Equal(data, retrieved) {
+		t.Fatalf("Received %s, expected %s", retrieved, data)
+	}
+
+	// Now retrieve info for the next update
+	flags = []string{
+		"--bzzapi", srv.URL,
+		"feed", "info",
+		"--topic", topic.Hex(),
+		"--user", address.Hex(),
+	}
+
+	log.Info(fmt.Sprintf("getting feed info with 'swarm feed info'"))
+	cmd = runSwarm(t, flags...)
+	_, matches := cmd.ExpectRegexp(`.*`) // regex hack to extract stdout
+	cmd.ExpectExit()
+
+	// verify we can deserialize the result as a valid JSON
+	var request feed.Request
+	err = json.Unmarshal([]byte(matches[0]), &request)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// make sure the retrieved feed is the same
+	if request.Feed != fd {
+		t.Fatalf("Expected feed to be: %s, got %s", fd, request.Feed)
+	}
+
+	// test publishing a manifest
+	flags = []string{
+		"--bzzapi", srv.URL,
+		"--bzzaccount", pkfile.Name(),
+		"feed", "create",
+		"--topic", topic.Hex(),
+	}
+
+	log.Info(fmt.Sprintf("Publishing manifest with 'swarm feed create'"))
+	cmd = runSwarm(t, flags...)
+	_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`) // regex hack to extract stdout
+	cmd.ExpectExit()
+
+	manifestAddress := matches[0] // read the received feed manifest
+
+	// now attempt to lookup the latest update using a manifest instead
+	reader, err = client.QueryFeed(nil, manifestAddress)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	retrieved, err = ioutil.ReadAll(reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(data, retrieved) {
+		t.Fatalf("Received %s, expected %s", retrieved, data)
+	}
+}

cmd/swarm/hash.go

@@ -39,7 +39,7 @@ func hash(ctx *cli.Context) {
 	defer f.Close()
 
 	stat, _ := f.Stat()
-	fileStore := storage.NewFileStore(storage.NewMapChunkStore(), storage.NewFileStoreParams())
+	fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams())
 	addr, _, err := fileStore.Store(context.TODO(), f, stat.Size(), false)
 	if err != nil {
 		utils.Fatalf("%v\n", err)

cmd/swarm/main.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"crypto/ecdsa"
+	"encoding/hex"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -37,7 +38,7 @@ import (
 	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/swarm"
 	bzzapi "github.com/ethereum/go-ethereum/swarm/api"
 	swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics"
@@ -115,6 +116,12 @@ var (
 		Usage:  "Duration for sync subscriptions update after no new peers are added (default 15s)",
 		EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
 	}
+	SwarmMaxStreamPeerServersFlag = cli.IntFlag{
+		Name:   "max-stream-peer-servers",
+		Usage:  "Limit of Stream peer servers, 0 denotes unlimited",
+		EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS,
+		Value:  10000, // A very large default value is possible as stream servers have very small memory footprint
+	}
 	SwarmLightNodeEnabled = cli.BoolFlag{
 		Name:   "lightnode",
 		Usage:  "Enable Swarm LightNode (default false)",
@@ -196,17 +203,29 @@ var (
 		Usage:  "Number of recent chunks cached in memory (default 5000)",
 		EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
 	}
-	SwarmResourceMultihashFlag = cli.BoolFlag{
-		Name:  "multihash",
-		Usage: "Determines how to interpret data for a resource update. If not present, data will be interpreted as raw, literal data that will be included in the resource",
+	SwarmCompressedFlag = cli.BoolFlag{
+		Name:  "compressed",
+		Usage: "Prints encryption keys in compressed form",
 	}
-	SwarmResourceNameFlag = cli.StringFlag{
+	SwarmFeedNameFlag = cli.StringFlag{
 		Name:  "name",
-		Usage: "User-defined name for the new resource",
+		Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name",
 	}
-	SwarmResourceDataOnCreateFlag = cli.StringFlag{
+	SwarmFeedTopicFlag = cli.StringFlag{
+		Name:  "topic",
+		Usage: "User-defined topic this feed is tracking, hex encoded. Limited to 64 hexadecimal characters",
+	}
+	SwarmFeedDataOnCreateFlag = cli.StringFlag{
 		Name:  "data",
-		Usage: "Initializes the resource with the given hex-encoded data. Data must be prefixed by 0x",
+		Usage: "Initializes the feed with the given hex-encoded data. Data must be prefixed by 0x",
+	}
+	SwarmFeedManifestFlag = cli.StringFlag{
+		Name:  "manifest",
+		Usage: "Refers to the feed through a manifest",
+	}
+	SwarmFeedUserFlag = cli.StringFlag{
+		Name:  "user",
+		Usage: "Indicates the user who updates the feed",
 	}
 )
 
@@ -237,12 +256,12 @@ func init() {
 	utils.ListenPortFlag.Value = 30399
 }
 
-var app = utils.NewApp(gitCommit, "Ethereum Swarm")
+var app = utils.NewApp("", "Ethereum Swarm")
 
 // This init function creates the cli.App.
 func init() {
 	app.Action = bzzd
-	app.HideVersion = true // we have a command to print the version
+	app.Version = sv.ArchiveVersion(gitCommit)
 	app.Copyright = "Copyright 2013-2016 The go-ethereum Authors"
 	app.Commands = []cli.Command{
 		{
@@ -252,6 +271,14 @@ func init() {
 			Usage:              "Print version numbers",
 			Description:        "The output of this command is supposed to be machine-readable",
 		},
+		{
+			Action:             keys,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "print-keys",
+			Flags:              []cli.Flag{SwarmCompressedFlag},
+			Usage:              "Print public key information",
+			Description:        "The output of this command is supposed to be machine-readable",
+		},
 		{
 			Action:             upload,
 			CustomHelpTemplate: helpTemplate,
@@ -306,6 +333,7 @@ func init() {
 							Flags: []cli.Flag{
 								SwarmAccessGrantKeysFlag,
 								SwarmDryRunFlag,
+								utils.PasswordFileFlag,
 							},
 							Name:        "act",
 							Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
@@ -318,36 +346,62 @@ func init() {
 		},
 		{
 			CustomHelpTemplate: helpTemplate,
-			Name:               "resource",
-			Usage:              "(Advanced) Create and update Mutable Resources",
+			Name:               "feed",
+			Usage:              "(Advanced) Create and update Swarm Feeds",
 			ArgsUsage:          "<create|update|info>",
-			Description:        "Works with Mutable Resource Updates",
+			Description:        "Works with Swarm Feeds",
 			Subcommands: []cli.Command{
 				{
-					Action:             resourceCreate,
+					Action:             feedCreateManifest,
 					CustomHelpTemplate: helpTemplate,
 					Name:               "create",
-					Usage:              "creates a new Mutable Resource",
-					ArgsUsage:          "<frequency>",
-					Description:        "creates a new Mutable Resource",
-					Flags:              []cli.Flag{SwarmResourceNameFlag, SwarmResourceDataOnCreateFlag, SwarmResourceMultihashFlag},
+					Usage:              "creates and publishes a new feed manifest",
+					Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic.
+					The feed topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						this feed tracks a discussion about that contract.
+					The --user flag allows to have this manifest refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)`,
+					Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
 				},
 				{
-					Action:             resourceUpdate,
+					Action:             feedUpdate,
 					CustomHelpTemplate: helpTemplate,
 					Name:               "update",
-					Usage:              "updates the content of an existing Mutable Resource",
-					ArgsUsage:          "<Manifest Address or ENS domain> <0x Hex data>",
-					Description:        "updates the content of an existing Mutable Resource",
-					Flags:              []cli.Flag{SwarmResourceMultihashFlag},
+					Usage:              "updates the content of an existing Swarm Feed",
+					ArgsUsage:          "<0x Hex data>",
+					Description: `publishes a new update on the specified topic
+					The feed topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						this feed tracks a discussion about that contract.
+					
+					If you have a manifest, you can specify it with --manifest to refer to the feed,
+					instead of using --topic / --name
+					`,
+					Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag},
 				},
 				{
-					Action:             resourceInfo,
+					Action:             feedInfo,
 					CustomHelpTemplate: helpTemplate,
 					Name:               "info",
-					Usage:              "obtains information about an existing Mutable Resource",
-					ArgsUsage:          "<Manifest Address or ENS domain>",
-					Description:        "obtains information about an existing Mutable Resource",
+					Usage:              "obtains information about an existing Swarm feed",
+					Description: `obtains information about an existing Swarm feed
+					The topic can be specified directly with the --topic flag as an hex string
+					If no topic is specified, the default topic (zero) will be used
+					The --name flag can be used to specify subtopics with a specific name.
+					The --user flag allows to refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)
+					If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user
+					to refer to the feed`,
+					Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
 				},
 			},
 		},
@@ -483,14 +537,6 @@ pv(1) tool to get a progress bar:
 
     pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
 				},
-				{
-					Action:             dbClean,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "clean",
-					Usage:              "remove corrupt entries from a local chunk database",
-					ArgsUsage:          "<chunkdb>",
-					Description:        "Remove corrupt entries from a local chunk database",
-				},
 			},
 		},
 
@@ -528,6 +574,7 @@ pv(1) tool to get a progress bar:
 		SwarmSwapAPIFlag,
 		SwarmSyncDisabledFlag,
 		SwarmSyncUpdateDelay,
+		SwarmMaxStreamPeerServersFlag,
 		SwarmLightNodeEnabled,
 		SwarmDeliverySkipCheckFlag,
 		SwarmListenAddrFlag,
@@ -580,6 +627,17 @@ func main() {
 	}
 }
 
+func keys(ctx *cli.Context) error {
+	privateKey := getPrivKey(ctx)
+	pub := hex.EncodeToString(crypto.FromECDSAPub(&privateKey.PublicKey))
+	pubCompressed := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey))
+	if !ctx.Bool(SwarmCompressedFlag.Name) {
+		fmt.Println(fmt.Sprintf("publicKey=%s", pub))
+	}
+	fmt.Println(fmt.Sprintf("publicKeyCompressed=%s", pubCompressed))
+	return nil
+}
+
 func version(ctx *cli.Context) error {
 	fmt.Println(strings.Title(clientIdentifier))
 	fmt.Println("Version:", sv.VersionWithMeta)
@@ -672,7 +730,7 @@ func getAccount(bzzaccount string, ctx *cli.Context, stack *node.Node) *ecdsa.Pr
 }
 
 // getPrivKey returns the private key of the specified bzzaccount
-// Used only by client commands, such as `resource`
+// Used only by client commands, such as `feed`
 func getPrivKey(ctx *cli.Context) *ecdsa.PrivateKey {
 	// booting up the swarm node just as we do in bzzd action
 	bzzconfig, err := buildConfig(ctx)
@@ -763,10 +821,10 @@ func setSwarmBootstrapNodes(ctx *cli.Context, cfg *node.Config) {
 		return
 	}
 
-	cfg.P2P.BootstrapNodes = []*discover.Node{}
+	cfg.P2P.BootstrapNodes = []*enode.Node{}
 
 	for _, url := range SwarmBootnodes {
-		node, err := discover.ParseNode(url)
+		node, err := enode.ParseV4(url)
 		if err != nil {
 			log.Error("Bootstrap URL invalid", "enode", url, "err", err)
 		}

cmd/swarm/manifest_test.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/swarm/api"
@@ -30,12 +31,20 @@ import (
 // TestManifestChange tests manifest add, update and remove
 // cli commands without encryption.
 func TestManifestChange(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
 	testManifestChange(t, false)
 }
 
 // TestManifestChange tests manifest add, update and remove
 // cli commands with encryption enabled.
 func TestManifestChangeEncrypted(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
 	testManifestChange(t, true)
 }
 
@@ -400,6 +409,10 @@ func testManifestChange(t *testing.T, encrypt bool) {
 // TestNestedDefaultEntryUpdate tests if the default entry is updated
 // if the file in nested manifest used for it is also updated.
 func TestNestedDefaultEntryUpdate(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
 	testNestedDefaultEntryUpdate(t, false)
 }
 
@@ -407,6 +420,10 @@ func TestNestedDefaultEntryUpdate(t *testing.T) {
 // of encrypted upload is updated if the file in nested manifest
 // used for it is also updated.
 func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
 	testNestedDefaultEntryUpdate(t, true)
 }
 

cmd/swarm/mimegen/generator.go

@@ -0,0 +1,124 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+package main
+
+// Standard "mime" package rely on system-settings, see mime.osInitMime
+// Swarm will run on many OS/Platform/Docker and must behave similar
+// This command generates code to add common mime types based on mime.types file
+//
+// mime.types file provided by mailcap, which follow https://www.iana.org/assignments/media-types/media-types.xhtml
+//
+// Get last version of mime.types file by:
+// docker run --rm -v $(pwd):/tmp alpine:edge /bin/sh -c "apk add -U mailcap; mv /etc/mime.types /tmp"
+
+import (
+	"bufio"
+	"bytes"
+	"flag"
+	"html/template"
+	"io/ioutil"
+	"strings"
+
+	"log"
+)
+
+var (
+	typesFlag   = flag.String("types", "", "Input mime.types file")
+	packageFlag = flag.String("package", "", "Golang package in output file")
+	outFlag     = flag.String("out", "", "Output file name for the generated mime types")
+)
+
+type mime struct {
+	Name string
+	Exts []string
+}
+
+type templateParams struct {
+	PackageName string
+	Mimes       []mime
+}
+
+func main() {
+	// Parse and ensure all needed inputs are specified
+	flag.Parse()
+	if *typesFlag == "" {
+		log.Fatalf("--types is required")
+	}
+	if *packageFlag == "" {
+		log.Fatalf("--types is required")
+	}
+	if *outFlag == "" {
+		log.Fatalf("--out is required")
+	}
+
+	params := templateParams{
+		PackageName: *packageFlag,
+	}
+
+	types, err := ioutil.ReadFile(*typesFlag)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	scanner := bufio.NewScanner(bytes.NewReader(types))
+	for scanner.Scan() {
+		txt := scanner.Text()
+		if strings.HasPrefix(txt, "#") || len(txt) == 0 {
+			continue
+		}
+		parts := strings.Fields(txt)
+		if len(parts) == 1 {
+			continue
+		}
+		params.Mimes = append(params.Mimes, mime{parts[0], parts[1:]})
+	}
+
+	if err = scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+
+	result := bytes.NewBuffer([]byte{})
+
+	if err := template.Must(template.New("_").Parse(tpl)).Execute(result, params); err != nil {
+		log.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile(*outFlag, result.Bytes(), 0600); err != nil {
+		log.Fatal(err)
+	}
+}
+
+var tpl = `// Code generated by github.com/ethereum/go-ethereum/cmd/swarm/mimegen. DO NOT EDIT.
+
+package {{ .PackageName }}
+
+import "mime"
+func init() {
+	var mimeTypes = map[string]string{
+{{- range .Mimes -}}
+	{{ $name := .Name -}}
+	{{- range .Exts }}
+		".{{ . }}": "{{ $name | html }}",
+	{{- end }}
+{{- end }}
+	}
+	for ext, name := range mimeTypes {
+		if err := mime.AddExtensionType(ext, name); err != nil {
+			panic(err)
+		}
+	}
+}
+`

cmd/swarm/mru.go

@@ -1,169 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// Command resource allows the user to create and update signed mutable resource updates
-package main
-
-import (
-	"fmt"
-	"strconv"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common/hexutil"
-
-	"github.com/ethereum/go-ethereum/cmd/utils"
-	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
-	"github.com/ethereum/go-ethereum/swarm/storage/mru"
-	"gopkg.in/urfave/cli.v1"
-)
-
-func NewGenericSigner(ctx *cli.Context) mru.Signer {
-	return mru.NewGenericSigner(getPrivKey(ctx))
-}
-
-// swarm resource create <frequency> [--name <name>] [--data <0x Hexdata> [--multihash=false]]
-// swarm resource update <Manifest Address or ENS domain> <0x Hexdata> [--multihash=false]
-// swarm resource info <Manifest Address or ENS domain>
-
-func resourceCreate(ctx *cli.Context) {
-	args := ctx.Args()
-
-	var (
-		bzzapi      = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client      = swarm.NewClient(bzzapi)
-		multihash   = ctx.Bool(SwarmResourceMultihashFlag.Name)
-		initialData = ctx.String(SwarmResourceDataOnCreateFlag.Name)
-		name        = ctx.String(SwarmResourceNameFlag.Name)
-	)
-
-	if len(args) < 1 {
-		fmt.Println("Incorrect number of arguments")
-		cli.ShowCommandHelpAndExit(ctx, "create", 1)
-		return
-	}
-	signer := NewGenericSigner(ctx)
-	frequency, err := strconv.ParseUint(args[0], 10, 64)
-	if err != nil {
-		fmt.Printf("Frequency formatting error: %s\n", err.Error())
-		cli.ShowCommandHelpAndExit(ctx, "create", 1)
-		return
-	}
-
-	metadata := mru.ResourceMetadata{
-		Name:      name,
-		Frequency: frequency,
-		Owner:     signer.Address(),
-	}
-
-	var newResourceRequest *mru.Request
-	if initialData != "" {
-		initialDataBytes, err := hexutil.Decode(initialData)
-		if err != nil {
-			fmt.Printf("Error parsing data: %s\n", err.Error())
-			cli.ShowCommandHelpAndExit(ctx, "create", 1)
-			return
-		}
-		newResourceRequest, err = mru.NewCreateUpdateRequest(&metadata)
-		if err != nil {
-			utils.Fatalf("Error creating new resource request: %s", err)
-		}
-		newResourceRequest.SetData(initialDataBytes, multihash)
-		if err = newResourceRequest.Sign(signer); err != nil {
-			utils.Fatalf("Error signing resource update: %s", err.Error())
-		}
-	} else {
-		newResourceRequest, err = mru.NewCreateRequest(&metadata)
-		if err != nil {
-			utils.Fatalf("Error creating new resource request: %s", err)
-		}
-	}
-
-	manifestAddress, err := client.CreateResource(newResourceRequest)
-	if err != nil {
-		utils.Fatalf("Error creating resource: %s", err.Error())
-		return
-	}
-	fmt.Println(manifestAddress) // output manifest address to the user in a single line (useful for other commands to pick up)
-
-}
-
-func resourceUpdate(ctx *cli.Context) {
-	args := ctx.Args()
-
-	var (
-		bzzapi    = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client    = swarm.NewClient(bzzapi)
-		multihash = ctx.Bool(SwarmResourceMultihashFlag.Name)
-	)
-
-	if len(args) < 2 {
-		fmt.Println("Incorrect number of arguments")
-		cli.ShowCommandHelpAndExit(ctx, "update", 1)
-		return
-	}
-	signer := NewGenericSigner(ctx)
-	manifestAddressOrDomain := args[0]
-	data, err := hexutil.Decode(args[1])
-	if err != nil {
-		utils.Fatalf("Error parsing data: %s", err.Error())
-		return
-	}
-
-	// Retrieve resource status and metadata out of the manifest
-	updateRequest, err := client.GetResourceMetadata(manifestAddressOrDomain)
-	if err != nil {
-		utils.Fatalf("Error retrieving resource status: %s", err.Error())
-	}
-
-	// set the new data
-	updateRequest.SetData(data, multihash)
-
-	// sign update
-	if err = updateRequest.Sign(signer); err != nil {
-		utils.Fatalf("Error signing resource update: %s", err.Error())
-	}
-
-	// post update
-	err = client.UpdateResource(updateRequest)
-	if err != nil {
-		utils.Fatalf("Error updating resource: %s", err.Error())
-		return
-	}
-}
-
-func resourceInfo(ctx *cli.Context) {
-	var (
-		bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client = swarm.NewClient(bzzapi)
-	)
-	args := ctx.Args()
-	if len(args) < 1 {
-		fmt.Println("Incorrect number of arguments.")
-		cli.ShowCommandHelpAndExit(ctx, "info", 1)
-		return
-	}
-	manifestAddressOrDomain := args[0]
-	metadata, err := client.GetResourceMetadata(manifestAddressOrDomain)
-	if err != nil {
-		utils.Fatalf("Error retrieving resource metadata: %s", err.Error())
-		return
-	}
-	encodedMetadata, err := metadata.MarshalJSON()
-	if err != nil {
-		utils.Fatalf("Error encoding metadata to JSON for display:%s", err)
-	}
-	fmt.Println(string(encodedMetadata))
-}

cmd/swarm/run_test.go

@@ -19,6 +19,7 @@ package main
 import (
 	"context"
 	"crypto/ecdsa"
+	"flag"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -41,6 +42,8 @@ import (
 	"github.com/ethereum/go-ethereum/swarm"
 )
 
+var loglevel = flag.Int("loglevel", 3, "verbosity of logs")
+
 func init() {
 	// Run the app if we've been exec'd as "swarm-test" in runSwarm.
 	reexec.Register("swarm-test", func() {
@@ -234,6 +237,7 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
 	// start the node
 	node.Cmd = runSwarm(t,
 		"--port", p2pPort,
+		"--nat", "extip:127.0.0.1",
 		"--nodiscover",
 		"--datadir", dir,
 		"--ipcpath", conf.IPCPath,
@@ -241,7 +245,7 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
 		"--bzzaccount", bzzaccount,
 		"--bzznetworkid", "321",
 		"--bzzport", httpPort,
-		"--verbosity", "6",
+		"--verbosity", fmt.Sprint(*loglevel),
 	)
 	node.Cmd.InputLine(testPassphrase)
 	defer func() {
@@ -284,8 +288,8 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
 	if err := node.Client.Call(&nodeInfo, "admin_nodeInfo"); err != nil {
 		t.Fatal(err)
 	}
-	node.Enode = fmt.Sprintf("enode://%s@127.0.0.1:%s", nodeInfo.ID, p2pPort)
-
+	node.Enode = nodeInfo.Enode
+	node.IpcPath = conf.IPCPath
 	return node
 }
 
@@ -309,6 +313,7 @@ func newTestNode(t *testing.T, dir string) *testNode {
 	// start the node
 	node.Cmd = runSwarm(t,
 		"--port", p2pPort,
+		"--nat", "extip:127.0.0.1",
 		"--nodiscover",
 		"--datadir", dir,
 		"--ipcpath", conf.IPCPath,
@@ -316,7 +321,7 @@ func newTestNode(t *testing.T, dir string) *testNode {
 		"--bzzaccount", account.Address.String(),
 		"--bzznetworkid", "321",
 		"--bzzport", httpPort,
-		"--verbosity", "6",
+		"--verbosity", fmt.Sprint(*loglevel),
 	)
 	node.Cmd.InputLine(testPassphrase)
 	defer func() {
@@ -359,9 +364,8 @@ func newTestNode(t *testing.T, dir string) *testNode {
 	if err := node.Client.Call(&nodeInfo, "admin_nodeInfo"); err != nil {
 		t.Fatal(err)
 	}
-	node.Enode = fmt.Sprintf("enode://%s@127.0.0.1:%s", nodeInfo.ID, p2pPort)
+	node.Enode = nodeInfo.Enode
 	node.IpcPath = conf.IPCPath
-
 	return node
 }
 

cmd/swarm/swarm-smoke/main.go

@@ -48,7 +48,7 @@ func main() {
 		cli.StringFlag{
 			Name:        "cluster-endpoint",
 			Value:       "testing",
-			Usage:       "cluster to point to (open, or testing)",
+			Usage:       "cluster to point to (local, open or testing)",
 			Destination: &cluster,
 		},
 		cli.IntFlag{
@@ -76,8 +76,8 @@ func main() {
 		},
 		cli.IntFlag{
 			Name:        "filesize",
-			Value:       1,
-			Usage:       "file size for generated random file in MB",
+			Value:       1024,
+			Usage:       "file size for generated random file in KB",
 			Destination: &filesize,
 		},
 	}

cmd/swarm/swarm-smoke/upload_and_sync.go

@@ -39,6 +39,11 @@ import (
 func generateEndpoints(scheme string, cluster string, from int, to int) {
 	if cluster == "prod" {
 		cluster = ""
+	} else if cluster == "local" {
+		for port := from; port <= to; port++ {
+			endpoints = append(endpoints, fmt.Sprintf("%s://localhost:%v", scheme, port))
+		}
+		return
 	} else {
 		cluster = cluster + "."
 	}
@@ -53,13 +58,13 @@ func generateEndpoints(scheme string, cluster string, from int, to int) {
 }
 
 func cliUploadAndSync(c *cli.Context) error {
-	defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size", filesize) }(time.Now())
+	defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
 
 	generateEndpoints(scheme, cluster, from, to)
 
 	log.Info("uploading to " + endpoints[0] + " and syncing")
 
-	f, cleanup := generateRandomFile(filesize * 1000000)
+	f, cleanup := generateRandomFile(filesize * 1000)
 	defer cleanup()
 
 	hash, err := upload(f, endpoints[0])
@@ -76,12 +81,7 @@ func cliUploadAndSync(c *cli.Context) error {
 
 	log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash))
 
-	if filesize < 10 {
-		time.Sleep(35 * time.Second)
-	} else {
-		time.Sleep(15 * time.Second)
-		time.Sleep(2 * time.Duration(filesize) * time.Second)
-	}
+	time.Sleep(3 * time.Second)
 
 	wg := sync.WaitGroup{}
 	for _, endpoint := range endpoints {
@@ -109,7 +109,7 @@ func cliUploadAndSync(c *cli.Context) error {
 // fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file
 func fetch(hash string, endpoint string, original []byte, ruid string) error {
 	log.Trace("sleeping", "ruid", ruid)
-	time.Sleep(5 * time.Second)
+	time.Sleep(3 * time.Second)
 
 	log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash)
 	res, err := http.Get(endpoint + "/bzz:/" + hash + "/")

cmd/swarm/upload.go

@@ -22,16 +22,15 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"mime"
-	"net/http"
 	"os"
 	"os/user"
 	"path"
 	"path/filepath"
 	"strings"
 
-	"github.com/ethereum/go-ethereum/cmd/utils"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -118,10 +117,9 @@ func upload(ctx *cli.Context) {
 				return "", fmt.Errorf("error opening file: %s", err)
 			}
 			defer f.Close()
-			if mimeType == "" {
-				mimeType = detectMimeType(file)
+			if mimeType != "" {
+				f.ContentType = mimeType
 			}
-			f.ContentType = mimeType
 			return client.Upload(f, "", toEncrypt)
 		}
 	}
@@ -138,6 +136,12 @@ func upload(ctx *cli.Context) {
 // 3. cleans the path, e.g. /a/b/../c -> /a/c
 // Note, it has limitations, e.g. ~someuser/tmp will not be expanded
 func expandPath(p string) string {
+	if i := strings.Index(p, ":"); i > 0 {
+		return p
+	}
+	if i := strings.Index(p, "@"); i > 0 {
+		return p
+	}
 	if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") {
 		if home := homeDir(); home != "" {
 			p = home + p[1:]
@@ -155,19 +159,3 @@ func homeDir() string {
 	}
 	return ""
 }
-
-func detectMimeType(file string) string {
-	if ext := filepath.Ext(file); ext != "" {
-		return mime.TypeByExtension(ext)
-	}
-	f, err := os.Open(file)
-	if err != nil {
-		return ""
-	}
-	defer f.Close()
-	buf := make([]byte, 512)
-	if n, _ := f.Read(buf); n > 0 {
-		return http.DetectContentType(buf)
-	}
-	return ""
-}

cmd/swarm/upload_test.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"bytes"
-	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -26,17 +25,16 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
-	colorable "github.com/mattn/go-colorable"
+	"github.com/mattn/go-colorable"
 )
 
-var loglevel = flag.Int("loglevel", 3, "verbosity of logs")
-
 func init() {
 	log.PrintOrigins(true)
 	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
@@ -45,18 +43,31 @@ func init() {
 // TestCLISwarmUp tests that running 'swarm up' makes the resulting file
 // available from all nodes via the HTTP API
 func TestCLISwarmUp(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
 	testCLISwarmUp(false, t)
 }
 func TestCLISwarmUpRecursive(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
 	testCLISwarmUpRecursive(false, t)
 }
 
 // TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file
 // available from all nodes via the HTTP API
 func TestCLISwarmUpEncrypted(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
 	testCLISwarmUp(true, t)
 }
 func TestCLISwarmUpEncryptedRecursive(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
 	testCLISwarmUpRecursive(true, t)
 }
 
@@ -277,6 +288,9 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 // TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute
 // default paths and with encryption.
 func TestCLISwarmUpDefaultPath(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
 	testCLISwarmUpDefaultPath(false, false, t)
 	testCLISwarmUpDefaultPath(false, true, t)
 	testCLISwarmUpDefaultPath(true, false, t)

cmd/utils/flags.go

@@ -51,8 +51,8 @@ import (
 	"github.com/ethereum/go-ethereum/metrics/influxdb"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 	"github.com/ethereum/go-ethereum/params"
@@ -329,21 +329,26 @@ var (
 	MinerGasTargetFlag = cli.Uint64Flag{
 		Name:  "miner.gastarget",
 		Usage: "Target gas floor for mined blocks",
-		Value: params.GenesisGasLimit,
+		Value: eth.DefaultConfig.MinerGasFloor,
 	}
 	MinerLegacyGasTargetFlag = cli.Uint64Flag{
 		Name:  "targetgaslimit",
 		Usage: "Target gas floor for mined blocks (deprecated, use --miner.gastarget)",
-		Value: params.GenesisGasLimit,
+		Value: eth.DefaultConfig.MinerGasFloor,
+	}
+	MinerGasLimitFlag = cli.Uint64Flag{
+		Name:  "miner.gaslimit",
+		Usage: "Target gas ceiling for mined blocks",
+		Value: eth.DefaultConfig.MinerGasCeil,
 	}
 	MinerGasPriceFlag = BigFlag{
 		Name:  "miner.gasprice",
-		Usage: "Minimal gas price for mining a transactions",
+		Usage: "Minimum gas price for mining a transaction",
 		Value: eth.DefaultConfig.MinerGasPrice,
 	}
 	MinerLegacyGasPriceFlag = BigFlag{
 		Name:  "gasprice",
-		Usage: "Minimal gas price for mining a transactions (deprecated, use --miner.gasprice)",
+		Usage: "Minimum gas price for mining a transaction (deprecated, use --miner.gasprice)",
 		Value: eth.DefaultConfig.MinerGasPrice,
 	}
 	MinerEtherbaseFlag = cli.StringFlag{
@@ -366,9 +371,13 @@ var (
 	}
 	MinerRecommitIntervalFlag = cli.DurationFlag{
 		Name:  "miner.recommit",
-		Usage: "Time interval to recreate the block being mined.",
+		Usage: "Time interval to recreate the block being mined",
 		Value: eth.DefaultConfig.MinerRecommit,
 	}
+	MinerNoVerfiyFlag = cli.BoolFlag{
+		Name:  "miner.noverify",
+		Usage: "Disable remote sealing verification",
+	}
 	// Account settings
 	UnlockedAccountFlag = cli.StringFlag{
 		Name:  "unlock",
@@ -558,6 +567,10 @@ var (
 		Usage: "Minimum POW accepted",
 		Value: whisper.DefaultMinimumPoW,
 	}
+	WhisperRestrictConnectionBetweenLightClientsFlag = cli.BoolFlag{
+		Name:  "shh.restrict-light",
+		Usage: "Restrict connection between two whisper light clients",
+	}
 
 	// Metrics flags
 	MetricsEnabledFlag = cli.BoolFlag{
@@ -597,6 +610,17 @@ var (
 		Usage: "InfluxDB `host` tag attached to all measurements",
 		Value: "localhost",
 	}
+
+	EWASMInterpreterFlag = cli.StringFlag{
+		Name:  "vm.ewasm",
+		Usage: "External ewasm configuration (default = built-in interpreter)",
+		Value: "",
+	}
+	EVMInterpreterFlag = cli.StringFlag{
+		Name:  "vm.evm",
+		Usage: "External EVM configuration (default = built-in interpreter)",
+		Value: "",
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -668,9 +692,9 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
 		return // already set, don't apply defaults.
 	}
 
-	cfg.BootstrapNodes = make([]*discover.Node, 0, len(urls))
+	cfg.BootstrapNodes = make([]*enode.Node, 0, len(urls))
 	for _, url := range urls {
-		node, err := discover.ParseNode(url)
+		node, err := enode.ParseV4(url)
 		if err != nil {
 			log.Crit("Bootstrap URL invalid", "enode", url, "err", err)
 		}
@@ -1061,11 +1085,14 @@ func checkExclusive(ctx *cli.Context, args ...interface{}) {
 		if i+1 < len(args) {
 			switch option := args[i+1].(type) {
 			case string:
-				// Extended flag, expand the name and shift the arguments
+				// Extended flag check, make sure value set doesn't conflict with passed in option
 				if ctx.GlobalString(flag.GetName()) == option {
 					name += "=" + option
+					set = append(set, "--"+name)
 				}
+				// shift arguments and continue
 				i++
+				continue
 
 			case cli.Flag:
 			default:
@@ -1090,6 +1117,9 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) {
 	if ctx.GlobalIsSet(WhisperMinPOWFlag.Name) {
 		cfg.MinimumAcceptedPOW = ctx.GlobalFloat64(WhisperMinPOWFlag.Name)
 	}
+	if ctx.GlobalIsSet(WhisperRestrictConnectionBetweenLightClientsFlag.Name) {
+		cfg.RestrictConnectionBetweenLightClients = true
+	}
 }
 
 // SetEthConfig applies eth-related command line flags to the config.
@@ -1130,12 +1160,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
 		cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
 	}
-	if ctx.GlobalIsSet(MinerLegacyThreadsFlag.Name) {
-		cfg.MinerThreads = ctx.GlobalInt(MinerLegacyThreadsFlag.Name)
-	}
-	if ctx.GlobalIsSet(MinerThreadsFlag.Name) {
-		cfg.MinerThreads = ctx.GlobalInt(MinerThreadsFlag.Name)
-	}
 	if ctx.GlobalIsSet(MinerNotifyFlag.Name) {
 		cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",")
 	}
@@ -1148,6 +1172,15 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	if ctx.GlobalIsSet(MinerExtraDataFlag.Name) {
 		cfg.MinerExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name))
 	}
+	if ctx.GlobalIsSet(MinerLegacyGasTargetFlag.Name) {
+		cfg.MinerGasFloor = ctx.GlobalUint64(MinerLegacyGasTargetFlag.Name)
+	}
+	if ctx.GlobalIsSet(MinerGasTargetFlag.Name) {
+		cfg.MinerGasFloor = ctx.GlobalUint64(MinerGasTargetFlag.Name)
+	}
+	if ctx.GlobalIsSet(MinerGasLimitFlag.Name) {
+		cfg.MinerGasCeil = ctx.GlobalUint64(MinerGasLimitFlag.Name)
+	}
 	if ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) {
 		cfg.MinerGasPrice = GlobalBig(ctx, MinerLegacyGasPriceFlag.Name)
 	}
@@ -1157,11 +1190,22 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	if ctx.GlobalIsSet(MinerRecommitIntervalFlag.Name) {
 		cfg.MinerRecommit = ctx.Duration(MinerRecommitIntervalFlag.Name)
 	}
+	if ctx.GlobalIsSet(MinerNoVerfiyFlag.Name) {
+		cfg.MinerNoverify = ctx.Bool(MinerNoVerfiyFlag.Name)
+	}
 	if ctx.GlobalIsSet(VMEnableDebugFlag.Name) {
 		// TODO(fjl): force-enable this in --dev mode
 		cfg.EnablePreimageRecording = ctx.GlobalBool(VMEnableDebugFlag.Name)
 	}
 
+	if ctx.GlobalIsSet(EWASMInterpreterFlag.Name) {
+		cfg.EWASMInterpreter = ctx.GlobalString(EWASMInterpreterFlag.Name)
+	}
+
+	if ctx.GlobalIsSet(EVMInterpreterFlag.Name) {
+		cfg.EVMInterpreter = ctx.GlobalString(EVMInterpreterFlag.Name)
+	}
+
 	// Override any default configs for hard coded networks.
 	switch {
 	case ctx.GlobalBool(TestnetFlag.Name):
@@ -1269,15 +1313,6 @@ func RegisterEthStatsService(stack *node.Node, url string) {
 	}
 }
 
-// SetupNetwork configures the system for either the main net or some test network.
-func SetupNetwork(ctx *cli.Context) {
-	// TODO(fjl): move target gas limit into config
-	params.TargetGasLimit = ctx.GlobalUint64(MinerLegacyGasTargetFlag.Name)
-	if ctx.GlobalIsSet(MinerGasTargetFlag.Name) {
-		params.TargetGasLimit = ctx.GlobalUint64(MinerGasTargetFlag.Name)
-	}
-}
-
 func SetupMetrics(ctx *cli.Context) {
 	if metrics.Enabled {
 		log.Info("Enabling metrics collection")
@@ -1351,7 +1386,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 				DatasetDir:     stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir),
 				DatasetsInMem:  eth.DefaultConfig.Ethash.DatasetsInMem,
 				DatasetsOnDisk: eth.DefaultConfig.Ethash.DatasetsOnDisk,
-			}, nil)
+			}, nil, false)
 		}
 	}
 	if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
@@ -1366,7 +1401,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 		cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
 	}
 	vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
-	chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg)
+	chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil)
 	if err != nil {
 		Fatalf("Can't create BlockChain: %v", err)
 	}

cmd/wnode/main.go

@@ -41,7 +41,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/whisper/mailserver"
 	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
@@ -175,7 +175,7 @@ func initialize() {
 	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*argVerbosity), log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
 
 	done = make(chan struct{})
-	var peers []*discover.Node
+	var peers []*enode.Node
 	var err error
 
 	if *generateKey {
@@ -203,7 +203,7 @@ func initialize() {
 		if len(*argEnode) == 0 {
 			argEnode = scanLineA("Please enter the peer's enode: ")
 		}
-		peer := discover.MustParseNode(*argEnode)
+		peer := enode.MustParseV4(*argEnode)
 		peers = append(peers, peer)
 	}
 
@@ -747,14 +747,14 @@ func requestExpiredMessagesLoop() {
 }
 
 func extractIDFromEnode(s string) []byte {
-	n, err := discover.ParseNode(s)
+	n, err := enode.ParseV4(s)
 	if err != nil {
 		utils.Fatalf("Failed to parse enode: %s", err)
 	}
-	return n.ID[:]
+	return n.ID().Bytes()
 }
 
-// obfuscateBloom adds 16 random bits to the the bloom
+// obfuscateBloom adds 16 random bits to the bloom
 // filter, in order to obfuscate the containing topics.
 // it does so deterministically within every session.
 // despite additional bits, it will match on average

common/bitutil/compress_test.go

@@ -117,7 +117,7 @@ func TestDecodingCycle(t *testing.T) {
 // TestCompression tests that compression works by returning either the bitset
 // encoded input, or the actual input if the bitset version is longer.
 func TestCompression(t *testing.T) {
-	// Check the the compression returns the bitset encoding is shorter
+	// Check the compression returns the bitset encoding is shorter
 	in := hexutil.MustDecode("0x4912385c0e7b64000000")
 	out := hexutil.MustDecode("0x80fe4912385c0e7b64")
 
@@ -127,7 +127,7 @@ func TestCompression(t *testing.T) {
 	if data, err := DecompressBytes(out, len(in)); err != nil || !bytes.Equal(data, in) {
 		t.Errorf("decoding mismatch for sparse data: have %x, want %x, error %v", data, in, err)
 	}
-	// Check the the compression returns the input if the bitset encoding is longer
+	// Check the compression returns the input if the bitset encoding is longer
 	in = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb")
 	out = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb")
 

common/bytes.go

@@ -100,7 +100,7 @@ func Hex2BytesFixed(str string, flen int) []byte {
 		return h[len(h)-flen:]
 	}
 	hh := make([]byte, flen)
-	copy(hh[flen-len(h):flen], h[:])
+	copy(hh[flen-len(h):flen], h)
 	return hh
 }
 

common/format.go

@@ -38,3 +38,45 @@ func (d PrettyDuration) String() string {
 	}
 	return label
 }
+
+// PrettyAge is a pretty printed version of a time.Duration value that rounds
+// the values up to a single most significant unit, days/weeks/years included.
+type PrettyAge time.Time
+
+// ageUnits is a list of units the age pretty printing uses.
+var ageUnits = []struct {
+	Size   time.Duration
+	Symbol string
+}{
+	{12 * 30 * 24 * time.Hour, "y"},
+	{30 * 24 * time.Hour, "mo"},
+	{7 * 24 * time.Hour, "w"},
+	{24 * time.Hour, "d"},
+	{time.Hour, "h"},
+	{time.Minute, "m"},
+	{time.Second, "s"},
+}
+
+// String implements the Stringer interface, allowing pretty printing of duration
+// values rounded to the most significant time unit.
+func (t PrettyAge) String() string {
+	// Calculate the time difference and handle the 0 cornercase
+	diff := time.Since(time.Time(t))
+	if diff < time.Second {
+		return "0"
+	}
+	// Accumulate a precision of 3 components before returning
+	result, prec := "", 0
+
+	for _, unit := range ageUnits {
+		if diff > unit.Size {
+			result = fmt.Sprintf("%s%d%s", result, diff/unit.Size, unit.Symbol)
+			diff %= unit.Size
+
+			if prec += 1; prec >= 3 {
+				break
+			}
+		}
+	}
+	return result
+}

common/types.go

@@ -34,7 +34,7 @@ import (
 const (
 	// HashLength is the expected length of the hash
 	HashLength = 32
-	// AddressLength is the expected length of the adddress
+	// AddressLength is the expected length of the address
 	AddressLength = 20
 )
 

consensus/clique/clique.go

@@ -93,27 +93,33 @@ var (
 
 	// errMissingSignature is returned if a block's extra-data section doesn't seem
 	// to contain a 65 byte secp256k1 signature.
-	errMissingSignature = errors.New("extra-data 65 byte suffix signature missing")
+	errMissingSignature = errors.New("extra-data 65 byte signature suffix missing")
 
 	// errExtraSigners is returned if non-checkpoint block contain signer data in
 	// their extra-data fields.
 	errExtraSigners = errors.New("non-checkpoint block contains extra signer list")
 
 	// errInvalidCheckpointSigners is returned if a checkpoint block contains an
-	// invalid list of signers (i.e. non divisible by 20 bytes, or not the correct
-	// ones).
+	// invalid list of signers (i.e. non divisible by 20 bytes).
 	errInvalidCheckpointSigners = errors.New("invalid signer list on checkpoint block")
 
+	// errMismatchingCheckpointSigners is returned if a checkpoint block contains a
+	// list of signers different than the one the local node calculated.
+	errMismatchingCheckpointSigners = errors.New("mismatching signer list on checkpoint block")
+
 	// errInvalidMixDigest is returned if a block's mix digest is non-zero.
 	errInvalidMixDigest = errors.New("non-zero mix digest")
 
 	// errInvalidUncleHash is returned if a block contains an non-empty uncle list.
 	errInvalidUncleHash = errors.New("non empty uncle hash")
 
-	// errInvalidDifficulty is returned if the difficulty of a block is not either
-	// of 1 or 2, or if the value does not match the turn of the signer.
+	// errInvalidDifficulty is returned if the difficulty of a block neither 1 or 2.
 	errInvalidDifficulty = errors.New("invalid difficulty")
 
+	// errWrongDifficulty is returned if the difficulty of a block doesn't match the
+	// turn of the signer.
+	errWrongDifficulty = errors.New("wrong difficulty")
+
 	// ErrInvalidTimestamp is returned if the timestamp of a block is lower than
 	// the previous block's timestamp + the minimum block period.
 	ErrInvalidTimestamp = errors.New("invalid timestamp")
@@ -122,13 +128,12 @@ var (
 	// be modified via out-of-range or non-contiguous headers.
 	errInvalidVotingChain = errors.New("invalid voting chain")
 
-	// errUnauthorized is returned if a header is signed by a non-authorized entity.
-	errUnauthorized = errors.New("unauthorized")
+	// errUnauthorizedSigner is returned if a header is signed by a non-authorized entity.
+	errUnauthorizedSigner = errors.New("unauthorized signer")
 
-	// errWaitTransactions is returned if an empty block is attempted to be sealed
-	// on an instant chain (0 second period). It's important to refuse these as the
-	// block reward is zero, so an empty block just bloats the chain... fast.
-	errWaitTransactions = errors.New("waiting for transactions")
+	// errRecentlySigned is returned if a header is signed by an authorized entity
+	// that already signed a header recently, thus is temporarily not allowed to.
+	errRecentlySigned = errors.New("recently signed")
 )
 
 // SignerFn is a signer callback function to request a hash to be signed by a
@@ -205,6 +210,9 @@ type Clique struct {
 	signer common.Address // Ethereum address of the signing key
 	signFn SignerFn       // Signer function to authorize hashes with
 	lock   sync.RWMutex   // Protects the signer fields
+
+	// The fields below are for testing only
+	fakeDiff bool // Skip difficulty verifications
 }
 
 // New creates a Clique proof-of-authority consensus engine with the initial
@@ -359,7 +367,7 @@ func (c *Clique) verifyCascadingFields(chain consensus.ChainReader, header *type
 		}
 		extraSuffix := len(header.Extra) - extraSeal
 		if !bytes.Equal(header.Extra[extraVanity:extraSuffix], signers) {
-			return errInvalidCheckpointSigners
+			return errMismatchingCheckpointSigners
 		}
 	}
 	// All basic checks passed, verify the seal and return
@@ -388,7 +396,7 @@ func (c *Clique) snapshot(chain consensus.ChainReader, number uint64, hash commo
 			}
 		}
 		// If we're at an checkpoint block, make a snapshot if it's known
-		if number%c.config.Epoch == 0 {
+		if number == 0 || (number%c.config.Epoch == 0 && chain.GetHeaderByNumber(number-1) == nil) {
 			checkpoint := chain.GetHeaderByNumber(number)
 			if checkpoint != nil {
 				hash := checkpoint.Hash()
@@ -481,23 +489,25 @@ func (c *Clique) verifySeal(chain consensus.ChainReader, header *types.Header, p
 		return err
 	}
 	if _, ok := snap.Signers[signer]; !ok {
-		return errUnauthorized
+		return errUnauthorizedSigner
 	}
 	for seen, recent := range snap.Recents {
 		if recent == signer {
 			// Signer is among recents, only fail if the current block doesn't shift it out
 			if limit := uint64(len(snap.Signers)/2 + 1); seen > number-limit {
-				return errUnauthorized
+				return errRecentlySigned
 			}
 		}
 	}
 	// Ensure that the difficulty corresponds to the turn-ness of the signer
-	inturn := snap.inturn(header.Number.Uint64(), signer)
-	if inturn && header.Difficulty.Cmp(diffInTurn) != 0 {
-		return errInvalidDifficulty
-	}
-	if !inturn && header.Difficulty.Cmp(diffNoTurn) != 0 {
-		return errInvalidDifficulty
+	if !c.fakeDiff {
+		inturn := snap.inturn(header.Number.Uint64(), signer)
+		if inturn && header.Difficulty.Cmp(diffInTurn) != 0 {
+			return errWrongDifficulty
+		}
+		if !inturn && header.Difficulty.Cmp(diffNoTurn) != 0 {
+			return errWrongDifficulty
+		}
 	}
 	return nil
 }
@@ -590,17 +600,18 @@ func (c *Clique) Authorize(signer common.Address, signFn SignerFn) {
 
 // Seal implements consensus.Engine, attempting to create a sealed block using
 // the local signing credentials.
-func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) {
+func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error {
 	header := block.Header()
 
 	// Sealing the genesis block is not supported
 	number := header.Number.Uint64()
 	if number == 0 {
-		return nil, errUnknownBlock
+		return errUnknownBlock
 	}
 	// For 0-period chains, refuse to seal empty blocks (no reward but would spin sealing)
 	if c.config.Period == 0 && len(block.Transactions()) == 0 {
-		return nil, errWaitTransactions
+		log.Info("Sealing paused, waiting for transactions")
+		return nil
 	}
 	// Don't hold the signer fields for the entire sealing procedure
 	c.lock.RLock()
@@ -610,10 +621,10 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch
 	// Bail out if we're unauthorized to sign a block
 	snap, err := c.snapshot(chain, number-1, header.ParentHash, nil)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	if _, authorized := snap.Signers[signer]; !authorized {
-		return nil, errUnauthorized
+		return errUnauthorizedSigner
 	}
 	// If we're amongst the recent signers, wait for the next block
 	for seen, recent := range snap.Recents {
@@ -621,8 +632,7 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch
 			// Signer is among recents, only wait if the current block doesn't shift it out
 			if limit := uint64(len(snap.Signers)/2 + 1); number < limit || seen > number-limit {
 				log.Info("Signed recently, must wait for others")
-				<-stop
-				return nil, nil
+				return nil
 			}
 		}
 	}
@@ -635,21 +645,29 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch
 
 		log.Trace("Out-of-turn signing requested", "wiggle", common.PrettyDuration(wiggle))
 	}
-	log.Trace("Waiting for slot to sign and propagate", "delay", common.PrettyDuration(delay))
-
-	select {
-	case <-stop:
-		return nil, nil
-	case <-time.After(delay):
-	}
 	// Sign all the things!
 	sighash, err := signFn(accounts.Account{Address: signer}, sigHash(header).Bytes())
 	if err != nil {
-		return nil, err
+		return err
 	}
 	copy(header.Extra[len(header.Extra)-extraSeal:], sighash)
+	// Wait until sealing is terminated or delay timeout.
+	log.Trace("Waiting for slot to sign and propagate", "delay", common.PrettyDuration(delay))
+	go func() {
+		select {
+		case <-stop:
+			return
+		case <-time.After(delay):
+		}
 
-	return block.WithSeal(header), nil
+		select {
+		case results <- block.WithSeal(header):
+		default:
+			log.Warn("Sealing result is not read by miner", "sealhash", c.SealHash(header))
+		}
+	}()
+
+	return nil
 }
 
 // CalcDifficulty is the difficulty adjustment algorithm. It returns the difficulty
@@ -673,6 +691,11 @@ func CalcDifficulty(snap *Snapshot, signer common.Address) *big.Int {
 	return new(big.Int).Set(diffNoTurn)
 }
 
+// SealHash returns the hash of a block prior to it being sealed.
+func (c *Clique) SealHash(header *types.Header) common.Hash {
+	return sigHash(header)
+}
+
 // Close implements consensus.Engine. It's a noop for clique as there is are no background threads.
 func (c *Clique) Close() error {
 	return nil

consensus/clique/snapshot.go

@@ -57,12 +57,12 @@ type Snapshot struct {
 	Tally   map[common.Address]Tally    `json:"tally"`   // Current vote tally to avoid recalculating
 }
 
-// signers implements the sort interface to allow sorting a list of addresses
-type signers []common.Address
+// signersAscending implements the sort interface to allow sorting a list of addresses
+type signersAscending []common.Address
 
-func (s signers) Len() int           { return len(s) }
-func (s signers) Less(i, j int) bool { return bytes.Compare(s[i][:], s[j][:]) < 0 }
-func (s signers) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s signersAscending) Len() int           { return len(s) }
+func (s signersAscending) Less(i, j int) bool { return bytes.Compare(s[i][:], s[j][:]) < 0 }
+func (s signersAscending) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 
 // newSnapshot creates a new snapshot with the specified startup parameters. This
 // method does not initialize the set of recent signers, so only ever use if for
@@ -214,11 +214,11 @@ func (s *Snapshot) apply(headers []*types.Header) (*Snapshot, error) {
 			return nil, err
 		}
 		if _, ok := snap.Signers[signer]; !ok {
-			return nil, errUnauthorized
+			return nil, errUnauthorizedSigner
 		}
 		for _, recent := range snap.Recents {
 			if recent == signer {
-				return nil, errUnauthorized
+				return nil, errRecentlySigned
 			}
 		}
 		snap.Recents[number] = signer
@@ -298,7 +298,7 @@ func (s *Snapshot) signers() []common.Address {
 	for sig := range s.Signers {
 		sigs = append(sigs, sig)
 	}
-	sort.Sort(signers(sigs))
+	sort.Sort(signersAscending(sigs))
 	return sigs
 }
 

consensus/clique/snapshot_test.go

@@ -19,24 +19,18 @@ package clique
 import (
 	"bytes"
 	"crypto/ecdsa"
-	"math/big"
+	"sort"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
 )
 
-type testerVote struct {
-	signer string
-	voted  string
-	auth   bool
-}
-
 // testerAccountPool is a pool to maintain currently active tester accounts,
 // mapped from textual names used in the tests below to actual Ethereum private
 // keys capable of signing transactions.
@@ -50,17 +44,26 @@ func newTesterAccountPool() *testerAccountPool {
 	}
 }
 
-func (ap *testerAccountPool) sign(header *types.Header, signer string) {
-	// Ensure we have a persistent key for the signer
-	if ap.accounts[signer] == nil {
-		ap.accounts[signer], _ = crypto.GenerateKey()
+// checkpoint creates a Clique checkpoint signer section from the provided list
+// of authorized signers and embeds it into the provided header.
+func (ap *testerAccountPool) checkpoint(header *types.Header, signers []string) {
+	auths := make([]common.Address, len(signers))
+	for i, signer := range signers {
+		auths[i] = ap.address(signer)
+	}
+	sort.Sort(signersAscending(auths))
+	for i, auth := range auths {
+		copy(header.Extra[extraVanity+i*common.AddressLength:], auth.Bytes())
 	}
-	// Sign the header and embed the signature in extra data
-	sig, _ := crypto.Sign(sigHash(header).Bytes(), ap.accounts[signer])
-	copy(header.Extra[len(header.Extra)-65:], sig)
 }
 
+// address retrieves the Ethereum address of a tester account by label, creating
+// a new account if no previous one exists yet.
 func (ap *testerAccountPool) address(account string) common.Address {
+	// Return the zero account for non-addresses
+	if account == "" {
+		return common.Address{}
+	}
 	// Ensure we have a persistent key for the account
 	if ap.accounts[account] == nil {
 		ap.accounts[account], _ = crypto.GenerateKey()
@@ -69,32 +72,38 @@ func (ap *testerAccountPool) address(account string) common.Address {
 	return crypto.PubkeyToAddress(ap.accounts[account].PublicKey)
 }
 
-// testerChainReader implements consensus.ChainReader to access the genesis
-// block. All other methods and requests will panic.
-type testerChainReader struct {
-	db ethdb.Database
-}
-
-func (r *testerChainReader) Config() *params.ChainConfig                 { return params.AllCliqueProtocolChanges }
-func (r *testerChainReader) CurrentHeader() *types.Header                { panic("not supported") }
-func (r *testerChainReader) GetHeader(common.Hash, uint64) *types.Header { panic("not supported") }
-func (r *testerChainReader) GetBlock(common.Hash, uint64) *types.Block   { panic("not supported") }
-func (r *testerChainReader) GetHeaderByHash(common.Hash) *types.Header   { panic("not supported") }
-func (r *testerChainReader) GetHeaderByNumber(number uint64) *types.Header {
-	if number == 0 {
-		return rawdb.ReadHeader(r.db, rawdb.ReadCanonicalHash(r.db, 0), 0)
+// sign calculates a Clique digital signature for the given block and embeds it
+// back into the header.
+func (ap *testerAccountPool) sign(header *types.Header, signer string) {
+	// Ensure we have a persistent key for the signer
+	if ap.accounts[signer] == nil {
+		ap.accounts[signer], _ = crypto.GenerateKey()
 	}
-	return nil
+	// Sign the header and embed the signature in extra data
+	sig, _ := crypto.Sign(sigHash(header).Bytes(), ap.accounts[signer])
+	copy(header.Extra[len(header.Extra)-extraSeal:], sig)
 }
 
-// Tests that voting is evaluated correctly for various simple and complex scenarios.
-func TestVoting(t *testing.T) {
+// testerVote represents a single block signed by a parcitular account, where
+// the account may or may not have cast a Clique vote.
+type testerVote struct {
+	signer     string
+	voted      string
+	auth       bool
+	checkpoint []string
+	newbatch   bool
+}
+
+// Tests that Clique signer voting is evaluated correctly for various simple and
+// complex scenarios, as well as that a few special corner cases fail correctly.
+func TestClique(t *testing.T) {
 	// Define the various voting scenarios to test
 	tests := []struct {
 		epoch   uint64
 		signers []string
 		votes   []testerVote
 		results []string
+		failure error
 	}{
 		{
 			// Single signer, no votes cast
@@ -322,10 +331,49 @@ func TestVoting(t *testing.T) {
 			votes: []testerVote{
 				{signer: "A", voted: "C", auth: true},
 				{signer: "B"},
-				{signer: "A"}, // Checkpoint block, (don't vote here, it's validated outside of snapshots)
+				{signer: "A", checkpoint: []string{"A", "B"}},
 				{signer: "B", voted: "C", auth: true},
 			},
 			results: []string{"A", "B"},
+		}, {
+			// An unauthorized signer should not be able to sign blocks
+			signers: []string{"A"},
+			votes: []testerVote{
+				{signer: "B"},
+			},
+			failure: errUnauthorizedSigner,
+		}, {
+			// An authorized signer that signed recenty should not be able to sign again
+			signers: []string{"A", "B"},
+			votes: []testerVote{
+				{signer: "A"},
+				{signer: "A"},
+			},
+			failure: errRecentlySigned,
+		}, {
+			// Recent signatures should not reset on checkpoint blocks imported in a batch
+			epoch:   3,
+			signers: []string{"A", "B", "C"},
+			votes: []testerVote{
+				{signer: "A"},
+				{signer: "B"},
+				{signer: "A", checkpoint: []string{"A", "B", "C"}},
+				{signer: "A"},
+			},
+			failure: errRecentlySigned,
+		}, {
+			// Recent signatures should not reset on checkpoint blocks imported in a new
+			// batch (https://github.com/ethereum/go-ethereum/issues/17593). Whilst this
+			// seems overly specific and weird, it was a Rinkeby consensus split.
+			epoch:   3,
+			signers: []string{"A", "B", "C"},
+			votes: []testerVote{
+				{signer: "A"},
+				{signer: "B"},
+				{signer: "A", checkpoint: []string{"A", "B", "C"}},
+				{signer: "A", newbatch: true},
+			},
+			failure: errRecentlySigned,
 		},
 	}
 	// Run through the scenarios and test them
@@ -356,28 +404,78 @@ func TestVoting(t *testing.T) {
 		genesis.Commit(db)
 
 		// Assemble a chain of headers from the cast votes
-		headers := make([]*types.Header, len(tt.votes))
-		for j, vote := range tt.votes {
-			headers[j] = &types.Header{
-				Number:   big.NewInt(int64(j) + 1),
-				Time:     big.NewInt(int64(j) * 15),
-				Coinbase: accounts.address(vote.voted),
-				Extra:    make([]byte, extraVanity+extraSeal),
+		config := *params.TestChainConfig
+		config.Clique = &params.CliqueConfig{
+			Period: 1,
+			Epoch:  tt.epoch,
+		}
+		engine := New(config.Clique, db)
+		engine.fakeDiff = true
+
+		blocks, _ := core.GenerateChain(&config, genesis.ToBlock(db), engine, db, len(tt.votes), func(j int, gen *core.BlockGen) {
+			// Cast the vote contained in this block
+			gen.SetCoinbase(accounts.address(tt.votes[j].voted))
+			if tt.votes[j].auth {
+				var nonce types.BlockNonce
+				copy(nonce[:], nonceAuthVote)
+				gen.SetNonce(nonce)
 			}
+		})
+		// Iterate through the blocks and seal them individually
+		for j, block := range blocks {
+			// Geth the header and prepare it for signing
+			header := block.Header()
 			if j > 0 {
-				headers[j].ParentHash = headers[j-1].Hash()
+				header.ParentHash = blocks[j-1].Hash()
 			}
-			if vote.auth {
-				copy(headers[j].Nonce[:], nonceAuthVote)
+			header.Extra = make([]byte, extraVanity+extraSeal)
+			if auths := tt.votes[j].checkpoint; auths != nil {
+				header.Extra = make([]byte, extraVanity+len(auths)*common.AddressLength+extraSeal)
+				accounts.checkpoint(header, auths)
 			}
-			accounts.sign(headers[j], vote.signer)
+			header.Difficulty = diffInTurn // Ignored, we just need a valid number
+
+			// Generate the signature, embed it into the header and the block
+			accounts.sign(header, tt.votes[j].signer)
+			blocks[j] = block.WithSeal(header)
+		}
+		// Split the blocks up into individual import batches (cornercase testing)
+		batches := [][]*types.Block{nil}
+		for j, block := range blocks {
+			if tt.votes[j].newbatch {
+				batches = append(batches, nil)
+			}
+			batches[len(batches)-1] = append(batches[len(batches)-1], block)
 		}
 		// Pass all the headers through clique and ensure tallying succeeds
-		head := headers[len(headers)-1]
-
-		snap, err := New(&params.CliqueConfig{Epoch: tt.epoch}, db).snapshot(&testerChainReader{db: db}, head.Number.Uint64(), head.Hash(), headers)
+		chain, err := core.NewBlockChain(db, nil, &config, engine, vm.Config{}, nil)
 		if err != nil {
-			t.Errorf("test %d: failed to create voting snapshot: %v", i, err)
+			t.Errorf("test %d: failed to create test chain: %v", i, err)
+			continue
+		}
+		failed := false
+		for j := 0; j < len(batches)-1; j++ {
+			if k, err := chain.InsertChain(batches[j]); err != nil {
+				t.Errorf("test %d: failed to import batch %d, block %d: %v", i, j, k, err)
+				failed = true
+				break
+			}
+		}
+		if failed {
+			continue
+		}
+		if _, err = chain.InsertChain(batches[len(batches)-1]); err != tt.failure {
+			t.Errorf("test %d: failure mismatch: have %v, want %v", i, err, tt.failure)
+		}
+		if tt.failure != nil {
+			continue
+		}
+		// No failure was produced or requested, generate the final voting snapshot
+		head := blocks[len(blocks)-1]
+
+		snap, err := engine.snapshot(chain, head.NumberU64(), head.Hash(), nil)
+		if err != nil {
+			t.Errorf("test %d: failed to retrieve voting snapshot: %v", i, err)
 			continue
 		}
 		// Verify the final list of signers against the expected ones

consensus/consensus.go

@@ -86,9 +86,15 @@ type Engine interface {
 	Finalize(chain ChainReader, header *types.Header, state *state.StateDB, txs []*types.Transaction,
 		uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error)
 
-	// Seal generates a new block for the given input block with the local miner's
-	// seal place on top.
-	Seal(chain ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error)
+	// Seal generates a new sealing request for the given input block and pushes
+	// the result into the given channel.
+	//
+	// Note, the method returns immediately and will send the result async. More
+	// than one result may also be returned depending on the consensus algorithm.
+	Seal(chain ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error
+
+	// SealHash returns the hash of a block prior to it being sealed.
+	SealHash(header *types.Header) common.Hash
 
 	// CalcDifficulty is the difficulty adjustment algorithm. It returns the difficulty
 	// that a new block should have.

consensus/ethash/algorithm_test.go

@@ -729,7 +729,7 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) {
 
 		go func(idx int) {
 			defer pend.Done()
-			ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal}, nil)
+			ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal}, nil, false)
 			defer ethash.Close()
 			if err := ethash.VerifySeal(nil, block.Header()); err != nil {
 				t.Errorf("proc %d: block verification failed: %v", idx, err)

consensus/ethash/consensus.go

@@ -31,15 +31,31 @@ import (
 	"github.com/ethereum/go-ethereum/consensus/misc"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // Ethash proof-of-work protocol constants.
 var (
-	FrontierBlockReward    *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block
-	ByzantiumBlockReward   *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium
-	maxUncles                       = 2                 // Maximum number of uncles allowed in a single block
-	allowedFutureBlockTime          = 15 * time.Second  // Max time from current time allowed for blocks, before they're considered future blocks
+	FrontierBlockReward       = big.NewInt(5e+18) // Block reward in wei for successfully mining a block
+	ByzantiumBlockReward      = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium
+	ConstantinopleBlockReward = big.NewInt(2e+18) // Block reward in wei for successfully mining a block upward from Constantinople
+	maxUncles                 = 2                 // Maximum number of uncles allowed in a single block
+	allowedFutureBlockTime    = 15 * time.Second  // Max time from current time allowed for blocks, before they're considered future blocks
+
+	// calcDifficultyConstantinople is the difficulty adjustment algorithm for Constantinople.
+	// It returns the difficulty that a new block should have when created at time given the
+	// parent block's time and difficulty. The calculation uses the Byzantium rules, but with
+	// bomb offset 5M.
+	// Specification EIP-1234: https://eips.ethereum.org/EIPS/eip-1234
+	calcDifficultyConstantinople = makeDifficultyCalculator(big.NewInt(5000000))
+
+	// calcDifficultyByzantium is the difficulty adjustment algorithm. It returns
+	// the difficulty that a new block should have when created at time given the
+	// parent block's time and difficulty. The calculation uses the Byzantium rules.
+	// Specification EIP-649: https://eips.ethereum.org/EIPS/eip-649
+	calcDifficultyByzantium = makeDifficultyCalculator(big.NewInt(3000000))
 )
 
 // Various error messages to mark blocks invalid. These should be private to
@@ -297,6 +313,8 @@ func (ethash *Ethash) CalcDifficulty(chain consensus.ChainReader, time uint64, p
 func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Header) *big.Int {
 	next := new(big.Int).Add(parent.Number, big1)
 	switch {
+	case config.IsConstantinople(next):
+		return calcDifficultyConstantinople(time, parent)
 	case config.IsByzantium(next):
 		return calcDifficultyByzantium(time, parent)
 	case config.IsHomestead(next):
@@ -314,66 +332,69 @@ var (
 	big9          = big.NewInt(9)
 	big10         = big.NewInt(10)
 	bigMinus99    = big.NewInt(-99)
-	big2999999    = big.NewInt(2999999)
 )
 
-// calcDifficultyByzantium is the difficulty adjustment algorithm. It returns
-// the difficulty that a new block should have when created at time given the
-// parent block's time and difficulty. The calculation uses the Byzantium rules.
-func calcDifficultyByzantium(time uint64, parent *types.Header) *big.Int {
-	// https://github.com/ethereum/EIPs/issues/100.
-	// algorithm:
-	// diff = (parent_diff +
-	//         (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99))
-	//        ) + 2^(periodCount - 2)
+// makeDifficultyCalculator creates a difficultyCalculator with the given bomb-delay.
+// the difficulty is calculated with Byzantium rules, which differs from Homestead in
+// how uncles affect the calculation
+func makeDifficultyCalculator(bombDelay *big.Int) func(time uint64, parent *types.Header) *big.Int {
+	// Note, the calculations below looks at the parent number, which is 1 below
+	// the block number. Thus we remove one from the delay given
+	bombDelayFromParent := new(big.Int).Sub(bombDelay, big1)
+	return func(time uint64, parent *types.Header) *big.Int {
+		// https://github.com/ethereum/EIPs/issues/100.
+		// algorithm:
+		// diff = (parent_diff +
+		//         (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99))
+		//        ) + 2^(periodCount - 2)
 
-	bigTime := new(big.Int).SetUint64(time)
-	bigParentTime := new(big.Int).Set(parent.Time)
+		bigTime := new(big.Int).SetUint64(time)
+		bigParentTime := new(big.Int).Set(parent.Time)
 
-	// holds intermediate values to make the algo easier to read & audit
-	x := new(big.Int)
-	y := new(big.Int)
+		// holds intermediate values to make the algo easier to read & audit
+		x := new(big.Int)
+		y := new(big.Int)
 
-	// (2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9
-	x.Sub(bigTime, bigParentTime)
-	x.Div(x, big9)
-	if parent.UncleHash == types.EmptyUncleHash {
-		x.Sub(big1, x)
-	} else {
-		x.Sub(big2, x)
-	}
-	// max((2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9, -99)
-	if x.Cmp(bigMinus99) < 0 {
-		x.Set(bigMinus99)
-	}
-	// parent_diff + (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99))
-	y.Div(parent.Difficulty, params.DifficultyBoundDivisor)
-	x.Mul(y, x)
-	x.Add(parent.Difficulty, x)
+		// (2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9
+		x.Sub(bigTime, bigParentTime)
+		x.Div(x, big9)
+		if parent.UncleHash == types.EmptyUncleHash {
+			x.Sub(big1, x)
+		} else {
+			x.Sub(big2, x)
+		}
+		// max((2 if len(parent_uncles) else 1) - (block_timestamp - parent_timestamp) // 9, -99)
+		if x.Cmp(bigMinus99) < 0 {
+			x.Set(bigMinus99)
+		}
+		// parent_diff + (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99))
+		y.Div(parent.Difficulty, params.DifficultyBoundDivisor)
+		x.Mul(y, x)
+		x.Add(parent.Difficulty, x)
 
-	// minimum difficulty can ever be (before exponential factor)
-	if x.Cmp(params.MinimumDifficulty) < 0 {
-		x.Set(params.MinimumDifficulty)
-	}
-	// calculate a fake block number for the ice-age delay:
-	// https://github.com/ethereum/EIPs/pull/669
-	// fake_block_number = max(0, block.number - 3_000_000)
-	fakeBlockNumber := new(big.Int)
-	if parent.Number.Cmp(big2999999) >= 0 {
-		fakeBlockNumber = fakeBlockNumber.Sub(parent.Number, big2999999) // Note, parent is 1 less than the actual block number
-	}
-	// for the exponential factor
-	periodCount := fakeBlockNumber
-	periodCount.Div(periodCount, expDiffPeriod)
+		// minimum difficulty can ever be (before exponential factor)
+		if x.Cmp(params.MinimumDifficulty) < 0 {
+			x.Set(params.MinimumDifficulty)
+		}
+		// calculate a fake block number for the ice-age delay
+		// Specification: https://eips.ethereum.org/EIPS/eip-1234
+		fakeBlockNumber := new(big.Int)
+		if parent.Number.Cmp(bombDelayFromParent) >= 0 {
+			fakeBlockNumber = fakeBlockNumber.Sub(parent.Number, bombDelayFromParent)
+		}
+		// for the exponential factor
+		periodCount := fakeBlockNumber
+		periodCount.Div(periodCount, expDiffPeriod)
 
-	// the exponential factor, commonly referred to as "the bomb"
-	// diff = diff + 2^(periodCount - 2)
-	if periodCount.Cmp(big1) > 0 {
-		y.Sub(periodCount, big2)
-		y.Exp(big2, y, nil)
-		x.Add(x, y)
+		// the exponential factor, commonly referred to as "the bomb"
+		// diff = diff + 2^(periodCount - 2)
+		if periodCount.Cmp(big1) > 0 {
+			y.Sub(periodCount, big2)
+			y.Exp(big2, y, nil)
+			x.Add(x, y)
+		}
+		return x
 	}
-	return x
 }
 
 // calcDifficultyHomestead is the difficulty adjustment algorithm. It returns
@@ -495,7 +516,7 @@ func (ethash *Ethash) verifySeal(chain consensus.ChainReader, header *types.Head
 	if fulldag {
 		dataset := ethash.dataset(number, true)
 		if dataset.generated() {
-			digest, result = hashimotoFull(dataset.dataset, header.HashNoNonce().Bytes(), header.Nonce.Uint64())
+			digest, result = hashimotoFull(dataset.dataset, ethash.SealHash(header).Bytes(), header.Nonce.Uint64())
 
 			// Datasets are unmapped in a finalizer. Ensure that the dataset stays alive
 			// until after the call to hashimotoFull so it's not unmapped while being used.
@@ -513,7 +534,7 @@ func (ethash *Ethash) verifySeal(chain consensus.ChainReader, header *types.Head
 		if ethash.config.PowMode == ModeTest {
 			size = 32 * 1024
 		}
-		digest, result = hashimotoLight(size, cache.cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64())
+		digest, result = hashimotoLight(size, cache.cache, ethash.SealHash(header).Bytes(), header.Nonce.Uint64())
 
 		// Caches are unmapped in a finalizer. Ensure that the cache stays alive
 		// until after the call to hashimotoLight so it's not unmapped while being used.
@@ -552,6 +573,29 @@ func (ethash *Ethash) Finalize(chain consensus.ChainReader, header *types.Header
 	return types.NewBlock(header, txs, uncles, receipts), nil
 }
 
+// SealHash returns the hash of a block prior to it being sealed.
+func (ethash *Ethash) SealHash(header *types.Header) (hash common.Hash) {
+	hasher := sha3.NewKeccak256()
+
+	rlp.Encode(hasher, []interface{}{
+		header.ParentHash,
+		header.UncleHash,
+		header.Coinbase,
+		header.Root,
+		header.TxHash,
+		header.ReceiptHash,
+		header.Bloom,
+		header.Difficulty,
+		header.Number,
+		header.GasLimit,
+		header.GasUsed,
+		header.Time,
+		header.Extra,
+	})
+	hasher.Sum(hash[:0])
+	return hash
+}
+
 // Some weird constants to avoid constant memory allocs for them.
 var (
 	big8  = big.NewInt(8)
@@ -567,6 +611,9 @@ func accumulateRewards(config *params.ChainConfig, state *state.StateDB, header
 	if config.IsByzantium(header.Number) {
 		blockReward = ByzantiumBlockReward
 	}
+	if config.IsConstantinople(header.Number) {
+		blockReward = ConstantinopleBlockReward
+	}
 	// Accumulate the rewards for the miner and any included uncles
 	reward := new(big.Int).Set(blockReward)
 	r := new(big.Int)

consensus/ethash/ethash.go

@@ -50,7 +50,7 @@ var (
 	two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0))
 
 	// sharedEthash is a full instance that can be shared between multiple users.
-	sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal}, nil)
+	sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal}, nil, false)
 
 	// algorithmRevision is the data structure version used for file naming.
 	algorithmRevision = 23
@@ -405,6 +405,12 @@ type Config struct {
 	PowMode        Mode
 }
 
+// sealTask wraps a seal block with relative result channel for remote sealer thread.
+type sealTask struct {
+	block   *types.Block
+	results chan<- *types.Block
+}
+
 // mineResult wraps the pow solution parameters for the specified block.
 type mineResult struct {
 	nonce     types.BlockNonce
@@ -444,12 +450,11 @@ type Ethash struct {
 	hashrate metrics.Meter // Meter tracking the average hashrate
 
 	// Remote sealer related fields
-	workCh       chan *types.Block // Notification channel to push new work to remote sealer
-	resultCh     chan *types.Block // Channel used by mining threads to return result
-	fetchWorkCh  chan *sealWork    // Channel used for remote sealer to fetch mining work
-	submitWorkCh chan *mineResult  // Channel used for remote sealer to submit their mining result
-	fetchRateCh  chan chan uint64  // Channel used to gather submitted hash rate for local or remote sealer.
-	submitRateCh chan *hashrate    // Channel used for remote sealer to submit their mining hashrate
+	workCh       chan *sealTask   // Notification channel to push new work and relative result channel to remote sealer
+	fetchWorkCh  chan *sealWork   // Channel used for remote sealer to fetch mining work
+	submitWorkCh chan *mineResult // Channel used for remote sealer to submit their mining result
+	fetchRateCh  chan chan uint64 // Channel used to gather submitted hash rate for local or remote sealer.
+	submitRateCh chan *hashrate   // Channel used for remote sealer to submit their mining hashrate
 
 	// The fields below are hooks for testing
 	shared    *Ethash       // Shared PoW verifier to avoid cache regeneration
@@ -464,7 +469,7 @@ type Ethash struct {
 // New creates a full sized ethash PoW scheme and starts a background thread for
 // remote mining, also optionally notifying a batch of remote services of new work
 // packages.
-func New(config Config, notify []string) *Ethash {
+func New(config Config, notify []string, noverify bool) *Ethash {
 	if config.CachesInMem <= 0 {
 		log.Warn("One ethash cache must always be in memory", "requested", config.CachesInMem)
 		config.CachesInMem = 1
@@ -480,37 +485,35 @@ func New(config Config, notify []string) *Ethash {
 		caches:       newlru("cache", config.CachesInMem, newCache),
 		datasets:     newlru("dataset", config.DatasetsInMem, newDataset),
 		update:       make(chan struct{}),
-		hashrate:     metrics.NewMeter(),
-		workCh:       make(chan *types.Block),
-		resultCh:     make(chan *types.Block),
+		hashrate:     metrics.NewMeterForced(),
+		workCh:       make(chan *sealTask),
 		fetchWorkCh:  make(chan *sealWork),
 		submitWorkCh: make(chan *mineResult),
 		fetchRateCh:  make(chan chan uint64),
 		submitRateCh: make(chan *hashrate),
 		exitCh:       make(chan chan error),
 	}
-	go ethash.remote(notify)
+	go ethash.remote(notify, noverify)
 	return ethash
 }
 
 // NewTester creates a small sized ethash PoW scheme useful only for testing
 // purposes.
-func NewTester(notify []string) *Ethash {
+func NewTester(notify []string, noverify bool) *Ethash {
 	ethash := &Ethash{
 		config:       Config{PowMode: ModeTest},
 		caches:       newlru("cache", 1, newCache),
 		datasets:     newlru("dataset", 1, newDataset),
 		update:       make(chan struct{}),
-		hashrate:     metrics.NewMeter(),
-		workCh:       make(chan *types.Block),
-		resultCh:     make(chan *types.Block),
+		hashrate:     metrics.NewMeterForced(),
+		workCh:       make(chan *sealTask),
 		fetchWorkCh:  make(chan *sealWork),
 		submitWorkCh: make(chan *mineResult),
 		fetchRateCh:  make(chan chan uint64),
 		submitRateCh: make(chan *hashrate),
 		exitCh:       make(chan chan error),
 	}
-	go ethash.remote(notify)
+	go ethash.remote(notify, noverify)
 	return ethash
 }
 

consensus/ethash/ethash_test.go

@@ -34,17 +34,23 @@ import (
 func TestTestMode(t *testing.T) {
 	header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)}
 
-	ethash := NewTester(nil)
+	ethash := NewTester(nil, false)
 	defer ethash.Close()
 
-	block, err := ethash.Seal(nil, types.NewBlockWithHeader(header), nil)
+	results := make(chan *types.Block)
+	err := ethash.Seal(nil, types.NewBlockWithHeader(header), results, nil)
 	if err != nil {
 		t.Fatalf("failed to seal block: %v", err)
 	}
-	header.Nonce = types.EncodeNonce(block.Nonce())
-	header.MixDigest = block.MixDigest()
-	if err := ethash.VerifySeal(nil, header); err != nil {
-		t.Fatalf("unexpected verification error: %v", err)
+	select {
+	case block := <-results:
+		header.Nonce = types.EncodeNonce(block.Nonce())
+		header.MixDigest = block.MixDigest()
+		if err := ethash.VerifySeal(nil, header); err != nil {
+			t.Fatalf("unexpected verification error: %v", err)
+		}
+	case <-time.NewTimer(time.Second).C:
+		t.Error("sealing result timeout")
 	}
 }
 
@@ -56,7 +62,7 @@ func TestCacheFileEvict(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(tmpdir)
-	e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}, nil)
+	e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}, nil, false)
 	defer e.Close()
 
 	workers := 8
@@ -85,7 +91,7 @@ func verifyTest(wg *sync.WaitGroup, e *Ethash, workerIndex, epochs int) {
 }
 
 func TestRemoteSealer(t *testing.T) {
-	ethash := NewTester(nil)
+	ethash := NewTester(nil, false)
 	defer ethash.Close()
 
 	api := &API{ethash}
@@ -94,37 +100,32 @@ func TestRemoteSealer(t *testing.T) {
 	}
 	header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)}
 	block := types.NewBlockWithHeader(header)
+	sealhash := ethash.SealHash(header)
 
 	// Push new work.
-	ethash.Seal(nil, block, nil)
+	results := make(chan *types.Block)
+	ethash.Seal(nil, block, results, nil)
 
 	var (
 		work [3]string
 		err  error
 	)
-	if work, err = api.GetWork(); err != nil || work[0] != block.HashNoNonce().Hex() {
+	if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() {
 		t.Error("expect to return a mining work has same hash")
 	}
 
-	if res := api.SubmitWork(types.BlockNonce{}, block.HashNoNonce(), common.Hash{}); res {
+	if res := api.SubmitWork(types.BlockNonce{}, sealhash, common.Hash{}); res {
 		t.Error("expect to return false when submit a fake solution")
 	}
 	// Push new block with same block number to replace the original one.
 	header = &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(1000)}
 	block = types.NewBlockWithHeader(header)
-	ethash.Seal(nil, block, nil)
+	sealhash = ethash.SealHash(header)
+	ethash.Seal(nil, block, results, nil)
 
-	if work, err = api.GetWork(); err != nil || work[0] != block.HashNoNonce().Hex() {
+	if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() {
 		t.Error("expect to return the latest pushed work")
 	}
-	// Push block with higher block number.
-	newHead := &types.Header{Number: big.NewInt(2), Difficulty: big.NewInt(100)}
-	newBlock := types.NewBlockWithHeader(newHead)
-	ethash.Seal(nil, newBlock, nil)
-
-	if res := api.SubmitWork(types.BlockNonce{}, block.HashNoNonce(), common.Hash{}); res {
-		t.Error("expect to return false when submit a stale solution")
-	}
 }
 
 func TestHashRate(t *testing.T) {
@@ -133,7 +134,7 @@ func TestHashRate(t *testing.T) {
 		expect   uint64
 		ids      = []common.Hash{common.HexToHash("a"), common.HexToHash("b"), common.HexToHash("c")}
 	)
-	ethash := NewTester(nil)
+	ethash := NewTester(nil, false)
 	defer ethash.Close()
 
 	if tot := ethash.Hashrate(); tot != 0 {
@@ -153,7 +154,7 @@ func TestHashRate(t *testing.T) {
 }
 
 func TestClosedRemoteSealer(t *testing.T) {
-	ethash := NewTester(nil)
+	ethash := NewTester(nil, false)
 	time.Sleep(1 * time.Second) // ensure exit channel is listening
 	ethash.Close()
 

consensus/ethash/sealer.go

@@ -35,6 +35,11 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 )
 
+const (
+	// staleThreshold is the maximum depth of the acceptable stale but valid ethash solution.
+	staleThreshold = 7
+)
+
 var (
 	errNoMiningWork      = errors.New("no mining work available yet")
 	errInvalidSealResult = errors.New("invalid or stale proof-of-work solution")
@@ -42,16 +47,21 @@ var (
 
 // Seal implements consensus.Engine, attempting to find a nonce that satisfies
 // the block's difficulty requirements.
-func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) {
+func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error {
 	// If we're running a fake PoW, simply return a 0 nonce immediately
 	if ethash.config.PowMode == ModeFake || ethash.config.PowMode == ModeFullFake {
 		header := block.Header()
 		header.Nonce, header.MixDigest = types.BlockNonce{}, common.Hash{}
-		return block.WithSeal(header), nil
+		select {
+		case results <- block.WithSeal(header):
+		default:
+			log.Warn("Sealing result is not read by miner", "mode", "fake", "sealhash", ethash.SealHash(block.Header()))
+		}
+		return nil
 	}
 	// If we're running a shared PoW, delegate sealing to it
 	if ethash.shared != nil {
-		return ethash.shared.Seal(chain, block, stop)
+		return ethash.shared.Seal(chain, block, results, stop)
 	}
 	// Create a runner and the multiple search threads it directs
 	abort := make(chan struct{})
@@ -62,7 +72,7 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop
 		seed, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64))
 		if err != nil {
 			ethash.lock.Unlock()
-			return nil, err
+			return err
 		}
 		ethash.rand = rand.New(rand.NewSource(seed.Int64()))
 	}
@@ -75,34 +85,45 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop
 	}
 	// Push new work to remote sealer
 	if ethash.workCh != nil {
-		ethash.workCh <- block
+		ethash.workCh <- &sealTask{block: block, results: results}
 	}
-	var pend sync.WaitGroup
+	var (
+		pend   sync.WaitGroup
+		locals = make(chan *types.Block)
+	)
 	for i := 0; i < threads; i++ {
 		pend.Add(1)
 		go func(id int, nonce uint64) {
 			defer pend.Done()
-			ethash.mine(block, id, nonce, abort, ethash.resultCh)
+			ethash.mine(block, id, nonce, abort, locals)
 		}(i, uint64(ethash.rand.Int63()))
 	}
 	// Wait until sealing is terminated or a nonce is found
-	var result *types.Block
-	select {
-	case <-stop:
-		// Outside abort, stop all miner threads
-		close(abort)
-	case result = <-ethash.resultCh:
-		// One of the threads found a block, abort all others
-		close(abort)
-	case <-ethash.update:
-		// Thread count was changed on user request, restart
-		close(abort)
+	go func() {
+		var result *types.Block
+		select {
+		case <-stop:
+			// Outside abort, stop all miner threads
+			close(abort)
+		case result = <-locals:
+			// One of the threads found a block, abort all others
+			select {
+			case results <- result:
+			default:
+				log.Warn("Sealing result is not read by miner", "mode", "local", "sealhash", ethash.SealHash(block.Header()))
+			}
+			close(abort)
+		case <-ethash.update:
+			// Thread count was changed on user request, restart
+			close(abort)
+			if err := ethash.Seal(chain, block, results, stop); err != nil {
+				log.Error("Failed to restart sealing after update", "err", err)
+			}
+		}
+		// Wait for all miners to terminate and return the block
 		pend.Wait()
-		return ethash.Seal(chain, block, stop)
-	}
-	// Wait for all miners to terminate and return the block
-	pend.Wait()
-	return result, nil
+	}()
+	return nil
 }
 
 // mine is the actual proof-of-work miner that searches for a nonce starting from
@@ -111,7 +132,7 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s
 	// Extract some data from the header
 	var (
 		header  = block.Header()
-		hash    = header.HashNoNonce().Bytes()
+		hash    = ethash.SealHash(header).Bytes()
 		target  = new(big.Int).Div(two256, header.Difficulty)
 		number  = header.Number.Uint64()
 		dataset = ethash.dataset(number, false)
@@ -165,11 +186,12 @@ search:
 }
 
 // remote is a standalone goroutine to handle remote mining related stuff.
-func (ethash *Ethash) remote(notify []string) {
+func (ethash *Ethash) remote(notify []string, noverify bool) {
 	var (
 		works = make(map[common.Hash]*types.Block)
 		rates = make(map[common.Hash]hashrate)
 
+		results      chan<- *types.Block
 		currentBlock *types.Block
 		currentWork  [3]string
 
@@ -213,7 +235,7 @@ func (ethash *Ethash) remote(notify []string) {
 	//   result[1], 32 bytes hex encoded seed hash used for DAG
 	//   result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty
 	makeWork := func(block *types.Block) {
-		hash := block.HashNoNonce()
+		hash := ethash.SealHash(block.Header())
 
 		currentWork[0] = hash.Hex()
 		currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex()
@@ -226,11 +248,15 @@ func (ethash *Ethash) remote(notify []string) {
 	// submitWork verifies the submitted pow solution, returning
 	// whether the solution was accepted or not (not can be both a bad pow as well as
 	// any other error, like no pending work or stale mining result).
-	submitWork := func(nonce types.BlockNonce, mixDigest common.Hash, hash common.Hash) bool {
+	submitWork := func(nonce types.BlockNonce, mixDigest common.Hash, sealhash common.Hash) bool {
+		if currentBlock == nil {
+			log.Error("Pending work without block", "sealhash", sealhash)
+			return false
+		}
 		// Make sure the work submitted is present
-		block := works[hash]
+		block := works[sealhash]
 		if block == nil {
-			log.Info("Work submitted but none pending", "hash", hash)
+			log.Warn("Work submitted but none pending", "sealhash", sealhash, "curnumber", currentBlock.NumberU64())
 			return false
 		}
 		// Verify the correctness of submitted result.
@@ -239,26 +265,36 @@ func (ethash *Ethash) remote(notify []string) {
 		header.MixDigest = mixDigest
 
 		start := time.Now()
-		if err := ethash.verifySeal(nil, header, true); err != nil {
-			log.Warn("Invalid proof-of-work submitted", "hash", hash, "elapsed", time.Since(start), "err", err)
-			return false
+		if !noverify {
+			if err := ethash.verifySeal(nil, header, true); err != nil {
+				log.Warn("Invalid proof-of-work submitted", "sealhash", sealhash, "elapsed", time.Since(start), "err", err)
+				return false
+			}
 		}
-		// Make sure the result channel is created.
-		if ethash.resultCh == nil {
+		// Make sure the result channel is assigned.
+		if results == nil {
 			log.Warn("Ethash result channel is empty, submitted mining result is rejected")
 			return false
 		}
-		log.Trace("Verified correct proof-of-work", "hash", hash, "elapsed", time.Since(start))
+		log.Trace("Verified correct proof-of-work", "sealhash", sealhash, "elapsed", time.Since(start))
 
 		// Solutions seems to be valid, return to the miner and notify acceptance.
-		select {
-		case ethash.resultCh <- block.WithSeal(header):
-			delete(works, hash)
-			return true
-		default:
-			log.Info("Work submitted is stale", "hash", hash)
-			return false
+		solution := block.WithSeal(header)
+
+		// The submitted solution is within the scope of acceptance.
+		if solution.NumberU64()+staleThreshold > currentBlock.NumberU64() {
+			select {
+			case results <- solution:
+				log.Debug("Work submitted is acceptable", "number", solution.NumberU64(), "sealhash", sealhash, "hash", solution.Hash())
+				return true
+			default:
+				log.Warn("Sealing result is not read by miner", "mode", "remote", "sealhash", sealhash)
+				return false
+			}
 		}
+		// The submitted block is too old to accept, drop it.
+		log.Warn("Work submitted is too old", "number", solution.NumberU64(), "sealhash", sealhash, "hash", solution.Hash())
+		return false
 	}
 
 	ticker := time.NewTicker(5 * time.Second)
@@ -266,14 +302,12 @@ func (ethash *Ethash) remote(notify []string) {
 
 	for {
 		select {
-		case block := <-ethash.workCh:
-			if currentBlock != nil && block.ParentHash() != currentBlock.ParentHash() {
-				// Start new round mining, throw out all previous work.
-				works = make(map[common.Hash]*types.Block)
-			}
+		case work := <-ethash.workCh:
 			// Update current work with new received block.
 			// Note same work can be past twice, happens when changing CPU threads.
-			makeWork(block)
+			results = work.results
+
+			makeWork(work.block)
 
 			// Notify and requested URLs of the new work availability
 			notifyWork()
@@ -315,6 +349,14 @@ func (ethash *Ethash) remote(notify []string) {
 					delete(rates, id)
 				}
 			}
+			// Clear stale pending blocks
+			if currentBlock != nil {
+				for hash, block := range works {
+					if block.NumberU64()+staleThreshold <= currentBlock.NumberU64() {
+						delete(works, hash)
+					}
+				}
+			}
 
 		case errc := <-ethash.exitCh:
 			// Exit remote loop if ethash is closed and return relevant error.

consensus/ethash/sealer_test.go

@@ -40,18 +40,30 @@ func TestRemoteNotify(t *testing.T) {
 
 	go server.Serve(listener)
 
+	// Wait for server to start listening
+	var tries int
+	for tries = 0; tries < 10; tries++ {
+		conn, _ := net.DialTimeout("tcp", listener.Addr().String(), 1*time.Second)
+		if conn != nil {
+			break
+		}
+	}
+	if tries == 10 {
+		t.Fatal("tcp listener not ready for more than 10 seconds")
+	}
+
 	// Create the custom ethash engine
-	ethash := NewTester([]string{"http://" + listener.Addr().String()})
+	ethash := NewTester([]string{"http://" + listener.Addr().String()}, false)
 	defer ethash.Close()
 
 	// Stream a work task and ensure the notification bubbles out
 	header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)}
 	block := types.NewBlockWithHeader(header)
 
-	ethash.Seal(nil, block, nil)
+	ethash.Seal(nil, block, nil, nil)
 	select {
 	case work := <-sink:
-		if want := header.HashNoNonce().Hex(); work[0] != want {
+		if want := ethash.SealHash(header).Hex(); work[0] != want {
 			t.Errorf("work packet hash mismatch: have %s, want %s", work[0], want)
 		}
 		if want := common.BytesToHash(SeedHash(header.Number.Uint64())).Hex(); work[1] != want {
@@ -61,12 +73,12 @@ func TestRemoteNotify(t *testing.T) {
 		if want := common.BytesToHash(target.Bytes()).Hex(); work[2] != want {
 			t.Errorf("work packet target mismatch: have %s, want %s", work[2], want)
 		}
-	case <-time.After(time.Second):
+	case <-time.After(3 * time.Second):
 		t.Fatalf("notification timed out")
 	}
 }
 
-// Tests that pushing work packages fast to the miner doesn't cause any daa race
+// Tests that pushing work packages fast to the miner doesn't cause any data race
 // issues in the notifications.
 func TestRemoteMultiNotify(t *testing.T) {
 	// Start a simple webserver to capture notifications
@@ -95,7 +107,7 @@ func TestRemoteMultiNotify(t *testing.T) {
 	go server.Serve(listener)
 
 	// Create the custom ethash engine
-	ethash := NewTester([]string{"http://" + listener.Addr().String()})
+	ethash := NewTester([]string{"http://" + listener.Addr().String()}, false)
 	defer ethash.Close()
 
 	// Stream a lot of work task and ensure all the notifications bubble out
@@ -103,13 +115,97 @@ func TestRemoteMultiNotify(t *testing.T) {
 		header := &types.Header{Number: big.NewInt(int64(i)), Difficulty: big.NewInt(100)}
 		block := types.NewBlockWithHeader(header)
 
-		ethash.Seal(nil, block, nil)
+		ethash.Seal(nil, block, nil, nil)
 	}
 	for i := 0; i < cap(sink); i++ {
 		select {
 		case <-sink:
-		case <-time.After(250 * time.Millisecond):
+		case <-time.After(3 * time.Second):
 			t.Fatalf("notification %d timed out", i)
 		}
 	}
 }
+
+// Tests whether stale solutions are correctly processed.
+func TestStaleSubmission(t *testing.T) {
+	ethash := NewTester(nil, true)
+	defer ethash.Close()
+	api := &API{ethash}
+
+	fakeNonce, fakeDigest := types.BlockNonce{0x01, 0x02, 0x03}, common.HexToHash("deadbeef")
+
+	testcases := []struct {
+		headers     []*types.Header
+		submitIndex int
+		submitRes   bool
+	}{
+		// Case1: submit solution for the latest mining package
+		{
+			[]*types.Header{
+				{ParentHash: common.BytesToHash([]byte{0xa}), Number: big.NewInt(1), Difficulty: big.NewInt(100000000)},
+			},
+			0,
+			true,
+		},
+		// Case2: submit solution for the previous package but have same parent.
+		{
+			[]*types.Header{
+				{ParentHash: common.BytesToHash([]byte{0xb}), Number: big.NewInt(2), Difficulty: big.NewInt(100000000)},
+				{ParentHash: common.BytesToHash([]byte{0xb}), Number: big.NewInt(2), Difficulty: big.NewInt(100000001)},
+			},
+			0,
+			true,
+		},
+		// Case3: submit stale but acceptable solution
+		{
+			[]*types.Header{
+				{ParentHash: common.BytesToHash([]byte{0xc}), Number: big.NewInt(3), Difficulty: big.NewInt(100000000)},
+				{ParentHash: common.BytesToHash([]byte{0xd}), Number: big.NewInt(9), Difficulty: big.NewInt(100000000)},
+			},
+			0,
+			true,
+		},
+		// Case4: submit very old solution
+		{
+			[]*types.Header{
+				{ParentHash: common.BytesToHash([]byte{0xe}), Number: big.NewInt(10), Difficulty: big.NewInt(100000000)},
+				{ParentHash: common.BytesToHash([]byte{0xf}), Number: big.NewInt(17), Difficulty: big.NewInt(100000000)},
+			},
+			0,
+			false,
+		},
+	}
+	results := make(chan *types.Block, 16)
+
+	for id, c := range testcases {
+		for _, h := range c.headers {
+			ethash.Seal(nil, types.NewBlockWithHeader(h), results, nil)
+		}
+		if res := api.SubmitWork(fakeNonce, ethash.SealHash(c.headers[c.submitIndex]), fakeDigest); res != c.submitRes {
+			t.Errorf("case %d submit result mismatch, want %t, get %t", id+1, c.submitRes, res)
+		}
+		if !c.submitRes {
+			continue
+		}
+		select {
+		case res := <-results:
+			if res.Header().Nonce != fakeNonce {
+				t.Errorf("case %d block nonce mismatch, want %s, get %s", id+1, fakeNonce, res.Header().Nonce)
+			}
+			if res.Header().MixDigest != fakeDigest {
+				t.Errorf("case %d block digest mismatch, want %s, get %s", id+1, fakeDigest, res.Header().MixDigest)
+			}
+			if res.Header().Difficulty.Uint64() != c.headers[c.submitIndex].Difficulty.Uint64() {
+				t.Errorf("case %d block difficulty mismatch, want %d, get %d", id+1, c.headers[c.submitIndex].Difficulty, res.Header().Difficulty)
+			}
+			if res.Header().Number.Uint64() != c.headers[c.submitIndex].Number.Uint64() {
+				t.Errorf("case %d block number mismatch, want %d, get %d", id+1, c.headers[c.submitIndex].Number.Uint64(), res.Header().Number.Uint64())
+			}
+			if res.Header().ParentHash != c.headers[c.submitIndex].ParentHash {
+				t.Errorf("case %d block parent hash mismatch, want %s, get %s", id+1, c.headers[c.submitIndex].ParentHash.Hex(), res.Header().ParentHash.Hex())
+			}
+		case <-time.NewTimer(time.Second).C:
+			t.Errorf("case %d fetch ethash result timeout", id+1)
+		}
+	}
+}

console/prompter.go

@@ -43,7 +43,7 @@ type UserPrompter interface {
 	// choice to be made, returning that choice.
 	PromptConfirm(prompt string) (bool, error)
 
-	// SetHistory sets the the input scrollback history that the prompter will allow
+	// SetHistory sets the input scrollback history that the prompter will allow
 	// the user to scroll back to.
 	SetHistory(history []string)
 
@@ -149,7 +149,7 @@ func (p *terminalPrompter) PromptConfirm(prompt string) (bool, error) {
 	return false, err
 }
 
-// SetHistory sets the the input scrollback history that the prompter will allow
+// SetHistory sets the input scrollback history that the prompter will allow
 // the user to scroll back to.
 func (p *terminalPrompter) SetHistory(history []string) {
 	p.State.ReadHistory(strings.NewReader(strings.Join(history, "\n")))

contracts/ens/ens.go

@@ -151,6 +151,38 @@ func (self *ENS) Resolve(name string) (common.Hash, error) {
 	return common.BytesToHash(ret[:]), nil
 }
 
+// Addr is a non-transactional call that returns the address associated with a name.
+func (self *ENS) Addr(name string) (common.Address, error) {
+	node := EnsNode(name)
+
+	resolver, err := self.getResolver(node)
+	if err != nil {
+		return common.Address{}, err
+	}
+
+	ret, err := resolver.Addr(node)
+	if err != nil {
+		return common.Address{}, err
+	}
+
+	return common.BytesToAddress(ret[:]), nil
+}
+
+// SetAddress sets the address associated with a name. Only works if the caller
+// owns the name, and the associated resolver implements a `setAddress` function.
+func (self *ENS) SetAddr(name string, addr common.Address) (*types.Transaction, error) {
+	node := EnsNode(name)
+
+	resolver, err := self.getResolver(node)
+	if err != nil {
+		return nil, err
+	}
+
+	opts := self.TransactOpts
+	opts.GasLimit = 200000
+	return resolver.Contract.SetAddr(&opts, node, addr)
+}
+
 // Register registers a new domain name for the caller, making them the owner of the new name.
 // Only works if the registrar for the parent domain implements the FIFS registrar protocol.
 func (self *ENS) Register(name string) (*types.Transaction, error) {

contracts/ens/ens_test.go

@@ -22,16 +22,18 @@ import (
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/contracts/ens/contract"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
 var (
-	key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
-	name   = "my name on ENS"
-	hash   = crypto.Keccak256Hash([]byte("my content"))
-	addr   = crypto.PubkeyToAddress(key.PublicKey)
+	key, _   = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	name     = "my name on ENS"
+	hash     = crypto.Keccak256Hash([]byte("my content"))
+	addr     = crypto.PubkeyToAddress(key.PublicKey)
+	testAddr = common.HexToAddress("0x1234123412341234123412341234123412341234")
 )
 
 func TestENS(t *testing.T) {
@@ -74,4 +76,19 @@ func TestENS(t *testing.T) {
 	if vhost != hash {
 		t.Fatalf("resolve error, expected %v, got %v", hash.Hex(), vhost.Hex())
 	}
+
+	// set the address for the name
+	if _, err = ens.SetAddr(name, testAddr); err != nil {
+		t.Fatalf("can't set address: %v", err)
+	}
+	contractBackend.Commit()
+
+	// Try to resolve the name to an address
+	recoveredAddr, err := ens.Addr(name)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	if vhost != hash {
+		t.Fatalf("resolve error, expected %v, got %v", testAddr.Hex(), recoveredAddr.Hex())
+	}
 }

core/asm/asm.go

@@ -109,9 +109,9 @@ func PrintDisassembled(code string) error {
 	it := NewInstructionIterator(script)
 	for it.Next() {
 		if it.Arg() != nil && 0 < len(it.Arg()) {
-			fmt.Printf("%06v: %v 0x%x\n", it.PC(), it.Op(), it.Arg())
+			fmt.Printf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg())
 		} else {
-			fmt.Printf("%06v: %v\n", it.PC(), it.Op())
+			fmt.Printf("%05x: %v\n", it.PC(), it.Op())
 		}
 	}
 	return it.Error()
@@ -124,9 +124,9 @@ func Disassemble(script []byte) ([]string, error) {
 	it := NewInstructionIterator(script)
 	for it.Next() {
 		if it.Arg() != nil && 0 < len(it.Arg()) {
-			instrs = append(instrs, fmt.Sprintf("%06v: %v 0x%x\n", it.PC(), it.Op(), it.Arg()))
+			instrs = append(instrs, fmt.Sprintf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg()))
 		} else {
-			instrs = append(instrs, fmt.Sprintf("%06v: %v\n", it.PC(), it.Op()))
+			instrs = append(instrs, fmt.Sprintf("%05x: %v\n", it.PC(), it.Op()))
 		}
 	}
 	if err := it.Error(); err != nil {

core/bench_test.go

@@ -111,7 +111,8 @@ func init() {
 func genTxRing(naccounts int) func(int, *BlockGen) {
 	from := 0
 	return func(i int, gen *BlockGen) {
-		gas := CalcGasLimit(gen.PrevBlock(i - 1))
+		block := gen.PrevBlock(i - 1)
+		gas := CalcGasLimit(block, block.GasLimit(), block.GasLimit())
 		for {
 			gas -= params.TxGas
 			if gas < params.TxGas {
@@ -174,7 +175,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
 
 	// Time the insertion of the new chain.
 	// State and blocks are stored in the same DB.
-	chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
+	chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	defer chainman.Stop()
 	b.ReportAllocs()
 	b.ResetTimer()
@@ -286,7 +287,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
 		if err != nil {
 			b.Fatalf("error opening database at %v: %v", dir, err)
 		}
-		chain, err := NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
+		chain, err := NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil)
 		if err != nil {
 			b.Fatalf("error creating chain: %v", err)
 		}

core/block_validator.go

@@ -45,7 +45,7 @@ func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engin
 	return validator
 }
 
-// ValidateBody validates the given block's uncles and verifies the the block
+// ValidateBody validates the given block's uncles and verifies the block
 // header's transaction and uncle roots. The headers are assumed to be already
 // validated at this point.
 func (v *BlockValidator) ValidateBody(block *types.Block) error {
@@ -101,9 +101,11 @@ func (v *BlockValidator) ValidateState(block, parent *types.Block, statedb *stat
 	return nil
 }
 
-// CalcGasLimit computes the gas limit of the next block after parent.
-// This is miner strategy, not consensus protocol.
-func CalcGasLimit(parent *types.Block) uint64 {
+// CalcGasLimit computes the gas limit of the next block after parent. It aims
+// to keep the baseline gas above the provided floor, and increase it towards the
+// ceil if the blocks are full. If the ceil is exceeded, it will always decrease
+// the gas allowance.
+func CalcGasLimit(parent *types.Block, gasFloor, gasCeil uint64) uint64 {
 	// contrib = (parentGasUsed * 3 / 2) / 1024
 	contrib := (parent.GasUsed() + parent.GasUsed()/2) / params.GasLimitBoundDivisor
 
@@ -121,12 +123,16 @@ func CalcGasLimit(parent *types.Block) uint64 {
 	if limit < params.MinGasLimit {
 		limit = params.MinGasLimit
 	}
-	// however, if we're now below the target (TargetGasLimit) we increase the
-	// limit as much as we can (parentGasLimit / 1024 -1)
-	if limit < params.TargetGasLimit {
+	// If we're outside our allowed gas range, we try to hone towards them
+	if limit < gasFloor {
 		limit = parent.GasLimit() + decay
-		if limit > params.TargetGasLimit {
-			limit = params.TargetGasLimit
+		if limit > gasFloor {
+			limit = gasFloor
+		}
+	} else if limit > gasCeil {
+		limit = parent.GasLimit() - decay
+		if limit < gasCeil {
+			limit = gasCeil
 		}
 	}
 	return limit

core/block_validator_test.go

@@ -42,7 +42,7 @@ func TestHeaderVerification(t *testing.T) {
 		headers[i] = block.Header()
 	}
 	// Run the header checker for blocks one-by-one, checking for both valid and invalid nonces
-	chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
+	chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil)
 	defer chain.Stop()
 
 	for i := 0; i < len(blocks); i++ {
@@ -106,11 +106,11 @@ func testHeaderConcurrentVerification(t *testing.T, threads int) {
 		var results <-chan error
 
 		if valid {
-			chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{})
+			chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil)
 			_, results = chain.engine.VerifyHeaders(chain, headers, seals)
 			chain.Stop()
 		} else {
-			chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{})
+			chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{}, nil)
 			_, results = chain.engine.VerifyHeaders(chain, headers, seals)
 			chain.Stop()
 		}
@@ -173,7 +173,7 @@ func testHeaderConcurrentAbortion(t *testing.T, threads int) {
 	defer runtime.GOMAXPROCS(old)
 
 	// Start the verifications and immediately abort
-	chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{})
+	chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{}, nil)
 	defer chain.Stop()
 
 	abort, results := chain.engine.VerifyHeaders(chain, headers, seals)

core/blockchain.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/mclock"
+	"github.com/ethereum/go-ethereum/common/prque"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
@@ -43,7 +44,6 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/hashicorp/golang-lru"
-	"gopkg.in/karalabe/cookiejar.v2/collections/prque"
 )
 
 var (
@@ -55,6 +55,7 @@ var (
 const (
 	bodyCacheLimit      = 256
 	blockCacheLimit     = 256
+	receiptsCacheLimit  = 32
 	maxFutureBlocks     = 256
 	maxTimeFutureBlocks = 30
 	badBlockLimit       = 10
@@ -111,11 +112,12 @@ type BlockChain struct {
 	currentBlock     atomic.Value // Current head of the block chain
 	currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!)
 
-	stateCache   state.Database // State database to reuse between imports (contains state cache)
-	bodyCache    *lru.Cache     // Cache for the most recent block bodies
-	bodyRLPCache *lru.Cache     // Cache for the most recent block bodies in RLP encoded format
-	blockCache   *lru.Cache     // Cache for the most recent entire blocks
-	futureBlocks *lru.Cache     // future blocks are blocks added for later processing
+	stateCache    state.Database // State database to reuse between imports (contains state cache)
+	bodyCache     *lru.Cache     // Cache for the most recent block bodies
+	bodyRLPCache  *lru.Cache     // Cache for the most recent block bodies in RLP encoded format
+	receiptsCache *lru.Cache     // Cache for the most recent receipts per block
+	blockCache    *lru.Cache     // Cache for the most recent entire blocks
+	futureBlocks  *lru.Cache     // future blocks are blocks added for later processing
 
 	quit    chan struct{} // blockchain quit channel
 	running int32         // running must be called atomically
@@ -128,13 +130,14 @@ type BlockChain struct {
 	validator Validator // block and state validator interface
 	vmConfig  vm.Config
 
-	badBlocks *lru.Cache // Bad block cache
+	badBlocks      *lru.Cache              // Bad block cache
+	shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block.
 }
 
 // NewBlockChain returns a fully initialised block chain using information
 // available in the database. It initialises the default Ethereum Validator and
 // Processor.
-func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config) (*BlockChain, error) {
+func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) {
 	if cacheConfig == nil {
 		cacheConfig = &CacheConfig{
 			TrieNodeLimit: 256 * 1024 * 1024,
@@ -143,24 +146,27 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 	}
 	bodyCache, _ := lru.New(bodyCacheLimit)
 	bodyRLPCache, _ := lru.New(bodyCacheLimit)
+	receiptsCache, _ := lru.New(receiptsCacheLimit)
 	blockCache, _ := lru.New(blockCacheLimit)
 	futureBlocks, _ := lru.New(maxFutureBlocks)
 	badBlocks, _ := lru.New(badBlockLimit)
 
 	bc := &BlockChain{
-		chainConfig:  chainConfig,
-		cacheConfig:  cacheConfig,
-		db:           db,
-		triegc:       prque.New(),
-		stateCache:   state.NewDatabase(db),
-		quit:         make(chan struct{}),
-		bodyCache:    bodyCache,
-		bodyRLPCache: bodyRLPCache,
-		blockCache:   blockCache,
-		futureBlocks: futureBlocks,
-		engine:       engine,
-		vmConfig:     vmConfig,
-		badBlocks:    badBlocks,
+		chainConfig:    chainConfig,
+		cacheConfig:    cacheConfig,
+		db:             db,
+		triegc:         prque.New(nil),
+		stateCache:     state.NewDatabase(db),
+		quit:           make(chan struct{}),
+		shouldPreserve: shouldPreserve,
+		bodyCache:      bodyCache,
+		bodyRLPCache:   bodyRLPCache,
+		receiptsCache:  receiptsCache,
+		blockCache:     blockCache,
+		futureBlocks:   futureBlocks,
+		engine:         engine,
+		vmConfig:       vmConfig,
+		badBlocks:      badBlocks,
 	}
 	bc.SetValidator(NewBlockValidator(chainConfig, bc, engine))
 	bc.SetProcessor(NewStateProcessor(chainConfig, bc, engine))
@@ -251,9 +257,9 @@ func (bc *BlockChain) loadLastState() error {
 	blockTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64())
 	fastTd := bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64())
 
-	log.Info("Loaded most recent local header", "number", currentHeader.Number, "hash", currentHeader.Hash(), "td", headerTd)
-	log.Info("Loaded most recent local full block", "number", currentBlock.Number(), "hash", currentBlock.Hash(), "td", blockTd)
-	log.Info("Loaded most recent local fast block", "number", currentFastBlock.Number(), "hash", currentFastBlock.Hash(), "td", fastTd)
+	log.Info("Loaded most recent local header", "number", currentHeader.Number, "hash", currentHeader.Hash(), "td", headerTd, "age", common.PrettyAge(time.Unix(currentHeader.Time.Int64(), 0)))
+	log.Info("Loaded most recent local full block", "number", currentBlock.Number(), "hash", currentBlock.Hash(), "td", blockTd, "age", common.PrettyAge(time.Unix(currentBlock.Time().Int64(), 0)))
+	log.Info("Loaded most recent local fast block", "number", currentFastBlock.Number(), "hash", currentFastBlock.Hash(), "td", fastTd, "age", common.PrettyAge(time.Unix(currentFastBlock.Time().Int64(), 0)))
 
 	return nil
 }
@@ -278,6 +284,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
 	// Clear out any stale content from the caches
 	bc.bodyCache.Purge()
 	bc.bodyRLPCache.Purge()
+	bc.receiptsCache.Purge()
 	bc.blockCache.Purge()
 	bc.futureBlocks.Purge()
 
@@ -601,11 +608,18 @@ func (bc *BlockChain) GetBlockByNumber(number uint64) *types.Block {
 
 // GetReceiptsByHash retrieves the receipts for all transactions in a given block.
 func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
+	if receipts, ok := bc.receiptsCache.Get(hash); ok {
+		return receipts.(types.Receipts)
+	}
+
 	number := rawdb.ReadHeaderNumber(bc.db, hash)
 	if number == nil {
 		return nil
 	}
-	return rawdb.ReadReceipts(bc.db, hash, *number)
+
+	receipts := rawdb.ReadReceipts(bc.db, hash, *number)
+	bc.receiptsCache.Add(hash, receipts)
+	return receipts
 }
 
 // GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors.
@@ -850,13 +864,16 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 	}
 	bc.mu.Unlock()
 
-	log.Info("Imported new block receipts",
-		"count", stats.processed,
-		"elapsed", common.PrettyDuration(time.Since(start)),
-		"number", head.Number(),
-		"hash", head.Hash(),
+	context := []interface{}{
+		"count", stats.processed, "elapsed", common.PrettyDuration(time.Since(start)),
+		"number", head.Number(), "hash", head.Hash(), "age", common.PrettyAge(time.Unix(head.Time().Int64(), 0)),
 		"size", common.StorageSize(bytes),
-		"ignored", stats.ignored)
+	}
+	if stats.ignored > 0 {
+		context = append(context, []interface{}{"ignored", stats.ignored}...)
+	}
+	log.Info("Imported new block receipts", context...)
+
 	return 0, nil
 }
 
@@ -915,7 +932,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 	} else {
 		// Full but not archive node, do proper garbage collection
 		triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
-		bc.triegc.Push(root, -float32(block.NumberU64()))
+		bc.triegc.Push(root, -int64(block.NumberU64()))
 
 		if current := block.NumberU64(); current > triesInMemory {
 			// If we exceeded our memory allowance, flush matured singleton nodes to disk
@@ -964,8 +981,17 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 	reorg := externTd.Cmp(localTd) > 0
 	currentBlock = bc.CurrentBlock()
 	if !reorg && externTd.Cmp(localTd) == 0 {
-		// Split same-difficulty blocks by number, then at random
-		reorg = block.NumberU64() < currentBlock.NumberU64() || (block.NumberU64() == currentBlock.NumberU64() && mrand.Float64() < 0.5)
+		// Split same-difficulty blocks by number, then preferentially select
+		// the block generated by the local miner as the canonical block.
+		if block.NumberU64() < currentBlock.NumberU64() {
+			reorg = true
+		} else if block.NumberU64() == currentBlock.NumberU64() {
+			var currentPreserve, blockPreserve bool
+			if bc.shouldPreserve != nil {
+				currentPreserve, blockPreserve = bc.shouldPreserve(currentBlock), bc.shouldPreserve(block)
+			}
+			reorg = !currentPreserve && (blockPreserve || mrand.Float64() < 0.5)
+		}
 	}
 	if reorg {
 		// Reorganise the chain if the parent is not the head block
@@ -1229,8 +1255,13 @@ func (st *insertStats) report(chain []*types.Block, index int, cache common.Stor
 		context := []interface{}{
 			"blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
 			"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
-			"number", end.Number(), "hash", end.Hash(), "cache", cache,
+			"number", end.Number(), "hash", end.Hash(),
 		}
+		if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
+			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
+		}
+		context = append(context, []interface{}{"cache", cache}...)
+
 		if st.queued > 0 {
 			context = append(context, []interface{}{"queued", st.queued}...)
 		}

core/blockchain_test.go

@@ -52,7 +52,7 @@ func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *B
 	)
 
 	// Initialize a fresh chain with only a genesis block
-	blockchain, _ := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil)
 	// Create and inject the requested chain
 	if n == 0 {
 		return db, blockchain, nil
@@ -523,7 +523,7 @@ func testReorgBadHashes(t *testing.T, full bool) {
 	blockchain.Stop()
 
 	// Create a new BlockChain and check that it rolled back the state.
-	ncm, err := NewBlockChain(blockchain.db, nil, blockchain.chainConfig, ethash.NewFaker(), vm.Config{})
+	ncm, err := NewBlockChain(blockchain.db, nil, blockchain.chainConfig, ethash.NewFaker(), vm.Config{}, nil)
 	if err != nil {
 		t.Fatalf("failed to create new chain manager: %v", err)
 	}
@@ -635,7 +635,7 @@ func TestFastVsFullChains(t *testing.T) {
 	// Import the chain as an archive node for the comparison baseline
 	archiveDb := ethdb.NewMemDatabase()
 	gspec.MustCommit(archiveDb)
-	archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
+	archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	defer archive.Stop()
 
 	if n, err := archive.InsertChain(blocks); err != nil {
@@ -644,7 +644,7 @@ func TestFastVsFullChains(t *testing.T) {
 	// Fast import the chain as a non-archive node to test
 	fastDb := ethdb.NewMemDatabase()
 	gspec.MustCommit(fastDb)
-	fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
+	fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	defer fast.Stop()
 
 	headers := make([]*types.Header, len(blocks))
@@ -722,7 +722,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	archiveDb := ethdb.NewMemDatabase()
 	gspec.MustCommit(archiveDb)
 
-	archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
+	archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	if n, err := archive.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to process block %d: %v", n, err)
 	}
@@ -735,7 +735,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	// Import the chain as a non-archive node and ensure all pointers are updated
 	fastDb := ethdb.NewMemDatabase()
 	gspec.MustCommit(fastDb)
-	fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
+	fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	defer fast.Stop()
 
 	headers := make([]*types.Header, len(blocks))
@@ -756,7 +756,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) {
 	lightDb := ethdb.NewMemDatabase()
 	gspec.MustCommit(lightDb)
 
-	light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
+	light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	if n, err := light.InsertHeaderChain(headers, 1); err != nil {
 		t.Fatalf("failed to insert header %d: %v", n, err)
 	}
@@ -825,7 +825,7 @@ func TestChainTxReorgs(t *testing.T) {
 		}
 	})
 	// Import the chain. This runs all block validation rules.
-	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	if i, err := blockchain.InsertChain(chain); err != nil {
 		t.Fatalf("failed to insert original chain[%d]: %v", i, err)
 	}
@@ -896,7 +896,7 @@ func TestLogReorgs(t *testing.T) {
 		signer  = types.NewEIP155Signer(gspec.Config.ChainID)
 	)
 
-	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	defer blockchain.Stop()
 
 	rmLogsCh := make(chan RemovedLogsEvent)
@@ -943,7 +943,7 @@ func TestReorgSideEvent(t *testing.T) {
 		signer  = types.NewEIP155Signer(gspec.Config.ChainID)
 	)
 
-	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	defer blockchain.Stop()
 
 	chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, gen *BlockGen) {})
@@ -1072,7 +1072,7 @@ func TestEIP155Transition(t *testing.T) {
 		genesis = gspec.MustCommit(db)
 	)
 
-	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	defer blockchain.Stop()
 
 	blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 4, func(i int, block *BlockGen) {
@@ -1179,7 +1179,7 @@ func TestEIP161AccountRemoval(t *testing.T) {
 		}
 		genesis = gspec.MustCommit(db)
 	)
-	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	defer blockchain.Stop()
 
 	blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, block *BlockGen) {
@@ -1254,7 +1254,7 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) {
 	diskdb := ethdb.NewMemDatabase()
 	new(Genesis).MustCommit(diskdb)
 
-	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
+	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
 	if err != nil {
 		t.Fatalf("failed to create tester chain: %v", err)
 	}
@@ -1298,7 +1298,7 @@ func TestTrieForkGC(t *testing.T) {
 	diskdb := ethdb.NewMemDatabase()
 	new(Genesis).MustCommit(diskdb)
 
-	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
+	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
 	if err != nil {
 		t.Fatalf("failed to create tester chain: %v", err)
 	}
@@ -1337,7 +1337,7 @@ func TestLargeReorgTrieGC(t *testing.T) {
 	diskdb := ethdb.NewMemDatabase()
 	new(Genesis).MustCommit(diskdb)
 
-	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
+	chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
 	if err != nil {
 		t.Fatalf("failed to create tester chain: %v", err)
 	}
@@ -1419,7 +1419,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in
 		diskdb := ethdb.NewMemDatabase()
 		gspec.MustCommit(diskdb)
 
-		chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
+		chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil)
 		if err != nil {
 			b.Fatalf("failed to create tester chain: %v", err)
 		}

core/chain_indexer.go

@@ -53,14 +53,14 @@ type ChainIndexerChain interface {
 	// CurrentHeader retrieves the latest locally known header.
 	CurrentHeader() *types.Header
 
-	// SubscribeChainEvent subscribes to new head header notifications.
-	SubscribeChainEvent(ch chan<- ChainEvent) event.Subscription
+	// SubscribeChainHeadEvent subscribes to new head header notifications.
+	SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription
 }
 
 // ChainIndexer does a post-processing job for equally sized sections of the
 // canonical chain (like BlooomBits and CHT structures). A ChainIndexer is
 // connected to the blockchain through the event system by starting a
-// ChainEventLoop in a goroutine.
+// ChainHeadEventLoop in a goroutine.
 //
 // Further child ChainIndexers can be added which use the output of the parent
 // section indexer. These child indexers receive new head notifications only
@@ -85,6 +85,9 @@ type ChainIndexer struct {
 	knownSections  uint64 // Number of sections known to be complete (block wise)
 	cascadedHead   uint64 // Block number of the last completed section cascaded to subindexers
 
+	checkpointSections uint64      // Number of sections covered by the checkpoint
+	checkpointHead     common.Hash // Section head belonging to the checkpoint
+
 	throttling time.Duration // Disk throttling to prevent a heavy upgrade from hogging resources
 
 	log  log.Logger
@@ -115,12 +118,19 @@ func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBacken
 	return c
 }
 
-// AddKnownSectionHead marks a new section head as known/processed if it is newer
-// than the already known best section head
-func (c *ChainIndexer) AddKnownSectionHead(section uint64, shead common.Hash) {
+// AddCheckpoint adds a checkpoint. Sections are never processed and the chain
+// is not expected to be available before this point. The indexer assumes that
+// the backend has sufficient information available to process subsequent sections.
+//
+// Note: knownSections == 0 and storedSections == checkpointSections until
+// syncing reaches the checkpoint
+func (c *ChainIndexer) AddCheckpoint(section uint64, shead common.Hash) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
+	c.checkpointSections = section + 1
+	c.checkpointHead = shead
+
 	if section < c.storedSections {
 		return
 	}
@@ -132,8 +142,8 @@ func (c *ChainIndexer) AddKnownSectionHead(section uint64, shead common.Hash) {
 // cascading background processing. Children do not need to be started, they
 // are notified about new events by their parents.
 func (c *ChainIndexer) Start(chain ChainIndexerChain) {
-	events := make(chan ChainEvent, 10)
-	sub := chain.SubscribeChainEvent(events)
+	events := make(chan ChainHeadEvent, 10)
+	sub := chain.SubscribeChainHeadEvent(events)
 
 	go c.eventLoop(chain.CurrentHeader(), events, sub)
 }
@@ -180,7 +190,7 @@ func (c *ChainIndexer) Close() error {
 // eventLoop is a secondary - optional - event loop of the indexer which is only
 // started for the outermost indexer to push chain head events into a processing
 // queue.
-func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainEvent, sub event.Subscription) {
+func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainHeadEvent, sub event.Subscription) {
 	// Mark the chain indexer as active, requiring an additional teardown
 	atomic.StoreUint32(&c.active, 1)
 
@@ -209,13 +219,13 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainE
 			}
 			header := ev.Block.Header()
 			if header.ParentHash != prevHash {
-				// Reorg to the common ancestor (might not exist in light sync mode, skip reorg then)
+				// Reorg to the common ancestor if needed (might not exist in light sync mode, skip reorg then)
 				// TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly?
 
-				// TODO(karalabe): This operation is expensive and might block, causing the event system to
-				// potentially also lock up. We need to do with on a different thread somehow.
-				if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, header); h != nil {
-					c.newHead(h.Number.Uint64(), true)
+				if rawdb.ReadCanonicalHash(c.chainDb, prevHeader.Number.Uint64()) != prevHash {
+					if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, header); h != nil {
+						c.newHead(h.Number.Uint64(), true)
+					}
 				}
 			}
 			c.newHead(header.Number.Uint64(), false)
@@ -233,16 +243,23 @@ func (c *ChainIndexer) newHead(head uint64, reorg bool) {
 	// If a reorg happened, invalidate all sections until that point
 	if reorg {
 		// Revert the known section number to the reorg point
-		changed := head / c.sectionSize
-		if changed < c.knownSections {
-			c.knownSections = changed
+		known := head / c.sectionSize
+		stored := known
+		if known < c.checkpointSections {
+			known = 0
+		}
+		if stored < c.checkpointSections {
+			stored = c.checkpointSections
+		}
+		if known < c.knownSections {
+			c.knownSections = known
 		}
 		// Revert the stored sections from the database to the reorg point
-		if changed < c.storedSections {
-			c.setValidSections(changed)
+		if stored < c.storedSections {
+			c.setValidSections(stored)
 		}
 		// Update the new head number to the finalized section end and notify children
-		head = changed * c.sectionSize
+		head = known * c.sectionSize
 
 		if head < c.cascadedHead {
 			c.cascadedHead = head
@@ -256,7 +273,18 @@ func (c *ChainIndexer) newHead(head uint64, reorg bool) {
 	var sections uint64
 	if head >= c.confirmsReq {
 		sections = (head + 1 - c.confirmsReq) / c.sectionSize
+		if sections < c.checkpointSections {
+			sections = 0
+		}
 		if sections > c.knownSections {
+			if c.knownSections < c.checkpointSections {
+				// syncing reached the checkpoint, verify section head
+				syncedHead := rawdb.ReadCanonicalHash(c.chainDb, c.checkpointSections*c.sectionSize-1)
+				if syncedHead != c.checkpointHead {
+					c.log.Error("Synced chain does not match checkpoint", "number", c.checkpointSections*c.sectionSize-1, "expected", c.checkpointHead, "synced", syncedHead)
+					return
+				}
+			}
 			c.knownSections = sections
 
 			select {
@@ -322,7 +350,6 @@ func (c *ChainIndexer) updateLoop() {
 						updating = false
 						c.log.Info("Finished upgrading chain index")
 					}
-
 					c.cascadedHead = c.storedSections*c.sectionSize - 1
 					for _, child := range c.children {
 						c.log.Trace("Cascading chain index update", "head", c.cascadedHead)
@@ -402,8 +429,14 @@ func (c *ChainIndexer) AddChildIndexer(indexer *ChainIndexer) {
 	c.children = append(c.children, indexer)
 
 	// Cascade any pending updates to new children too
-	if c.storedSections > 0 {
-		indexer.newHead(c.storedSections*c.sectionSize-1, false)
+	sections := c.storedSections
+	if c.knownSections < sections {
+		// if a section is "stored" but not "known" then it is a checkpoint without
+		// available chain data so we should not cascade it yet
+		sections = c.knownSections
+	}
+	if sections > 0 {
+		indexer.newHead(sections*c.sectionSize-1, false)
 	}
 }
 
@@ -412,7 +445,7 @@ func (c *ChainIndexer) AddChildIndexer(indexer *ChainIndexer) {
 func (c *ChainIndexer) loadValidSections() {
 	data, _ := c.indexDb.Get([]byte("count"))
 	if len(data) == 8 {
-		c.storedSections = binary.BigEndian.Uint64(data[:])
+		c.storedSections = binary.BigEndian.Uint64(data)
 	}
 }
 

core/chain_makers.go

@@ -67,6 +67,11 @@ func (b *BlockGen) SetExtra(data []byte) {
 	b.header.Extra = data
 }
 
+// SetNonce sets the nonce field of the generated block.
+func (b *BlockGen) SetNonce(nonce types.BlockNonce) {
+	b.header.Nonce = nonce
+}
+
 // AddTx adds a transaction to the generated block. If no coinbase has
 // been set, the block's coinbase is set to the zero address.
 //
@@ -172,7 +177,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 	genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
 		// TODO(karalabe): This is needed for clique, which depends on multiple blocks.
 		// It's nonetheless ugly to spin up a blockchain here. Get rid of this somehow.
-		blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{})
+		blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{}, nil)
 		defer blockchain.Stop()
 
 		b := &BlockGen{i: i, parent: parent, chain: blocks, chainReader: blockchain, statedb: statedb, config: config, engine: engine}
@@ -190,13 +195,14 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		if config.DAOForkSupport && config.DAOForkBlock != nil && config.DAOForkBlock.Cmp(b.header.Number) == 0 {
 			misc.ApplyDAOHardFork(statedb)
 		}
-		// Execute any user modifications to the block and finalize it
+		// Execute any user modifications to the block
 		if gen != nil {
 			gen(i, b)
 		}
-
 		if b.engine != nil {
+			// Finalize and seal the block
 			block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts)
+
 			// Write state changes to db
 			root, err := statedb.Commit(config.IsEIP158(b.header.Number))
 			if err != nil {
@@ -240,7 +246,7 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S
 			Difficulty: parent.Difficulty(),
 			UncleHash:  parent.UncleHash(),
 		}),
-		GasLimit: CalcGasLimit(parent),
+		GasLimit: CalcGasLimit(parent, parent.GasLimit(), parent.GasLimit()),
 		Number:   new(big.Int).Add(parent.Number(), common.Big1),
 		Time:     time,
 	}

core/chain_makers_test.go

@@ -79,7 +79,7 @@ func ExampleGenerateChain() {
 	})
 
 	// Import the chain. This runs all block validation rules.
-	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil)
 	defer blockchain.Stop()
 
 	if i, err := blockchain.InsertChain(chain); err != nil {

core/dao_test.go

@@ -45,7 +45,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	proConf.DAOForkBlock = forkBlock
 	proConf.DAOForkSupport = true
 
-	proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{})
+	proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil)
 	defer proBc.Stop()
 
 	conDb := ethdb.NewMemDatabase()
@@ -55,7 +55,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	conConf.DAOForkBlock = forkBlock
 	conConf.DAOForkSupport = false
 
-	conBc, _ := NewBlockChain(conDb, nil, &conConf, ethash.NewFaker(), vm.Config{})
+	conBc, _ := NewBlockChain(conDb, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil)
 	defer conBc.Stop()
 
 	if _, err := proBc.InsertChain(prefix); err != nil {
@@ -69,7 +69,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		// Create a pro-fork block, and try to feed into the no-fork chain
 		db = ethdb.NewMemDatabase()
 		gspec.MustCommit(db)
-		bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{})
+		bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil)
 		defer bc.Stop()
 
 		blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()))
@@ -94,7 +94,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		// Create a no-fork block, and try to feed into the pro-fork chain
 		db = ethdb.NewMemDatabase()
 		gspec.MustCommit(db)
-		bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{})
+		bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil)
 		defer bc.Stop()
 
 		blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()))
@@ -120,7 +120,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	// Verify that contra-forkers accept pro-fork extra-datas after forking finishes
 	db = ethdb.NewMemDatabase()
 	gspec.MustCommit(db)
-	bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{})
+	bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil)
 	defer bc.Stop()
 
 	blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64()))
@@ -140,7 +140,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	// Verify that pro-forkers accept contra-fork extra-datas after forking finishes
 	db = ethdb.NewMemDatabase()
 	gspec.MustCommit(db)
-	bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{})
+	bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil)
 	defer bc.Stop()
 
 	blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64()))

core/genesis.go

@@ -355,7 +355,7 @@ func DeveloperGenesisBlock(period uint64, faucet common.Address) *Genesis {
 			common.BytesToAddress([]byte{6}): {Balance: big.NewInt(1)}, // ECAdd
 			common.BytesToAddress([]byte{7}): {Balance: big.NewInt(1)}, // ECScalarMul
 			common.BytesToAddress([]byte{8}): {Balance: big.NewInt(1)}, // ECPairing
-			faucet: {Balance: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(9))},
+			faucet:                           {Balance: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(9))},
 		},
 	}
 }

core/genesis_test.go

@@ -120,7 +120,7 @@ func TestSetupGenesis(t *testing.T) {
 				// Advance to block #4, past the homestead transition block of customg.
 				genesis := oldcustomg.MustCommit(db)
 
-				bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{})
+				bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{}, nil)
 				defer bc.Stop()
 
 				blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil)

core/headerchain.go

@@ -281,8 +281,18 @@ func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, writeHeader WhCa
 	}
 	// Report some public statistics so the user has a clue what's going on
 	last := chain[len(chain)-1]
-	log.Info("Imported new block headers", "count", stats.processed, "elapsed", common.PrettyDuration(time.Since(start)),
-		"number", last.Number, "hash", last.Hash(), "ignored", stats.ignored)
+
+	context := []interface{}{
+		"count", stats.processed, "elapsed", common.PrettyDuration(time.Since(start)),
+		"number", last.Number, "hash", last.Hash(),
+	}
+	if timestamp := time.Unix(last.Time.Int64(), 0); time.Since(timestamp) > time.Minute {
+		context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
+	}
+	if stats.ignored > 0 {
+		context = append(context, []interface{}{"ignored", stats.ignored}...)
+	}
+	log.Info("Imported new block headers", context...)
 
 	return 0, nil
 }